COMPRESSION, PYTHON: Add support for LZ4 compression.
Signed-off-by: Grzegorz Kowal <grzegorz@amuncode.org>
This commit is contained in:
parent
2012a5eb92
commit
eb7c8dbd76
@ -82,3 +82,13 @@ if(ENABLE_ZSTD)
|
||||
target_link_libraries(amun.x ${ZSTD_LIBRARIES})
|
||||
endif()
|
||||
endif()
|
||||
|
||||
option(ENABLE_LZ4 "Enable LZ4 compression support" ON)
|
||||
if(ENABLE_LZ4)
|
||||
include(FindPkgConfig)
|
||||
pkg_search_module(LZ4 QUIET liblz4)
|
||||
if(LZ4_FOUND)
|
||||
target_compile_definitions(amun.x PRIVATE LZ4)
|
||||
target_link_libraries(amun.x ${LZ4_LIBRARIES})
|
||||
endif()
|
||||
endif()
|
||||
|
||||
@ -55,6 +55,11 @@ try:
|
||||
zstd_available = True
|
||||
except ImportError:
|
||||
zstd_available = False
|
||||
try:
|
||||
import lz4.block as lz4
|
||||
lz4_available = True
|
||||
except ImportError:
|
||||
lz4_available = False
|
||||
|
||||
|
||||
class AmunXML:
|
||||
@ -196,7 +201,6 @@ class AmunXML:
|
||||
if cformat == 'zstd':
|
||||
if zstd_available:
|
||||
dctx = zstd.ZstdDecompressor()
|
||||
size = dctx.memory_size()
|
||||
with open(dfile, mode ='rb') as f:
|
||||
compressed = f.read()
|
||||
if 'compressed_digest' in self.binaries[aname]:
|
||||
@ -206,7 +210,20 @@ class AmunXML:
|
||||
self.check_digest(dfile, data, self.binaries[aname]['digest'])
|
||||
return np.frombuffer(data, dtype = dtype)
|
||||
else:
|
||||
print('ZSTD unavailable. Please, install Zstandard for Python.')
|
||||
print('ZSTD module unavailable. Please, install Zstandard for Python.')
|
||||
return None
|
||||
elif cformat == 'lz4':
|
||||
if lz4_available:
|
||||
with open(dfile, mode ='rb') as f:
|
||||
compressed = f.read()
|
||||
if 'compressed_digest' in self.binaries[aname]:
|
||||
self.check_digest(dfile, compressed, self.binaries[aname]['compressed_digest'])
|
||||
data = lz4.decompress(compressed)
|
||||
if 'digest' in self.binaries[aname]:
|
||||
self.check_digest(dfile, data, self.binaries[aname]['digest'])
|
||||
return np.frombuffer(data, dtype = dtype)
|
||||
else:
|
||||
print('LZ4 module unavailable. Please, install LZ4 for Python.')
|
||||
return None
|
||||
else:
|
||||
print('Compression format {} unavailable. Please, install corresponding Python module'.format(cformat))
|
||||
@ -232,7 +249,6 @@ class AmunXML:
|
||||
if cformat == 'zstd':
|
||||
if zstd_available:
|
||||
dctx = zstd.ZstdDecompressor()
|
||||
size = dctx.memory_size()
|
||||
with open(dfile, mode ='rb') as f:
|
||||
compressed = f.read()
|
||||
if 'compressed_digest' in self.chunks[n][var]:
|
||||
@ -242,7 +258,20 @@ class AmunXML:
|
||||
self.check_digest(dfile, data, self.chunks[n][var]['digest'])
|
||||
return np.frombuffer(data, dtype = dtype)
|
||||
else:
|
||||
print('ZSTD unavailable. Please, install Zstandard for Python.')
|
||||
print('ZSTD module unavailable. Please, install Zstandard for Python.')
|
||||
return None
|
||||
elif cformat == 'lz4':
|
||||
if lz4_available:
|
||||
with open(dfile, mode ='rb') as f:
|
||||
compressed = f.read()
|
||||
if 'compressed_digest' in self.chunks[n][var]:
|
||||
self.check_digest(dfile, compressed, self.chunks[n][var]['compressed_digest'])
|
||||
data = lz4.decompress(compressed)
|
||||
if 'digest' in self.chunks[n][var]:
|
||||
self.check_digest(dfile, data, self.chunks[n][var]['digest'])
|
||||
return np.frombuffer(data, dtype = dtype)
|
||||
else:
|
||||
print('LZ4 module unavailable. Please, install LZ4 for Python.')
|
||||
return None
|
||||
else:
|
||||
print('Compression format {} unavailable. Please, install corresponding Python module'.format(cformat))
|
||||
|
||||
@ -29,8 +29,6 @@
|
||||
!
|
||||
module compression
|
||||
|
||||
use iso_c_binding, only: c_int, c_loc
|
||||
|
||||
! module variables are not implicit by default
|
||||
!
|
||||
implicit none
|
||||
@ -49,6 +47,23 @@ module compression
|
||||
end function zstd_compress
|
||||
end interface
|
||||
#endif /* ZSTD */
|
||||
#ifdef LZ4
|
||||
interface
|
||||
integer(c_int) function lz4_compress(src, dst, srcSize, dstCapacity) &
|
||||
bind(C, name="LZ4_compress_default")
|
||||
use iso_c_binding, only: c_int, c_ptr
|
||||
implicit none
|
||||
type(c_ptr), value :: src, dst
|
||||
integer(kind=c_int), value :: srcSize, dstCapacity
|
||||
end function lz4_compress
|
||||
integer(c_int) function lz4_compressBound(srcSize) &
|
||||
bind(C, name="LZ4_compressBound")
|
||||
use iso_c_binding, only: c_int
|
||||
implicit none
|
||||
integer(kind=c_int), value :: srcSize
|
||||
end function lz4_compressBound
|
||||
end interface
|
||||
#endif /* LZ4 */
|
||||
|
||||
! compression parameters
|
||||
!
|
||||
@ -57,7 +72,9 @@ module compression
|
||||
|
||||
! supported compression formats
|
||||
!
|
||||
integer, parameter :: compression_none = 0, compression_zstd = 1
|
||||
integer, parameter :: compression_none = 0
|
||||
integer, parameter :: compression_zstd = 1
|
||||
integer, parameter :: compression_lz4 = 2
|
||||
|
||||
! by default everything is private
|
||||
!
|
||||
@ -112,6 +129,13 @@ module compression
|
||||
compression_level = max(0, min(19, clevel))
|
||||
suffix = ".zst"
|
||||
#endif /* ZSTD */
|
||||
#ifdef LZ4
|
||||
case("lz4", "LZ4")
|
||||
cformat = "lz4"
|
||||
compression_format = compression_lz4
|
||||
compression_level = 0
|
||||
suffix = ".lz4"
|
||||
#endif /* LZ4 */
|
||||
case default
|
||||
cformat = "none"
|
||||
compression_format = compression_none
|
||||
@ -161,6 +185,8 @@ module compression
|
||||
!
|
||||
subroutine compress(input, output, csize)
|
||||
|
||||
use iso_c_binding, only: c_int, c_loc
|
||||
|
||||
implicit none
|
||||
|
||||
! subroutine arguments
|
||||
@ -169,6 +195,10 @@ module compression
|
||||
integer(kind=1), dimension(:), target, intent(out) :: output
|
||||
integer(kind=8) , intent(out) :: csize
|
||||
|
||||
! compression buffer
|
||||
!
|
||||
integer(kind=1), dimension(:), allocatable, target :: buffer
|
||||
|
||||
!-------------------------------------------------------------------------------
|
||||
!
|
||||
csize = min(size(input), size(output))
|
||||
@ -178,6 +208,20 @@ module compression
|
||||
csize = zstd_compress(c_loc(output), sizeof(output), &
|
||||
c_loc(input), sizeof(input), compression_level)
|
||||
#endif /* ZSTD */
|
||||
#ifdef LZ4
|
||||
case(compression_lz4)
|
||||
allocate(buffer(lz4_compressBound(size(input))))
|
||||
csize = lz4_compress(c_loc(input), c_loc(buffer), &
|
||||
size(input), size(buffer))
|
||||
if (csize > 0 .and. csize <= size(output) - 4) then
|
||||
output(1:4) = transfer(1_4 * size(input), [1_1])
|
||||
output(5:csize+4) = buffer(1:csize)
|
||||
csize = csize + 4
|
||||
else
|
||||
csize = -1
|
||||
end if
|
||||
deallocate(buffer)
|
||||
#endif /* LZ4 */
|
||||
case default
|
||||
output(1:csize) = input(1:csize)
|
||||
end select
|
||||
|
||||
@ -3029,7 +3029,7 @@ module io
|
||||
allocate(buffer(data_bytes), stat = status)
|
||||
if (status == 0) then
|
||||
call compress(data, buffer, compressed_bytes)
|
||||
if (compressed_bytes < data_bytes) then
|
||||
if (compressed_bytes > 0) then
|
||||
open(newunit = lun, file = fname, form = 'unformatted', &
|
||||
access = 'direct', recl = compressed_bytes, &
|
||||
status = 'replace')
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user