gkowal/amun-code
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

IO: Add SZIP compression to HDF5 snapshots.

Browse Source 
Signed-off-by: Grzegorz Kowal <grzegorz@amuncode.org>
This commit is contained in:
Grzegorz Kowal 2021-12-01 21:52:41 -03:00
parent 1664a37e10
commit 43ad4a7af3
2 changed files with 25 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
README.md
View File

@ -27,7 +27,7 @@ following features are already implemented:
* data stored in an internal XML+binary or the HDF5 format, * data stored in an internal XML+binary or the HDF5 format,
* data integrity of the XML+binary format guaranteed by the XXH64 or XXH3 hashes; * data integrity of the XML+binary format guaranteed by the XXH64 or XXH3 hashes;
* support for Zstandard, LZ4, and LZMA compressions in the XML+binary format, * support for Zstandard, LZ4, and LZMA compressions in the XML+binary format,
* support for Deflate, Zstandard, and ZFP compressions in the HDF5 format, * support for Deflate, SZIP, Zstandard, and ZFP compressions in the HDF5 format,
* easy and consistend Python interface to read snapshots in both formats, * easy and consistend Python interface to read snapshots in both formats,
* MPI parallelization, * MPI parallelization,
* completely written in Fortran 2008, * completely written in Fortran 2008,
@ -80,6 +80,7 @@ Requirements
will need these libraries. will need these libraries.
* Deflate compression is natively supported in HDF5 libraries, however, * Deflate compression is natively supported in HDF5 libraries, however,
optionally these compression formats are supported through filters: optionally these compression formats are supported through filters:
[SZIP](https://support.hdfgroup.org/doc_resource/SZIP/)
[HDF5Plugin-Zstandard](https://github.com/gkowal/HDF5Plugin-Zstandard), [HDF5Plugin-Zstandard](https://github.com/gkowal/HDF5Plugin-Zstandard),
[H5Z-ZFP](https://github.com/LLNL/H5Z-ZFP). [H5Z-ZFP](https://github.com/LLNL/H5Z-ZFP).

30
sources/io.F90
View File

@ -124,7 +124,10 @@ module io
#ifdef HDF5 #ifdef HDF5
! supported compression types ! supported compression types
! !
integer, parameter :: H5Z_DEFLATE = 1, H5Z_ZFP = 32013, H5Z_ZSTANDARD = 32015 integer, parameter :: H5Z_DEFLATE = 1
integer, parameter :: H5Z_SZIP = 4
integer, parameter :: H5Z_ZFP = 32013
integer, parameter :: H5Z_ZSTANDARD = 32015
! used compression type and level ! used compression type and level
! !
@ -329,6 +332,17 @@ module io
call print_message(loc, & call print_message(loc, &
"Could not check if the filter is available!") "Could not check if the filter is available!")
end if end if
case("szip")
call h5zfilter_avail_f(H5Z_FILTER_SZIP_F, test, status)
if (status == 0) then
if (test) then
hcformat = H5Z_SZIP
call h5pset_szip_f(prp_id, 32, 32, status)
end if
else
call print_message(loc, &
"Could not check if the filter is available!")
end if
case("zstd", "zstandard") case("zstd", "zstandard")
call h5zfilter_avail_f(H5Z_ZSTANDARD, test, status) call h5zfilter_avail_f(H5Z_ZSTANDARD, test, status)
if (status == 0) then if (status == 0) then
@ -478,13 +492,15 @@ module io
#ifdef HDF5 #ifdef HDF5
select case(hcformat) select case(hcformat)
case(H5Z_DEFLATE) case(H5Z_DEFLATE)
call print_parameter(verbose, "HDF5 compression" , "deflate") call print_parameter(verbose, "HDF5 compression", "deflate")
call print_parameter(verbose, "compression level", hclevel ) call print_parameter(verbose, "compression level", hclevel)
case(H5Z_SZIP)
call print_parameter(verbose, "HDF5 compression", "szip")
case(H5Z_ZSTANDARD) case(H5Z_ZSTANDARD)
call print_parameter(verbose, "HDF5 compression" , "zstd" ) call print_parameter(verbose, "HDF5 compression", "zstd")
call print_parameter(verbose, "compression level", hclevel ) call print_parameter(verbose, "compression level", hclevel)
case(H5Z_ZFP) case(H5Z_ZFP)
call print_parameter(verbose, "HDF5 compression" , "zfp") call print_parameter(verbose, "HDF5 compression", "zfp")
call print_parameter(verbose, "ZFP mode", zfpmode) call print_parameter(verbose, "ZFP mode", zfpmode)
select case(trim(zfpmode)) select case(trim(zfpmode))
case('rate') case('rate')
@ -495,7 +511,7 @@ module io
call print_parameter(verbose, "ZFP accuracy" , zfpaccu) call print_parameter(verbose, "ZFP accuracy" , zfpaccu)
end select end select
case default case default
call print_parameter(verbose, "HDF5 compression" , "none" ) call print_parameter(verbose, "HDF5 compression" , "none")
end select end select
call print_parameter(verbose, "generate XDMF files", with_xdmf, "on") call print_parameter(verbose, "generate XDMF files", with_xdmf, "on")
#endif /* HDF5 */ #endif /* HDF5 */