gkowal/amun-code
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

PYTHON: Add support for XXH3 hash.

Browse Source 
Signed-off-by: Grzegorz Kowal <grzegorz@amuncode.org>
This commit is contained in:
Grzegorz Kowal 2021-11-30 19:02:32 -03:00
parent 0ac1fec2d7
commit 657a608ce0
1 changed files with 26 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
python/amunpy/src/amunpy/amunxml.py
View File

@ -208,14 +208,18 @@ class AmunXML(Amun):
return self.__swap__(dset) return self.__swap__(dset)
def __check_digest(self, filename, digest, data): def __check_digest(self, filename, hash_type, digest, data):
''' '''
Verifies if the provided digest matches the XXH64 hash of data Verifies if the provided digest matches the data.
stored in the given filename.
''' '''
import xxhash import xxhash
if digest.lower() != xxhash.xxh64(data).hexdigest(): failed = False
if hash_type == 'xxh64':
failed = digest.lower() != xxhash.xxh64(data).hexdigest()
elif hash_type == 'xxh3':
failed = digest.lower() != xxhash.xxh3_64(data).hexdigest()
if failed:
print("File '{}' seems to be corrupted! Proceeding anyway...".format(filename)) print("File '{}' seems to be corrupted! Proceeding anyway...".format(filename))
@ -235,7 +239,9 @@ class AmunXML(Amun):
if 'compression_format' in self.binaries[dataset]: if 'compression_format' in self.binaries[dataset]:
if 'compressed_digest' in self.binaries[dataset]: if 'compressed_digest' in self.binaries[dataset]:
self.__check_digest(fname, self.binaries[dataset]['compressed_digest'], stream) htype = self.binaries[dataset]['digest_type']
dhash = self.binaries[dataset]['compressed_digest']
self.__check_digest(fname, htype, dhash, stream)
comp = self.binaries[dataset]['compression_format'] comp = self.binaries[dataset]['compression_format']
if comp == 'zstd': if comp == 'zstd':
@ -248,12 +254,16 @@ class AmunXML(Amun):
raise Exception("Binary file '{}' compressed in unsupported format {}!".format(fname, comp)) raise Exception("Binary file '{}' compressed in unsupported format {}!".format(fname, comp))
if 'digest' in self.binaries[dataset]: if 'digest' in self.binaries[dataset]:
self.__check_digest(fname, self.binaries[dataset]['digest'], data) htype = self.binaries[dataset]['digest_type']
dhash = self.binaries[dataset]['digest']
self.__check_digest(fname, htype, dhash, data)
return numpy.frombuffer(data, dtype=dtype) return numpy.frombuffer(data, dtype=dtype)
else: else:
if 'digest' in self.binaries[dataset]: if 'digest' in self.binaries[dataset]:
self.__check_digest(fname, self.binaries[dataset]['digest'], stream) htype = self.binaries[dataset]['digest_type']
dhash = self.binaries[dataset]['digest']
self.__check_digest(fname, htype, dhash, stream)
return numpy.frombuffer(stream, dtype=dtype) return numpy.frombuffer(stream, dtype=dtype)
else: else:
@ -276,7 +286,9 @@ class AmunXML(Amun):
if 'compression_format' in self.chunks[chunk_number][dataset_name]: if 'compression_format' in self.chunks[chunk_number][dataset_name]:
if 'compressed_digest' in self.chunks[chunk_number][dataset_name]: if 'compressed_digest' in self.chunks[chunk_number][dataset_name]:
self.__check_digest(fname, self.chunks[chunk_number][dataset_name]['compressed_digest'], stream) htype = self.chunks[chunk_number][dataset_name]['digest_type']
dhash = self.chunks[chunk_number][dataset_name]['compressed_digest']
self.__check_digest(fname, htype, dhash, stream)
comp = self.chunks[chunk_number][dataset_name]['compression_format'] comp = self.chunks[chunk_number][dataset_name]['compression_format']
if comp == 'zstd': if comp == 'zstd':
@ -290,12 +302,16 @@ class AmunXML(Amun):
raise Exception("Binary file '{}' compressed in unsupported format {}!".format(fname, comp)) raise Exception("Binary file '{}' compressed in unsupported format {}!".format(fname, comp))
if 'digest' in self.chunks[chunk_number][dataset_name]: if 'digest' in self.chunks[chunk_number][dataset_name]:
self.__check_digest(fname, self.chunks[chunk_number][dataset_name]['digest'], data) htype = self.chunks[chunk_number][dataset_name]['digest_type']
dhash = self.chunks[chunk_number][dataset_name]['digest']
self.__check_digest(fname, htype, dhash, data)
return numpy.frombuffer(data, dtype=dtype) return numpy.frombuffer(data, dtype=dtype)
else: else:
if 'digest' in self.chunks[chunk_number][dataset_name]: if 'digest' in self.chunks[chunk_number][dataset_name]:
self.__check_digest(fname, self.chunks[chunk_number][dataset_name]['digest'], stream) htype = self.chunks[chunk_number][dataset_name]['digest_type']
dhash = self.chunks[chunk_number][dataset_name]['digest']
self.__check_digest(fname, htype, dhash, stream)
return numpy.frombuffer(stream, dtype=dtype) return numpy.frombuffer(stream, dtype=dtype)
else: else: