Source code for pyquickhelper.filehelper.transfer_api

"""
API to move files


:githublink:`%|py|5`
"""
import json
import hashlib
from io import StringIO
from ..loghelper.flog import noLOG
from ..loghelper.convert_helper import str2datetime, datetime2str


[docs]class TransferAPI_FileInfo: """ keeps tracks of transferred files :githublink:`%|py|15` """
[docs] def __init__(self, name, pieces, last_update): """ information about a transferred file :param name: name of the file :param pieces: list of pieces contributing to the file :param last_update: last_update :githublink:`%|py|24` """ self.name = name self.pieces = pieces self.last_update = last_update
[docs] def __str__(self): """ usual :githublink:`%|py|32` """ mes = "[%s,#%d,%s]" % (self.name, len(self.pieces), self.last_update) return mes
[docs] def add_piece(self, piece): """ add a piece :param piece: add piece :githublink:`%|py|41` """ self.pieces.append(piece)
[docs] @staticmethod def read_json(s): """ retrieves information from a json string :githublink:`%|py|48` """ st = StringIO(s) js = json.load(st) js[2] = str2datetime(js[2]) return TransferAPI_FileInfo(*js)
[docs] def to_json(self): """ serializes this class info JSON :githublink:`%|py|57` """ li = [self.name, self.pieces, datetime2str(self.last_update)] return json.dumps(li)
[docs]class TransferAPI: """ defines an API to transfer files over a remote location :githublink:`%|py|65` """
[docs] def __init__(self, fLOG=noLOG): """ constructor :param fLOG: logging function :githublink:`%|py|72` """ self.fLOG = fLOG if fLOG else noLOG
[docs] def transfer(self, path, data): """ we assume a data holds in memory, tansfer data to path :param data: bytes :param path: path to remove location :return: boolean :githublink:`%|py|83` """ raise NotImplementedError()
[docs] def retrieve(self, path, exc=True): """ retrieve data from path :param path: remove location :param exc: keep exception :return: data :githublink:`%|py|93` """ raise NotImplementedError()
[docs] def retrieve_mapping(self, decrypt): """ returns the mapping :param decrypt: decrypt function :return: list of key,value pair :githublink:`%|py|102` """ m = self.retrieve("__mapping__", exc=False) if m is None: return {} else: return TransferAPI.bytes2mapping(m)
[docs] def transfer_mapping(self, mapping, encrypt, filename=None): """ transfer the mapping :param mapping: mapping :param encrypt: encryption function :param filename: local filename :return: boolean :githublink:`%|py|117` """ b = TransferAPI.mapping2bytes(mapping) if filename is not None: with open(filename, "wb") as f: f.write(b) return self.transfer("__mapping__", b)
[docs] @staticmethod def mapping2bytes(mapping): """ serializes a mapping :param mapping: dictionary { str, :class:`TransferAPI_FileInfo <pyquickhelper.filehelper.transfer_api.TransferAPI_FileInfo>` } :return: bytes :githublink:`%|py|131` """ rows = [] for k, v in sorted(mapping.items()): r = "{0}\t{1}".format(k, v.to_json()) rows.append(r) return "\n".join(rows).encode()
[docs] @staticmethod def bytes2mapping(byt): """ deserializes a mapping :param byt: bytes :return: dictionary { str, :class:`TransferAPI_FileInfo <pyquickhelper.filehelper.transfer_api.TransferAPI_FileInfo>` } :githublink:`%|py|145` """ lines = byt.decode().split("\n") res = {} for line in lines: spl = line.split("\t") res[spl[0]] = TransferAPI_FileInfo.read_json("\n".join(spl[1:])) return res
[docs] @staticmethod def checksum_md5(data): """ computes MD5 for a file :param data: some data :return: string :githublink:`%|py|160` """ zero = hashlib.md5() zero.update(data) return zero.hexdigest()
[docs] def get_remote_path(self, data, name, piece=0): """ produces a remote path :param data: binary data to transfer (to be hashed) :param name: local name :param piece: pieces :return: remote path hash of everything :githublink:`%|py|175` """ m1 = TransferAPI.checksum_md5(name.encode() + str(piece).encode()) m2 = TransferAPI.checksum_md5(data) return m1 + "_" + m2
[docs]class MockTransferAPI(TransferAPI): """ class used for unit test purposes, simple key, value storage :githublink:`%|py|184` """
[docs] def __init__(self, fLOG=noLOG): """ constructor :param fLOG: logging function :githublink:`%|py|191` """ TransferAPI.__init__(self, fLOG) self._storage = {}
[docs] def transfer(self, path, data): """ we assume a data holds in memory, tansfer data to path :param data: bytes :param path: path to remove location :return: boolean :githublink:`%|py|203` """ self._storage[path] = data return True
[docs] def retrieve(self, path, exc=True): """ retrieve data from path :param path: remove location :param exc: keep exception :return: data :githublink:`%|py|214` """ if exc: return self._storage[path] else: try: return self._storage[path] except KeyError: return None