Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# -*- coding: utf-8 -*-
2"""
3@file
4@brief Copy files
5"""
6import os
7import shutil
8import datetime
9from pyquickhelper.loghelper import fLOG
10from .clean_python_script_before_exporting_outside import cleanFileFromtohtmlreplace
11from .utils_file import checksum_md5
12from .latex_svg_gif import replace_file
15class FileToCopy:
17 def __init__(self, filename, size, date, mdate, checksum):
18 """constructor"""
19 self.filename = filename
20 self.size = size
21 self.date = date
22 self.mdate = mdate # modification date
23 self.checksum = checksum
24 if date is not None and not isinstance(self.date, datetime.datetime):
25 raise ValueError( # pragma: no cover
26 "mismatch for date (%s) and file %s" %
27 (str(type(date)), filename))
28 if mdate is not None and not isinstance(self.mdate, datetime.datetime):
29 raise ValueError( # pragma: no cover
30 "mismatch for mdate (%s) and file %s" %
31 (str(type(mdate)), filename))
32 if not isinstance(size, int):
33 raise ValueError( # pragma: no cover
34 "mismatch for size (%s) and file %s" %
35 (str(type(size)), filename))
36 if checksum is not None and not isinstance(checksum, str):
37 raise ValueError( # pragma: no cover
38 "mismatch for checksum (%s) and file %s" %
39 (str(type(checksum)), filename))
40 if date is not None and mdate is not None:
41 if mdate > date:
42 raise ValueError(
43 "expecting mdate <= date for file " + filename)
45 def __str__(self):
46 return "File[name=%s, size=%d (%s), mdate=%s (%s), date=%s (%s), md5=%s (%s)]" % \
47 (self.filename,
48 self.size, str(type(self.size)),
49 str(self.mdate), str(type(self.mdate)),
50 str(self.date), str(type(self.date)),
51 self.checksum, str(type(self.checksum)))
53 def set_date(self, date):
54 self.date = date
55 if not isinstance(self.date, datetime.datetime):
56 raise ValueError("mismatch for date (%s) and file %s" %
57 (str(type(date)), self.filename))
59 def set_mdate(self, mdate):
60 self.mdate = mdate
61 if not isinstance(self.mdate, datetime.datetime):
62 raise ValueError("mismatch for date (%s) and file %s" %
63 (str(type(mdate)), self.filename))
65 def set_md5(self, checksum):
66 self.checksum = checksum
67 if not isinstance(checksum, str):
68 raise ValueError("mismatch for checksum (%s) and file %s" %
69 (str(type(checksum)), self.filename))
72class CopyFileForFtp:
73 """
74 This classes maintains a list of files
75 and does some verifications in order to check if a file
76 was modified or not (if yes, then it will be updated to the website).
77 """
78 @staticmethod
79 def convert_st_date_to_datetime(t):
80 if isinstance(t, str):
81 if "." in t:
82 return datetime.datetime.strptime(t, "%Y-%m-%d %H:%M:%S.%f")
83 return datetime.datetime.strptime(t, "%Y-%m-%d %H:%M:%S")
84 return datetime.datetime.fromtimestamp(t)
86 def __init__(self, file,
87 logfunction=fLOG,
88 bloggiflatex="blog/giflatex/",
89 giflatex="giflatex",
90 giflatextemp="giflatex/temp",
91 specificTrigger=False):
92 self.copyFiles = {}
93 self.fileKeep = file
94 self.LOG = logfunction
95 self.bloggiflatex = bloggiflatex
96 self.giflatex = giflatex
97 self.giflatextemp = giflatextemp
98 self.specificTrigger = specificTrigger
100 if os.path.exists(self.fileKeep): # pragma: no cover
101 f = open(self.fileKeep, "r")
102 for _ in f.readlines():
103 spl = _.strip("\r\n ").split("\t")
104 try:
105 if len(spl) >= 2:
106 a, b = spl[:2]
107 obj = FileToCopy(a, int(b), None, None, None)
108 if len(spl) > 2 and len(spl[2]) > 0:
109 obj.set_date(
110 CopyFileForFtp.convert_st_date_to_datetime(spl[2]))
111 if len(spl) > 3 and len(spl[3]) > 0:
112 obj.set_mdate(
113 CopyFileForFtp.convert_st_date_to_datetime(spl[3]))
114 if len(spl) > 4 and len(spl[4]) > 0:
115 obj.set_md5(spl[4])
116 self.copyFiles[a] = obj
117 else:
118 raise ValueError( # pragma: no cover
119 "expecting a filename and a date on this line: " + _)
120 except Exception as e: # pragma: no cover
121 fLOG("issue with line", _, spl)
122 raise e
124 f.close()
126 # contains all file to update
127 self.modifiedFile = []
129 def save_dates(self, checkfile=None):
130 """
131 Saves the status of the copy.
133 @param checkfile check the status for file checkfile
134 """
135 if checkfile is None:
136 checkfile = []
137 rows = []
138 for k in sorted(self.copyFiles):
139 obj = self.copyFiles[k]
140 da = "" if obj.date is None else str(obj.date)
141 mda = "" if obj.mdate is None else str(obj.mdate)
142 sum5 = "" if obj.checksum is None else str(obj.checksum)
144 if k in checkfile and len(da) == 0:
145 raise ValueError( # pragma: no cover
146 "there should be a date for file " + k + "\n" + str(obj))
147 if k in checkfile and len(mda) == 0:
148 raise ValueError( # pragma: no cover
149 "there should be a mdate for file " + k + "\n" + str(obj))
150 if k in checkfile and len(sum5) <= 10:
151 raise ValueError( # pragma: no cover
152 "there should be a checksum( for file " + k + "\n" + str(obj))
154 values = [k, str(obj.size), da, mda, sum5]
155 sval = "%s\n" % "\t".join(values)
156 if "\tNone" in sval:
157 raise AssertionError(
158 "this case should happen " + sval + "\n" + str(obj))
160 rows.append(sval)
162 f = open(self.fileKeep, "w")
163 for r in rows:
164 f.write(r)
165 f.close()
167 def has_been_modified_and_reason(self, file):
168 """
169 Returns True, reason if a file was modified or False, None if not.
171 @param file filename
172 @return True,reason or False,None
173 """
174 res = True
175 reason = None
177 if file not in self.copyFiles:
178 reason = "new"
179 res = True
180 else:
181 obj = self.copyFiles[file]
182 st = os.stat(file)
183 if st.st_size != obj.size:
184 reason = "size %s != old size %s" % (
185 str(st.st_size), str(obj.size))
186 res = True
187 else:
188 l_ = obj.mdate
189 _m = st.st_mtime
190 d = CopyFileForFtp.convert_st_date_to_datetime(_m)
191 if d != l_:
192 # les dates sont différentes mais les fichiers peuvent être
193 # différents
194 if obj.checksum is not None:
195 ch = checksum_md5(file)
196 if ch != obj.checksum:
197 reason = "date/md5 %s != old date %s md5 %s != %s" % (
198 str(l_), str(d), obj.checksum, ch)
199 res = True
200 else:
201 res = False
202 else:
203 # on ne peut pas savoir, dans le doute, on s'abstient
204 res = False
205 else:
206 # mda.... mais pas sûr (la date n'a pas changé)
207 res = False
209 if res:
210 self.modifiedFile.append((file, reason))
211 return res, reason
213 def add_if_modified(self, file):
214 """
215 Adds a file to self.modifiedList if it was modified.
217 @param file filename
218 @return True or False
219 """
220 res, reason = self.has_been_modified_and_reason(file)
221 if res:
222 memo = [_ for _ in self.modifiedFile if _[0] == file]
223 if len(memo) == 0:
224 # not already added
225 self.modifiedFile.append((file, reason))
226 return res
228 def update_copied_file(self, file):
229 """
230 Updates the file in copyFiles (before saving), update all field.
232 @param file filename
233 @return file object
234 """
235 st = os.stat(file)
236 size = st.st_size
237 mdate = CopyFileForFtp.convert_st_date_to_datetime(st.st_mtime)
238 date = datetime.datetime.now()
239 md = checksum_md5(file)
240 obj = FileToCopy(file, size, date, mdate, md)
241 self.copyFiles[file] = obj
242 return obj
244 def copy_file(self, file, to, doFTP=True, doClean=False, to_is_a_file=False):
245 """
246 Processes a file copy.
248 @param file file to copy
249 @param to destination (folder)
250 @param doFTP if True, does some latex modifications (creates an image)
251 @param doClean if True, does some cleaning before the copy
252 (for script in pyhome having section such as the one in tableformula.py)
253 @param to_is_a_file it means to is a file, not a folder
254 """
255 if doClean and doFTP:
256 raise AssertionError(
257 "this case is not meant to happen, doClean and doFTP, set up at the same time")
258 if len(to) == 0:
259 raise ValueError("an empty folder is not allowed for parameter to")
261 folder = to
262 if not os.path.exists(folder):
263 ffff, last = os.path.split(to)
264 if to_is_a_file:
265 folder = ffff
266 elif "." in last:
267 raise ValueError("are you sure to is not a file :" + to + "?")
269 if not os.path.exists(folder):
270 self.LOG("[copy_file] creating folder ", folder)
271 os.makedirs(folder)
273 if doFTP:
274 if file not in self.copyFiles or \
275 os.stat(file).st_size != self.copyFiles[file].size:
277 # some exception for latex
278 if self.specificTrigger and "2013-02-04" not in file and \
279 file.endswith(".html") and "-" in file:
280 fLOG("[copy_file] latex exception for: ", file)
281 replace_file(file, file, self.bloggiflatex,
282 self.giflatex, self.LOG, self.giflatextemp)
284 if not os.path.exists(to):
285 self.LOG("[copy_file] creating directory ", to)
286 os.mkdir(to)
288 self.LOG("[copy_file] copy of '{}' to '{}'.".format(file, to))
289 reason = "new" if file not in self.copyFiles else \
290 ("new size %s != old size %s" % (str(os.stat(file).st_size),
291 str(self.copyFiles[file].size)))
293 try:
294 try:
295 shutil.copy(file, to)
296 except shutil.SameFileError: # pragma: no cover
297 pass
298 self.modifiedFile.append((file, reason))
299 return to
301 except Exception as e: # pragma: no cover
302 self.LOG(
303 "[copy_file] issue with '{}' copy to '{}'.".format(file, to))
304 self.LOG(
305 "[copy_file] message d'erreur {}: {}".format(type(e), e))
306 return to
307 else:
308 try:
309 shutil.copy(file, to)
310 if not os.path.isfile(to):
311 to = os.path.join(to, os.path.split(file)[-1])
312 fLOG("copy ", file, " as ", to)
313 except Exception as e: # pragma: no cover
314 self.LOG("issue with ", file, " copy to ", to)
315 self.LOG("message d'erreur ", e)
317 if doClean:
318 f = open(to, "r")
319 content = f.read()
320 f.close()
322 newcontent = cleanFileFromtohtmlreplace(content)
324 if newcontent != content:
325 fLOG(" cleaning python script ", to)
326 f = open(to, "w")
327 f.write(newcontent)
328 f.close()
330 return to
332 def copy_file_ext(self, file, exte, to, doFTP=True, doClean=False):
333 """
334 @see me copy_file,
335 """
336 res = []
337 if not os.path.exists(file):
338 raise FileNotFoundError(file)
339 nb = 0
340 fi = os.listdir(file)
341 for f in fi:
342 if not os.path.isfile(file + "/" + f):
343 continue
344 ext = os.path.splitext(f)[1]
345 if exte is None or ext[1:] == exte:
346 self.copy_file(file + "/" + f, to, doFTP, doClean)
347 res.append(file + "/" + f)
348 nb += 1
349 if nb == 0:
350 raise RuntimeError("No file found in '{}'.".format(file))
351 return res
353 def copy_file_contains(self, file, pattern, to, doFTP=True, doClean=False):
354 """
355 @see me copy_file
356 """
357 fi = os.listdir(file)
358 nb = 0
359 for f in fi:
360 if not os.path.isfile(file + "/" + f):
361 continue
362 if pattern in f:
363 self.copy_file(file + "/" + f, to, doFTP, doClean)
364 nb += 1
365 if nb == 0:
366 raise RuntimeError("No file found in '{}'.".format(file))