Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# -*- coding: utf-8 -*- 

2""" 

3@file 

4@brief Copy files 

5""" 

6import os 

7import shutil 

8import datetime 

9from pyquickhelper.loghelper import fLOG 

10from .clean_python_script_before_exporting_outside import cleanFileFromtohtmlreplace 

11from .utils_file import checksum_md5 

12from .latex_svg_gif import replace_file 

13 

14 

15class FileToCopy: 

16 

17 def __init__(self, filename, size, date, mdate, checksum): 

18 """constructor""" 

19 self.filename = filename 

20 self.size = size 

21 self.date = date 

22 self.mdate = mdate # modification date 

23 self.checksum = checksum 

24 if date is not None and not isinstance(self.date, datetime.datetime): 

25 raise ValueError( # pragma: no cover 

26 "mismatch for date (%s) and file %s" % 

27 (str(type(date)), filename)) 

28 if mdate is not None and not isinstance(self.mdate, datetime.datetime): 

29 raise ValueError( # pragma: no cover 

30 "mismatch for mdate (%s) and file %s" % 

31 (str(type(mdate)), filename)) 

32 if not isinstance(size, int): 

33 raise ValueError( # pragma: no cover 

34 "mismatch for size (%s) and file %s" % 

35 (str(type(size)), filename)) 

36 if checksum is not None and not isinstance(checksum, str): 

37 raise ValueError( # pragma: no cover 

38 "mismatch for checksum (%s) and file %s" % 

39 (str(type(checksum)), filename)) 

40 if date is not None and mdate is not None: 

41 if mdate > date: 

42 raise ValueError( 

43 "expecting mdate <= date for file " + filename) 

44 

45 def __str__(self): 

46 return "File[name=%s, size=%d (%s), mdate=%s (%s), date=%s (%s), md5=%s (%s)]" % \ 

47 (self.filename, 

48 self.size, str(type(self.size)), 

49 str(self.mdate), str(type(self.mdate)), 

50 str(self.date), str(type(self.date)), 

51 self.checksum, str(type(self.checksum))) 

52 

53 def set_date(self, date): 

54 self.date = date 

55 if not isinstance(self.date, datetime.datetime): 

56 raise ValueError("mismatch for date (%s) and file %s" % 

57 (str(type(date)), self.filename)) 

58 

59 def set_mdate(self, mdate): 

60 self.mdate = mdate 

61 if not isinstance(self.mdate, datetime.datetime): 

62 raise ValueError("mismatch for date (%s) and file %s" % 

63 (str(type(mdate)), self.filename)) 

64 

65 def set_md5(self, checksum): 

66 self.checksum = checksum 

67 if not isinstance(checksum, str): 

68 raise ValueError("mismatch for checksum (%s) and file %s" % 

69 (str(type(checksum)), self.filename)) 

70 

71 

72class CopyFileForFtp: 

73 """ 

74 This classes maintains a list of files 

75 and does some verifications in order to check if a file 

76 was modified or not (if yes, then it will be updated to the website). 

77 """ 

78 @staticmethod 

79 def convert_st_date_to_datetime(t): 

80 if isinstance(t, str): 

81 if "." in t: 

82 return datetime.datetime.strptime(t, "%Y-%m-%d %H:%M:%S.%f") 

83 return datetime.datetime.strptime(t, "%Y-%m-%d %H:%M:%S") 

84 return datetime.datetime.fromtimestamp(t) 

85 

86 def __init__(self, file, 

87 logfunction=fLOG, 

88 bloggiflatex="blog/giflatex/", 

89 giflatex="giflatex", 

90 giflatextemp="giflatex/temp", 

91 specificTrigger=False): 

92 self.copyFiles = {} 

93 self.fileKeep = file 

94 self.LOG = logfunction 

95 self.bloggiflatex = bloggiflatex 

96 self.giflatex = giflatex 

97 self.giflatextemp = giflatextemp 

98 self.specificTrigger = specificTrigger 

99 

100 if os.path.exists(self.fileKeep): # pragma: no cover 

101 f = open(self.fileKeep, "r") 

102 for _ in f.readlines(): 

103 spl = _.strip("\r\n ").split("\t") 

104 try: 

105 if len(spl) >= 2: 

106 a, b = spl[:2] 

107 obj = FileToCopy(a, int(b), None, None, None) 

108 if len(spl) > 2 and len(spl[2]) > 0: 

109 obj.set_date( 

110 CopyFileForFtp.convert_st_date_to_datetime(spl[2])) 

111 if len(spl) > 3 and len(spl[3]) > 0: 

112 obj.set_mdate( 

113 CopyFileForFtp.convert_st_date_to_datetime(spl[3])) 

114 if len(spl) > 4 and len(spl[4]) > 0: 

115 obj.set_md5(spl[4]) 

116 self.copyFiles[a] = obj 

117 else: 

118 raise ValueError( # pragma: no cover 

119 "expecting a filename and a date on this line: " + _) 

120 except Exception as e: # pragma: no cover 

121 fLOG("issue with line", _, spl) 

122 raise e 

123 

124 f.close() 

125 

126 # contains all file to update 

127 self.modifiedFile = [] 

128 

129 def save_dates(self, checkfile=None): 

130 """ 

131 Saves the status of the copy. 

132 

133 @param checkfile check the status for file checkfile 

134 """ 

135 if checkfile is None: 

136 checkfile = [] 

137 rows = [] 

138 for k in sorted(self.copyFiles): 

139 obj = self.copyFiles[k] 

140 da = "" if obj.date is None else str(obj.date) 

141 mda = "" if obj.mdate is None else str(obj.mdate) 

142 sum5 = "" if obj.checksum is None else str(obj.checksum) 

143 

144 if k in checkfile and len(da) == 0: 

145 raise ValueError( # pragma: no cover 

146 "there should be a date for file " + k + "\n" + str(obj)) 

147 if k in checkfile and len(mda) == 0: 

148 raise ValueError( # pragma: no cover 

149 "there should be a mdate for file " + k + "\n" + str(obj)) 

150 if k in checkfile and len(sum5) <= 10: 

151 raise ValueError( # pragma: no cover 

152 "there should be a checksum( for file " + k + "\n" + str(obj)) 

153 

154 values = [k, str(obj.size), da, mda, sum5] 

155 sval = "%s\n" % "\t".join(values) 

156 if "\tNone" in sval: 

157 raise AssertionError( 

158 "this case should happen " + sval + "\n" + str(obj)) 

159 

160 rows.append(sval) 

161 

162 f = open(self.fileKeep, "w") 

163 for r in rows: 

164 f.write(r) 

165 f.close() 

166 

167 def has_been_modified_and_reason(self, file): 

168 """ 

169 Returns True, reason if a file was modified or False, None if not. 

170 

171 @param file filename 

172 @return True,reason or False,None 

173 """ 

174 res = True 

175 reason = None 

176 

177 if file not in self.copyFiles: 

178 reason = "new" 

179 res = True 

180 else: 

181 obj = self.copyFiles[file] 

182 st = os.stat(file) 

183 if st.st_size != obj.size: 

184 reason = "size %s != old size %s" % ( 

185 str(st.st_size), str(obj.size)) 

186 res = True 

187 else: 

188 l_ = obj.mdate 

189 _m = st.st_mtime 

190 d = CopyFileForFtp.convert_st_date_to_datetime(_m) 

191 if d != l_: 

192 # les dates sont différentes mais les fichiers peuvent être 

193 # différents 

194 if obj.checksum is not None: 

195 ch = checksum_md5(file) 

196 if ch != obj.checksum: 

197 reason = "date/md5 %s != old date %s md5 %s != %s" % ( 

198 str(l_), str(d), obj.checksum, ch) 

199 res = True 

200 else: 

201 res = False 

202 else: 

203 # on ne peut pas savoir, dans le doute, on s'abstient 

204 res = False 

205 else: 

206 # mda.... mais pas sûr (la date n'a pas changé) 

207 res = False 

208 

209 if res: 

210 self.modifiedFile.append((file, reason)) 

211 return res, reason 

212 

213 def add_if_modified(self, file): 

214 """ 

215 Adds a file to self.modifiedList if it was modified. 

216 

217 @param file filename 

218 @return True or False 

219 """ 

220 res, reason = self.has_been_modified_and_reason(file) 

221 if res: 

222 memo = [_ for _ in self.modifiedFile if _[0] == file] 

223 if len(memo) == 0: 

224 # not already added 

225 self.modifiedFile.append((file, reason)) 

226 return res 

227 

228 def update_copied_file(self, file): 

229 """ 

230 Updates the file in copyFiles (before saving), update all field. 

231 

232 @param file filename 

233 @return file object 

234 """ 

235 st = os.stat(file) 

236 size = st.st_size 

237 mdate = CopyFileForFtp.convert_st_date_to_datetime(st.st_mtime) 

238 date = datetime.datetime.now() 

239 md = checksum_md5(file) 

240 obj = FileToCopy(file, size, date, mdate, md) 

241 self.copyFiles[file] = obj 

242 return obj 

243 

244 def copy_file(self, file, to, doFTP=True, doClean=False, to_is_a_file=False): 

245 """ 

246 Processes a file copy. 

247 

248 @param file file to copy 

249 @param to destination (folder) 

250 @param doFTP if True, does some latex modifications (creates an image) 

251 @param doClean if True, does some cleaning before the copy 

252 (for script in pyhome having section such as the one in tableformula.py) 

253 @param to_is_a_file it means to is a file, not a folder 

254 """ 

255 if doClean and doFTP: 

256 raise AssertionError( 

257 "this case is not meant to happen, doClean and doFTP, set up at the same time") 

258 if len(to) == 0: 

259 raise ValueError("an empty folder is not allowed for parameter to") 

260 

261 folder = to 

262 if not os.path.exists(folder): 

263 ffff, last = os.path.split(to) 

264 if to_is_a_file: 

265 folder = ffff 

266 elif "." in last: 

267 raise ValueError("are you sure to is not a file :" + to + "?") 

268 

269 if not os.path.exists(folder): 

270 self.LOG("[copy_file] creating folder ", folder) 

271 os.makedirs(folder) 

272 

273 if doFTP: 

274 if file not in self.copyFiles or \ 

275 os.stat(file).st_size != self.copyFiles[file].size: 

276 

277 # some exception for latex 

278 if self.specificTrigger and "2013-02-04" not in file and \ 

279 file.endswith(".html") and "-" in file: 

280 fLOG("[copy_file] latex exception for: ", file) 

281 replace_file(file, file, self.bloggiflatex, 

282 self.giflatex, self.LOG, self.giflatextemp) 

283 

284 if not os.path.exists(to): 

285 self.LOG("[copy_file] creating directory ", to) 

286 os.mkdir(to) 

287 

288 self.LOG("[copy_file] copy of '{}' to '{}'.".format(file, to)) 

289 reason = "new" if file not in self.copyFiles else \ 

290 ("new size %s != old size %s" % (str(os.stat(file).st_size), 

291 str(self.copyFiles[file].size))) 

292 

293 try: 

294 try: 

295 shutil.copy(file, to) 

296 except shutil.SameFileError: # pragma: no cover 

297 pass 

298 self.modifiedFile.append((file, reason)) 

299 return to 

300 

301 except Exception as e: # pragma: no cover 

302 self.LOG( 

303 "[copy_file] issue with '{}' copy to '{}'.".format(file, to)) 

304 self.LOG( 

305 "[copy_file] message d'erreur {}: {}".format(type(e), e)) 

306 return to 

307 else: 

308 try: 

309 shutil.copy(file, to) 

310 if not os.path.isfile(to): 

311 to = os.path.join(to, os.path.split(file)[-1]) 

312 fLOG("copy ", file, " as ", to) 

313 except Exception as e: # pragma: no cover 

314 self.LOG("issue with ", file, " copy to ", to) 

315 self.LOG("message d'erreur ", e) 

316 

317 if doClean: 

318 f = open(to, "r") 

319 content = f.read() 

320 f.close() 

321 

322 newcontent = cleanFileFromtohtmlreplace(content) 

323 

324 if newcontent != content: 

325 fLOG(" cleaning python script ", to) 

326 f = open(to, "w") 

327 f.write(newcontent) 

328 f.close() 

329 

330 return to 

331 

332 def copy_file_ext(self, file, exte, to, doFTP=True, doClean=False): 

333 """ 

334 @see me copy_file, 

335 """ 

336 res = [] 

337 if not os.path.exists(file): 

338 raise FileNotFoundError(file) 

339 nb = 0 

340 fi = os.listdir(file) 

341 for f in fi: 

342 if not os.path.isfile(file + "/" + f): 

343 continue 

344 ext = os.path.splitext(f)[1] 

345 if exte is None or ext[1:] == exte: 

346 self.copy_file(file + "/" + f, to, doFTP, doClean) 

347 res.append(file + "/" + f) 

348 nb += 1 

349 if nb == 0: 

350 raise RuntimeError("No file found in '{}'.".format(file)) 

351 return res 

352 

353 def copy_file_contains(self, file, pattern, to, doFTP=True, doClean=False): 

354 """ 

355 @see me copy_file 

356 """ 

357 fi = os.listdir(file) 

358 nb = 0 

359 for f in fi: 

360 if not os.path.isfile(file + "/" + f): 

361 continue 

362 if pattern in f: 

363 self.copy_file(file + "/" + f, to, doFTP, doClean) 

364 nb += 1 

365 if nb == 0: 

366 raise RuntimeError("No file found in '{}'.".format(file))