Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1""" 

2@file 

3@brief Class to transfer files to a website using FTP, it only transfers updated files 

4""" 

5from __future__ import print_function 

6import re 

7import os 

8import warnings 

9import sys 

10import ftplib 

11from io import BytesIO 

12from time import sleep 

13from random import random 

14from .files_status import FilesStatus 

15from ..loghelper.flog import noLOG 

16from .ftp_transfer import CannotCompleteWithoutNewLoginException 

17 

18 

19class FolderTransferFTPException(Exception): 

20 

21 """ 

22 custom exception for @see cl FolderTransferFTP 

23 """ 

24 pass 

25 

26 

27_text_extensions = {".ipynb", ".html", ".py", ".cpp", ".h", ".hpp", ".c", 

28 ".cs", ".txt", ".csv", ".xml", ".css", ".js", ".r", ".doc", 

29 ".ind", ".buildinfo", ".rst", ".aux", ".out", ".log", ".cc", 

30 '.tmpl'} 

31 

32 

33def content_as_binary(filename): 

34 """ 

35 determines if filename is binary or None before transfering it 

36 

37 @param filename filename 

38 @return boolean 

39 """ 

40 global _text_extensions 

41 ext = os.path.splitext(filename)[-1].lower() 

42 if ext in _text_extensions: 

43 return False 

44 else: 

45 return True 

46 

47 

48class FolderTransferFTP: 

49 

50 """ 

51 This class aims at transfering a folder to a FTP website, 

52 it checks that a file was updated before transfering, 

53 @see cl TransferFTP . 

54 

55 .. exref:: 

56 :title: Transfer updated files to a website 

57 

58 The following code shows how to transfer the content of a folder to 

59 website through FTP protocol. 

60 

61 :: 

62 

63 ftn = FileTreeNode("c:/somefolder") 

64 ftp = TransferFTP("ftp.website.fr", "login", "password", fLOG=print) 

65 fftp = FolderTransferFTP (ftn, ftp, "status_file.txt", 

66 root_web = "/www/htdocs/app/pyquickhelper/helpsphinx") 

67 

68 fftp.start_transfering() 

69 ftp.close() 

70 

71 The following example is more complete: 

72 

73 :: 

74 

75 import sys, os 

76 from pyquickhelper.filehelper import TransferFTP, FileTreeNode, FolderTransferFTP 

77 import keyring 

78 

79 user = keyring.get_password("webtransfer", "user") 

80 pwd = keyring.get_password("webtransfer", "pwd") 

81 

82 ftp = TransferFTP("ftp.website.fr", user, pwd, fLOG=print) 

83 

84 location = r"local_location/GitHub/%s/dist/html" 

85 this = os.path.abspath(os.path.dirname(__file__)) 

86 rootw = "/root/subfolder/%s/helpsphinx" 

87 

88 for module in ["pyquickhelper", "pyensae"] : 

89 root = location % module 

90 

91 # documentation 

92 sfile = os.path.join(this, "status_%s.txt" % module) 

93 ftn = FileTreeNode(root) 

94 fftp = FolderTransferFTP (ftn, ftp, sfile, 

95 root_web = rootw % module, 

96 fLOG=print) 

97 

98 fftp.start_transfering() 

99 

100 # setup, wheels 

101 ftn = FileTreeNode(os.path.join(root,".."), filter = lambda root, path, f, dir: not dir) 

102 fftp = FolderTransferFTP (ftn, ftp, sfile, 

103 root_web = (rootw % module).replace("helpsphinx",""), 

104 fLOG=print) 

105 

106 fftp.start_transfering() 

107 

108 ftp.close() 

109 """ 

110 

111 def __init__(self, file_tree_node, ftp_transfer, file_status, root_local=None, 

112 root_web=None, footer_html=None, content_filter=None, 

113 is_binary=content_as_binary, text_transform=None, filter_out=None, 

114 exc=False, force_allow=None, fLOG=noLOG): 

115 """ 

116 @param file_tree_node @see cl FileTreeNode 

117 @param ftp_transfer @see cl TransferFTP 

118 @param file_status file keeping the status for each file (date, hash of the content for the last upload) 

119 @param root_local local root 

120 @param root_web remote root on the website 

121 @param footer_html append this HTML code to any uploaded page (such a javascript code to count the audience) 

122 at the end of the file (before tag ``</body>``) 

123 @param content_filter function which transform the content if a specific string is found 

124 in the file, if the result is None, it raises an exception 

125 indicating the file cannot be transfered (applies only on text files) 

126 @param is_binary function which determines if content of a files is binary or not 

127 @param text_transform function to transform the content of a text file before uploading it 

128 @param filter_out regular expression to exclude some files, it can also be a function. 

129 @param exc raise exception if not able to transfer 

130 @param force_allow the class does not transfer a file containing a set of specific strings 

131 except if they are in the list 

132 @param fLOG logging function 

133 

134 Function *text_transform(self, filename, content)* returns the modified content. 

135 

136 If *filter_out* is a function, the signature is:: 

137 

138 def filter_out(full_file_name, filename): 

139 # ... 

140 return True # if the file is filtered out, False otherwise 

141 

142 Function *filter_out* receives another parameter (filename) 

143 to give more information when raising an exception. 

144 """ 

145 self._ftn = file_tree_node 

146 self._ftp = ftp_transfer 

147 self._status = file_status 

148 self._root_local = root_local if root_local is not None else file_tree_node.root 

149 self._root_web = root_web if root_web is not None else "" 

150 self.fLOG = fLOG 

151 self._footer_html = footer_html 

152 self._content_filter = content_filter 

153 self._is_binary = is_binary 

154 self._exc = exc 

155 self._force_allow = force_allow 

156 if filter_out is not None and not isinstance(filter_out, str): 

157 self._filter_out = filter_out 

158 else: 

159 self._filter_out_reg = None if filter_out is None else re.compile( 

160 filter_out) 

161 self._filter_out = (lambda f: False) if filter_out is None else ( 

162 lambda f: self._filter_out_reg.search(f) is not None) 

163 

164 self._ft = FilesStatus(file_status) 

165 self._text_transform = text_transform 

166 

167 def __str__(self): 

168 """ 

169 usual 

170 """ 

171 mes = ["FolderTransferFTP"] 

172 mes += [" local root: {0}".format(self._root_local)] 

173 mes += [" remote root: {0}".format(self._root_web)] 

174 return "\n".join(mes) 

175 

176 def iter_eligible_files(self): 

177 """ 

178 Iterates on eligible file for transfering 

179 (if they have been modified). 

180 

181 @return iterator on file name 

182 """ 

183 for f in self._ftn: 

184 if f.isfile(): 

185 if self._filter_out(f.fullname): 

186 continue 

187 n = self._ft.has_been_modified_and_reason(f.fullname)[0] 

188 if n: 

189 yield f 

190 

191 def update_status(self, file): 

192 """ 

193 Updates the status of a file. 

194 

195 @param file filename 

196 @return @see cl FileInfo 

197 """ 

198 r = self._ft.update_copied_file(file) 

199 self._ft.save_dates() 

200 return r 

201 

202 def preprocess_before_transfering(self, path, force_binary=False, force_allow=None): 

203 """ 

204 Applies some preprocessing to the file to transfer. 

205 It adds the footer for example. 

206 It returns a stream which should be closed by 

207 using method @see me close_stream. 

208 

209 @param path file name 

210 @param force_binary impose a binary transfer 

211 @param force_allow allow these strings even if they seem to be credentials 

212 @return binary stream, size 

213 

214 Bypass utf-8 encoding checking when the extension is ``.rst.txt``. 

215 """ 

216 if force_binary or self._is_binary(path): 

217 size = os.stat(path).st_size 

218 return open(path, "rb"), size 

219 else: 

220 if self._footer_html is None and self._content_filter is None: 

221 size = os.stat(path).st_size 

222 return open(path, "rb"), size 

223 else: 

224 size = os.stat(path).st_size 

225 with open(path, "r", encoding="utf8") as f: 

226 try: 

227 content = f.read() 

228 except UnicodeDecodeError as e: 

229 ext = os.path.splitext(path)[-1] 

230 if ext in {".js"} or path.endswith(".rst.txt"): 

231 # just a warning 

232 warnings.warn( 

233 "FTP transfer, encoding issue with '{0}'".format(path), UserWarning) 

234 return self.preprocess_before_transfering(path, True) 

235 else: 

236 stex = str(e).split("\n") 

237 stex = "\n ".join(stex) 

238 raise FolderTransferFTPException( 

239 'Unable to transfer:\n File "{0}", line 1\nEXC:\n{1}'.format(path, stex)) from e 

240 

241 # footer 

242 if self._footer_html is not None and os.path.splitext( 

243 path)[-1].lower() in (".htm", ".html"): 

244 spl = content.split("</body>") 

245 if len(spl) > 1: 

246 if len(spl) != 2: 

247 spl = ["</body>".join(spl[:-1]), spl[-1]] 

248 

249 content = spl[0] + self._footer_html + \ 

250 "</body>" + spl[-1] 

251 

252 # filter 

253 try: 

254 content = self._content_filter( 

255 content, path, force_allow=force_allow) 

256 except Exception as e: # pragma: no cover 

257 import traceback 

258 exc_type, exc_value, exc_traceback = sys.exc_info() 

259 trace = traceback.format_exception( 

260 exc_type, exc_value, exc_traceback) 

261 if isinstance(trace, list): 

262 trace = "\n".join(trace) 

263 raise FolderTransferFTPException( 

264 "File '{0}' cannot be transferred (filtering exception)\nfunction:\n{1}\nEXC\n{2}\nStackTrace:\n{3}".format( 

265 path, self._content_filter, e, trace)) from e 

266 if content is None: 

267 raise FolderTransferFTPException( 

268 "File '{0}' cannot be transferred due to its content.".format(path)) 

269 

270 # transform 

271 if self._text_transform is not None: 

272 content = self._text_transform(self, path, content) 

273 

274 # to binary 

275 bcont = content.encode("utf8") 

276 return BytesIO(bcont), len(bcont) 

277 

278 def close_stream(self, stream): 

279 """ 

280 Closes a stream opened by @see me preprocess_before_transfering. 

281 

282 @param stream stream to close 

283 """ 

284 if isinstance(stream, BytesIO): 

285 pass 

286 else: 

287 stream.close() 

288 

289 def start_transfering(self, max_errors=20, delay=None): 

290 """ 

291 Starts transfering files to a remote :epkg:`FTP` website. 

292 

293 :param max_errors: stops after this number of errors 

294 :param delay: delay between two files 

295 :return: list of transferred @see cl FileInfo 

296 :raises FolderTransferFTPException: the class raises 

297 an exception (@see cl FolderTransferFTPException) 

298 more than *max_errors* issues happened 

299 """ 

300 issues = [] 

301 done = [] 

302 total = list(self.iter_eligible_files()) 

303 sum_bytes = 0 

304 for i, file in enumerate(total): 

305 if i % 20 == 0: 

306 self.fLOG("#### transfering %d/%d (so far %d bytes)" % 

307 (i, len(total), sum_bytes)) 

308 relp = os.path.relpath(file.fullname, self._root_local) 

309 if ".." in relp: 

310 raise ValueError( # pragma: no cover 

311 "The local root is not accurate:\n{0}\nFILE:\n{1}" 

312 "\nRELPATH:\n{2}".format(self, file.fullname, relp)) 

313 path = self._root_web + "/" + os.path.split(relp)[0] 

314 path = path.replace("\\", "/") 

315 

316 size = os.stat(file.fullname).st_size 

317 self.fLOG("[upload % 8d bytes name=%s -- fullname=%s -- to=%s]" % ( 

318 size, os.path.split(file.fullname)[-1], file.fullname, path)) 

319 

320 if self._exc: 

321 data, size = self.preprocess_before_transfering( 

322 file.fullname, force_allow=self._force_allow) 

323 else: 

324 try: 

325 data, size = self.preprocess_before_transfering( 

326 file.fullname, force_allow=self._force_allow) 

327 except FolderTransferFTPException as ex: # pragma: no cover 

328 stex = str(ex).split("\n") 

329 stex = "\n ".join(stex) 

330 warnings.warn( 

331 "Unable to transfer '{0}' due to [{1}].".format(file.fullname, stex), ResourceWarning) 

332 issues.append( 

333 (file.fullname, "FolderTransferFTPException", ex)) 

334 continue 

335 

336 if size > 2**20: 

337 blocksize = 2**20 

338 transfered = 0 

339 

340 def callback_function_(*args, **kwargs): 

341 "local function" 

342 private_p = kwargs.get('private_p', None) 

343 if private_p is None: 

344 raise ValueError("private_p cannot be None") 

345 private_p[1] += private_p[0] 

346 private_p[1] = min(private_p[1], size) 

347 self.fLOG(" transferred: %1.3f - %d/%d" % 

348 (1.0 * private_p[1] / private_p[2], private_p[1], private_p[2])) 

349 

350 tp_ = [blocksize, transfered, size] 

351 cb = lambda *args2, **kwargs2: callback_function_( 

352 *args2, private_p=tp_, **kwargs2) 

353 else: 

354 blocksize = None 

355 cb = None 

356 

357 if self._exc: 

358 r = self._ftp.transfer( 

359 data, path, os.path.split(file.fullname)[-1], blocksize=blocksize, callback=cb) 

360 else: 

361 try: 

362 r = self._ftp.transfer( 

363 data, path, os.path.split(file.fullname)[-1], blocksize=blocksize, callback=cb) 

364 except FileNotFoundError as e: # pragma: no cover 

365 r = False 

366 issues.append((file.fullname, "not found", e)) 

367 self.fLOG("[FolderTransferFTP] - issue", e) 

368 except ftplib.error_perm as ee: # pragma: no cover 

369 r = False 

370 issues.append((file.fullname, str(ee), ee)) 

371 self.fLOG("[FolderTransferFTP] - issue", ee) 

372 except TimeoutError as eee: # pragma: no cover 

373 r = False 

374 issues.append((file.fullname, "TimeoutError", eee)) 

375 self.fLOG("[FolderTransferFTP] - issue", eee) 

376 except EOFError as eeee: # pragma: no cover 

377 r = False 

378 issues.append((file.fullname, "EOFError", eeee)) 

379 self.fLOG("[FolderTransferFTP] - issue", eeee) 

380 except ConnectionAbortedError as eeeee: # pragma: no cover 

381 r = False 

382 issues.append( 

383 (file.fullname, "ConnectionAbortedError", eeeee)) 

384 self.fLOG(" issue", eeeee) 

385 except ConnectionResetError as eeeeee: # pragma: no cover 

386 r = False 

387 issues.append( 

388 (file.fullname, "ConnectionResetError", eeeeee)) 

389 self.fLOG("[FolderTransferFTP] - issue", eeeeee) 

390 except CannotCompleteWithoutNewLoginException as e8: # pragma: no cover 

391 r = False 

392 issues.append( 

393 (file.fullname, "CannotCompleteWithoutNewLoginException", e8)) 

394 self.fLOG("[FolderTransferFTP] - issue", e8) 

395 except Exception as e7: # pragma: no cover 

396 try: 

397 import paramiko 

398 except ImportError: 

399 raise e7 

400 if isinstance(e7, paramiko.sftp.SFTPError): 

401 r = False 

402 issues.append( 

403 (file.fullname, "ConnectionResetError", e7)) 

404 self.fLOG("[FolderTransferFTP] - issue", e7) 

405 else: 

406 raise e7 

407 

408 self.close_stream(data) 

409 

410 sum_bytes += size 

411 

412 if r: 

413 fi = self.update_status(file.fullname) 

414 done.append(fi) 

415 

416 if len(issues) >= max_errors: 

417 raise FolderTransferFTPException( # pragma: no cover 

418 "Too many issues:\n{0}".format( 

419 "\n".join("{0} -- {1} --- {2}".format( 

420 a, b, str(c).replace('\n', ' ')) for a, b, c in issues))) 

421 

422 if delay is not None and delay > 0: 

423 h = random() 

424 delta = (h - 0.5) * delay * 0.1 

425 delay_rnd = delay + delta 

426 sleep(delay_rnd) 

427 

428 return done