Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

""" 

@file 

@brief Various function to clean the code. 

""" 

 

import os 

import autopep8 

from ..filehelper.synchelper import explore_folder 

 

 

def remove_extra_spaces_and_pep8(filename, apply_pep8=True, aggressive=False, is_string=None): 

""" 

Removes extra spaces in a filename, replaces the file in place. 

 

@param filename file name or string (but it assumes it is python). 

@param apply_pep8 if True, calls :epkg:`autopep8` on the file 

@param aggressive more aggressive 

@param is_string force *filename* to be a string 

@return number of removed extra spaces 

""" 

encoding = None 

initial_content = None 

if "\n" in filename or (is_string is not None and is_string): 

ext = ".py" 

lines = filename.replace("\r", "").split("\n") 

filename = None 

else: 

ext = os.path.splitext(filename)[-1] 

if ext in (".bat", ".py", ".sh", ".pyx", ".pxd"): 

try: 

with open(filename, "r") as f: 

lines = f.readlines() 

encoding = None 

except PermissionError as e: 

raise PermissionError(filename) from e 

except UnicodeDecodeError as e: 

try: 

with open(filename, "r", encoding="utf-8") as f: 

lines = f.readlines() 

encoding = "utf-8" 

except Exception: 

raise Exception( 

"unable to load file {} due to unicode errors".format(filename)) from e 

initial_content = "".join(lines) 

else: 

try: 

with open(filename, "r", encoding="utf-8-sig") as f: 

lines = f.readlines() 

encoding = "utf-8" 

except PermissionError as e: 

raise PermissionError(filename) from e 

except UnicodeDecodeError as e: 

try: 

with open(filename, "r") as f: 

lines = f.readlines() 

encoding = None 

except Exception: 

raise Exception( 

"unable to load file {} due to unicode errors".format(filename)) from e 

initial_content = "".join(lines) 

 

if filename is not None and len(lines) == 0 and not filename.endswith("__init__.py"): 

raise ValueError( 

"File '{0}' is empty, encoding='{1}'.".format(filename, encoding)) 

 

if filename is not None and ext in (".py", ".pyx", ".pxd"): 

if encoding is not None and len(lines) > 0 and "#-*-coding:utf-8-*-" in lines[0].replace(" ", ""): 

with open(filename, "r", encoding="utf8") as f: 

try: 

lines = f.readlines() 

except UnicodeDecodeError as e: 

raise Exception("unable to read: " + filename) from e 

encoding = "utf8" 

else: 

encoding = None 

 

def cdiff(lines): 

lines2 = [_.rstrip(" \r\n") for _ in lines] 

last = len(lines2) - 1 

while last >= 0 and len(lines2[last]) == 0: 

last -= 1 

last += 1 

lines2 = lines2[:last] 

 

diff = len("".join(lines)) - len("\n".join(lines2)) + len(lines) 

return diff, lines2 

 

diff, lines2 = cdiff(lines) 

 

if filename is not None: 

ext = os.path.splitext(filename)[-1] 

if ext in (".py", ) and apply_pep8: 

options = ['', '-a'] if aggressive else [''] 

options.extend(["--ignore=E402"]) 

r = autopep8.fix_code( 

"\n".join(lines2), options=autopep8.parse_args(options)) 

 

if len(lines) > 0 and (len(lines2) == 0 or len(lines2) < len(lines) // 2): 

raise ValueError("Resulting file is empty for '{3}',\ninitial number of lines {0} encoding='{1}' diff={2}".format( 

len(lines), encoding, diff, filename)) 

if filename is None: 

return r 

elif r != initial_content: 

if encoding is None: 

with open(filename, "w") as f: 

f.write(r) 

else: 

with open(filename, "w", encoding="utf8") as f: 

f.write(r) 

if r != "".join(lines): 

diff, lines2 = cdiff(r.split("\n")) 

else: 

diff = 0 

elif ext in (".rst", ".md", ".pyx", ".pxd"): 

lines2 = [_.replace("\r", "").rstrip("\n ") for _ in lines] 

rem = set() 

for i, line in enumerate(lines2): 

if i >= 1 and line == lines2[i - 1] == "": 

rem.add(i) 

lines2 = [_ for i, _ in enumerate(lines2) if i not in rem] 

if len(lines) > 0 and len(lines2[-1]) > 0: 

lines2.append("") 

if len(lines) > 0 and len(lines2) == 0: 

begin = 5 if len(lines) > 5 else len(lines) 

mes = "Resulting file is empty for '{4}',\ninitial number of lines {0} encoding='{1}' len(rem)={2} diff={3}\nBeginning:\n{5}" 

raise ValueError(mes.format(len(lines), encoding, len( 

rem), diff, filename, "".join(lines[:begin]))) 

if len(lines2) < len(lines) // 2: 

lines2_ = [_ for _ in lines2 if _ and _ != "\n"] 

lines_ = [_ for _ in lines if _ and _ != "\n"] 

if len(lines2_) < len(lines_) // 2: 

begin = 5 if len(lines_) > 5 else len(lines_) 

mes = "Resulting file is almost empty for '{4}',\ninitial number of lines {0} encoding='{1}' " + \ 

"len(rem)={2} diff={3}\nBeginning:\n{5}" 

raise ValueError(mes.format(len(lines_), encoding, len( 

rem), diff, filename, "".join(lines_[:begin]))) 

rl = "".join(lines) 

r2 = "\n".join(lines2) 

if r2 != rl: 

if encoding is None: 

with open(filename, "w") as f: 

f.write("\n".join(lines2)) 

else: 

with open(filename, "w", encoding="utf8") as f: 

f.write("\n".join(lines2)) 

diff = max(1, diff) 

else: 

diff = 0 

elif diff != 0: 

if len(lines) > 0 and (len(lines2) == 0 or len(lines2) < len(lines) // 2): 

raise ValueError("Resulting file is empty for '{3}',\ninitial number of lines {0} encoding='{1}' diff={2}".format( 

len(lines), encoding, diff, filename)) 

r1 = "".join(lines) 

r2 = "\n".join(lines2) 

if r2 != r1: 

if encoding is None: 

with open(filename, "w") as f: 

f.write("\n".join(lines2)) 

else: 

with open(filename, "w", encoding="utf8") as f: 

f.write("\n".join(lines2)) 

 

if not os.path.exists(filename): 

raise FileNotFoundError( 

"issue when applying autopep8 with filename: {0}".format(filename)) 

return diff 

 

 

def remove_extra_spaces_folder( 

folder, extensions=(".py", ".rst", ".md", ".pyx", ".pxd"), apply_pep8=True, 

file_filter=None): 

""" 

Removes extra files in a folder for specific file extensions. 

 

@param folder folder to explore 

@param extensions list of file extensions to process 

@param apply_pep8 if True, calls :epkg:`autopep8` on the file 

@param file_filter None of function which filters based on the filename 

@return the list of modified files 

 

The function does not check files having 

``/build/`` or ``/dist/`` or ``temp_`` 

or ``/build2/`` or ``/build3/`` 

in their name. 

 

The signature of *file_filter* is the following: 

 

:: 

 

def file_filter(filename): 

return True or False 

""" 

neg_pattern = "|".join("[/\\\\]{0}[/\\\\]".format(_) for _ in ["build", "build2", "build3", 

"dist", "_venv", "_todo", "dist_module27", "_virtualenv"]) 

files = explore_folder(folder, neg_pattern=neg_pattern, fullname=True)[1] 

mod = [] 

for f in files: 

fl = f.lower().replace("\\", "/") 

if "/temp_" not in fl and "/build/" not in fl \ 

and "/dist/" not in fl \ 

and "/build2/" not in fl \ 

and "/build3/" not in fl \ 

and "/_virtualenv/" not in fl \ 

and ".egg/" not in fl \ 

and "/_venv/" not in fl \ 

and "/_todo/" not in fl \ 

and "/dist_module27" not in fl \ 

and "automation_done.rst" not in fl \ 

and "auto_import.rst" not in fl \ 

and os.stat(f).st_size < 200000 \ 

and (file_filter is None or file_filter(f)): 

ext = os.path.splitext(f)[-1] 

if ext in extensions: 

d = remove_extra_spaces_and_pep8(f, apply_pep8=apply_pep8) 

if d != 0: 

mod.append(f) 

return mod