Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1""" 

2@file 

3@brief Helpers to build grammars 

4This module requires `antlr4 <http://www.antlr.org/>`_. 

5and `antlr4-python3-runtime <https://pypi.python.org/pypi/antlr4-python3-runtime/>`_. 

6""" 

7import os 

8import sys 

9from pyquickhelper.loghelper import noLOG 

10 

11 

12def _is_syntax_is_missing(language): 

13 """ 

14 Downloads the grammar for a specific language if 

15 the files is missing. 

16 

17 @param language language: python, sqlite, ... 

18 @return grammar file 

19 """ 

20 locations = { 

21 "R": "https://github.com/antlr/grammars-v4/tree/master/r/", 

22 "SQLite": "https://github.com/antlr/grammars-v4/blob/master/sqlite/", 

23 "Pig": "http://wiki.apache.org/pig/", 

24 "CSharp": "https://github.com/antlr/grammars-v4/tree/master/csharp", 

25 } 

26 

27 folder = os.path.dirname(__file__) 

28 filename = os.path.join(folder, language + ".g4") 

29 if os.path.exists(filename): 

30 return filename 

31 if language in locations: 

32 raise FileNotFoundError( 

33 "The grammar '{0}' is not available, you should get " 

34 "it from {1}".format(language, locations[language])) 

35 raise KeyError( 

36 "Unexpected language: '{0}', not in '{1}'".format( 

37 language, ",".join(locations.keys()))) 

38 

39 

40def build_grammar(g4, version="4.9", fLOG=noLOG): 

41 """ 

42 Compiles the grammar for a specific file. 

43 

44 @param g4 grammar format antlr4 

45 @param version version of *antlr4* to use, 4.9 

46 @param fLOG logging function 

47 @return list of files 

48 

49 The compilation must be done with `antlr4 <http://www.antlr.org/>`_. 

50 It generates a lexer and a parser which can be imported in Python. 

51 The options for the command line are described at: 

52 `antlr4 options 

53 <https://theantlrguy.atlassian.net/wiki/display/ANTLR4/Options>`_. 

54 

55 .. exref:: 

56 :title: Builds a Antlr4 grammar 

57 

58 See `grammars-v4 <https://github.com/antlr/grammars-v4>`_ 

59 

60 :: 

61 

62 build_grammar("R.g4") 

63 """ 

64 if not g4.endswith(".g4"): 

65 fold = os.path.abspath(os.path.dirname(__file__)) 

66 g4 = os.path.join(fold, g4 + ".g4") 

67 

68 url = "http://www.antlr.org/download/antlr-{0}-complete.jar".format( 

69 version) 

70 spl = url.split("/") 

71 domain, name = "/".join(spl[:-1]) + "/", spl[-1] 

72 folder = os.path.abspath(os.path.dirname(__file__)) 

73 final = os.path.join(folder, name) 

74 

75 if not os.path.exists(final): 

76 from ..datasource.http_retrieve import download_data 

77 name = download_data(name, website=domain, whereTo=folder) 

78 if not os.path.exists(name): 

79 raise FileNotFoundError("unable to download: " + url) 

80 

81 path = os.environ.get("CLASSPATH", "") 

82 if name not in path: 

83 path = ".;{0}\\antlr-{1}-complete.jar".format(folder, version) 

84 else: 

85 path = ".;{0}\\antlr-{1}-complete.jar;{2}".format( 

86 folder, 

87 version, 

88 os.environ["CLASSPATH"]) 

89 

90 os.environ["CLASSPATH"] = path 

91 fLOG("CLASSPATH", os.environ["CLASSPATH"]) 

92 

93 # we remove -rc... 

94 version = version.split("-")[0] 

95 

96 cmd = "org.antlr.v4.Tool " 

97 if "Lexer" not in g4: 

98 cmd += "-Dlanguage=Python3 " 

99 cmd += g4 

100 from pyquickhelper.loghelper import run_cmd 

101 out, err = run_cmd("java " + cmd, wait=True, fLOG=fLOG) 

102 

103 def compiled(): 

104 if "Lexer" in g4: 

105 lexer = g4.replace(".g4", ".tokens") 

106 else: 

107 lexer = g4.replace(".g4", ".py") 

108 return os.path.exists(lexer) 

109 

110 if not compiled() or (len(err) > 0 and "error" in err): 

111 

112 javapath = r'C:\Program Files\Java\jre7\bin\java.exe' 

113 os.environ["PATH"] = os.environ["PATH"] + ";" + javapath 

114 if sys.platform.startswith("win") and os.path.exists(javapath): 

115 out, err = run_cmd( 

116 '"' + javapath + '" ' + cmd, wait=True, fLOG=fLOG) 

117 if not compiled() or (len(err) > 0 and "error" in err): 

118 raise Exception( 

119 "unable to compile: " + 

120 final + 

121 "\nCLASSPATH:\n" + 

122 os.environ["CLASSPATH"] + 

123 "\nERR:\n" + 

124 err + 

125 "\nCMD:\njava " + 

126 cmd + 

127 "\nYou should do it manually.") 

128 elif err: 

129 err_lines = err.split(err) 

130 err_lines = [_ for _ in err_lines if not _.startswith("warning(")] 

131 err2 = "\n".join(err_lines).strip("\n ") 

132 if len(err2) > 0: 

133 raise Exception( 

134 "unable to compile: " + 

135 final + 

136 "\nCLASSPATH:\n" + 

137 os.environ["CLASSPATH"] + 

138 "\nERR:\n" + 

139 err + 

140 "\nCMD:\njava " + 

141 cmd) 

142 

143 if os.environ.get("USERNAME", os.environ.get("USER", "")) in g4: 

144 dest = os.path.dirname(g4) 

145 for name in os.listdir(dest): 

146 if "Parser" not in name and "Lexer" not in name and \ 

147 "Token" not in name and "Listener" not in name: 

148 continue 

149 full = os.path.join(dest, name) 

150 with open(full, "r", encoding="utf-8") as f: 

151 content = f.read() 

152 content1 = content.replace(dest, "") 

153 if content1 != content: 

154 fLOG("[build_grammar] modified", name) 

155 with open(full, "w", encoding="utf-8") as f: 

156 f.write(content1) 

157 

158 return out + "\n---ERR---\n" + err