Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2@file
3@brief Helpers to build grammars
4This module requires `antlr4 <http://www.antlr.org/>`_.
5and `antlr4-python3-runtime <https://pypi.python.org/pypi/antlr4-python3-runtime/>`_.
6"""
7import os
8import sys
9from pyquickhelper.loghelper import noLOG
12def _is_syntax_is_missing(language):
13 """
14 Downloads the grammar for a specific language if
15 the files is missing.
17 @param language language: python, sqlite, ...
18 @return grammar file
19 """
20 locations = {
21 "R": "https://github.com/antlr/grammars-v4/tree/master/r/",
22 "SQLite": "https://github.com/antlr/grammars-v4/blob/master/sqlite/",
23 "Pig": "http://wiki.apache.org/pig/",
24 "CSharp": "https://github.com/antlr/grammars-v4/tree/master/csharp",
25 }
27 folder = os.path.dirname(__file__)
28 filename = os.path.join(folder, language + ".g4")
29 if os.path.exists(filename):
30 return filename
31 if language in locations:
32 raise FileNotFoundError(
33 "The grammar '{0}' is not available, you should get "
34 "it from {1}".format(language, locations[language]))
35 raise KeyError(
36 "Unexpected language: '{0}', not in '{1}'".format(
37 language, ",".join(locations.keys())))
40def build_grammar(g4, version="4.9", fLOG=noLOG):
41 """
42 Compiles the grammar for a specific file.
44 @param g4 grammar format antlr4
45 @param version version of *antlr4* to use, 4.9
46 @param fLOG logging function
47 @return list of files
49 The compilation must be done with `antlr4 <http://www.antlr.org/>`_.
50 It generates a lexer and a parser which can be imported in Python.
51 The options for the command line are described at:
52 `antlr4 options
53 <https://theantlrguy.atlassian.net/wiki/display/ANTLR4/Options>`_.
55 .. exref::
56 :title: Builds a Antlr4 grammar
58 See `grammars-v4 <https://github.com/antlr/grammars-v4>`_
60 ::
62 build_grammar("R.g4")
63 """
64 if not g4.endswith(".g4"):
65 fold = os.path.abspath(os.path.dirname(__file__))
66 g4 = os.path.join(fold, g4 + ".g4")
68 url = "http://www.antlr.org/download/antlr-{0}-complete.jar".format(
69 version)
70 spl = url.split("/")
71 domain, name = "/".join(spl[:-1]) + "/", spl[-1]
72 folder = os.path.abspath(os.path.dirname(__file__))
73 final = os.path.join(folder, name)
75 if not os.path.exists(final):
76 from ..datasource.http_retrieve import download_data
77 name = download_data(name, website=domain, whereTo=folder)
78 if not os.path.exists(name):
79 raise FileNotFoundError("unable to download: " + url)
81 path = os.environ.get("CLASSPATH", "")
82 if name not in path:
83 path = ".;{0}\\antlr-{1}-complete.jar".format(folder, version)
84 else:
85 path = ".;{0}\\antlr-{1}-complete.jar;{2}".format(
86 folder,
87 version,
88 os.environ["CLASSPATH"])
90 os.environ["CLASSPATH"] = path
91 fLOG("CLASSPATH", os.environ["CLASSPATH"])
93 # we remove -rc...
94 version = version.split("-")[0]
96 cmd = "org.antlr.v4.Tool "
97 if "Lexer" not in g4:
98 cmd += "-Dlanguage=Python3 "
99 cmd += g4
100 from pyquickhelper.loghelper import run_cmd
101 out, err = run_cmd("java " + cmd, wait=True, fLOG=fLOG)
103 def compiled():
104 if "Lexer" in g4:
105 lexer = g4.replace(".g4", ".tokens")
106 else:
107 lexer = g4.replace(".g4", ".py")
108 return os.path.exists(lexer)
110 if not compiled() or (len(err) > 0 and "error" in err):
112 javapath = r'C:\Program Files\Java\jre7\bin\java.exe'
113 os.environ["PATH"] = os.environ["PATH"] + ";" + javapath
114 if sys.platform.startswith("win") and os.path.exists(javapath):
115 out, err = run_cmd(
116 '"' + javapath + '" ' + cmd, wait=True, fLOG=fLOG)
117 if not compiled() or (len(err) > 0 and "error" in err):
118 raise Exception(
119 "unable to compile: " +
120 final +
121 "\nCLASSPATH:\n" +
122 os.environ["CLASSPATH"] +
123 "\nERR:\n" +
124 err +
125 "\nCMD:\njava " +
126 cmd +
127 "\nYou should do it manually.")
128 elif err:
129 err_lines = err.split(err)
130 err_lines = [_ for _ in err_lines if not _.startswith("warning(")]
131 err2 = "\n".join(err_lines).strip("\n ")
132 if len(err2) > 0:
133 raise Exception(
134 "unable to compile: " +
135 final +
136 "\nCLASSPATH:\n" +
137 os.environ["CLASSPATH"] +
138 "\nERR:\n" +
139 err +
140 "\nCMD:\njava " +
141 cmd)
143 if os.environ.get("USERNAME", os.environ.get("USER", "")) in g4:
144 dest = os.path.dirname(g4)
145 for name in os.listdir(dest):
146 if "Parser" not in name and "Lexer" not in name and \
147 "Token" not in name and "Listener" not in name:
148 continue
149 full = os.path.join(dest, name)
150 with open(full, "r", encoding="utf-8") as f:
151 content = f.read()
152 content1 = content.replace(dest, "")
153 if content1 != content:
154 fLOG("[build_grammar] modified", name)
155 with open(full, "w", encoding="utf-8") as f:
156 f.write(content1)
158 return out + "\n---ERR---\n" + err