Coverage for pyquickhelper/server/filestore_fastapi.py: 90%

1# -*- coding:utf-8 -*- 

2""" 

3@file 

4@brief Simple class to store and retrieve files through an API. 

5""" 

6import os 

7import io 

8from typing import Optional 

9from fastapi import FastAPI, Request, HTTPException 

10from pydantic import BaseModel # pylint: disable=E0611 

11from .filestore_sqlite import SqlLite3FileStore 

12 

13 

14class Item(BaseModel): 

15 name: Optional[str] # pylint: disable=E1136 

16 format: Optional[str] # pylint: disable=E1136 

17 team: Optional[str] # pylint: disable=E1136 

18 project: Optional[str] # pylint: disable=E1136 

19 version: Optional[int] # pylint: disable=E1136 

20 content: Optional[str] # pylint: disable=E1136 

21 password: str 

22 

23 

24class Metric(BaseModel): 

25 name: Optional[str] 

26 project: Optional[str] 

27 password: str 

28 

29 

30class Query(BaseModel): 

31 name: Optional[str] # pylint: disable=E1136 

32 team: Optional[str] # pylint: disable=E1136 

33 project: Optional[str] # pylint: disable=E1136 

34 version: Optional[int] # pylint: disable=E1136 

35 password: str 

36 

37 

38class QueryL(BaseModel): 

39 name: Optional[str] # pylint: disable=E1136 

40 team: Optional[str] # pylint: disable=E1136 

41 project: Optional[str] # pylint: disable=E1136 

42 version: Optional[int] # pylint: disable=E1136 

43 limit: Optional[int] # pylint: disable=E1136 

44 password: str 

45 

46 

47def create_fast_api_app(db_path, password): 

48 """ 

49 Creates a :epkg:`REST` application based on :epkg:`FastAPI`. 

50 

51 :return: app 

52 """ 

53 store = SqlLite3FileStore(db_path) 

54 

55 async def get_root(): 

56 return {"pyquickhelper": "FastAPI to load and query files"} 

57 

58 async def submit(item: Item, request: Request): 

59 if item.password != password: 

60 raise HTTPException(status_code=401, detail="Wrong password") 

61 kwargs = dict(name=item.name, format=item.format, 

62 team=item.team, project=item.project, 

63 version=item.version, content=item.content) 

64 kwargs['metadata'] = dict(client=request.client) 

65 res = store.submit(**kwargs) 

66 if 'content' in res: 

67 del res['content'] 

68 return res 

69 

70 async def metrics(query: Metric, request: Request): 

71 if query.password != password: 

72 raise HTTPException(status_code=401, detail="Wrong password") 

73 res = list(store.enumerate_data( 

74 name=query.name, project=query.project, join=True)) 

75 return res 

76 

77 async def query(query: Query, request: Request): 

78 if query.password != password: 

79 raise HTTPException(status_code=401, detail="Wrong password") 

80 res = list(store.enumerate(name=query.name, team=query.team, 

81 project=query.project, version=query.version)) 

82 return res 

83 

84 async def content(query: QueryL, request: Request): 

85 if query.password != password: 

86 raise HTTPException(status_code=401, detail="Wrong password") 

87 if query.limit is None: 

88 limit = 5 

89 else: 

90 limit = query.limit 

91 res = [] 

92 for r in store.enumerate_content( 

93 name=query.name, team=query.team, project=query.project, 

94 version=query.version): 

95 if len(res) >= limit: 

96 break 

97 if "content" in r: 

98 content = r['content'] 

99 if hasattr(content, 'to_csv'): 

100 st = io.StringIO() 

101 content.to_csv(st, index=False, encoding="utf-8") 

102 r['content'] = st.getvalue() 

103 res.append(r) 

104 return res 

105 

106 app = FastAPI() 

107 app.get("/")(get_root) 

108 app.post("/submit/")(submit) 

109 app.post("/content/")(content) 

110 app.post("/metrics/")(metrics) 

111 app.post("/query/")(query) 

112 return app 

113 

114 

115def create_app(): 

116 """ 

117 Creates an instance of application class returned 

118 by @see fn create_fast_api_app. It checks that 

119 environment variables ``PYQUICKHELPER_FASTAPI_PWD`` 

120 and ``PYQUICKHELPER_FASTAPI_PATH`` are set up with 

121 a password and a filename. Otherwise, the function 

122 raised an exception. 

123 

124 Inspired from the guidelines 

125 `uvicorn/deployment <https://www.uvicorn.org/deployment/>`_, 

126 `(2) <https://www.uvicorn.org/deployment/#running-programmatically>`_. 

127 Some command lines: 

128 

129 :: 

130 

131 uvicorn --factory pyquickhelper.server.filestore_fastapi:create_app --port 8798 

132 --ssl-keyfile=./key.pem --ssl-certfile=./cert.pem 

133 gunicorn --keyfile=./key.pem --certfile=./cert.pem -k uvicorn.workers.UvicornWorker 

134 --factory pyquickhelper.server.filestore_fastapi:create_app --port 8798 

135 

136 :: 

137 

138 nohup python -m uvicorn --factory pyquickhelper.server.filestore_fastapi:create_app 

139 --port xxxx --ssl-keyfile=./key.pem --ssl-certfile=./cert.pem 

140 --host xx.xxx.xx.xxx --ssl-keyfile-password xxxx > fastapi.log & 

141 

142 :: 

143 

144 uvicorn.run("pyquickhelper.server.filestore_fastapi:create_app", 

145 host="127.0.0.1", port=8798, log_level="info", factory=True) 

146 

147 :: 

148 

149 openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -days 365 

150 """ 

151 if "PYQUICKHELPER_FASTAPI_PWD" not in os.environ: 

152 raise RuntimeError( 

153 "Environment variable PYQUICKHELPER_FASTAPI_PWD is missing.") 

154 if "PYQUICKHELPER_FASTAPI_PATH" not in os.environ: 

155 raise RuntimeError( 

156 "Environment variable PYQUICKHELPER_FASTAPI_PATH is missing.") 

157 app = create_fast_api_app(os.environ['PYQUICKHELPER_FASTAPI_PATH'], 

158 os.environ['PYQUICKHELPER_FASTAPI_PWD']) 

159 return app 

160 

161 

162def _get_password(password, env="PYQUICKHELPER_FASTAPI_PWD"): 

163 if password is None: 

164 password = os.environ.get(env, None) 

165 if password is None: 

166 raise RuntimeError( 

167 "password must be specified or environement variable " 

168 "'PYQUICKHELPER_FASTAPI_PWD'.") 

169 return password 

170 

171 

172def _post_request(client, url, data, suffix, timeout=None): 

173 if client is None: 

174 import requests 

175 resp = requests.post(f"{url.strip('/')}/{suffix}", data=data, 

176 timeout=timeout) 

177 else: 

178 resp = client.post(f"/{suffix}/", json=data) 

179 if resp.status_code != 200: 

180 del data['content'] 

181 del data['password'] 

182 raise RuntimeError( 

183 f"Post request failed due to {resp!r}\ndata={data!r}.") 

184 return resp 

185 

186 

187def fast_api_submit(df, client=None, url=None, name=None, team=None, 

188 project=None, version=None, password=None): 

189 """ 

190 Stores a dataframe into a local stores. 

191 

192 :param df: dataframe 

193 :param client: for unittest purpose 

194 :param url: API url (can be None if client is not) 

195 :param name: name 

196 :param team: team 

197 :param project: project 

198 :param version: version 

199 :param password: password for the submission 

200 :return: response 

201 """ 

202 password = _get_password(password) 

203 st = io.StringIO() 

204 df.to_csv(st, index=False, encoding="utf-8") 

205 data = dict(team=team, project=project, version=version, 

206 password=password, content=st.getvalue(), 

207 name=name, format="df") 

208 return _post_request(client, url, data, "submit") 

209 

210 

211def fast_api_query(client=None, url=None, name=None, team=None, 

212 project=None, version=None, password=None, 

213 as_df=False): 

214 """ 

215 Retrieves the list of dataframe based on partial information. 

216 

217 :param client: for unittest purpose 

218 :param url: API url (can be None if client is not) 

219 :param name: name 

220 :param team: team 

221 :param project: project 

222 :param version: version 

223 :param password: password for the submission 

224 :return: response 

225 """ 

226 password = _get_password(password) 

227 data = dict(team=team, project=project, version=version, 

228 password=password, name=name) 

229 resp = _post_request(client, url, data, "query") 

230 if as_df: 

231 import pandas 

232 return pandas.DataFrame(resp.json()) 

233 return resp.json() 

234 

235 

236def fast_api_content(client=None, url=None, name=None, team=None, 

237 project=None, version=None, limit=5, 

238 password=None, as_df=True): 

239 """ 

240 Retrieves the dataframes based on partial information. 

241 Enumerates a list of dataframes. 

242 

243 :param client: for unittest purpose 

244 :param url: API url (can be None if client is not) 

245 :param name: name 

246 :param team: team 

247 :param project: project 

248 :param version: version 

249 :param limit: maximum number of dataframes to retrieve 

250 :param as_df: returns the content as a dataframe 

251 :param password: password for the submission 

252 :return: list of dictionary, content is a dataframe 

253 """ 

254 password = _get_password(password) 

255 data = dict(team=team, project=project, version=version, 

256 password=password, name=name, limit=limit) 

257 resp = _post_request(client, url, data, "content") 

258 res = resp.json() 

259 if as_df: 

260 import pandas 

261 

262 for r in res: 

263 content = r.get('content', None) 

264 if content is None: 

265 continue 

266 if 'format' in r and r['format'] == 'df': 

267 st = io.StringIO(r['content']) 

268 df = pandas.read_csv(st, encoding="utf-8") 

269 r['content'] = df 

270 return res