1""" 

2@file 

3@brief Command line about validation of prediction runtime. 

4""" 

5from pandas import DataFrame 

6 

7 

8def benchmark_replay(folder, runtime='python', time_kwargs=None, 

9 skip_long_test=True, time_kwargs_fact=None, 

10 time_limit=4, out=None, verbose=1, fLOG=print): 

11 """ 

12 The command rerun a benchmark if models were stored by 

13 command line `vaidate_runtime`. 

14 

15 :param folder: where to find pickled files 

16 :param runtime: runtimes, comma separated list 

17 :param verbose: integer from 0 (None) to 2 (full verbose) 

18 :param out: output raw results into this file (excel format) 

19 :param time_kwargs: a dictionary which defines the number of rows and 

20 the parameter *number* and *repeat* when benchmarking a model, 

21 the value must follow :epkg:`json` format 

22 :param skip_long_test: skips tests for high values of N if 

23 they seem too long 

24 :param time_kwargs_fact: to multiply number and repeat in 

25 *time_kwargs* depending on the model 

26 (see :func:`_multiply_time_kwargs <mlprodict.onnxrt.validate.validate_helper._multiply_time_kwargs>`) 

27 :param time_limit: to stop benchmarking after this limit of time 

28 :param fLOG: logging function 

29 

30 .. cmdref:: 

31 :title: Replay a benchmark of stored converted models by validate_runtime 

32 :cmd: -m mlprodict benchmark_replay --help 

33 :lid: l-cmd-benchmark_replay 

34 

35 The command rerun a benchmark if models were stored by 

36 command line `vaidate_runtime`. 

37 

38 Example:: 

39 

40 python -m mlprodict benchmark_replay --folder dumped --out bench_results.xlsx 

41 

42 Parameter ``--time_kwargs`` may be used to reduce or increase 

43 bencharmak precisions. The following value tells the function 

44 to run a benchmarks with datasets of 1 or 10 number, to repeat 

45 a given number of time *number* predictions in one row. 

46 The total time is divided by :math:`number \\times repeat``. 

47 Parameter ``--time_kwargs_fact`` may be used to increase these 

48 number for some specific models. ``'lin'`` multiplies 

49 by 10 number when the model is linear. 

50 

51 :: 

52 

53 -t "{\\"1\\":{\\"number\\":10,\\"repeat\\":10},\\"10\\":{\\"number\\":5,\\"repeat\\":5}}" 

54 """ 

55 from ..onnxrt.validate.validate_benchmark_replay import enumerate_benchmark_replay # pylint: disable=E0402 

56 

57 rows = list(enumerate_benchmark_replay( 

58 folder=folder, runtime=runtime, time_kwargs=time_kwargs, 

59 skip_long_test=skip_long_test, time_kwargs_fact=time_kwargs_fact, 

60 time_limit=time_limit, verbose=verbose, fLOG=fLOG)) 

61 if out is not None: 

62 df = DataFrame(rows) 

63 df.to_excel(out, index=False) 

64 return rows