Note
Click here to download the full example code
Benchmark of onnxruntime on LogisticRegression¶
The example uses what pymlbenchmark implements, in particular
class OnnxRuntimeBenchPerfTestBinaryClassification
which defines a side-by-side benchmark to compare the prediction
function between scikit-learn, onnxruntime and
a simple numpy implementation.
Benchmark function¶
from time import perf_counter as time
import numpy
import pandas
import matplotlib.pyplot as plt
import sklearn
from sklearn.linear_model import LogisticRegression
try:
from sklearn.utils._testing import ignore_warnings
except ImportError:
from sklearn.utils.testing import ignore_warnings
from scipy.special import expit
from pymlbenchmark.context import machine_information
from pymlbenchmark.benchmark import BenchPerf
from pymlbenchmark.external import OnnxRuntimeBenchPerfTestBinaryClassification
from pymlbenchmark.plotting import plot_bench_results
class OnnxRuntimeBenchPerfTestBinaryClassification3(
OnnxRuntimeBenchPerfTestBinaryClassification):
"""
Overwrites the class to add a pure python implementation
of the logistic regression.
"""
def fcts(self, dim=None, **kwargs):
def predict_py_predict(X, model=self.skl):
coef = model.coef_
intercept = model.intercept_
pred = numpy.dot(X, coef.T) + intercept
return (pred >= 0).astype(numpy.int32)
def predict_py_predict_proba(X, model=self.skl):
coef = model.coef_
intercept = model.intercept_
pred = numpy.dot(X, coef.T) + intercept
decision_2d = numpy.c_[-pred, pred]
return expit(decision_2d)
res = OnnxRuntimeBenchPerfTestBinaryClassification.fcts(
self, dim=dim, **kwargs)
res.extend([
{'method': 'predict', 'lib': 'py', 'fct': predict_py_predict},
{'method': 'predict_proba', 'lib': 'py',
'fct': predict_py_predict_proba},
])
return res
@ignore_warnings(category=FutureWarning)
def run_bench(repeat=100, verbose=False):
pbefore = dict(dim=[1, 5, 10, 20, 50, 100, 150],
fit_intercept=[True, False])
pafter = dict(N=[1, 10, 100, 1000, 10000])
test = lambda dim=None, **opts: (
OnnxRuntimeBenchPerfTestBinaryClassification3(
LogisticRegression, dim=dim, **opts))
bp = BenchPerf(pbefore, pafter, test)
with sklearn.config_context(assume_finite=True):
start = time()
results = list(bp.enumerate_run_benchs(repeat=repeat, verbose=verbose))
end = time()
results_df = pandas.DataFrame(results)
print("Total time = %0.3f sec\n" % (end - start))
return results_df
Run the benchmark¶
df = run_bench(verbose=True)
df.to_csv("bench_plot_onnxruntime_logistic_regression.perf.csv", index=False)
print(df.head(n=4).T)
0%| | 0/70 [00:00<?, ?it/s]
1%|1 | 1/70 [00:06<07:04, 6.16s/it]
3%|2 | 2/70 [00:07<03:57, 3.50s/it]
4%|4 | 3/70 [00:14<05:43, 5.13s/it]
6%|5 | 4/70 [01:15<29:43, 27.02s/it]
7%|7 | 5/70 [01:17<19:21, 17.87s/it]
9%|8 | 6/70 [01:18<12:58, 12.16s/it]
10%|# | 7/70 [01:19<09:08, 8.71s/it]
11%|#1 | 8/70 [01:26<08:25, 8.15s/it]
13%|#2 | 9/70 [02:25<24:31, 24.12s/it]
14%|#4 | 10/70 [02:27<17:13, 17.23s/it]
16%|#5 | 11/70 [02:28<12:04, 12.29s/it]
17%|#7 | 12/70 [02:30<08:44, 9.04s/it]
19%|#8 | 13/70 [02:37<07:58, 8.40s/it]
20%|## | 14/70 [03:38<22:46, 24.41s/it]
21%|##1 | 15/70 [03:40<16:07, 17.58s/it]
23%|##2 | 16/70 [03:41<11:21, 12.62s/it]
24%|##4 | 17/70 [03:43<08:13, 9.31s/it]
26%|##5 | 18/70 [03:50<07:27, 8.61s/it]
27%|##7 | 19/70 [04:50<20:35, 24.22s/it]
29%|##8 | 20/70 [04:52<14:35, 17.51s/it]
30%|### | 21/70 [04:53<10:16, 12.58s/it]
31%|###1 | 22/70 [04:55<07:25, 9.29s/it]
33%|###2 | 23/70 [05:02<06:44, 8.60s/it]
34%|###4 | 24/70 [06:03<18:36, 24.28s/it]
36%|###5 | 25/70 [06:05<13:10, 17.56s/it]
37%|###7 | 26/70 [06:06<09:15, 12.62s/it]
39%|###8 | 27/70 [06:07<06:40, 9.32s/it]
40%|#### | 28/70 [06:14<06:03, 8.65s/it]
41%|####1 | 29/70 [07:15<16:39, 24.37s/it]
43%|####2 | 30/70 [07:18<11:49, 17.73s/it]
44%|####4 | 31/70 [07:19<08:16, 12.74s/it]
46%|####5 | 32/70 [07:20<05:57, 9.41s/it]
47%|####7 | 33/70 [07:28<05:22, 8.72s/it]
49%|####8 | 34/70 [08:29<14:41, 24.49s/it]
50%|##### | 35/70 [08:31<10:21, 17.74s/it]
51%|#####1 | 36/70 [08:32<07:13, 12.75s/it]
53%|#####2 | 37/70 [08:34<05:10, 9.41s/it]
54%|#####4 | 38/70 [08:41<04:39, 8.72s/it]
56%|#####5 | 39/70 [09:42<12:42, 24.59s/it]
57%|#####7 | 40/70 [09:45<08:58, 17.96s/it]
59%|#####8 | 41/70 [09:46<06:14, 12.90s/it]
60%|###### | 42/70 [09:48<04:26, 9.53s/it]
61%|######1 | 43/70 [09:55<04:02, 8.97s/it]
63%|######2 | 44/70 [11:00<11:10, 25.79s/it]
64%|######4 | 45/70 [11:03<07:53, 18.95s/it]
66%|######5 | 46/70 [11:04<05:26, 13.60s/it]
67%|######7 | 47/70 [11:06<03:50, 10.03s/it]
69%|######8 | 48/70 [11:14<03:24, 9.29s/it]
70%|####### | 49/70 [12:19<09:07, 26.09s/it]
71%|#######1 | 50/70 [12:23<06:28, 19.44s/it]
73%|#######2 | 51/70 [12:24<04:24, 13.94s/it]
74%|#######4 | 52/70 [12:26<03:04, 10.27s/it]
76%|#######5 | 53/70 [12:34<02:43, 9.60s/it]
77%|#######7 | 54/70 [13:44<07:24, 27.75s/it]
79%|#######8 | 55/70 [13:47<05:04, 20.30s/it]
80%|######## | 56/70 [13:48<03:23, 14.54s/it]
81%|########1 | 57/70 [13:50<02:19, 10.70s/it]
83%|########2 | 58/70 [13:58<01:58, 9.90s/it]
84%|########4 | 59/70 [15:08<05:07, 27.97s/it]
86%|########5 | 60/70 [15:12<03:27, 20.73s/it]
87%|########7 | 61/70 [15:13<02:13, 14.86s/it]
89%|########8 | 62/70 [15:15<01:27, 10.93s/it]
90%|######### | 63/70 [15:23<01:11, 10.24s/it]
91%|#########1| 64/70 [16:37<02:55, 29.29s/it]
93%|#########2| 65/70 [16:41<01:48, 21.61s/it]
94%|#########4| 66/70 [16:42<01:01, 15.45s/it]
96%|#########5| 67/70 [16:43<00:34, 11.37s/it]
97%|#########7| 68/70 [16:52<00:20, 10.50s/it]
99%|#########8| 69/70 [18:04<00:29, 29.04s/it]
99%|#########8| 69/70 [18:04<00:15, 15.72s/it]
Total time = 1085.073 sec
0 ... 3
method predict ... predict_proba
lib skl ... skl
skl_nb_base_estimators 1.0 ... 1.0
N 1 ... 1
dim 1 ... 1
fit_intercept True ... True
repeat 100 ... 100
number 1 ... 1
min 0.000388 ... 0.000458
max 0.012667 ... 0.000678
min3 0.000388 ... 0.000458
max3 0.000496 ... 0.000529
mean 0.000641 ... 0.000471
lower 0.000388 ... 0.000458
upper 0.003974 ... 0.000521
count 100 ... 100
median 0.000392 ... 0.000462
error_c 0 ... 0
onnx_nodes NaN ... NaN
onnx_opset NaN ... NaN
ort_size NaN ... NaN
[21 rows x 4 columns]
Extract information about the machine used¶
pkgs = ['numpy', 'pandas', 'sklearn', 'skl2onnx',
'onnxruntime', 'onnx', 'mlprodict']
dfi = pandas.DataFrame(machine_information(pkgs))
dfi.to_csv("bench_plot_onnxruntime_logistic_regression.time.csv", index=False)
print(dfi)
name ... value
0 date ... NaN
1 python ... 3.9.1 (default, Jan 18 2021, 16:35:58) \n[GCC ...
2 platform ... linux
3 OS ... Linux-4.19.0-23-amd64-x86_64-with-glibc2.28
4 machine ... x86_64
5 processor ...
6 release ... 4.19.0-23-amd64
7 architecture ... (64bit, ELF)
8 arch ... X86_64
9 brand_raw ... Intel(R) Atom(TM) CPU C2750 @ 2.40GHz
10 count ... 8
11 flags ... 3dnowprefetch acpi aes aperfmperf apic arat ar...
12 hz_advertised ... [2400000000, 0]
13 l1_data_cache_size ... 24576
14 l1_instruction_cache_size ... 32768
15 l2_cache_associativity ... 8
16 l2_cache_line_size ... 1024
17 l2_cache_size ... 1048576
18 l3_cache_size ... 1048576
19 stepping ... 8
20 mlprodict ... NaN
21 numpy ... openblas, language=c
22 onnx ... opset=17
23 onnxruntime ... CPU
24 pandas ... NaN
25 skl2onnx ... NaN
26 sklearn ... NaN
[27 rows x 3 columns]
Plot the results¶
def label_fct(la):
la = la.replace("onxpython_compiled", "opy")
la = la.replace("onxpython", "opy")
la = la.replace("onxonnxruntime1", "ort")
la = la.replace("fit_intercept", "fi")
la = la.replace("True", "1")
la = la.replace("False", "0")
return la
def color_fct(la, col):
if "onxpython_compiled" in la:
return "red"
if "onxpython" in la:
return "red"
return col
plot_bench_results(
df, row_cols=['N', 'fit_intercept'], col_cols='method', x_value='dim',
title="LogisticRegression\nBenchmark scikit-learn / onnxruntime",
label_fct=label_fct, color_fct=color_fct)
plt.show()
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:143: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
piv = ds.pivot(*y_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:153: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
lower_piv = ds.pivot(*lower_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:164: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
upper_piv = ds.pivot(*upper_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:143: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
piv = ds.pivot(*y_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:153: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
lower_piv = ds.pivot(*lower_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:164: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
upper_piv = ds.pivot(*upper_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:143: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
piv = ds.pivot(*y_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:153: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
lower_piv = ds.pivot(*lower_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:164: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
upper_piv = ds.pivot(*upper_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:143: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
piv = ds.pivot(*y_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:153: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
lower_piv = ds.pivot(*lower_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:164: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
upper_piv = ds.pivot(*upper_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:143: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
piv = ds.pivot(*y_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:153: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
lower_piv = ds.pivot(*lower_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:164: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
upper_piv = ds.pivot(*upper_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:143: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
piv = ds.pivot(*y_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:153: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
lower_piv = ds.pivot(*lower_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:164: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
upper_piv = ds.pivot(*upper_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:143: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
piv = ds.pivot(*y_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:153: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
lower_piv = ds.pivot(*lower_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:164: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
upper_piv = ds.pivot(*upper_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:143: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
piv = ds.pivot(*y_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:153: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
lower_piv = ds.pivot(*lower_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:164: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
upper_piv = ds.pivot(*upper_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:143: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
piv = ds.pivot(*y_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:153: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
lower_piv = ds.pivot(*lower_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:164: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
upper_piv = ds.pivot(*upper_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:143: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
piv = ds.pivot(*y_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:153: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
lower_piv = ds.pivot(*lower_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:164: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
upper_piv = ds.pivot(*upper_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:143: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
piv = ds.pivot(*y_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:153: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
lower_piv = ds.pivot(*lower_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:164: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
upper_piv = ds.pivot(*upper_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:143: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
piv = ds.pivot(*y_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:153: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
lower_piv = ds.pivot(*lower_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:164: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
upper_piv = ds.pivot(*upper_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:143: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
piv = ds.pivot(*y_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:153: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
lower_piv = ds.pivot(*lower_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:164: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
upper_piv = ds.pivot(*upper_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:143: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
piv = ds.pivot(*y_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:153: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
lower_piv = ds.pivot(*lower_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:164: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
upper_piv = ds.pivot(*upper_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:143: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
piv = ds.pivot(*y_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:153: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
lower_piv = ds.pivot(*lower_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:164: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
upper_piv = ds.pivot(*upper_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:143: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
piv = ds.pivot(*y_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:153: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
lower_piv = ds.pivot(*lower_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:164: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
upper_piv = ds.pivot(*upper_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:143: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
piv = ds.pivot(*y_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:153: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
lower_piv = ds.pivot(*lower_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:164: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
upper_piv = ds.pivot(*upper_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:143: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
piv = ds.pivot(*y_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:153: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
lower_piv = ds.pivot(*lower_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:164: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
upper_piv = ds.pivot(*upper_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:143: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
piv = ds.pivot(*y_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:153: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
lower_piv = ds.pivot(*lower_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:164: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
upper_piv = ds.pivot(*upper_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:143: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
piv = ds.pivot(*y_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:153: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
lower_piv = ds.pivot(*lower_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:164: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
upper_piv = ds.pivot(*upper_cols)
Total running time of the script: ( 18 minutes 55.759 seconds)