ONNX and Python Runtime

This package implements a python runtime for ONNX in class OnnxInference. It does not depend on scikit-learn, only numpy and this module. However, this module was not really developped to get the fastest python runtime but mostly to easily develop converters.

Python Runtime for ONNX

Class OnnxInference implements a python runtime for a subset of ONNX operators needed to convert many scikit-learn models.

<<<

import numpy
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.cluster import KMeans
from skl2onnx import to_onnx
from mlprodict.onnxrt import OnnxInference

iris = load_iris()
X = iris.data.astype(numpy.float32)
X_train, X_test = train_test_split(X)
clr = KMeans(n_clusters=3)
clr.fit(X_train)

model_def = to_onnx(clr, X_train.astype(numpy.float32),
                    target_opset=12)

oinf = OnnxInference(model_def, runtime='python')
print(oinf.run({'X': X_test[:5]}))

>>>

    {'label': array([0, 0, 0, 0, 0]), 'scores': array([[0.448, 3.01 , 1.958],
           [0.688, 3.45 , 1.491],
           [0.769, 3.005, 2.334],
           [0.489, 3.359, 1.613],
           [0.861, 4.123, 1.152]], dtype=float32)}

It is usually useful to get information on intermediate results in the graph itself to understand where the discrepencies begin.

<<<

import numpy
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.cluster import KMeans
from skl2onnx import to_onnx
from mlprodict.onnxrt import OnnxInference

iris = load_iris()
X = iris.data.astype(numpy.float32)
X_train, X_test = train_test_split(X)
clr = KMeans(n_clusters=3)
clr.fit(X_train)

model_def = to_onnx(clr, X_train.astype(numpy.float32),
                    target_opset=12)

oinf = OnnxInference(model_def, runtime='python')
print(oinf.run({'X': X_test[:5]}, verbose=1, fLOG=print))

>>>

    +ki='Ad_Addcst': (3,) (dtype=float32 min=39.611202239990234 max=94.24984741210938)
    +ki='Ge_Gemmcst': (3, 4) (dtype=float32 min=0.25454550981521606 max=6.855999946594238)
    +ki='Mu_Mulcst': (1,) (dtype=float32 min=0.0 max=0.0)
    -- OnnxInference: run 7 nodes
    Onnx-ReduceSumSquare(X) -> Re_reduced0    (name='Re_ReduceSumSquare')
    +kr='Re_reduced0': (5, 1) (dtype=float32 min=51.65999984741211 max=86.2199935913086)
    Onnx-Mul(Re_reduced0, Mu_Mulcst) -> Mu_C0    (name='Mu_Mul')
    +kr='Mu_C0': (5, 1) (dtype=float32 min=0.0 max=0.0)
    Onnx-Gemm(X, Ge_Gemmcst, Mu_C0) -> Ge_Y0    (name='Ge_Gemm')
    +kr='Ge_Y0': (5, 3) (dtype=float32 min=-180.04959106445312 max=-83.78909301757812)
    Onnx-Add(Re_reduced0, Ge_Y0) -> Ad_C01    (name='Ad_Add')
    +kr='Ad_C01': (5, 3) (dtype=float32 min=-93.82959747314453 max=-18.54999542236328)
    Onnx-Add(Ad_Addcst, Ad_C01) -> Ad_C0    (name='Ad_Add1')
    +kr='Ad_C0': (5, 3) (dtype=float32 min=0.15047454833984375 max=21.061206817626953)
    Onnx-ArgMin(Ad_C0) -> label    (name='Ar_ArgMin')
    +kr='label': (5,) (dtype=int64 min=1 max=2)
    Onnx-Sqrt(Ad_C0) -> scores    (name='Sq_Sqrt')
    +kr='scores': (5, 3) (dtype=float32 min=0.38791048526763916 max=4.589249134063721)
    {'label': array([2, 1, 1, 1, 1]), 'scores': array([[4.589, 1.442, 0.648],
           [2.98 , 0.388, 2.222],
           [2.501, 0.876, 2.663],
           [3.731, 0.643, 1.411],
           [2.735, 0.873, 2.693]], dtype=float32)}

The verbosity can be increased.

<<<

import numpy
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.cluster import KMeans
from skl2onnx import to_onnx
from mlprodict.onnxrt import OnnxInference

iris = load_iris()
X = iris.data.astype(numpy.float32)
X_train, X_test = train_test_split(X)
clr = KMeans(n_clusters=3)
clr.fit(X_train)

model_def = to_onnx(clr, X_train.astype(numpy.float32),
                    target_opset=12)

oinf = OnnxInference(model_def, runtime='python')
print(oinf.run({'X': X_test[:5]}, verbose=3, fLOG=print))

>>>

    +ki='Ad_Addcst': (3,) (dtype=float32 min=38.033653259277344 max=93.627685546875
    [63.452 93.628 38.034]
    +ki='Ge_Gemmcst': (3, 4) (dtype=float32 min=0.23636382818222046 max=6.889655113220215
    [[5.87  2.752 4.396 1.448]
     [6.89  3.086 5.71  2.007]
     [4.945 3.376 1.458 0.236]]
    +ki='Mu_Mulcst': (1,) (dtype=float32 min=0.0 max=0.0
    [0.]
    -kv='X' shape=(5, 4) dtype=float32 min=0.4000000059604645 max=6.800000190734863
    -- OnnxInference: run 7 nodes
    Onnx-ReduceSumSquare(X) -> Re_reduced0    (name='Re_ReduceSumSquare')
    +kr='Re_reduced0': (5, 1) (dtype=float32 min=54.2599983215332 max=91.6199951171875)
    [[79.08]
     [84.58]
     [91.62]
     [70.55]
     [54.26]]
    Onnx-Mul(Re_reduced0, Mu_Mulcst) -> Mu_C0    (name='Mu_Mul')
    +kr='Mu_C0': (5, 1) (dtype=float32 min=0.0 max=0.0)
    [[0.]
     [0.]
     [0.]
     [0.]
     [0.]]
    Onnx-Gemm(X, Ge_Gemmcst, Mu_C0) -> Ge_Y0    (name='Ge_Gemm')
    +kr='Ge_Y0': (5, 3) (dtype=float32 min=-185.04481506347656 max=-90.64666748046875)
    [[-141.499 -171.421 -100.817]
     [-146.007 -177.701 -101.444]
     [-151.906 -185.045 -104.658]
     [-133.294 -162.457  -91.362]
     [-105.482 -124.437  -90.647]]
    Onnx-Add(Re_reduced0, Ge_Y0) -> Ad_C01    (name='Ad_Add')
    +kr='Ad_C01': (5, 3) (dtype=float32 min=-93.42481994628906 max=-13.038177490234375)
    [[-62.419 -92.341 -21.737]
     [-61.427 -93.121 -16.864]
     [-60.286 -93.425 -13.038]
     [-62.744 -91.907 -20.812]
     [-51.222 -70.177 -36.387]]
    Onnx-Add(Ad_Addcst, Ad_C01) -> Ad_C0    (name='Ad_Add1')
    +kr='Ad_C0': (5, 3) (dtype=float32 min=0.2028656005859375 max=24.99547576904297)
    [[ 1.033  1.287 16.297]
     [ 2.025  0.507 21.169]
     [ 3.166  0.203 24.995]
     [ 0.708  1.72  17.222]
     [12.23  23.45   1.647]]
    Onnx-ArgMin(Ad_C0) -> label    (name='Ar_ArgMin')
    +kr='label': (5,) (dtype=int64 min=0 max=2)
    [0 1 1 0 2]
    Onnx-Sqrt(Ad_C0) -> scores    (name='Sq_Sqrt')
    +kr='scores': (5, 3) (dtype=float32 min=0.4504060447216034 max=4.999547481536865)
    [[1.016 1.134 4.037]
     [1.423 0.712 4.601]
     [1.779 0.45  5.   ]
     [0.841 1.312 4.15 ]
     [3.497 4.843 1.283]]
    {'label': array([0, 1, 1, 0, 2]), 'scores': array([[1.016, 1.134, 4.037],
           [1.423, 0.712, 4.601],
           [1.779, 0.45 , 5.   ],
           [0.841, 1.312, 4.15 ],
           [3.497, 4.843, 1.283]], dtype=float32)}

Other runtimes with OnnxInference

OnnxInference can also call onnxruntime to compute the predictions by using runtime='onnxruntime1'.

<<<

import numpy
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.cluster import KMeans
from skl2onnx import to_onnx
from mlprodict.onnxrt import OnnxInference

iris = load_iris()
X = iris.data.astype(numpy.float32)
X_train, X_test = train_test_split(X)
clr = KMeans(n_clusters=3)
clr.fit(X_train)

model_def = to_onnx(clr, X_train.astype(numpy.float32),
                    target_opset=12)

oinf = OnnxInference(model_def, runtime='onnxruntime1')
print(oinf.run({'X': X_test[:5]}))

>>>

    {'label': array([1, 0, 2, 1, 0], dtype=int64), 'scores': array([[3.598, 0.834, 5.319],
           [0.867, 4.051, 1.548],
           [1.551, 4.731, 0.331],
           [3.315, 0.099, 4.954],
           [1.149, 3.479, 2.412]], dtype=float32)}

Intermediate cannot be seen but the class may decompose the ONNX graph into smaller graphs, one per operator, to look into intermediate results.

<<<

import numpy
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.cluster import KMeans
from skl2onnx import to_onnx
from mlprodict.onnxrt import OnnxInference

iris = load_iris()
X = iris.data.astype(numpy.float32)
X_train, X_test = train_test_split(X)
clr = KMeans(n_clusters=3)
clr.fit(X_train)

model_def = to_onnx(clr, X_train.astype(numpy.float32),
                    target_opset=12)

oinf = OnnxInference(model_def, runtime='onnxruntime2')
print(oinf.run({'X': X_test[:5]}, verbose=1, fLOG=print))

>>>

    +ki='Ad_Addcst': (3,) (dtype=float32 min=38.842979431152344 max=94.03599548339844)
    +ki='Ge_Gemmcst': (3, 4) (dtype=float32 min=0.23684221506118774 max=6.851515293121338)
    +ki='Mu_Mulcst': (1,) (dtype=float32 min=0.0 max=0.0)
    -- OnnxInference: run 7 nodes
    Onnx-ReduceSumSquare(X) -> Re_reduced0    (name='Re_ReduceSumSquare')
    +kr='Re_reduced0': (5, 1) (dtype=float32 min=40.2599983215332 max=121.80999755859375)
    Onnx-Mul(Re_reduced0, Mu_Mulcst) -> Mu_C0    (name='Mu_Mul')
    +kr='Mu_C0': (5, 1) (dtype=float32 min=0.0 max=0.0)
    Onnx-Gemm(X, Ge_Gemmcst, Mu_C0) -> Ge_Y0    (name='Ge_Gemm')
    +kr='Ge_Y0': (5, 3) (dtype=float32 min=-213.79698181152344 max=-79.08473205566406)
    Onnx-Add(Re_reduced0, Ge_Y0) -> Ad_C01    (name='Ad_Add')
    +kr='Ad_C01': (5, 3) (dtype=float32 min=-91.98698425292969 max=-2.5368499755859375)
    Onnx-Add(Ad_Addcst, Ad_C01) -> Ad_C0    (name='Ad_Add1')
    +kr='Ad_C0': (5, 3) (dtype=float32 min=0.018245697021484375 max=36.306129455566406)
    Onnx-ArgMin(Ad_C0) -> label    (name='Ar_ArgMin')
    +kr='label': (5,) (dtype=int64 min=0 max=2)
    Onnx-Sqrt(Ad_C0) -> scores    (name='Sq_Sqrt')
    +kr='scores': (5, 3) (dtype=float32 min=0.1350766271352768 max=6.02545690536499)
    {'label': array([2, 1, 2, 2, 0], dtype=int64), 'scores': array([[1.554, 4.098, 0.824],
           [5.099, 0.135, 3.423],
           [2.429, 3.521, 1.086],
           [2.174, 3.263, 0.5  ],
           [1.431, 6.025, 3.063]], dtype=float32)}

Finally, a last runtime ‘python_compiled’ converts some part of the class OnnxInference into python code then dynamically compiled. As a consequence, interdiate results cannot be seen anymore.

<<<

import numpy
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.cluster import KMeans
from skl2onnx import to_onnx
from mlprodict.onnxrt import OnnxInference

iris = load_iris()
X = iris.data.astype(numpy.float32)
X_train, X_test = train_test_split(X)
clr = KMeans(n_clusters=3)
clr.fit(X_train)

model_def = to_onnx(clr, X_train.astype(numpy.float32),
                    target_opset=12)

oinf = OnnxInference(model_def, runtime='python_compiled')
print(oinf.run({'X': X_test[:5]}))

>>>

    {'label': array([2, 2, 1, 2, 1]), 'scores': array([[4.837, 1.579, 0.643],
           [4.324, 1.251, 0.796],
           [3.309, 0.415, 1.8  ],
           [4.478, 1.202, 0.813],
           [2.811, 0.792, 2.644]], dtype=float32)}

From scikit-learn to ONNX

Function skl2onnx.to_onnx is the main entrypoint to convert a scikit-learn pipeline into ONNX. The same function was extended in this package into to_onnx to handle dataframes, an extended list of supported converters, scorers. It works exactly the same:

<<<

import numpy
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.cluster import KMeans
from mlprodict.onnx_conv import to_onnx
from mlprodict.onnxrt import OnnxInference

iris = load_iris()
X = iris.data.astype(numpy.float32)
X_train, X_test = train_test_split(X)
clr = KMeans(n_clusters=3)
clr.fit(X_train)

model_def = to_onnx(clr, X_train.astype(numpy.float32),
                    target_opset=12)

oinf = OnnxInference(model_def, runtime='python')
print(oinf.run({'X': X_test[:5]}))

>>>

    {'label': array([1, 0, 0, 1, 0]), 'scores': array([[3.908, 0.981, 5.65 ],
           [0.773, 3.515, 1.511],
           [0.348, 3.13 , 1.902],
           [3.939, 0.688, 5.595],
           [0.846, 2.774, 2.596]], dtype=float32)}

previous

Tutorial

next

From numpy to ONNX