ONNX and Python Runtime

This package implements a python runtime for ONNX in class OnnxInference. It does not depend on scikit-learn, only numpy and this module. However, this module was not really developped to get the fastest python runtime but mostly to easily develop converters.

Python Runtime for ONNX

Class OnnxInference implements a python runtime for a subset of ONNX operators needed to convert many scikit-learn models.

<<<

import numpy
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.cluster import KMeans
from skl2onnx import to_onnx
from mlprodict.onnxrt import OnnxInference

iris = load_iris()
X = iris.data.astype(numpy.float32)
X_train, X_test = train_test_split(X)
clr = KMeans(n_clusters=3)
clr.fit(X_train)

model_def = to_onnx(clr, X_train.astype(numpy.float32),
                    target_opset=12)

oinf = OnnxInference(model_def, runtime='python')
print(oinf.run({'X': X_test[:5]}))

>>>

    {'label': array([2, 2, 0, 0, 2]), 'scores': array([[5.299, 2.155, 0.535],
           [4.62 , 1.507, 0.829],
           [0.458, 3.254, 4.99 ],
           [0.497, 3.184, 4.912],
           [4.833, 1.602, 0.272]], dtype=float32)}

It is usually useful to get information on intermediate results in the graph itself to understand where the discrepencies begin.

<<<

import numpy
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.cluster import KMeans
from skl2onnx import to_onnx
from mlprodict.onnxrt import OnnxInference

iris = load_iris()
X = iris.data.astype(numpy.float32)
X_train, X_test = train_test_split(X)
clr = KMeans(n_clusters=3)
clr.fit(X_train)

model_def = to_onnx(clr, X_train.astype(numpy.float32),
                    target_opset=12)

oinf = OnnxInference(model_def, runtime='python')
print(oinf.run({'X': X_test[:5]}, verbose=1, fLOG=print))

>>>

    +ki='Ad_Addcst': (3,) (dtype=float32 min=38.63642883300781 max=93.46731567382812)
    +ki='Ge_Gemmcst': (3, 4) (dtype=float32 min=0.25499993562698364 max=6.90333366394043)
    +ki='Mu_Mulcst': (1,) (dtype=float32 min=0.0 max=0.0)
    -- OnnxInference: run 7 nodes
    Onnx-ReduceSumSquare(X) -> Re_reduced0    (name='Re_ReduceSumSquare')
    +kr='Re_reduced0': (5, 1) (dtype=float32 min=35.01000213623047 max=90.79000091552734)
    Onnx-Mul(Re_reduced0, Mu_Mulcst) -> Mu_C0    (name='Mu_Mul')
    +kr='Mu_C0': (5, 1) (dtype=float32 min=0.0 max=0.0)
    Onnx-Gemm(X, Ge_Gemmcst, Mu_C0) -> Ge_Y0    (name='Ge_Gemm')
    +kr='Ge_Y0': (5, 3) (dtype=float32 min=-184.13133239746094 max=-73.47999572753906)
    Onnx-Add(Re_reduced0, Ge_Y0) -> Ad_C01    (name='Ad_Add')
    +kr='Ad_C01': (5, 3) (dtype=float32 min=-93.3413314819336 max=-16.0364990234375)
    Onnx-Add(Ad_Addcst, Ad_C01) -> Ad_C0    (name='Ad_Add1')
    +kr='Ad_C0': (5, 3) (dtype=float32 min=0.06017303466796875 max=26.542648315429688)
    Onnx-Sqrt(Ad_C0) -> scores    (name='Sq_Sqrt')
    +kr='scores': (5, 3) (dtype=float32 min=0.24530193209648132 max=5.151955604553223)
    Onnx-ArgMin(Ad_C0) -> label    (name='Ar_ArgMin')
    +kr='label': (5,) (dtype=int64 min=0 max=2)
    {'label': array([1, 2, 1, 2, 0]), 'scores': array([[5.084, 0.408, 3.355],
           [1.115, 4.005, 1.061],
           [5.152, 0.287, 3.489],
           [1.857, 3.338, 0.245],
           [0.355, 4.754, 1.662]], dtype=float32)}

The verbosity can be increased.

<<<

import numpy
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.cluster import KMeans
from skl2onnx import to_onnx
from mlprodict.onnxrt import OnnxInference

iris = load_iris()
X = iris.data.astype(numpy.float32)
X_train, X_test = train_test_split(X)
clr = KMeans(n_clusters=3)
clr.fit(X_train)

model_def = to_onnx(clr, X_train.astype(numpy.float32),
                    target_opset=12)

oinf = OnnxInference(model_def, runtime='python')
print(oinf.run({'X': X_test[:5]}, verbose=3, fLOG=print))

>>>

    +ki='Ad_Addcst': (3,) (dtype=float32 min=38.59067153930664 max=93.73397827148438
    [64.799 38.591 93.734]
    +ki='Ge_Gemmcst': (3, 4) (dtype=float32 min=0.23055553436279297 max=6.885185241699219
    [[5.912 2.79  4.459 1.476]
     [4.986 3.406 1.442 0.231]
     [6.885 3.059 5.722 2.056]]
    +ki='Mu_Mulcst': (1,) (dtype=float32 min=0.0 max=0.0
    [0.]
    -kv='X' shape=(5, 4) dtype=float32 min=0.4000000059604645 max=7.199999809265137
    -- OnnxInference: run 7 nodes
    Onnx-ReduceSumSquare(X) -> Re_reduced0    (name='Re_ReduceSumSquare')
    +kr='Re_reduced0': (5, 1) (dtype=float32 min=46.220001220703125 max=101.31999969482422)
    [[ 54.42]
     [101.32]
     [ 84.06]
     [ 46.22]
     [ 58.74]]
    Onnx-Mul(Re_reduced0, Mu_Mulcst) -> Mu_C0    (name='Mu_Mul')
    +kr='Mu_C0': (5, 1) (dtype=float32 min=0.0 max=0.0)
    [[0.]
     [0.]
     [0.]
     [0.]
     [0.]]
    Onnx-Gemm(X, Ge_Gemmcst, Mu_C0) -> Ge_Y0    (name='Ge_Gemm')
    +kr='Ge_Y0': (5, 3) (dtype=float32 min=-194.79258728027344 max=-84.34611511230469)
    [[-118.34   -86.576 -141.402]
     [-161.813 -111.726 -194.793]
     [-147.336 -101.625 -177.442]
     [ -98.387  -84.346 -114.744]
     [-123.163  -88.512 -147.421]]
    Onnx-Add(Re_reduced0, Ge_Y0) -> Ad_C01    (name='Ad_Add')
    +kr='Ad_C01': (5, 3) (dtype=float32 min=-93.47258758544922 max=-10.405555725097656)
    [[-63.92  -32.156 -86.982]
     [-60.493 -10.406 -93.473]
     [-63.276 -17.565 -93.382]
     [-52.167 -38.126 -68.524]
     [-64.423 -29.772 -88.681]]
    Onnx-Add(Ad_Addcst, Ad_C01) -> Ad_C0    (name='Ad_Add1')
    +kr='Ad_C0': (5, 3) (dtype=float32 min=0.26139068603515625 max=28.185115814208984)
    [[ 0.879  6.435  6.752]
     [ 4.306 28.185  0.261]
     [ 1.523 21.026  0.352]
     [12.632  0.465 25.21 ]
     [ 0.376  8.819  5.052]]
    Onnx-ArgMin(Ad_C0) -> label    (name='Ar_ArgMin')
    +kr='label': (5,) (dtype=int64 min=0 max=2)
    [0 2 2 1 0]
    Onnx-Sqrt(Ad_C0) -> scores    (name='Sq_Sqrt')
    +kr='scores': (5, 3) (dtype=float32 min=0.5112637877464294 max=5.308965682983398)
    [[0.937 2.537 2.598]
     [2.075 5.309 0.511]
     [1.234 4.585 0.593]
     [3.554 0.682 5.021]
     [0.613 2.97  2.248]]
    {'label': array([0, 2, 2, 1, 0]), 'scores': array([[0.937, 2.537, 2.598],
           [2.075, 5.309, 0.511],
           [1.234, 4.585, 0.593],
           [3.554, 0.682, 5.021],
           [0.613, 2.97 , 2.248]], dtype=float32)}

Other runtimes with OnnxInference

OnnxInference can also call onnxruntime to compute the predictions by using runtime='onnxruntime1'.

<<<

import numpy
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.cluster import KMeans
from skl2onnx import to_onnx
from mlprodict.onnxrt import OnnxInference

iris = load_iris()
X = iris.data.astype(numpy.float32)
X_train, X_test = train_test_split(X)
clr = KMeans(n_clusters=3)
clr.fit(X_train)

model_def = to_onnx(clr, X_train.astype(numpy.float32),
                    target_opset=12)

oinf = OnnxInference(model_def, runtime='onnxruntime1')
print(oinf.run({'X': X_test[:5]}))

>>>

    {'label': array([0, 2, 0, 2, 2], dtype=int64), 'scores': array([[1.035, 4.143, 1.165],
           [1.324, 3.844, 0.602],
           [0.772, 5.609, 2.381],
           [1.632, 3.481, 0.692],
           [3.305, 2.586, 1.546]], dtype=float32)}

Intermediate cannot be seen but the class may decompose the ONNX graph into smaller graphs, one per operator, to look into intermediate results.

<<<

import numpy
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.cluster import KMeans
from skl2onnx import to_onnx
from mlprodict.onnxrt import OnnxInference

iris = load_iris()
X = iris.data.astype(numpy.float32)
X_train, X_test = train_test_split(X)
clr = KMeans(n_clusters=3)
clr.fit(X_train)

model_def = to_onnx(clr, X_train.astype(numpy.float32),
                    target_opset=12)

oinf = OnnxInference(model_def, runtime='onnxruntime2')
print(oinf.run({'X': X_test[:5]}, verbose=1, fLOG=print))

>>>

    +ki='Ad_Addcst': (3,) (dtype=float32 min=39.003353118896484 max=90.87934875488281)
    +ki='Ge_Gemmcst': (3, 4) (dtype=float32 min=0.23243272304534912 max=6.805714130401611)
    +ki='Mu_Mulcst': (1,) (dtype=float32 min=0.0 max=0.0)
    -- OnnxInference: run 7 nodes
    Onnx-ReduceSumSquare(X) -> Re_reduced0    (name='Re_ReduceSumSquare')
    +kr='Re_reduced0': (5, 1) (dtype=float32 min=37.88999938964844 max=114.72998809814453)
    Onnx-Mul(Re_reduced0, Mu_Mulcst) -> Mu_C0    (name='Mu_Mul')
    +kr='Mu_C0': (5, 1) (dtype=float32 min=0.0 max=0.0)
    Onnx-Gemm(X, Ge_Gemmcst, Mu_C0) -> Ge_Y0    (name='Ge_Gemm')
    +kr='Ge_Y0': (5, 3) (dtype=float32 min=-203.92800903320312 max=-76.87081146240234)
    Onnx-Add(Re_reduced0, Ge_Y0) -> Ad_C01    (name='Ad_Add')
    +kr='Ad_C01': (5, 3) (dtype=float32 min=-90.14028930664062 max=-2.1273040771484375)
    Onnx-Add(Ad_Addcst, Ad_C01) -> Ad_C0    (name='Ad_Add1')
    +kr='Ad_C0': (5, 3) (dtype=float32 min=0.022541046142578125 max=36.87604904174805)
    Onnx-ArgMin(Ad_C0) -> label    (name='Ar_ArgMin')
    +kr='label': (5,) (dtype=int64 min=0 max=1)
    Onnx-Sqrt(Ad_C0) -> scores    (name='Sq_Sqrt')
    +kr='scores': (5, 3) (dtype=float32 min=0.15013675391674042 max=6.072564601898193)
    {'label': array([1, 0, 1, 0, 1], dtype=int64), 'scores': array([[4.502, 0.398, 2.818],
           [0.86 , 5.243, 2.233],
           [4.966, 0.183, 3.305],
           [1.297, 6.073, 3.079],
           [4.902, 0.15 , 3.21 ]], dtype=float32)}

Finally, a last runtime ‘python_compiled’ converts some part of the class OnnxInference into python code then dynamically compiled. As a consequence, interdiate results cannot be seen anymore.

<<<

import numpy
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.cluster import KMeans
from skl2onnx import to_onnx
from mlprodict.onnxrt import OnnxInference

iris = load_iris()
X = iris.data.astype(numpy.float32)
X_train, X_test = train_test_split(X)
clr = KMeans(n_clusters=3)
clr.fit(X_train)

model_def = to_onnx(clr, X_train.astype(numpy.float32),
                    target_opset=12)

oinf = OnnxInference(model_def, runtime='python_compiled')
print(oinf.run({'X': X_test[:5]}))

>>>

    {'label': array([0, 2, 0, 0, 1]), 'scores': array([[0.279, 3.078, 4.592],
           [4.805, 1.736, 0.386],
           [1.076, 3.456, 4.787],
           [0.473, 3.12 , 4.588],
           [2.962, 0.356, 1.891]], dtype=float32)}

From scikit-learn to ONNX

Function skl2onnx.to_onnx is the main entrypoint to convert a scikit-learn pipeline into ONNX. The same function was extended in this package into to_onnx to handle dataframes, an extended list of supported converters, scorers. It works exactly the same:

<<<

import numpy
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.cluster import KMeans
from mlprodict.onnx_conv import to_onnx
from mlprodict.onnxrt import OnnxInference

iris = load_iris()
X = iris.data.astype(numpy.float32)
X_train, X_test = train_test_split(X)
clr = KMeans(n_clusters=3)
clr.fit(X_train)

model_def = to_onnx(clr, X_train.astype(numpy.float32),
                    target_opset=12)

oinf = OnnxInference(model_def, runtime='python')
print(oinf.run({'X': X_test[:5]}))

>>>

    {'label': array([2, 2, 0, 2, 0]), 'scores': array([[1.47 , 4.594, 0.825],
           [2.271, 5.51 , 0.755],
           [0.557, 2.934, 2.153],
           [2.826, 6.059, 1.193],
           [0.762, 3.018, 2.39 ]], dtype=float32)}