Benchmark Random Forests, Tree Ensemble#

The following script benchmarks different libraries implementing random forests and boosting trees. This benchmark can be replicated by installing the following packages:

python -m virtualenv env
cd env
pip install -i https://test.pypi.org/simple/ ort-nightly
pip install git+https://github.com/microsoft/onnxconverter-common.git@jenkins
pip install git+https://https://github.com/xadupre/sklearn-onnx.git@jenkins
pip install mlprodict matplotlib scikit-learn pandas threadpoolctl
pip install mlprodict lightgbm xgboost jinja2

Import#

import os
import pickle
from pprint import pprint
import numpy
import pandas
import matplotlib.pyplot as plt
from xgboost import XGBClassifier
from lightgbm import LGBMClassifier
from onnxruntime import InferenceSession
from sklearn.ensemble import HistGradientBoostingClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.datasets import make_classification
from skl2onnx import to_onnx
from mlprodict.onnx_conv import register_converters
from mlprodict.onnxrt.validate.validate_helper import measure_time
from mlprodict.onnxrt import OnnxInference

Registers new converters for sklearn-onnx.

register_converters()

Out:

[<class 'lightgbm.sklearn.LGBMClassifier'>, <class 'lightgbm.sklearn.LGBMRegressor'>, <class 'lightgbm.basic.Booster'>, <class 'mlprodict.onnx_conv.operator_converters.parse_lightgbm.WrappedLightGbmBooster'>, <class 'mlprodict.onnx_conv.operator_converters.parse_lightgbm.WrappedLightGbmBoosterClassifier'>, <class 'xgboost.sklearn.XGBClassifier'>, <class 'xgboost.sklearn.XGBRegressor'>, <class 'mlinsights.mlmodel.transfer_transformer.TransferTransformer'>, <class 'skl2onnx.sklapi.woe_transformer.WOETransformer'>, <class 'mlprodict.onnx_conv.scorers.register.CustomScorerTransform'>]

Problem#

max_depth = 7
n_classes = 20
n_estimators = 500
n_features = 100
REPEAT = 3
NUMBER = 1
train, test = 1000, 10000

print('dataset')
X_, y_ = make_classification(n_samples=train + test, n_features=n_features,
                             n_classes=n_classes, n_informative=n_features - 3)
X_ = X_.astype(numpy.float32)
y_ = y_.astype(numpy.int64)
X_train, X_test = X_[:train], X_[train:]
y_train, y_test = y_[:train], y_[train:]

compilation = []


def train_cache(model, X_train, y_train, max_depth, n_estimators, n_classes):
    name = "cache-{}-N{}-f{}-d{}-e{}-cl{}.pkl".format(
        model.__class__.__name__, X_train.shape[0], X_train.shape[1],
        max_depth, n_estimators, n_classes)
    if os.path.exists(name):
        with open(name, 'rb') as f:
            return pickle.load(f)
    else:
        model.fit(X_train, y_train)
        with open(name, 'wb') as f:
            pickle.dump(model, f)
        return model

Out:

dataset

RandomForestClassifier#

rf = RandomForestClassifier(n_estimators=n_estimators, max_depth=max_depth)
print('train')
rf = train_cache(rf, X_train, y_train, max_depth, n_estimators, n_classes)

res = measure_time(rf.predict_proba, X_test[:10],
                   repeat=REPEAT, number=NUMBER,
                   div_by_number=True, first_run=True)
res['model'], res['runtime'] = rf.__class__.__name__, 'INNER'
pprint(res)

Out:

train
{'average': 0.18777668566569142,
 'context_size': 64,
 'deviation': 0.0007722039125760551,
 'max_exec': 0.18880734899721574,
 'min_exec': 0.18694870699982857,
 'model': 'RandomForestClassifier',
 'number': 1,
 'repeat': 3,
 'runtime': 'INNER',
 'ttime': 0.5633300569970743}

ONNX#

def measure_onnx_runtime(model, xt, repeat=REPEAT, number=NUMBER,
                         verbose=True):
    if verbose:
        print(model.__class__.__name__)

    res = measure_time(model.predict_proba, xt,
                       repeat=repeat, number=number,
                       div_by_number=True, first_run=True)
    res['model'], res['runtime'] = model.__class__.__name__, 'INNER'
    res['N'] = X_test.shape[0]
    res["max_depth"] = max_depth
    res["n_estimators"] = n_estimators
    res["n_features"] = n_features
    if verbose:
        pprint(res)
    yield res

    onx = to_onnx(model, X_train[:1], options={id(model): {'zipmap': False}})

    oinf = OnnxInference(onx)
    res = measure_time(lambda x: oinf.run({'X': x}), xt,
                       repeat=repeat, number=number,
                       div_by_number=True, first_run=True)
    res['model'], res['runtime'] = model.__class__.__name__, 'NPY/C++'
    res['N'] = X_test.shape[0]
    res['size'] = len(onx.SerializeToString())
    res["max_depth"] = max_depth
    res["n_estimators"] = n_estimators
    res["n_features"] = n_features
    if verbose:
        pprint(res)
    yield res

    sess = InferenceSession(onx.SerializeToString())
    res = measure_time(lambda x: sess.run(None, {'X': x}), xt,
                       repeat=repeat, number=number,
                       div_by_number=True, first_run=True)
    res['model'], res['runtime'] = model.__class__.__name__, 'ORT'
    res['N'] = X_test.shape[0]
    res['size'] = len(onx.SerializeToString())
    res["max_depth"] = max_depth
    res["n_estimators"] = n_estimators
    res["n_features"] = n_features
    if verbose:
        pprint(res)
    yield res


compilation.extend(list(measure_onnx_runtime(rf, X_test)))

Out:

RandomForestClassifier
{'N': 10000,
 'average': 3.029850146335472,
 'context_size': 64,
 'deviation': 0.025470749012883022,
 'max_depth': 7,
 'max_exec': 3.0597273470048094,
 'min_exec': 2.997485897001752,
 'model': 'RandomForestClassifier',
 'n_estimators': 500,
 'n_features': 100,
 'number': 1,
 'repeat': 3,
 'runtime': 'INNER',
 'ttime': 9.089550439006416}
somewhere/workspace/mlprodict/mlprodict_UT_39_std/_venv/lib/python3.9/site-packages/sklearn/utils/deprecation.py:103: FutureWarning: The attribute `n_features_` is deprecated in 1.0 and will be removed in 1.2. Use `n_features_in_` instead.
  warnings.warn(msg, category=FutureWarning)
somewhere/workspace/mlprodict/mlprodict_UT_39_std/_venv/lib/python3.9/site-packages/sklearn/utils/deprecation.py:103: FutureWarning: Attribute `n_features_` was deprecated in version 1.0 and will be removed in 1.2. Use `n_features_in_` instead.
  warnings.warn(msg, category=FutureWarning)
{'N': 10000,
 'average': 0.27055083133260877,
 'context_size': 64,
 'deviation': 0.00659535853935804,
 'max_depth': 7,
 'max_exec': 0.2797015079995617,
 'min_exec': 0.26441121399693657,
 'model': 'RandomForestClassifier',
 'n_estimators': 500,
 'n_features': 100,
 'number': 1,
 'repeat': 3,
 'runtime': 'NPY/C++',
 'size': 7219478,
 'ttime': 0.8116524939978262}
{'N': 10000,
 'average': 0.26056272233108757,
 'context_size': 64,
 'deviation': 0.0022717566216635533,
 'max_depth': 7,
 'max_exec': 0.2624742329935543,
 'min_exec': 0.25737069499882637,
 'model': 'RandomForestClassifier',
 'n_estimators': 500,
 'n_features': 100,
 'number': 1,
 'repeat': 3,
 'runtime': 'ORT',
 'size': 7219478,
 'ttime': 0.7816881669932627}

HistGradientBoostingClassifier#

hist = HistGradientBoostingClassifier(
    max_iter=n_estimators, max_depth=max_depth)
print('train')
hist = train_cache(hist, X_train, y_train, max_depth, n_estimators, n_classes)

compilation.extend(list(measure_onnx_runtime(hist, X_test)))

Out:

train
HistGradientBoostingClassifier
{'N': 10000,
 'average': 5.853270884663895,
 'context_size': 64,
 'deviation': 2.91627723886048,
 'max_depth': 7,
 'max_exec': 9.976832280997769,
 'min_exec': 3.7267564329958986,
 'model': 'HistGradientBoostingClassifier',
 'n_estimators': 500,
 'n_features': 100,
 'number': 1,
 'repeat': 3,
 'runtime': 'INNER',
 'ttime': 17.559812653991685}
{'N': 10000,
 'average': 2.3716306276667942,
 'context_size': 64,
 'deviation': 0.031040325571987402,
 'max_depth': 7,
 'max_exec': 2.4072191110026324,
 'min_exec': 2.3315799800038803,
 'model': 'HistGradientBoostingClassifier',
 'n_estimators': 500,
 'n_features': 100,
 'number': 1,
 'repeat': 3,
 'runtime': 'NPY/C++',
 'size': 4192000,
 'ttime': 7.114891883000382}
{'N': 10000,
 'average': 3.3973181686645453,
 'context_size': 64,
 'deviation': 0.0056440300577018515,
 'max_depth': 7,
 'max_exec': 3.405253183998866,
 'min_exec': 3.3926028229980147,
 'model': 'HistGradientBoostingClassifier',
 'n_estimators': 500,
 'n_features': 100,
 'number': 1,
 'repeat': 3,
 'runtime': 'ORT',
 'size': 4192000,
 'ttime': 10.191954505993635}

LightGBM#

lgb = LGBMClassifier(n_estimators=n_estimators,
                     max_depth=max_depth, pred_early_stop=False)
print('train')
lgb = train_cache(lgb, X_train, y_train, max_depth, n_estimators, n_classes)

compilation.extend(list(measure_onnx_runtime(lgb, X_test)))

Out:

train
LGBMClassifier
{'N': 10000,
 'average': 3.7533199123330028,
 'context_size': 64,
 'deviation': 0.058524118023433554,
 'max_depth': 7,
 'max_exec': 3.8282686270031263,
 'min_exec': 3.6854378879943397,
 'model': 'LGBMClassifier',
 'n_estimators': 500,
 'n_features': 100,
 'number': 1,
 'repeat': 3,
 'runtime': 'INNER',
 'ttime': 11.259959736999008}
{'N': 10000,
 'average': 2.2685407123329546,
 'context_size': 64,
 'deviation': 0.00613489004533332,
 'max_depth': 7,
 'max_exec': 2.2740087030033465,
 'min_exec': 2.2599730969959637,
 'model': 'LGBMClassifier',
 'n_estimators': 500,
 'n_features': 100,
 'number': 1,
 'repeat': 3,
 'runtime': 'NPY/C++',
 'size': 4422938,
 'ttime': 6.805622136998863}
{'N': 10000,
 'average': 3.463639162000618,
 'context_size': 64,
 'deviation': 0.0021855480046899336,
 'max_depth': 7,
 'max_exec': 3.46551479199843,
 'min_exec': 3.4605738040045253,
 'model': 'LGBMClassifier',
 'n_estimators': 500,
 'n_features': 100,
 'number': 1,
 'repeat': 3,
 'runtime': 'ORT',
 'size': 4422938,
 'ttime': 10.390917486001854}

XGBoost#

xgb = XGBClassifier(n_estimators=n_estimators, max_depth=max_depth)
print('train')
xgb = train_cache(xgb, X_train, y_train, max_depth, n_estimators, n_classes)

compilation.extend(list(measure_onnx_runtime(xgb, X_test)))

Out:

train
XGBClassifier
{'N': 10000,
 'average': 0.22472329266990224,
 'context_size': 64,
 'deviation': 0.005312912232971719,
 'max_depth': 7,
 'max_exec': 0.23219809900183463,
 'min_exec': 0.22032558100181632,
 'model': 'XGBClassifier',
 'n_estimators': 500,
 'n_features': 100,
 'number': 1,
 'repeat': 3,
 'runtime': 'INNER',
 'ttime': 0.6741698780097067}
{'N': 10000,
 'average': 2.295039069336781,
 'context_size': 64,
 'deviation': 0.0270457606567268,
 'max_depth': 7,
 'max_exec': 2.3319733060052386,
 'min_exec': 2.267963445003261,
 'model': 'XGBClassifier',
 'n_estimators': 500,
 'n_features': 100,
 'number': 1,
 'repeat': 3,
 'runtime': 'NPY/C++',
 'size': 1482345,
 'ttime': 6.885117208010342}
{'N': 10000,
 'average': 3.3440230636673127,
 'context_size': 64,
 'deviation': 0.004760641214420695,
 'max_depth': 7,
 'max_exec': 3.349822153999412,
 'min_exec': 3.3381615140024223,
 'model': 'XGBClassifier',
 'n_estimators': 500,
 'n_features': 100,
 'number': 1,
 'repeat': 3,
 'runtime': 'ORT',
 'size': 1482345,
 'ttime': 10.032069191001938}

Summary#

All data

name = 'plot_time_tree_ensemble'
df = pandas.DataFrame(compilation)
df.to_csv('%s.csv' % name, index=False)
df.to_excel('%s.xlsx' % name, index=False)
df
average deviation min_exec max_exec repeat number ttime context_size model runtime N max_depth n_estimators n_features size
0 3.029850 0.025471 2.997486 3.059727 3 1 9.089550 64 RandomForestClassifier INNER 10000 7 500 100 NaN
1 0.270551 0.006595 0.264411 0.279702 3 1 0.811652 64 RandomForestClassifier NPY/C++ 10000 7 500 100 7219478.0
2 0.260563 0.002272 0.257371 0.262474 3 1 0.781688 64 RandomForestClassifier ORT 10000 7 500 100 7219478.0
3 5.853271 2.916277 3.726756 9.976832 3 1 17.559813 64 HistGradientBoostingClassifier INNER 10000 7 500 100 NaN
4 2.371631 0.031040 2.331580 2.407219 3 1 7.114892 64 HistGradientBoostingClassifier NPY/C++ 10000 7 500 100 4192000.0
5 3.397318 0.005644 3.392603 3.405253 3 1 10.191955 64 HistGradientBoostingClassifier ORT 10000 7 500 100 4192000.0
6 3.753320 0.058524 3.685438 3.828269 3 1 11.259960 64 LGBMClassifier INNER 10000 7 500 100 NaN
7 2.268541 0.006135 2.259973 2.274009 3 1 6.805622 64 LGBMClassifier NPY/C++ 10000 7 500 100 4422938.0
8 3.463639 0.002186 3.460574 3.465515 3 1 10.390917 64 LGBMClassifier ORT 10000 7 500 100 4422938.0
9 0.224723 0.005313 0.220326 0.232198 3 1 0.674170 64 XGBClassifier INNER 10000 7 500 100 NaN
10 2.295039 0.027046 2.267963 2.331973 3 1 6.885117 64 XGBClassifier NPY/C++ 10000 7 500 100 1482345.0
11 3.344023 0.004761 3.338162 3.349822 3 1 10.032069 64 XGBClassifier ORT 10000 7 500 100 1482345.0


Time per model and runtime.

piv = df.pivot("model", "runtime", "average")
piv
runtime INNER NPY/C++ ORT
model
HistGradientBoostingClassifier 5.853271 2.371631 3.397318
LGBMClassifier 3.753320 2.268541 3.463639
RandomForestClassifier 3.029850 0.270551 0.260563
XGBClassifier 0.224723 2.295039 3.344023


Graphs.

ax = piv.T.plot(kind="bar")
ax.set_title("Computation time ratio for %d observations and %d features\n"
             "lower is better for onnx runtimes" % X_test.shape)
plt.savefig('%s.png' % name)
Computation time ratio for 10000 observations and 100 features lower is better for onnx runtimes

Available optimisation on this machine:

from mlprodict.testing.experimental_c_impl.experimental_c import code_optimisation
print(code_optimisation())

plt.show()

Out:

AVX-omp=8

Total running time of the script: ( 21 minutes 53.511 seconds)

Gallery generated by Sphinx-Gallery