.. _2018-10-09ensemblegradientboostingrst:

========================================
2018-10-09 Ensemble, Gradient, Boosting…
========================================


.. only:: html

    **Links:** :download:`notebook <2018-10-09_ensemble_gradient_boosting.ipynb>`, :downloadlink:`html <2018-10-09_ensemble_gradient_boosting2html.html>`, :download:`python <2018-10-09_ensemble_gradient_boosting.py>`, :downloadlink:`slides <2018-10-09_ensemble_gradient_boosting.slides.html>`, :githublink:`GitHub|_doc/notebooks/notebook_eleves/2018-2019/2018-10-09_ensemble_gradient_boosting.ipynb|*`


Le noteboook explore quelques particularités des algorithmes
d’apprentissage pour expliquer certains résultats numériques.
L’algoithme `AdaBoost <https://fr.wikipedia.org/wiki/AdaBoost>`__
surpondère les exemples sur lequel un modèle fait des erreurs.

.. code:: ipython3

    from jyquickhelper import add_notebook_menu
    add_notebook_menu()






.. contents::
    :local:





.. code:: ipython3

    %matplotlib inline

Skewed split train test
-----------------------

Lorsqu’une classe est sous représentée, il est difficile de prédire les
résultats d’un modèle de machine learning.

.. code:: ipython3

    import numpy, numpy.random
    from sklearn.model_selection import train_test_split
    from sklearn.linear_model import LogisticRegression
    from sklearn.tree import DecisionTreeClassifier
    from sklearn.neural_network import MLPClassifier
    from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier
    from sklearn.metrics import confusion_matrix
    
    N = 1000
    
    res = []
    
    for n in [1, 2, 5, 10, 20, 50, 80, 90, 100, 110]:
        print("n=", n)
        for k in range(10):
    
            X = numpy.zeros((N, 2))
            X[:, 0] = numpy.random.randint(0, 2, (N,))
            X[:, 1] = numpy.random.randint(0, n+1, (N,))
            Y = X[:, 0] + X[:, 1] + numpy.random.normal(size=(N,)) / 2
            Y[Y < 1.5] = 0
            Y[Y >= 1.5] = 1
    
            X_train, X_test, y_train, y_test = train_test_split(X, Y)
    
            stat = dict(N=N, n=n, ratio_train=y_train.sum()/y_train.shape[0],
                        k=k, ratio_test=y_test.sum()/y_test.shape[0])
            
            for model in [LogisticRegression(solver="liblinear"),
                          MLPClassifier(max_iter=500),
                          RandomForestClassifier(n_estimators=10),
                          AdaBoostClassifier(DecisionTreeClassifier(), n_estimators=10)]:
                obs = stat.copy()
                obs["model"] = model.__class__.__name__
                if obs["model"] == "AdaBoostClassifier":
                    obs["model"] = "AdaB-" + model.base_estimator.__class__.__name__
                try:
                    model.fit(X_train, y_train)
                except ValueError as e:
                    obs["erreur"] = str(e)
                    res.append(obs)
                    continue
                sc = model.score(X_test, y_test)
                obs["accuracy"] = sc
                conf = confusion_matrix(y_test, model.predict(X_test))
                try:
                    obs["Error-0|1"] = conf[0, 1] / conf[0, :].sum()
                    obs["Error-1|0"] = conf[1, 0] / conf[1, :].sum()
                except Exception:
                    pass
                res.append(obs)


.. parsed-literal::
    n= 1
    n= 2
    n= 5
    n= 10
    n= 20
    n= 50
    n= 80
    n= 90
    n= 100
    n= 110


.. code:: ipython3

    from pandas import DataFrame
    df = DataFrame(res)
    df = df.sort_values(['n', 'model', 'model', "k"]).reset_index(drop=True)
    df["diff_ratio"] = (df["ratio_test"] - df["ratio_train"]).abs()
    df.head(n=5)






.. raw:: html

    <div>
    <style scoped>
        .dataframe tbody tr th:only-of-type {
            vertical-align: middle;
        }

        .dataframe tbody tr th {
            vertical-align: top;
        }

        .dataframe thead th {
            text-align: right;
        }
    </style>
    <table border="1" class="dataframe">
      <thead>
        <tr style="text-align: right;">
          <th></th>
          <th>N</th>
          <th>n</th>
          <th>ratio_train</th>
          <th>k</th>
          <th>ratio_test</th>
          <th>model</th>
          <th>accuracy</th>
          <th>Error-0|1</th>
          <th>Error-1|0</th>
          <th>diff_ratio</th>
        </tr>
      </thead>
      <tbody>
        <tr>
          <th>0</th>
          <td>1000</td>
          <td>1</td>
          <td>0.273333</td>
          <td>0</td>
          <td>0.300</td>
          <td>AdaB-DecisionTreeClassifier</td>
          <td>0.860</td>
          <td>0.062857</td>
          <td>0.320000</td>
          <td>0.026667</td>
        </tr>
        <tr>
          <th>1</th>
          <td>1000</td>
          <td>1</td>
          <td>0.274667</td>
          <td>1</td>
          <td>0.328</td>
          <td>AdaB-DecisionTreeClassifier</td>
          <td>0.916</td>
          <td>0.029762</td>
          <td>0.195122</td>
          <td>0.053333</td>
        </tr>
        <tr>
          <th>2</th>
          <td>1000</td>
          <td>1</td>
          <td>0.304000</td>
          <td>2</td>
          <td>0.284</td>
          <td>AdaB-DecisionTreeClassifier</td>
          <td>0.860</td>
          <td>0.072626</td>
          <td>0.309859</td>
          <td>0.020000</td>
        </tr>
        <tr>
          <th>3</th>
          <td>1000</td>
          <td>1</td>
          <td>0.285333</td>
          <td>3</td>
          <td>0.268</td>
          <td>AdaB-DecisionTreeClassifier</td>
          <td>0.896</td>
          <td>0.027322</td>
          <td>0.313433</td>
          <td>0.017333</td>
        </tr>
        <tr>
          <th>4</th>
          <td>1000</td>
          <td>1</td>
          <td>0.297333</td>
          <td>4</td>
          <td>0.256</td>
          <td>AdaB-DecisionTreeClassifier</td>
          <td>0.888</td>
          <td>0.053763</td>
          <td>0.281250</td>
          <td>0.041333</td>
        </tr>
      </tbody>
    </table>
    </div>



.. code:: ipython3

    df.tail(n=5)






.. raw:: html

    <div>
    <style scoped>
        .dataframe tbody tr th:only-of-type {
            vertical-align: middle;
        }

        .dataframe tbody tr th {
            vertical-align: top;
        }

        .dataframe thead th {
            text-align: right;
        }
    </style>
    <table border="1" class="dataframe">
      <thead>
        <tr style="text-align: right;">
          <th></th>
          <th>N</th>
          <th>n</th>
          <th>ratio_train</th>
          <th>k</th>
          <th>ratio_test</th>
          <th>model</th>
          <th>accuracy</th>
          <th>Error-0|1</th>
          <th>Error-1|0</th>
          <th>diff_ratio</th>
        </tr>
      </thead>
      <tbody>
        <tr>
          <th>395</th>
          <td>1000</td>
          <td>110</td>
          <td>0.982667</td>
          <td>5</td>
          <td>0.996</td>
          <td>RandomForestClassifier</td>
          <td>0.996</td>
          <td>0.0</td>
          <td>0.004016</td>
          <td>0.013333</td>
        </tr>
        <tr>
          <th>396</th>
          <td>1000</td>
          <td>110</td>
          <td>0.990667</td>
          <td>6</td>
          <td>0.980</td>
          <td>RandomForestClassifier</td>
          <td>0.996</td>
          <td>0.2</td>
          <td>0.000000</td>
          <td>0.010667</td>
        </tr>
        <tr>
          <th>397</th>
          <td>1000</td>
          <td>110</td>
          <td>0.985333</td>
          <td>7</td>
          <td>0.988</td>
          <td>RandomForestClassifier</td>
          <td>1.000</td>
          <td>0.0</td>
          <td>0.000000</td>
          <td>0.002667</td>
        </tr>
        <tr>
          <th>398</th>
          <td>1000</td>
          <td>110</td>
          <td>0.985333</td>
          <td>8</td>
          <td>0.992</td>
          <td>RandomForestClassifier</td>
          <td>1.000</td>
          <td>0.0</td>
          <td>0.000000</td>
          <td>0.006667</td>
        </tr>
        <tr>
          <th>399</th>
          <td>1000</td>
          <td>110</td>
          <td>0.985333</td>
          <td>9</td>
          <td>0.992</td>
          <td>RandomForestClassifier</td>
          <td>0.996</td>
          <td>0.5</td>
          <td>0.000000</td>
          <td>0.006667</td>
        </tr>
      </tbody>
    </table>
    </div>



La répartition train/test est loin d’être statisfaisante lorsqu’il
existe une classe sous représentée.

.. code:: ipython3

    df[df.n==100][["n", "ratio_test", "ratio_train"]].head(n=10)






.. raw:: html

    <div>
    <style scoped>
        .dataframe tbody tr th:only-of-type {
            vertical-align: middle;
        }

        .dataframe tbody tr th {
            vertical-align: top;
        }

        .dataframe thead th {
            text-align: right;
        }
    </style>
    <table border="1" class="dataframe">
      <thead>
        <tr style="text-align: right;">
          <th></th>
          <th>n</th>
          <th>ratio_test</th>
          <th>ratio_train</th>
        </tr>
      </thead>
      <tbody>
        <tr>
          <th>320</th>
          <td>100</td>
          <td>0.980</td>
          <td>0.992000</td>
        </tr>
        <tr>
          <th>321</th>
          <td>100</td>
          <td>0.984</td>
          <td>0.980000</td>
        </tr>
        <tr>
          <th>322</th>
          <td>100</td>
          <td>0.988</td>
          <td>0.984000</td>
        </tr>
        <tr>
          <th>323</th>
          <td>100</td>
          <td>0.988</td>
          <td>0.986667</td>
        </tr>
        <tr>
          <th>324</th>
          <td>100</td>
          <td>0.976</td>
          <td>0.986667</td>
        </tr>
        <tr>
          <th>325</th>
          <td>100</td>
          <td>0.984</td>
          <td>0.985333</td>
        </tr>
        <tr>
          <th>326</th>
          <td>100</td>
          <td>0.984</td>
          <td>0.981333</td>
        </tr>
        <tr>
          <th>327</th>
          <td>100</td>
          <td>0.988</td>
          <td>0.982667</td>
        </tr>
        <tr>
          <th>328</th>
          <td>100</td>
          <td>0.984</td>
          <td>0.989333</td>
        </tr>
        <tr>
          <th>329</th>
          <td>100</td>
          <td>0.992</td>
          <td>0.989333</td>
        </tr>
      </tbody>
    </table>
    </div>



.. code:: ipython3

    #df.to_excel("data.xlsx")

.. code:: ipython3

    columns = ["n", "N", "model"]
    agg = df.groupby(columns, as_index=False).mean().sort_values(["n", "model"]).reset_index(drop=True)
    agg.tail()






.. raw:: html

    <div>
    <style scoped>
        .dataframe tbody tr th:only-of-type {
            vertical-align: middle;
        }

        .dataframe tbody tr th {
            vertical-align: top;
        }

        .dataframe thead th {
            text-align: right;
        }
    </style>
    <table border="1" class="dataframe">
      <thead>
        <tr style="text-align: right;">
          <th></th>
          <th>n</th>
          <th>N</th>
          <th>model</th>
          <th>ratio_train</th>
          <th>k</th>
          <th>ratio_test</th>
          <th>accuracy</th>
          <th>Error-0|1</th>
          <th>Error-1|0</th>
          <th>diff_ratio</th>
        </tr>
      </thead>
      <tbody>
        <tr>
          <th>35</th>
          <td>100</td>
          <td>1000</td>
          <td>RandomForestClassifier</td>
          <td>0.985733</td>
          <td>4.5</td>
          <td>0.9848</td>
          <td>0.9956</td>
          <td>0.185000</td>
          <td>0.001216</td>
          <td>0.004933</td>
        </tr>
        <tr>
          <th>36</th>
          <td>110</td>
          <td>1000</td>
          <td>AdaB-DecisionTreeClassifier</td>
          <td>0.986533</td>
          <td>4.5</td>
          <td>0.9900</td>
          <td>0.9972</td>
          <td>0.130000</td>
          <td>0.000810</td>
          <td>0.007200</td>
        </tr>
        <tr>
          <th>37</th>
          <td>110</td>
          <td>1000</td>
          <td>LogisticRegression</td>
          <td>0.986533</td>
          <td>4.5</td>
          <td>0.9900</td>
          <td>0.9960</td>
          <td>0.346667</td>
          <td>0.000402</td>
          <td>0.007200</td>
        </tr>
        <tr>
          <th>38</th>
          <td>110</td>
          <td>1000</td>
          <td>MLPClassifier</td>
          <td>0.986533</td>
          <td>4.5</td>
          <td>0.9900</td>
          <td>0.9956</td>
          <td>0.346667</td>
          <td>0.000810</td>
          <td>0.007200</td>
        </tr>
        <tr>
          <th>39</th>
          <td>110</td>
          <td>1000</td>
          <td>RandomForestClassifier</td>
          <td>0.986533</td>
          <td>4.5</td>
          <td>0.9900</td>
          <td>0.9980</td>
          <td>0.090000</td>
          <td>0.000810</td>
          <td>0.007200</td>
        </tr>
      </tbody>
    </table>
    </div>



.. code:: ipython3

    import matplotlib.pyplot as plt
    fig, ax = plt.subplots(1, 2, figsize=(10,4))
    agg.plot(x="n", y="diff_ratio", ax=ax[0])
    agg.plot(x="n", y="ratio_train", ax=ax[1])
    agg.plot(x="n", y="ratio_test", ax=ax[1])
    ax[0].set_title("Maximum difference between\nratio of first class on train and test")
    ax[1].set_title("Ratio of first class on train and test")
    ax[0].legend();



.. image:: 2018-10-09_ensemble_gradient_boosting_11_0.png


Une astuce pour éviter les doublons avant d’effecturer un pivot.

.. code:: ipython3

    agg2 = agg.copy()
    agg2["ratio_test2"] = agg2["ratio_test"] + agg2["n"] / 100000

.. code:: ipython3

    import matplotlib.pyplot as plt
    fig, ax = plt.subplots(1, 3, figsize=(14,4))
    agg2.pivot("ratio_test2", "model", "accuracy").plot(ax=ax[0])
    agg2.pivot("ratio_test2", "model", "Error-0|1").plot(ax=ax[1])
    agg2.pivot("ratio_test2", "model", "Error-1|0").plot(ax=ax[2])
    ax[0].plot([0.5, 1.0], [0.5, 1.0], '--', label="constant")
    ax[0].set_title("Accuracy")
    ax[1].set_title("Error-0|1")
    ax[2].set_title("Error-1|0")
    ax[0].legend();



.. image:: 2018-10-09_ensemble_gradient_boosting_14_0.png


.. code:: ipython3

    agg2.pivot("ratio_test2", "model", "Error-0|1")






.. raw:: html

    <div>
    <style scoped>
        .dataframe tbody tr th:only-of-type {
            vertical-align: middle;
        }

        .dataframe tbody tr th {
            vertical-align: top;
        }

        .dataframe thead th {
            text-align: right;
        }
    </style>
    <table border="1" class="dataframe">
      <thead>
        <tr style="text-align: right;">
          <th>model</th>
          <th>AdaB-DecisionTreeClassifier</th>
          <th>LogisticRegression</th>
          <th>MLPClassifier</th>
          <th>RandomForestClassifier</th>
        </tr>
        <tr>
          <th>ratio_test2</th>
          <th></th>
          <th></th>
          <th></th>
          <th></th>
        </tr>
      </thead>
      <tbody>
        <tr>
          <th>0.29721</th>
          <td>0.052249</td>
          <td>0.052249</td>
          <td>0.052249</td>
          <td>0.052249</td>
        </tr>
        <tr>
          <th>0.50682</th>
          <td>0.110686</td>
          <td>0.110686</td>
          <td>0.110686</td>
          <td>0.110686</td>
        </tr>
        <tr>
          <th>0.75525</th>
          <td>0.119578</td>
          <td>0.119578</td>
          <td>0.119578</td>
          <td>0.119578</td>
        </tr>
        <tr>
          <th>0.86690</th>
          <td>0.099333</td>
          <td>0.099333</td>
          <td>0.099333</td>
          <td>0.099333</td>
        </tr>
        <tr>
          <th>0.92900</th>
          <td>0.088095</td>
          <td>0.113095</td>
          <td>0.113095</td>
          <td>0.088095</td>
        </tr>
        <tr>
          <th>0.96970</th>
          <td>0.106349</td>
          <td>0.253968</td>
          <td>0.220635</td>
          <td>0.163492</td>
        </tr>
        <tr>
          <th>0.98120</th>
          <td>0.125000</td>
          <td>0.310000</td>
          <td>0.200000</td>
          <td>0.175000</td>
        </tr>
        <tr>
          <th>0.98490</th>
          <td>0.110000</td>
          <td>0.155000</td>
          <td>0.155000</td>
          <td>0.170000</td>
        </tr>
        <tr>
          <th>0.98580</th>
          <td>0.185000</td>
          <td>0.335000</td>
          <td>0.268333</td>
          <td>0.185000</td>
        </tr>
        <tr>
          <th>0.99110</th>
          <td>0.130000</td>
          <td>0.346667</td>
          <td>0.346667</td>
          <td>0.090000</td>
        </tr>
      </tbody>
    </table>
    </div>



Le modèle
`AdaBoost <http://scikit-learn.org/stable/modules/generated/sklearn.ensemble.AdaBoostClassifier.html>`__
construit 10 arbres tout comme la forêt aléatoire à ceci près que le
poids associé à chacun des arbres des différents et non uniforme.

Apprentissage continu
---------------------

Apprendre une forêt aléatoire, puis ajouter un arbre, encore un tout en
gardant le résultat des apprentissages précédents.

.. code:: ipython3

    from sklearn.datasets import load_diabetes
    data = load_diabetes()
    X, y = data.data, data.target

.. code:: ipython3

    X_train, X_test, y_train, y_test = train_test_split(X, y)

.. code:: ipython3

    from sklearn.ensemble import RandomForestRegressor
    
    model = None
    res = []
    for i in range(0, 20):
        if model is None:
            model = RandomForestRegressor(n_estimators=1, warm_start=True)
        else:
            model.set_params(**dict(n_estimators=model.n_estimators+1))
        model.fit(X_train, y_train)
        score = model.score(X_test, y_test)
        res.append(dict(n_estimators=model.n_estimators, score=score))

.. code:: ipython3

    df = DataFrame(res)
    df.head()






.. raw:: html

    <div>
    <style scoped>
        .dataframe tbody tr th:only-of-type {
            vertical-align: middle;
        }

        .dataframe tbody tr th {
            vertical-align: top;
        }

        .dataframe thead th {
            text-align: right;
        }
    </style>
    <table border="1" class="dataframe">
      <thead>
        <tr style="text-align: right;">
          <th></th>
          <th>n_estimators</th>
          <th>score</th>
        </tr>
      </thead>
      <tbody>
        <tr>
          <th>0</th>
          <td>1</td>
          <td>0.128906</td>
        </tr>
        <tr>
          <th>1</th>
          <td>2</td>
          <td>0.323854</td>
        </tr>
        <tr>
          <th>2</th>
          <td>3</td>
          <td>0.352876</td>
        </tr>
        <tr>
          <th>3</th>
          <td>4</td>
          <td>0.389476</td>
        </tr>
        <tr>
          <th>4</th>
          <td>5</td>
          <td>0.429992</td>
        </tr>
      </tbody>
    </table>
    </div>



.. code:: ipython3

    ax = df.plot(x="n_estimators", y="score")
    ax.set_title("Apprentissage continu\nmesure de la performance à chaque itération");



.. image:: 2018-10-09_ensemble_gradient_boosting_22_0.png