.. _uploaddonneesrst:

===========
Upload data
===========


.. only:: html

    **Links:** :download:`notebook <upload_donnees.ipynb>`, :downloadlink:`html <upload_donnees2html.html>`, :download:`PDF <upload_donnees.pdf>`, :download:`python <upload_donnees.py>`, :downloadlink:`slides <upload_donnees.slides.html>`, :githublink:`GitHub|_doc/notebooks/hackathon_2015/upload_donnees.ipynb|*`


Material for the hackathon ENSAE / Red Cross / 2015. Upload data to a
blob storage.

.. code:: ipython3

    from jyquickhelper import add_notebook_menu
    add_notebook_menu()






.. contents::
    :local:





Connexion to the blob storage
-----------------------------

.. code:: ipython3

    from pyquickhelper.loghelper import get_password
    hackathon = {}
    hackathon["blob_storage"] = get_password("CRCREDENTIALS", "blobstorage")
    hackathon["password"] = get_password("CRCREDENTIALS", "password")

.. code:: ipython3

    blobstorage = hackathon["blob_storage"]
    blobpassword = hackathon["password"]

.. code:: ipython3

    %load_ext pyensae

.. code:: ipython3

    cl, bs = %blob_open
    cl, bs




.. parsed-literal::
    (<pyensae.remote.azure_connection.AzureClient at 0x925c2e8>,
     <azure.storage.blob.blobservice.BlobService at 0x925c320>)



.. code:: ipython3

    %blob_containers




.. parsed-literal::
    ['croix-rouge']



Upload data
-----------

.. code:: ipython3

    %%file readme.txt
    
    This folder conains data for the Red Cross.


.. parsed-literal::
    Writing readme.txt


.. code:: ipython3

    %blob_up readme.txt croix-rouge/readme.txt




.. parsed-literal::
    'readme.txt'



.. code:: ipython3

    %blob_ls croix-rouge






.. raw:: html

    <div>
    <table border="1" class="dataframe">
      <thead>
        <tr style="text-align: right;">
          <th></th>
          <th>name</th>
          <th>last_modified</th>
          <th>content_type</th>
          <th>content_length</th>
          <th>blob_type</th>
        </tr>
      </thead>
      <tbody>
        <tr>
          <th>0</th>
          <td>build/SINVOICE_M.csv</td>
          <td>Wed, 18 Nov 2015 18:56:27 GMT</td>
          <td>application/octet-stream</td>
          <td>533771533</td>
          <td>BlockBlob</td>
        </tr>
        <tr>
          <th>1</th>
          <td>data/ITMMASTER.schema.txt</td>
          <td>Mon, 16 Nov 2015 23:00:34 GMT</td>
          <td>application/octet-stream</td>
          <td>5658</td>
          <td>BlockBlob</td>
        </tr>
        <tr>
          <th>2</th>
          <td>data/ITMMASTER.txt</td>
          <td>Mon, 09 Nov 2015 21:41:00 GMT</td>
          <td>application/octet-stream</td>
          <td>103096479</td>
          <td>BlockBlob</td>
        </tr>
        <tr>
          <th>3</th>
          <td>data/SINVOICE.schema.txt</td>
          <td>Mon, 16 Nov 2015 23:00:35 GMT</td>
          <td>application/octet-stream</td>
          <td>10252</td>
          <td>BlockBlob</td>
        </tr>
        <tr>
          <th>4</th>
          <td>data/SINVOICE.txt</td>
          <td>Mon, 09 Nov 2015 21:42:32 GMT</td>
          <td>application/octet-stream</td>
          <td>1362433753</td>
          <td>BlockBlob</td>
        </tr>
        <tr>
          <th>5</th>
          <td>data/SINVOICEV.schema.txt</td>
          <td>Mon, 16 Nov 2015 23:00:35 GMT</td>
          <td>application/octet-stream</td>
          <td>7999</td>
          <td>BlockBlob</td>
        </tr>
        <tr>
          <th>6</th>
          <td>data/SINVOICEV.txt</td>
          <td>Mon, 09 Nov 2015 21:44:08 GMT</td>
          <td>application/octet-stream</td>
          <td>1252461865</td>
          <td>BlockBlob</td>
        </tr>
        <tr>
          <th>7</th>
          <td>data/enseignes_france.csv</td>
          <td>Mon, 09 Nov 2015 21:40:54 GMT</td>
          <td>application/octet-stream</td>
          <td>6303836</td>
          <td>BlockBlob</td>
        </tr>
        <tr>
          <th>8</th>
          <td>data/stojou.csv</td>
          <td>Mon, 09 Nov 2015 21:55:23 GMT</td>
          <td>application/octet-stream</td>
          <td>8821375868</td>
          <td>BlockBlob</td>
        </tr>
        <tr>
          <th>9</th>
          <td>readme.txt</td>
          <td>Sun, 22 Nov 2015 18:00:34 GMT</td>
          <td>application/octet-stream</td>
          <td>45</td>
          <td>BlockBlob</td>
        </tr>
      </tbody>
    </table>
    </div>



.. code:: ipython3

    import os
    files = ["ITMMASTER.schema.txt", "SINVOICE.schema.txt", "SINVOICEV.schema.txt"]
    for file in files:
        print("upload", file, os.stat(file).st_size)
        cl.upload(bs, "croix-rouge", "data/" + file, file)


.. parsed-literal::
    upload ITMMASTER.schema.txt 5658
    upload SINVOICE.schema.txt 10252
    upload SINVOICEV.schema.txt 7999


.. code:: ipython3

    import os
    files = ["enseignes_france.csv", "ITMMASTER.txt", "SINVOICE.txt", "SINVOICEV.txt",
             "stojou.csv"]
    for file in files:
        print("upload", file, os.stat(file).st_size)
        cl.upload(bs, "croix-rouge", "data/" + file, file)


.. parsed-literal::
    upload enseignes_france.csv 6303836
    upload ITMMASTER.txt 103096479
    upload SINVOICE.txt 1362433753
    upload SINVOICEV.txt 1252461865
    upload stojou.csv 8821375868


.. code:: ipython3

    import os
    files = ["SINVOICE_.txt", "SINVOICEV_.txt"]
    for file in files:
        print("upload", file, os.stat(file).st_size)
        cl.upload(bs, "croix-rouge", "data/" + file, file)


.. parsed-literal::
    upload SINVOICE_.txt 1362433753
    upload SINVOICEV_.txt 1252461865


.. code:: ipython3

    %blob_ls croix-rouge/data






.. raw:: html

    <div>
    <table border="1" class="dataframe">
      <thead>
        <tr style="text-align: right;">
          <th></th>
          <th>name</th>
          <th>last_modified</th>
          <th>content_type</th>
          <th>content_length</th>
          <th>blob_type</th>
        </tr>
      </thead>
      <tbody>
        <tr>
          <th>0</th>
          <td>data/ITMMASTER.schema.txt</td>
          <td>Mon, 16 Nov 2015 23:00:34 GMT</td>
          <td>application/octet-stream</td>
          <td>5658</td>
          <td>BlockBlob</td>
        </tr>
        <tr>
          <th>1</th>
          <td>data/ITMMASTER.txt</td>
          <td>Mon, 09 Nov 2015 21:41:00 GMT</td>
          <td>application/octet-stream</td>
          <td>103096479</td>
          <td>BlockBlob</td>
        </tr>
        <tr>
          <th>2</th>
          <td>data/SINVOICE.schema.txt</td>
          <td>Mon, 16 Nov 2015 23:00:35 GMT</td>
          <td>application/octet-stream</td>
          <td>10252</td>
          <td>BlockBlob</td>
        </tr>
        <tr>
          <th>3</th>
          <td>data/SINVOICE.txt</td>
          <td>Mon, 09 Nov 2015 21:42:32 GMT</td>
          <td>application/octet-stream</td>
          <td>1362433753</td>
          <td>BlockBlob</td>
        </tr>
        <tr>
          <th>4</th>
          <td>data/SINVOICEV.schema.txt</td>
          <td>Mon, 16 Nov 2015 23:00:35 GMT</td>
          <td>application/octet-stream</td>
          <td>7999</td>
          <td>BlockBlob</td>
        </tr>
        <tr>
          <th>5</th>
          <td>data/SINVOICEV.txt</td>
          <td>Mon, 09 Nov 2015 21:44:08 GMT</td>
          <td>application/octet-stream</td>
          <td>1252461865</td>
          <td>BlockBlob</td>
        </tr>
        <tr>
          <th>6</th>
          <td>data/enseignes_france.csv</td>
          <td>Mon, 09 Nov 2015 21:40:54 GMT</td>
          <td>application/octet-stream</td>
          <td>6303836</td>
          <td>BlockBlob</td>
        </tr>
        <tr>
          <th>7</th>
          <td>data/stojou.csv</td>
          <td>Mon, 09 Nov 2015 21:55:23 GMT</td>
          <td>application/octet-stream</td>
          <td>8821375868</td>
          <td>BlockBlob</td>
        </tr>
      </tbody>
    </table>
    </div>



Upload preprocessed data
------------------------

.. code:: ipython3

    %blob_up SINVOICE_M.csv croix-rouge/build/SINVOICE_M.csv




.. parsed-literal::
    'build/SINVOICE_M.csv'



.. code:: ipython3

    %blob_up Test_CRFFOR.GACCTMPD.csv croix-rouge/build/Test_CRFFOR.GACCTMPD.csv




.. parsed-literal::
    'build/Test_CRFFOR.GACCTMPD.csv'



.. code:: ipython3

    %blob_ls croix-rouge/build






.. raw:: html

    <div>
    <table border="1" class="dataframe">
      <thead>
        <tr style="text-align: right;">
          <th></th>
          <th>name</th>
          <th>last_modified</th>
          <th>content_type</th>
          <th>content_length</th>
          <th>blob_type</th>
        </tr>
      </thead>
      <tbody>
        <tr>
          <th>0</th>
          <td>build/SINVOICE_M.csv</td>
          <td>Wed, 18 Nov 2015 18:56:27 GMT</td>
          <td>application/octet-stream</td>
          <td>533771533</td>
          <td>BlockBlob</td>
        </tr>
        <tr>
          <th>1</th>
          <td>build/Test_CRFFOR.GACCTMPD.csv</td>
          <td>Sun, 22 Nov 2015 21:53:38 GMT</td>
          <td>application/octet-stream</td>
          <td>822231942</td>
          <td>BlockBlob</td>
        </tr>
      </tbody>
    </table>
    </div>



Close connection
----------------

.. code:: ipython3

    %blob_close




.. parsed-literal::
    True