{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "outputs": [
    {
     "ename": "ImportError",
     "evalue": "cannot import name 'get_config' from 'tensorflow.python.eager.context' (C:\\Users\\zhaojh\\AppData\\Roaming\\Python\\Python37\\site-packages\\tensorflow\\python\\eager\\context.py)",
     "output_type": "error",
     "traceback": [
      "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[1;31mImportError\u001B[0m                               Traceback (most recent call last)",
      "\u001B[1;32m~\\AppData\\Local\\Temp\\ipykernel_7812\\4169542727.py\u001B[0m in \u001B[0;36m<module>\u001B[1;34m\u001B[0m\n\u001B[0;32m      1\u001B[0m \u001B[1;32mimport\u001B[0m \u001B[0mnumpy\u001B[0m \u001B[1;32mas\u001B[0m \u001B[0mnp\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m      2\u001B[0m \u001B[1;32mimport\u001B[0m \u001B[0mpandas\u001B[0m \u001B[1;32mas\u001B[0m \u001B[0mpd\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m----> 3\u001B[1;33m \u001B[1;32mimport\u001B[0m \u001B[0mkeras\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m      4\u001B[0m \u001B[1;32mfrom\u001B[0m \u001B[0mkeras\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mlayers\u001B[0m \u001B[1;32mimport\u001B[0m \u001B[0mDense\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mConv1D\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mInput\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mBidirectional\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mLSTM\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mMultiply\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mDropout\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mFlatten\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mSoftmax\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mLambda\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m      5\u001B[0m \u001B[1;32mfrom\u001B[0m \u001B[0mkeras\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mmodels\u001B[0m \u001B[1;32mimport\u001B[0m \u001B[0mModel\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32m~\\AppData\\Roaming\\Python\\Python37\\site-packages\\keras\\__init__.py\u001B[0m in \u001B[0;36m<module>\u001B[1;34m\u001B[0m\n\u001B[0;32m     23\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     24\u001B[0m \u001B[1;31m# See b/110718070#comment18 for more details about this import.\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m---> 25\u001B[1;33m \u001B[1;32mfrom\u001B[0m \u001B[0mkeras\u001B[0m \u001B[1;32mimport\u001B[0m \u001B[0mmodels\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m     26\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     27\u001B[0m \u001B[1;32mfrom\u001B[0m \u001B[0mkeras\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mengine\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0minput_layer\u001B[0m \u001B[1;32mimport\u001B[0m \u001B[0mInput\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32m~\\AppData\\Roaming\\Python\\Python37\\site-packages\\keras\\models.py\u001B[0m in \u001B[0;36m<module>\u001B[1;34m\u001B[0m\n\u001B[0;32m     17\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     18\u001B[0m \u001B[1;32mimport\u001B[0m \u001B[0mtensorflow\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mcompat\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mv2\u001B[0m \u001B[1;32mas\u001B[0m \u001B[0mtf\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m---> 19\u001B[1;33m \u001B[1;32mfrom\u001B[0m \u001B[0mkeras\u001B[0m \u001B[1;32mimport\u001B[0m \u001B[0mbackend\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m     20\u001B[0m \u001B[1;32mfrom\u001B[0m \u001B[0mkeras\u001B[0m \u001B[1;32mimport\u001B[0m \u001B[0mmetrics\u001B[0m \u001B[1;32mas\u001B[0m \u001B[0mmetrics_module\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     21\u001B[0m \u001B[1;32mfrom\u001B[0m \u001B[0mkeras\u001B[0m \u001B[1;32mimport\u001B[0m \u001B[0moptimizer_v1\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32m~\\AppData\\Roaming\\Python\\Python37\\site-packages\\keras\\backend.py\u001B[0m in \u001B[0;36m<module>\u001B[1;34m\u001B[0m\n\u001B[0;32m     35\u001B[0m \u001B[1;32mfrom\u001B[0m \u001B[0mtensorflow\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mpython\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mdistribute\u001B[0m \u001B[1;32mimport\u001B[0m \u001B[0mdistribute_coordinator\u001B[0m \u001B[1;32mas\u001B[0m \u001B[0mdc\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     36\u001B[0m \u001B[1;32mfrom\u001B[0m \u001B[0mtensorflow\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mpython\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mdistribute\u001B[0m \u001B[1;32mimport\u001B[0m \u001B[0mdistribute_coordinator_context\u001B[0m \u001B[1;32mas\u001B[0m \u001B[0mdc_context\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m---> 37\u001B[1;33m \u001B[1;32mfrom\u001B[0m \u001B[0mtensorflow\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mpython\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0meager\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mcontext\u001B[0m \u001B[1;32mimport\u001B[0m \u001B[0mget_config\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m     38\u001B[0m \u001B[1;32mfrom\u001B[0m \u001B[0mtensorflow\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mpython\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mframework\u001B[0m \u001B[1;32mimport\u001B[0m \u001B[0mconfig\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     39\u001B[0m \u001B[1;32mfrom\u001B[0m \u001B[0mkeras\u001B[0m \u001B[1;32mimport\u001B[0m \u001B[0mbackend_config\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;31mImportError\u001B[0m: cannot import name 'get_config' from 'tensorflow.python.eager.context' (C:\\Users\\zhaojh\\AppData\\Roaming\\Python\\Python37\\site-packages\\tensorflow\\python\\eager\\context.py)"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import keras\n",
    "from keras.layers import Dense, Conv1D, Input, Bidirectional, LSTM, Multiply, Dropout, Flatten, Softmax, Lambda\n",
    "from keras.models import Model"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "data = pd.read_csv('./train_data_processed.csv')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [],
   "source": [
    "obj_cols = data.columns[-32:]\n",
    "num_cols = [x for x in data.columns if x not in obj_cols]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [],
   "source": [
    "maxs = data[num_cols].max()\n",
    "mins = data[num_cols].min()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [],
   "source": [
    "for col in num_cols:\n",
    "    data[col] = (data[col] - mins[col]) / (maxs[col] - mins[col])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [],
   "source": [
    "optim = keras.optimizers.Adam(learning_rate=5e-4)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [],
   "source": [
    "def build_model(n_features, n_outs):\n",
    "    inputs = Input(shape=(1, n_features))\n",
    "    x = Conv1D(filters=64, kernel_size=1, activation='relu')(inputs)\n",
    "    x = Dropout(rate=0.1)(x)\n",
    "    lstm_out = Bidirectional(LSTM(units=128, return_sequences=True))(x)\n",
    "    attention_pre = Dense(1, name='attention_vec')(lstm_out)\n",
    "    attention_probs = Softmax()(attention_pre)\n",
    "    attention_mul = Multiply()([attention_probs, lstm_out])\n",
    "    attention_mul = Flatten()(attention_mul)\n",
    "    output = Dense(32, activation='relu')(attention_mul)\n",
    "    output = Dense(n_outs, activation='sigmoid')(output)\n",
    "    model = Model(inputs=[inputs], outputs=output)\n",
    "    model.summary()\n",
    "    model.compile(loss='mse', optimizer=optim,)\n",
    "    return model"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_1\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 1, 251)       0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_1 (Conv1D)               (None, 1, 64)        16128       input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, 1, 64)        0           conv1d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_1 (Bidirectional) (None, 1, 256)       197632      dropout_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "attention_vec (Dense)           (None, 1, 1)         257         bidirectional_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "softmax_1 (Softmax)             (None, 1, 1)         0           attention_vec[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "multiply_1 (Multiply)           (None, 1, 256)       0           softmax_1[0][0]                  \n",
      "                                                                 bidirectional_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "flatten_1 (Flatten)             (None, 256)          0           multiply_1[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 32)           8224        flatten_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 1)            33          dense_1[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 222,274\n",
      "Trainable params: 222,274\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = build_model(len(data.columns) - 1, 1)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "outputs": [],
   "source": [
    "from tensorflow.python.keras.utils.vis_utils import plot_model"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARYAAAOoCAIAAACbcEw0AAAABmJLR0QA/wD/AP+gvaeTAAAgAElEQVR4nO3de3wTZb4/8GfSpoDciisFEXDdg8jFFZVVKB4uYnWR43QVWgqVwnJeyklf6/Gg67rucfLD1+JxdTd1OR6PQALHw7LHtAV1bbxrC9t1m8IuEgRWiywwBZEE1ASQW9rO749HhjFJ00meJE8y83n/0VdmMpfvTOYzzzPTXARFUQgAJMvCuwCA3IYIATBBhACYIEIATPJ5F5AWzz77rNfr5V0FZNrDDz9cXFyc4ZUasxXyer2tra28q4CM2rRp06FDhzK/XmO2QoSQyZMnb9y4kXcVkDmCIHBZrzFbIYCMQYQAmCBCAEwQIQAmiBAAE0QIgAkiBMAEEQJggggBMEGEAJggQgBMECEAJogQABNECICJqSNkt9vtdjvvKiC3mTpC6RYKhXR+iCUUCrW2trpcrtLSUv3LF6IkW2kPtalLzswac4thP3Knx4oVK9K6/ObmZp1TOhwOQsiTTz6Z0PIVRQmFQoWFhYSQYDA4cODARCvUQ7sViqIEAoEhQ4akdY25xdQRSqtQKORyuXROTMOcaIQIIepBnKajOXorioqK0rrGnGPejlwgEKitrVU7TtpBj8cjCEJpaWl7ezt9yuPx0KdcLpcgCNXV1Xv37iWajg1diHbQ4XB4PB51ZNJ1JnTBlg1bQVNHp7fb7YFAoKamRl1mTU0NnUwdqZZHx5SWljY1NWkLDoVC1dXV2XvVqhhRWVlZWVlZ/GlEUdTuAXXQ6/UqiiLLMiHEZrMpmi9Mpk8Fg0GbzUYIaWtr8/v92oXQudTBRPdwzOklSZIkSedcmdmK+NtFF+v3+7Vrp1+oRB+rRFH0+/2Kovj9flEU3W63oiiNjY2EEJ/Pp90Wn88XMW/M/VBXVxd/mnQwb4SUqEMhzmDEUz6fjxDicDgSmivRepKbKwNbEb9OSZLUw107Jb3ek2VZXTvNjKIobrc7Yu30rEFnDwaDOvcDIpQy6Y6QdkzuRijprdBTpyzLNDPqlDSxTqeTDjocDjVOaoOjpXNF2qq4RMi810KQPi6X64EHHogIxoQJE2w229KlS0OhUCgU2rdv38iRI+lT9HIrOhI5ARFKHu3057oUbkV1dTUhpLa2dunSpc8///zo0aNjruutt95qbm5evHhxxLP03kbOQYSSQV/s2bNn8y6ESWq3orW1dfr06YSQBQsWEELUFkaLNkQLFixwuVyTJ09WxzudTkLIhg0bQqEQuXB3LiVVZYB5IxQIBLQP1EH6KtK/2vGEkNraWvrUhg0bRFGkHRV6ZqWHo/otxPR8TCfQeUCoa1QfUPFvakfMlYGt0C5K1draWlxcPHbsWHX69vZ2tVXRzkIbn4g+3o9+9CNCyJNPPllYWCgIwpAhQ8rLy2OuKBtl/vIrA/TcTojYCXoG1TutTqdTvU0kyzId2dDQoCgKvTlL79XSC2hJkuigzmIiXpc4N7V7fGVTvhXx10iXpp2e3p1TbxtQoii2tbVFbIssy5IkEULU6dXFiqIYf++pe4PL7QRByZ3rNv3Ky8sJISn8Tm36X8Vc31fZsBWhUOixxx5btWpVypcsCEJdXd28efNSvuT4zNuRAy7q6+vpCc4wEKGeRVw15Si+W2G329W388ycOTPzBaQP3mbaM/rGZPog6V5Q/DeYZaBzlZKtSBq9Qed0Ou+///4MrzrdEKGepeSA434dxbeA+++/33jhodCRA2CCCAEwQYQAmCBCAEwQIQAmiBAAE0QIgAkiBMAEEQJggggBMEGEAJggQgBMECEAJoZ9p3Zra6vBPtoF2cmYESouLuZdAh/Hjh37+OOPp02bxrsQDsrKykaMGJH59RrzuxNMq76+vqKiAq9pJuFaCIAJIgTABBECYIIIATBBhACYIEIATBAhACaIEAATRAiACSIEwAQRAmCCCAEwQYQAmCBCAEwQIQAmiBAAE0QIgAkiBMAEEQJggggBMEGEAJggQgBMECEAJogQABNECIAJIgTABBECYIIIATBBhACYIEIATBAhACaIEAATRAiAiTF/KNI8Dh8+vHjx4s7OTjp4/Pjx/Pz8GTNmqBNcc801a9as4VOcOSBCuW348OEHDx7cv3+/duQf//hH9fHUqVMzXpS5oCOX8xYtWmS1Wrt7dv78+ZksxoTwc8U5b9++fVdffXXMp8aNG7dnz54M12M2aIVy3qhRo6677jpBECLGW63WxYsXcynJVBAhI1i0aFFeXl7EyI6Ojnnz5nGpx1TQkTOCI0eOjBgxoqurSx0jCMKkSZO8Xi/HqkwCrZARDBs2bMqUKRbLxVczLy9v0aJFHEsyD0TIIKqqqrSDiqLMnTuXVzGmgggZRHl5udoK5eXllZSUFBUV8S3JJBAhgxg0aNAdd9xBbyooirJw4ULeFZkFImQcCxcupHcU8vPzS0tLeZdjFoiQcZSWlvbq1Ys+GDBgAO9yzMKw75E7fPhwS0sL7yoy7cYbb2xpabnqqqvq6+t515JpvP4JZtj/C9XX11dUVPCuAjKH15Fs8I6cYjLnz59/9NFHeVeRaXV1dRyPMYNHyGysVusTTzzBuwpzQYSMpk+fPrxLMBdECIAJIgTABBECYIIIATBBhACYIEIATBAhACaIEAATRAiACSIEwAQRAmCCCAEwQYSyTiAQqK2txSe3cwUilCGhUKi1tdXlcvWYjeXLly9YsMDj8aRqgd1pbW212+2CIAiCYLfbd+7cGQgEor9YOCW6q1aIpaamxuPxhEKhdFSSerw/LpUu9GNYvKu4SJIkSZJ07nM9kyW0wJiz22y2trY2Ouj3+xsaGtJ3SMSp1u/30/HBYJCO8fl8oiiKouj3+/UsnO9rnUUHWWplW4SoFEYo0Sm1JEkSRTF6PP0C4USXpl931UaP9/v9NEVqruLg+1qjI0dCoVBtbS3tQrhcru7GBwIB8u0LFY/HIwhCaWlpe3t7a2urtitCl1BTU0MH29vbddZQWlq6d+9exi2y2+12u727Z1tbW5988sl///d/j35q8uTJ0SVlYPOjFRUVLVu2zOPxNDc3JzpvpvHKbrrpPzOJoihJEn1ss9nUx6IoOp1O5dtnRFEU6X7zer2KosiyTAix2WyKojQ2NhJC1NkpSZJ8Pp862N0+F0XRZrPRM67b7db/0sSckvaaupuFdqh67CNlcvNjjg8Gg+rC40NHLi107lZ6vKrHk9frpT0cekBoxxNC3G63EvV6awfp0an2PYLBYMQhFfNYoRch6mUJPXRYIsQ+SyY3P4nxEdCR4+mll14ihKhfPz158mR6QG/cuFE7fuzYserEcZSVlRFC3nrrLTq4fft2Oia+N998kxAyevRoOjhw4MCENyPVMrn5OY9XdtNN55mpu50QPV4dE/FUxCDt89DH0b2pmKuLs66k64/DZrMRTVuhc7Hp2/zuxtPWOE6PVIVWiCfaud+5c2fM8fQaWkUPvvgqKys9Hk9ra2t7e/vNN9+cukpTZvbs2YSQgwcPxpkmGzZ/+/bthJBbb72VZSEZgAiJhJDVq1fTf+S1t7dXV1cTQiorKwkh6o/R02fLy8t7XODMmTMJIevXr29paZk2bZqeGpxOJ4kV4zShDcXq1aujn2pvb6+pqSGZ3fyYAoHAypUrRVGkC8xqvJq/dNPZuNPbTereUP/bSO8+qf/dc7vd9NZQxP8B1Ut/7Q0uelXtcDgi1qVOHNGJove1RFGUZVm5cClPdNyM6m6B8e/IqVut/dcqLUPd3kxufvR4/Gs1K+jfrX6/n77qkiRpDym/30/bB0KI2+2mL3DE2Sfmycjn8xHNHTYq/slLlmXaTbLZbPT4drvd8Q+gOAvsMUKKogSDwYaGBrVvRm9h0wxncvOjxxNCHA4HvWmuE98IGfxr6Y26daDF97U2+7UQACNECICJYX+iywDif+4AfdQsgQhlL4QkJ6AjB8AEEQJggggBMEGEAJggQgBMECEAJogQABNECIAJIgTABBECYIIIATBBhACYIEIATAz+Tu36+nreJUDa0W+K5MXgEaqoqOBdAhicYb87wZzwjRGZh2shACaIEAATRAiACSIEwAQRAmCCCAEwQYQAmCBCAEwQIQAmiBAAE0QIgAkiBMAEEQJggggBMEGEAJggQgBMECEAJogQABNECIAJIgTABBECYIIIATBBhACYIEIATBAhACaIEAATRAiACSIEwAQRAmCCCAEwQYQAmCBCAEwQIQAmBv+hSMM7duzYq6++qg7+9a9/JYQ4nU51TL9+/SorKzlUZhr4ocjcdu7cucGDB3/99dd5eXmEEEVRFEWxWL7pXITD4UWLFq1fv55rjQaHjlxu69WrV3l5eX5+fjgcDofDHR0dnZ2d4QsIIWiC0g2tUM5rbGwsKSmJ+VRhYeGxY8fy89FdTyO0Qjnv1ltvHTx4cPR4q9W6cOFC5CfdEKGcZ7FYKisrCwoKIsaHw+EFCxZwKclU0JEzgq1bt06ePDli5OWXX/7ZZ58JgsClJPNAK2QEkyZNuvLKK7VjrFbr4sWLkZ8MQIQMoqqqymq1qoPoxWUMOnIG8cknn4wdO1YdHDVq1KeffsqxHvNAK2QQY8aMGTduHO25Wa3WJUuW8K7ILBAh41i0aBF9j0I4HJ43bx7vcswCHTnjkGX5qquuUhRl4sSJ9M1ykAFohYzjyiuvvOmmmwghixYt4l2LmShGVFZWxnu/Agd1dXWZP9gM++6PyZMnP/TQQ7yryLQTJ0688MILjz32GO9COKioqOCyXsNGaPjw4ea8pJ4+ffrVV1/NuwoOeEUI10JGY878cIQIATBBhACYIEIATBAhACaIEAATRAiACSIEwAQRAmCCCAEwQYQAmCBCAEwQIQAmiBAAE0TookAgUFtbW1payrsQyCWG/bxQEpYvX7569WreVRBCSCgUKiwsVHR8rUUoFPr444937drl8XgaGhp0Lj/mVzQ6HI7Ro0dPmzZt4MCBiZWbNvr3A0dohS5atWoV7xK+0dzcrHNKh8PxxhtvLF261OPx6F++oih+v58+DgaD9APMJSUlLperqqoqEAgkXHF66N8PHCFCWScUCrlcLp0Tr1ixYsWKFUmspaioiD5Q25wJEyasXbuWEHLfffeFQqEklplaCe0HjsweoVAoVFtbKwhCaWnp3r176chAIODxeEpLS0OhUHV1td1uj5hYEASXy0XP1urEhBCXyyUIQnV1tbqo7uYSLqDTaAcdDgdtUrQTJMFut6uV61RUVLRs2TKPx0NP/8bYD2mX+W88yYCysrKysjI9U4qiaLPZaGfG7XbTfSKKIn3g9Xp9Pp/NZlMndjqdiqL4/X5RFEVRDAaD6p70er2KogSDQZvNRghpa2uLM5faj6LTyLKsHUz0pYk5vSRJkiQlOhfdIrrJubUfCKdv8DF1hOj1t/oaq8eBcuHFU68TFEVpbGwkhPj9fjro9XoJIW63W4l6pX0+HyHE4XAkNFfKI5T0XNGV5MR+QIRSSWeE6GlSOyYiQnEmpnkTRTHmxOoY/XNlc4S0z2btfkCEUklnhOK85HGeSmhi/U9lSYTowa32AHNoP/CKkNlvJ+hHLwwibvjSk2tM9KlE5+Ju+/bthJBbb721uwlMsh/0M3WEnE4nIWTnzp16Jqa/Pr9//346SG/7lpeXR09Jb0PNnj07obmyQSAQWLlypSiKM2fO7G4aM+yHxGS+4csAnR05ev9HFEVZlpULl7yEkDlz5kTvnGAwSO8j0Wtit9ut3qGiE9OL42AwKEkS7eXHn0t7w4peXpNv3wrz+/30Wjw+9S6I9qJf6emOXPRcPp9PW6qi+fdrTuwHgmuhFNJ/U1uWZfoS2mw2erNVvbVNLlzvqvx+P2246IGiHnx0DD0ECSFOp1N7NHc3lyzLdPqGhgZFUeiq6RFG72VJkqQezd2Jc06ME6GYJ1OHw0HvR0dPlv37QeEXIWP+vhDtIWzcuDEzq6P/+DPknkwI3/0gCEJdXV3mv0jd1NdCAOwQIVbqXabseXcmF6bdD/iwA6shQ4aoD9LRh4n/9rDs6T2mez9kLUSIVboPl1w5HHOlzpRDRw6ACSIEwAQRAmCCCAEwQYQAmCBCAEwQIQAmiBAAE0QIgAkiBMAEEQJggggBMEGEAJgY9p3amzZtyupvkQWjMOYHv71e76FDh3hXwYHX6125cmVdXR3vQviYMmXK8OHDM7xSY0bItOrr6ysqKvCaZhKuhQCYIEIATBAhACaIEAATRAiACSIEwAQRAmCCCAEwQYQAmCBCAEwQIQAmiBAAE0QIgAkiBMAEEQJggggBMEGEAJggQgBMECEAJogQABNECIAJIgTABBECYIIIATBBhACYIEIATBAhACaIEAATRAiACSIEwAQRAmCCCAEwMewPRZpEOBw+deqUOvj1118TQr766it1jCAIhYWFHCozDfzKXW47evTo8OHDOzs7u5tgxowZmzdvzmRJZoOOXG4bOnTotGnTLJbYr6MgCAsWLMhwSWaDCOW8qqqq7n7b3GKxzJ07N8P1mA0ilPPmzp2bl5cXPT4vL2/WrFnf+c53Ml+SqSBCOW/AgAGzZs3Kz4+8M6QoysKFC7mUZCqIkBEsXLgw+o5CQUHBXXfdxaUeU0GEjEAUxUsuuUQ7Jj8//5577unXrx+vkswDETKC3r17z5kzx2q1qmM6OjruvfdejiWZByJkEJWVleFwWB0cMGDA7bffzrEe80CEDKKkpOTSSy+lj61W6/z58wsKCviWZBKIkEHk5+fPnz+f9uXC4XBlZSXviswCb/Axjg8++GDq1KmEkCFDhhw5cqS7tyxAamEvG8ctt9wybNgwQkhVVRXykzGZfqf2s88+6/V6M7xS8+jfvz8hZMeOHeXl5bxrMayHH364uLhYHcz0ucrr9ba2tmZ4peYxcuTI/v37Dxo0iHchhrVp06ZDhw5px3D4vNDkyZM3btyY+fWaRH19/bx583hXYVjR7+hFj9lokJ8MQ4QAmCBCAEwQIQAmiBAAE0QIgAkiBMAEEQJggggBMEGEAJggQgBMECEAJogQABNECIBJNkYoEAjU1taWlpbGfNZut9vtdj3zxp+SXfqWH38PZBL2do+y8feFli9fvnr16szP26NQKFRYWJiBb5tIaCtCodDHH3+8a9cuj8fT0NCgc66Y32TvcDgGDBhw//33p6PORGXn3o5ByayysrKysrIeJ2OpLX3bRQ/QdCw5mv6tkCRJkqQkttrv90fM1djYSAhxu93pqDNR2bm3CSF1dXXaMdnYkctOoVDI5XLxriKGFStWrFixIokZi4qKIsbMnDmTEPLSSy+loCw2Wbu3o2V1hAKBQE1NjSAI1dXV7e3tpJtuaygUqq2tFQShtLR079696rzaKQOBgMfjKS0tDYVC1dXVaq9aXUVpaWlTU1P0MgVBoK+lw+HweDyEEDoyfiV0rkAgEFGJx+Oh66KbQy4cK3QWu91OZ0mV5K4f6GYS7G2d0tMwdiuhjpzX61UUxe/3i6JICFEfRJQtiqLNZgsGg4qiuN1uOkHElOqg1+v1+Xw2m01dMu230D6Mz+dTp5ckiT622Wz0ccwFRlTidDrVJYuiGAwGtatWFEWWZUIILYAunG5axPhEX52Y09M+XkJzEU1HDns75h6L6MhldYTUwba2NkII3V8RT9Eec1tbGx0MBoPqBBFT0kH62lP0CNBOQF88Ot7v99PxXq9XFMXuFqgO0sNCO5d6OMaZUZKkmC9kSiKkcy4tSZK0uwh7O3qP5WSEtGMinqLnFT1TRi9TPWNpqeN7rCp+JfTw0nM0KIoiy7LD4UjuRU1u+phz+f1+SZJEUVQPTezt6KqMFqGUTNnj2mOOT1UlTqdTFEXa0ibxoiY3fXdz0dt0arcKezu6qhyOEG2CU/6iqt0SFT0vqj317pYQMahesCVUM+3GyLIcMZ5XhOKXgb0dHaGsviOn2rlzJyFk+vTp0U85nU51gkTReTds2BAKhciF+0XkwsXx6tWr6fj29vbq6uoel0Z/TGH//v10kM7b4xfz0h+1HzlyZBL1pwO9eUW7SdGwt2PQk7wU0tkK0d3a2NioXLjf4nA4FM1/A9XTD721IooiPbXQq0xCyJw5c7RTRv8bUTtSRReivfVHCLHZbPTcqZ75HA5HdCX0dpB6IeF2u9U7UXRKem2tXoLTyegyZVlWuxZ+vz964fGpy9Revis93ZGLKExRlLa2NvpfWrq92NvRSK505BRFaWxspBtss9lolpRv30FSp5RlmZ41bTabeuc0Ykr1Mb3i1M5LDxqbzUZfUYpeWBNCJElS+x4+n4+OiTgatHPRcy0hxO1201exu0rooHaZ9H4RPUyjF94dEkV9Kk6EoueiO8fpdKr7AXs75n6LiFCmf1+INrX4Tm3IUYIg1NXVab91OTeuhQCyFiIEwCQbP+wAEWJ+MEGV4a44RECEcgBCks3QkQNggggBMEGEAJggQgBMECEAJogQABNECIAJIgTABBECYIIIATBBhACYIEIATBAhACYc3qnd2tra49dEgEpRlPgfdgC+Mt0KFRcXT548OcMrzV2KomzZsuXAgQNpWn44HN6zZ09nZ2ealm88ZWVlI0aM+NaoHr9vATh6+umne/XqtXv37jQt//Dhw/369fvlL3+ZpuWbASKUvT755JM+ffr86le/SutannrqqT59+hw4cCCtazGwTH+DD+jU1dU1ffr0r7/+euvWrVarNX0rOn/+/HXXXXfttddu2rQpfWsxMNyRy1K//e1vt23btn79+rTmhxBSUFDwX//1Xy+//PLbb7+d1hUZFVqhbLR3797rr7/+8ccff/zxxzOzxrvvvnvv3r07d+5Md2KNBxHKOl1dXTNmzDh58uS2bdsydkDv379/3LhxDofjgQceyMwaDQMduazz3HPPeb3edevWZbJB+N73vvev//qvy5cv//LLLzO2UmNAhLLLgQMH7Ha7JEk33nhjhlf9+OOPWyyWX/3qVxleb65DRy6LdHV1zZw586uvvvrLX/5SUFCQ+QKef/75n/70p7t377766qszv/YchQhlkeeff/6hhx5qbW2dOHEilwI6Ojquv/76sWPH4ncDEsD1v1Jw0YEDB/r372+32/mWQX+QmP5cNuiBVigrKIoya9asQ4cOffjhh7179+ZbzIwZMxRF+eMf/8i3jFyB2wlZYc2aNU1NTevXr+eeH0LIk08+2dzc/P777/MuJDegFeJPluXvf//7DzzwwFNPPcW7lm/Mnj372LFj27Ztw+cseoQIcaYoyp133inL8o4dO7KhCaI++uijG264YdOmTffccw/vWrIdIsSZy+Wy2WzNzc233HIL71q+ZcGCBR999NFHH32Ul5fHu5ashgjxdOTIkfHjxy9duvSZZ57hXUukvXv3jhs3bsOGDfSH5qE7iBBPs2fP3r9//44dO/r06cO7lhjuvffeHTt27N6922LBbaduYddw8+KLL77zzjtr167NzvyQCz9h/4c//IF3IVkNrRAfR44cufbaa5csWVJTU8O7lnjKy8v37dv34Ycf4tZcdxAhPu655x56sd63b1/etcSzc+fOG264oaGh4a677uJdS5ZChDjYsGHDj3/8482bN0+bNo13LT370Y9+dPTo0a1bt/IuJEshQpl29OjR8ePHL1y48D//8z9516LLtm3bJk2atHnz5hkzZvCuJRshQpk2d+7cHTt2fPTRR/369eNdi15Tp0699NJLX3vtNd6FZCPckcuol1566dVXX3U6nTmUH0LIQw895PF4PvnkE96FZCO0Qplz/Pjx8ePHl5eXP//887xrSUxXV9fo0aNnzZqVc5VnAFqhzKmurqZfrci7kIRZLJYHH3zwxRdf/OKLL3jXknUQoQypr69/+eWXnU5n//79edeSjCVLlhQUFLhcLt6FZB105DLh+PHj11577d1337169WretSTvZz/7WV1d3YEDB/DGUy1EKBMqKir+/Oc/7969u7CwkHctydu/f/+oUaNef/312bNn864liyBCadfQ0HD33Xe/+eabs2bN4l0Lq5kzZw4cOPDVV1/lXUgWQYTS64svvhg/frwoisa4inC73YsWLZJlediwYbxryRa4nZBeDz74YF5e3q9//WvehaTG3LlzBw0atH79et6FZBFEKGUOHjwY8W26r7/++ksvvfTCCy8MGjSIV1WpVVBQcO+997pcrq6uLt61ZAtEKGXWr18/ZswY9V0woVDIZrP9+Mc//tGPfsS3sNS6//77Dxw4sHnzZt6FZAtEKGXee++948eP33333fPnz//iiy8efPDBrq6uZ599lnddKTZu3LibbrrppZde4l1ItsDthNQ4e/bsgAEDwuEwIcRqtfbt27erq2vDhg2lpaW8S0s9h8PxH//xH36/n8sXf2cbtEKp0dLSQvNDCAmHwydOnDhx4sSaNWuOHDnCt7B0qKioCIVC7777Lu9CsgIilBpbtmzRnpLp1fZ77703ZswYp9PJr660GDFiRHFxcV1dHe9CsgIilBrvvvvu+fPnI0aGw+FTp079y7/8S2VlpcFuYVVUVLz22mtnzpzhXQh/iFAKnD59evv27TGfysvLGz58+MMPP2yw75GaN2/e6dOn33rrLd6F8Geo15WXP//5zx0dHdHjLRbLHXfc8dFHH/3gBz/IfFVpNXTo0FtuuQWfYyWIUEps3rw54t5UXl6exWKx2+0ej8cw/1eNMGvWrHfffRd3dBGhFIi4ELJarZdddllzc/MTTzxhsP6b1g9/+MOjR4/6fD7ehXBm2Bc4Y06dOqU9jCwWS3Fx8c6dO7Pta+ZT7oYbbrj88svffvtt3oVwhgix+tOf/tTZ2UkIsVgstPO2efPmIUOG8K4r7QRBKCkpeeedd3gXwhkixGrLli0WiyU/P3/QoEHvv/++sTtvEX74wx+2tLSEQiHehfD0rTf4HD58uKWlhWM1uejnP//5wYMHx4wZ89BDD22s1kEAACAASURBVOX0h1Kj0X+hxpng2LFjQ4cO/cMf/iCKYsaqyjra3y7G/5tBq6ysrMffux41atQTTzyRgh/Ozln50TtOMf1tSv1aW1uDwaABPtEdrby8XM9kEyZM2LlzZ7qLyWYxIgT6TZ48mXcJnF133XW/+93veFfBk1kufCFNJkyYsH///pMnT/IuhBtECJiMGTNGUZS///3vvAvhBhECJiNGjCCEtLe38y6EG0QImFxyySWXXXYZIgSQvJEjRx46dIh3FdwgQsBq5MiRaIUAkjdixAi0QgDJGzZs2Oeff867Cm4QIWA1ePDgY8eO8a6CG0QIWA0ePPjkyZNnz57lXQgfiBCwuuyyywghx48f510IH4gQsBo8eDAhxLR9OUQIWNEIoRXKAXa73W63864CIg0cONBqtaIVSoFQKCQIQneD7As0BiGWmpoaj8eTo5+gFgThsssuQyuUAs3NzXEG2Re4YsWKFStWMC6TO0VR/H4/fRwMBuknH0tKSlwuV1VVVSAQ4FtecgYOHJij+WeXsgiFQiHtz4lGDLIv0EiKiorog4EDB9IHEyZMWLt2LSHkvvvuy8VjERFKDD24aQ/EbrfTE6fD4fB4PORCRyVikM4YCARqamoEQSgtLW1qaqJjamtr6Y/weDwe+hR9w1XEErRTaiupra2lE7hcLlpJnGV2p7W1VdutoiNpqYIg0Hmji49ZAx2ZxGVbUVHRsmXLPB6Ptu1NaI9py6Z7I86eT63CwkLTRijG14/0+H0LNpuNEOL3+2VZJoTYbDY6PmKBEYN+v18URbfbrShKY2MjIcTn86nf/OL1ehVFibNAdUptJaIoOp1OdeGiKAaDwfjL7A4tSZIk7UhJknw+X3fFqzWoc9lsNvpYkqSIRUWI3hZFUYLBoLbUJPaYw+GQZZkuSpIkcqHf2F3xcZSVlen5+hFq3rx55eXlOic2mGQiJElSzKM8foTcbnfEs/QgizNX/AXSo8Hv99NBr9dLCKEHSvwZ42wX0Vyf0KMwfvF0vLYGURR7XFGckrTjk9tjajH0iivOcuJLKEL333//HXfcoXNig0kmQpQsyw6HQ/8RH/OrxuLPFX+BtDFUB+kpnB7ByUWIfq8vDaGiKI2NjdqmJmbxdHyPS46mJ0JJ7DG6T9xut3oiiLOc+BKK0COPPDJp0iSdExtMkhFyOp2iKLa1tek/4vUcNEoiEYpeoJ6DLD7aG6SPtadqncXrF3NGehZQ15vEHmtra1MD43A4WIpMKEIrVqygX6JgQsncTqitrV26dOnzzz8/evToROfdu3dvEmuMiR4rEXeB6Wk4aZWVlR6Pp7W1tb29/eabb454Nrp4WkOqvkiN/s7XrbfeGn+lcYwePbqhocHn89lstkceeaSmpia55SQKd+QSs2DBAkLIyJEjE5qL/uTohg0b6L6m94iSWLuqsrKSELJ//346SBer8wsEuzNz5kxCyPr161taWqZNm6aO7654GqHVq1fT8e3t7dXV1cmtOhAIrFy5UhRFWkOclcYhCEIoFJowYcKqVat8Pt8jjzyS3HIS1b9//xMnTqR2mTlD2yTp7MjR40aWZbUjRy9h6Xi/30+7EBGD6v8TVbIsR/yTkfZkYi5QnVK9XKY330RRpGPcbje9yRF/mT2iNxXUXhAVs3jlws0udaTNZmtra1N6uiOnlqResdBbbeq2xFlp/K0jhEiSRGujF6txio8voY5cbW2txWLRObHBJBMhetktSZLf76d35+hLoh0fPagoiizL9ABVZ4kIc8SgdgkRT1F+v1/9PW31Gjr+MnVuHU2CVnTxag10vCRJ6lxxIkRicTgc9CZ1jyuNv3XkwhmHfPss0F3xcSQUIfqLkefOndM5vZF865cd6uvrKyoqlG5eZjAV2iXeuHGjnonfe++9O+64IxgMqu+3MI9ceqc2ZK3evXsTQsz5wVVECFKgT58+hJAzZ87wLoQDc/2yQ/yPTqAHmzREyCwQkjShHTlzRggdOUgB2grhWgggSWbuyCFCkALoyAEw6d27tyAIiBBAkgRB6NWrFyIEkLw+ffrgdgJA8nr37o1WCCB5ffr0QYQAkoeOHAATdOQAmKAjB8AEEQJgYtoIxXindn19febrgGxz+PDh4cOH658+Pz+/o6MjffVkrRgRqqioyHwdkIXKysr0T2yxWLq6utJXTNYS8BGaaPPmzSNojRM0f/78cDj88ssv8y4k03AtFENeXp45+yQsTNsKIUIx5Ofnd3Z28q4ix+Tl5ZlzpyFCMaAVSkJeXh5aIfiGaW8usbBYLGiF4BvoyCUBrRBchI5cEtAKwUVohZKAVgguwrVQEtAKwUXoyCUBrRBchI5cEtAKwUVohZKAVgguMu0/2lmgFYKLcDshCWiF4CJEKAloheAidOSSgFYILkIrlAS0QnAR7sglAZ8Xgovwf6EkmLb3iwjFgI5cEtAKwUWmPaGywO0EuAitUBI6Ojry8vJ4V8EBIhQDIpSEzs5ORAi+gY5cEhAhuCg/P7+rqwvfsJeQzs7O/PwY3+xpeIhQDPRsir5cQnAtBBfRsykilBC0QnARPRRwOZQQtEJwETpySUArBBehI5cEtEJwETpyScBNbbgIHbkkoCMHF6EVSgI6cnARWqEkmLYVMuM2RwuHw2+++eb58+fp4MGDBwkh//d//zdkyBBCiKIooVDo4YcfNuchopNpWyH8UOQ3xo4d+8knn6iDFovFYrEIgkAI6ejouPnmm1tbW/lVlwMqKio6Ozs3bdrEu5BMQ0fuG4sXL9Y2Ml1dXR0dHeFwOBwO5+XlLViwgGNtOcG0HTlE6BtVVVXdfWKss7Nzzpw5Ga4n55i2I4cIfeOKK66YOnVq9EEgCMLNN988YsQILlXlELRCQJYsWRJ9ZZiXl1dZWcmlntyCVghIWVlZQUFBxMjOzs65c+dyqSe34N0JQPr27Tt37lyr1aqOEQRh0qRJV1xxBceqckVHRwc6ckAWLVoUDofVQfTi9EMrBIQQUlJSUlRUpA7iXpx+iBAQQojFYlm8eDHtywmCMHnyZPTidDp79myfPn14V8EBIhRp8eLFtC+HXlxCzpw507t3b95VcIAIRRo/fvy1115L0ItL0NmzZxEh+MaSJUsIIcXFxcOGDeNdS844c+aMOTty3d6FpO+wNLOWlhaT74SE3oJs2o5cvBv5y5YtKy4uzlgpWeWZZ55ZunTpoEGDeBfCh9frXblyZUKzmPZ2QrwIFRcXz5s3L2OlZJUbbrjh6quv5l0FTwlFSFGUc+fOmbMVwrVQbCbPT6LOnj2rKIo5WyFECFLg7NmzhBC0QgBJOnPmDCEErRBAktAKATA5deoUIaRv3768C+EAEYIUoBHq378/70I4QIQgBWiE+vXrx7sQDhAhSIGTJ08SRAggaadOnerTpw8+tQqQpFOnTpmzCSKIEKTEqVOnzHkvgSBCkBJohQCYoBUCYHLy5Em0QpnT2tpaXV0tCEJ1dfXOnTszX0BWMcbeOHnyJFqhDGlqaiouLv7FL36hKMr06dMffPDBLPlkaCgUam1tdblcpaWlic6rjUFTU1MoFNK5UVm7NxL15ZdfXnrppbyr4CPTN/I3btxICBk5ciQhZP78+X379m1ubs5wDTE5HA5CyJNPPpnojK2trcXFxW63e9WqVYSQnTt3VlVV6Zw3a/dGor788ssf/OAHvKvgROkGIaSurq67Z5OmXWkwGBRFMU4NmRd/n8Rks9kiZvH5fDoXkrV7o66uLqFKvvvd7z7zzDPpqyebsXbkampqBEFwuVyBQEDthIRCodraWkEQ1KcIIXSQTkAfOxwOj8ejDgYCgdraWtqP8ng8tF/U3t5OCKFLUwfVtbhcLjqv3W7XrkVdV8Rgcux2u91u7+7Zzz77jBCivYyZMGGCdgKD7Y2YvvrqK9N25JhaIYfDIcuyoijBYFCSJHVpoig6nU5FUfx+vyiKoigGg0F1sdqVagfpOZgQ4vP5FEXxer2EEJvN5vV6FUWRZZkOqvPS07/f7494yul00vFqAXSBesTcJ5IkSZLU3Sy0zSGEOJ1OdTO1cnFvJNQKhcNhQRA2bdqkc3qDYYqQ+tooiuL3++lOb2xs1I6nr73b7VZn6e6gSXRQkiT1QIl4Sj2eHA6HWoke8U8r3Wlra6NrpFuqDVKO7o2EIkRbvKamJp3TGwxThOhrE3HQRFwbBINBQogoiupiU3XQULIs0zsB2qdonkVRbGtri78JEZKLEOX1etUgNTQ00JE5ujcSilBbWxu50FqaEFOE2tra1P6Gw+FQZ4zY+9oxqT1onE4nPTKin3K73YQQ2u3RjyVClNfrpfuEpihH90ZCEaJNK+3Sm1AK7sj5fD56rqUpogeQtsNANF3zFB409LCgr1z0edfhcNDzcbo7coSQiEsgejVCl5OjeyOhCL3xxhuEkJMnT+qc3mBYr4XUo0e9kxtxwqNdl8bGRnWWVB00cU7nNMz0NrH2mrtHyUVI3TrtSNpby9G9kVCEfve73/Xq1UvnxMbDGiFJkuiZj/bClQsvlSiK9ITndrvVl029eaV2ytWTNL3Spc/SWKqD6t0k7aA6ryzLatfF7/fTe4NqsOkhG+d+mhadmES1KvHvyNFZGhsb6VzBYJDGhl4b5OjeSChCTz/99He/+12dExtPCu7I0R6Cei2kKIrf76f3UonmZgOJolw4jCRJUo8J9an4gxHz0vtRag8q4nysHRNnW6LLo3qMkKIobW1t6iZLkqS9cM/FvZFQhJYtWzZ58mSdExuPoHTz7f2CINTV1Zn2O7VNrr6+vqKiortjI8KCBQvOnTv3yiuvpLuq7IQPOwCro0ePDh06lHcV3CBCwOrzzz8fMmQI7yq4MdFXrsR/Y5jOTgtEO3r06OWXX867Cm5MFCGEJB3OnDkTCoXQkQNI0tGjRwkhiBBAkg4ePEgufGrQnBAhYLJ///5LLrnEzLcTECFgsn///u9973s5+pUPKYEIAZP9+/f/wz/8A+8qeEKEgMnf//73733ve7yr4AkRAia0I8e7Cp4QIUjeiRMnvvjiC0QIIEn0gxWjRo3iXQhPiBAkb/v27f369UOEAJK0Y8eOG2+80WIx9VFk6o0HRh9++OGNN97IuwrOECFIUjgc3r179w033MC7EM7ifWo1w6VAton/3nafz3fDDTfs2rXr2muvzVhJWajbDzvQT8+DHq+88sobb7zx7LPPDhw4kHctmdPa2tqvX78xY8bwLoSzblsh0O/06dPf//73J06cWF9fz7uWzJk7d+65c+def/113oVwhmuhFLjkkktcLtemTZteffVV3rVkSGdn55YtW26//XbehfCHCKXGzJkzFy9e/JOf/OSrr77iXUsm/OUvf/nyyy8RIYIIpdDKlSstFsujjz7Ku5BMeP/996+44opx48bxLoQ/RChlBg4cuHLlynXr1r333nu8a0m7119/fdasWbyryAq4nZBic+fO/fDDD3ft2mXgH5E/cuTI8OHDGxoa7rrrLt618IdWKMX++7//OxQKLV++nHchafSHP/yhb9++JSUlvAvJCohQig0dOvQ3v/nNypUrW1paeNeSLq+99tqsWbN69+7Nu5CsgI5c6imKMmvWrPb2dp/P16tXL97lpFgwGBwyZMjatWurqqp415IV0AqlniAIa9asOXz48NNPP827ltR7+eWXLRaL+uuGgFYoXX77298+9thj27dvN9hbyG677bZBgwZt2rSJdyHZAhFKl66urmnTpoXD4ZaWlry8PN7lpMbnn38+YsSI+vr6OXPm8K4lW6Ajly4Wi2Xt2rU7d+587rnneNeSMm63u2/fvnfeeSfvQrIIIpRGY8aMeeyxxyRJ+vvf/867ltT4/e9/X1ZW1qdPH96FZBF05NKro6PjpptuGjRoUGNjY65/BOvDDz+cOHHiBx98cMstt/CuJYugFUqv/Pz8NWvWNDc3/8///A/vWlitW7fummuumTJlCu9CsgsilHY333zzv/3bv/30pz/97LPPeNeSvDNnzrjd7vvuuy/X29KUQ0cuE06fPj1hwoSxY8c2NDTwriVJv//97//5n//50KFDZv4Rh5gQoQzZsmXLzJkz6+vry8rKeNeSjOLi4uHDh2/cuJF3IVkHEcqc++67r6Gh4W9/+9tll13Gu5bEeL3eKVOm4EZCTIhQ5oRCofHjx99+++0vvvgi71oSM3/+/H379v31r3/lXUg2wu2EzBk4cODq1av/93//95133uFdSwI+++yzV1555aGHHuJdSJZCK5Rp8+bN27Zt265du/r378+7Fl1+/vOf//73vz9w4EBBQQHvWrIRWqFMe+GFF06fPi1JEu9CdDl9+vS6det+8pOfID/dQYQy7bLLLvvNb37z/PPPf/DBB7xr6dmLL754+vTppUuX8i4ke6Ejx8edd9554MABn8+XzZ/9VBRl3Lhx06ZNW7NmDe9ashdaIT7WrFlz5MiRp556inch8bz55puffPLJT37yE96FZDW0Qtw899xzP/3pT7dt25a1P44wbdq0Sy655O233+ZdSFZDhLjp6uqaPn36qVOntm3bZrVaeZcTafPmzTNnzmxubp46dSrvWrIaIsRTW1vb9ddf/8tf/vJnP/sZ71oi3XrrrXl5ee+//z7vQrKeAlytWLGiT58+n376Ke9CvoXeLdyyZQvvQnIAWiHOOjo6Jk2a1L9//82bN2fP5whuu+22rq6uzZs38y4kB+COHGf5+fnr1q1raWlxuVy8a/mG1+ttamoy9heyphAixN/111//0EMP/exnPzt8+LA68rPPPsvMOxi+/vrr4uLiP/3pT+qY5cuXT5kyZcaMGRlYuxHw7kmCoijK2bNnx4wZ80//9E+KonR1da1Zs6Zv3755eXmhUCjdq966dSs9EubMmbNv3z6v10sIef/999O9XsNAhLLFli1bBEH47W9/+4//+I8Wi4VeF7311lvpXu+6devo19xZrdb8/PwJEyZMmjQp3Ss1EnTkssXUqVPvvPPORx99dOvWrV1dXYqiFBQUaPtXabJnzx4aoXA43NHR8be//W3Pnj3PPPPM2bNn071qY0CEssKuXbt+8IMfvP322+FwOBwO05Hnz59vbGxM96p9Pt/58+fVwXA4fOrUqccff/yqq6763e9+p+CGbY94N4Nmd/bsWUmS8vLyYr5BwWq1nj59Oq0FdPcpdNqT/H//7/+lde0GgFaIszfffPPXv/51V1eX2vhohcNh9XI/Hb788svjx4939+ySJUvsdnv61m4MiBBn99xzT0tLyxVXXBGzFSooKGhubk7f2nfv3t3dU48++ui6devy8/PTt3ZjQIT4mzhxos/nmz59usUS+XKEw+Gmpqb0rXr37t0RIREEwWKxrFq16umnn86ed0tkM0QoK3znO9959913n3rqKXoEq+MVRWltbdVe7qfWnj17tDnJy8srKCh47bXXbDZbmtZoPIhQthAE4ec///nrr7/er18/bafu3Llz27dvT9NKd+zYoV6D5efn9+3bt7GxEb/jnRBEKLvMnj17165d3//+99X+VVovh/bs2UMfWK3WoUOHbtu2DV+2mChEKOuMHDmypaWlurqaECIIQkdHx5YtW9Kxos8///zEiROEEKvVOmrUqNbW1muuuSYdKzI2RCgb9erV67nnnlu7dq3ValUU5U9/+lNnZ2fK10JvxwmCMH369G3btl1xxRUpX4UZGOTzQl6v99lnn+VdReoFg8E///nPZ86cKSkpKSwsTO3CP/300507d1555ZUTJ06MvhloAJn5En2D7LhDhw4Z8jeoCwsLb7/99ssvv/zYsWMpX/iJEyfGjh170003GS8/hw8fztjxYKh/nBn1pzsURdm6devkyZNTu9iWlhaj/mRdfX19RUVFZtZltNOPIQmCkPL8EEKMmp8MQ4QAmCBCAEwQIQAmiBAAE0QIgAkiBMAEEQJggggBMEGEAJggQgBMECEAJogQABNECIAJIpSwQCBQW1tbWlra3QR2u13PNxj2uJwcYuZ9YqjPC2XG8uXLV69erR0TCoUKCwsT/fxv9HKSEwqFPv744127dnk8noaGBp1zab/7yuv1Rn+YorW1tbi4WB2Mv3XZtk8yiuOXEadQXV1dJrclYtfRA5d9OcmRJEn9Ma+EZpRlmc5ls9min1W/Ss7v9+tZWlbtk0weD+jIsQqFQnx/43HFihUrVqxIYsaRI0cSQhwOx+rVq9vb27VPtbe3jxo1ij4uKipKdMnc90kmmShC2n62x+MRBKG6upoeOrW1tdpBQohwQcxBLYfD4fF41GkievOBQMDj8dBBl8tF17J3796IhdTU1KirqKmpiRgZcXzrp+cKpKSkhBDS0tKiHdnS0kLHaxljn6ReZhq7dNPTcIuiSDfZ5/MpikJ/EdFms3m9XuVCr0bt0vj9fu3+Ufs8dDBi12kH1bVonyKE0LUEg0HaQWpra4uYUa0noubk+lEU7ePFn0u50GfTjqdlRCwzh/ZJJjtyJoqQEvdlTmiQZTk+n48Q4nA4op9yOByEEFmW1SndbnePGxVzLfrnUhSF/hAYPaDpehsbG2MuM1f2Ca6FjGzChAmEkEceeST6Kdp3euedd+jg+++/n5lvCJk5cyYhZP369XRw06ZNdEzGZOE+0Q8RyiITJkyw2WxLly4NhUKhUGjfvn30ij8D3G43vakQCATGjx+fmZXqwXGf6IQI8dHdr4/Q8W+99VZzc/PixYszVg89tbe0tDQ1NfE6zWfbPtErM/3FdMuha6G2tjZCSENDQ/RTFD1iRFHscXPibJr+udTH9J9L9IKku2Xmyj7BtVBaBAIB+iAUCmkH6YOIQXLh5Edvtra2ttKR1dXV0VPSO06BQKCmpib6Waq2tpauesOGDaIoiqLY3ZT0RKvexdKDbpH2ARX/pnbEhpeVlZEL1x4k1g4hObVPMiczSU03PWediK2OP6goiizL9DWjZ0dRFN1ut3pjVzslvaEkSVLMZ+ljn89Hl+Z0OoPBYMw1qkRRpHd49Yjzmsa5qR1zFvX2cXfLzJV9gpvaCcvwG3wSEvOAiIP+nyR99WSDdO8TdORMrb6+vry8nHcV2SWb9wkilF7dde6j2e129a0rGf63TIYZbJ/gww7pNWTIEPWBEved//TfHU6n8/777494Kv6P18dfbBZKyT7JHgb5lTv6ezLG2BZgl8njAR05ACaIEAATRAiACSIEwAQRAmCCCAEwQYQAmCBCAEwQIQAmiBAAE0QIgAkiBMAEEQJgYqgPO2Ttp7Igww4fPpyxdRmkFRoxYgT99gyTO3bsWHNzM+8q+Bs+fHjGjgeDfF4IKHxuKvMM0goB8IIIATBBhACYIEIATBAhACaIEAATRAiACSIEwAQRAmCCCAEwQYQAmCBCAEwQIQAmiBAAE0QIgAkiBMAEEQJggggBMEGEAJggQgBMECEAJogQABNECIAJIgTABBECYIIIATBBhACYIEIATBAhACaIEAATRAiACSIEwAQRAmBiqN9aNaHDhw8vXry4s7OTDh4/fjw/P3/GjBnqBNdcc82aNWv4FGcOiFBuGz58+MGDB/fv368d+cc//lF9PHXq1IwXZS7oyOW8RYsWWa3W7p6dP39+JosxIfxccc7bt2/f1VdfHfOpcePG7dmzJ8P1mA1aoZw3atSo6667ThCEiPFWq3Xx4sVcSjIVRMgIFi1alJeXFzGyo6Nj3rx5XOoxFXTkjODIkSMjRozo6upSxwiCMGnSJK/Xy7Eqk0ArZATDhg2bMmWKxXLx1czLy1u0aBHHkswDETKIqqoq7aCiKHPnzuVVjKkgQgZRXl6utkJ5eXklJSVFRUV8SzIJRMggBg0adMcdd9CbCoqiLFy4kHdFZoEIGcfChQvpHYX8/PzS0lLe5ZgFImQcpaWlvXr1og8GDBjAuxyzMON75Lxe76FDh3hXkRY33nhjS0vLVVddVV9fz7uWtJgyZcrw4cN5V/FtivmUlZXx3uuQpLq6Ot6HTySTduTKysp47/m0OH/+/KOPPsq7inThfdTEZtIIGZXVan3iiSd4V2EuiJDR9OnTh3cJ5oIIATBBhACYIEIATBAhACaIEAATRAiACSIEwAQRAmCCCAEwQYQAmCBCAEwQIQAmiFAPWltbq6urBUGYO3fuL37xCzN8oDoQCNTW1pphS1PCjJ9a1a+pqem2226TZXnVqlWDBg165ZVX9MwVCoUKCwvVz7dEDKZWKBT6+OOPd+3a5fF4GhoadM4V/e3BWjabbfXq1TrXnrEtzVpoheLZuHEjIWTkyJGEkK+++krnXM3NzXEGU8vhcLzxxhtLly71eDz651IUJRgMqo9VjY2NhJBVq1bpXE4mtzR7cfjwIW9lZWU6P7UasYv07LFgMCiKojpZxGCaJPdSxpyLjsnOLSX44HcOEQRB7e1oH2uFQiGXy0WftdvtgUCAEOJwOGiDQMdHDNIZA4FATU2NIAilpaVNTU3k25cfHo+HPtXe3s6yCXa73W63J7TJRJOfHNpSznhnmINUtUI2m40Q4vf7ZVkmhNhsNj1z+f1+URTdbrdyoePk8/no+ZsQ4vV6FUWJWGCidVKSJEmSpHMuusYs31KSla0QIhRP/ENEkqSYB1P8udxud8Sz9ECPP1dCdSY0V8zzaXZuKSKULVJ7LSTLssPh0H9gqafhiGOXV4TUrcj+Lc3OCOFaiInL5XrggQdiHivdoRcMES9D2grUi951jMMwW5py+L9Q8mpra5cuXSrLco/HX7S9e/eOHj06HVUlLc7xbbAtTS20QslbsGAB0XH+juB0OgkhGzZsCIVC5MI9q3SUl0Lm2dJkZKjDmE10Xgv5fD66i9ra2hRF8fv9dNDv99MJaK9GluW2tjbtU3S83+93OBzRg+pyVLIsqyODwaCi+b+nuq441InpvKr4d+S6myubt5Rk5bUQIhSbSE3iqgAADGZJREFUnvMOzZgkSX6/n96zkmU5Ynz0oKIosixLkkQIUWeJWHj0uvTXqT4VJ0Jx5oq59mzYUiVbI2TGnysuLy8nF968AzlEEIS6urps+xlzXAsBMEGEAJjgpna2i//BBBP2w7MNIpTtEJIsh44cABNECIAJIgTABBECYIIIATBBhACYIEIATBAhACaIEAATRAiACSIEwAQRAmCCCAEwMek7tQ8fPlxfX8+7CjACk0aotbW1oqKCdxVgBGb87gQDq6+vr6iowGuaSbgWAmCCCAEwQYQAmCBCAEwQIQAmiBAAE0QIgAkiBMAEEQJggggBMEGEAJggQgBMECEAJogQABNECIAJIgTABBECYIIIATBBhACYIEIATBAhACaIEAATRAiACSIEwAQRAmCCCAEwQYQAmCBCAEwQIQAmiBAAE0QIgAkiBMAEEQJgYtIfijSMY8eOvfrqq+rgX//6V0KI0+lUx/Tr16+yspJDZaaBH4rMbefOnRs8ePDXX3+dl5dHCFEURVEUi+WbzkU4HF60aNH69eu51mhw6Mjltl69epWXl+fn54fD4XA43NHR0dnZGb6AEIImKN3QCuW8xsbGkpKSmE8VFhYeO3YsPx/d9TRCK5Tzbr311sGDB0ePt1qtCxcuRH7SDRHKeRaLpbKysqCgIGJ8OBxesGABl5JMBR05I9i6devkyZMjRl5++eWfffaZIAhcSjIPtEJGMGnSpCuvvFI7xmq1Ll68GPnJAETIIKqqqqxWqzqIXlzGoCNnEJ988snYsWPVwVGjRn366acc6zEPtEIGMWbMmHHjxtGem9VqXbJkCe+KzAIRMo5FixbR9yiEw+F58+bxLscs0JEzDlmWr7rqKkVRJk6cSN8sBxmAVsg4rrzyyptuuokQsmjRIt61mAhaIVJfX19RUcG7Cogt+49PvPvjG3V1dbxLSIETJ0688MILjz32GO9CUsDr9a5cuZJ3FT1DhL5hmOvv6dOnX3311byrSI2ciBCuhYzGMPnJFYgQABNECIAJIgTABBECYIIIATBBhACYIEIATBAhACaIEAATRAiACSIEwAQRAmCCCAEwQYSSFAgEamtrS0tLeRcCnOHzQklavnz56tWreVdBQqHQxx9/vGvXLo/H09DQoHOumF/R6HA4Ro8ePW3atIEDB6a0RoNDK5SkVatW8S6BEEIcDscbb7yxdOlSj8ejfy5FUfx+P30cDAbprxKVlJS4XK6qqqpAIJCeYo0JEcptK1asWLFiRRIzFhUV0QdqmzNhwoS1a9cSQu67775QKJSqCg0PEUpAKBSqra0VBKG0tHTv3r3apwKBQE1NDX2qqamJfPtiyePx0Kfa29vVWej0LpcrEAioPavo5STNbrfb7faEZikqKlq2bJnH42lubs7mTcsuiunRLx7RM6UoijabjfZ83G63ugP9fr8oim63W1GUxsZGQojP5xNFkU7g9XoVRZFlmRBis9noohwOhyzLiqIEg0FJkuIsR09hMV9KSZIkSUp0rmAwqK2T46bpf134yoES003nS0Uv1tva2uggPdTojDRO6pSEEHrsRhyj2kFCiN/vp4/pZUmc5fQoubNhd3Npx3PcNEQoZ+h8qWw2W8Rk6nGjnpUjmvc4xxldmtvtVq/m4yynR+mLEMdNQ4Ryhs6XKvpV7+546m4W7WBbW5t6VDkcju5WoVMKI0RbV7WJ4LhpiFDOSFWE1D5ed7NEL8Hn89FzNj3UultOErUlPRe9UGlsbNROw2XTEKGcofOlcjqd5NsXwepxQ5+SJIl2Xfx+v/a4iZ6ePlb7OT6fL/5yepSqCNGLflEUI7aay6YhQjlD50tF7zuJokhvN9GzNSHEZrOp/6ZUybIc8b9L9fYDvdSmxxNdlCzL9HiKuZweC1OXrL32UHq6Ixc9F73VJoqiejOgu5Iys2mIUM7Q/1LJskw7JzQ29C4tPW5kWaY3cG02Gz04tIdLzEF6JiaaC4aYy4mPRFGfihOh6LloGfQmdfRWc9m0XIkQftnhm192wH7INrnyuuDdCQBMECEAJviwQ7aL+cEEVfb3cwwPEcp2CEmWQ0cOgAkiBMAEEQJggggBMEGEAJggQgBMECEAJogQABNECIAJIgTABBECYIIIATBBhACY4J3a34j/mQKA7iBCZMqUKfRj+gbg9XpXrlxpmM3JCfjuBEPJle8bMBJcCwEwQYQAmCBCAEwQIQAmiBAAE0QIgAkiBMAEEQJggggBMEGEAJggQgBMECEAJogQABNECIAJIgTABBECYIIIATBBhACYIEIATBAhACaIEAATRAiACSIEwAQRAmCCCAEwQYQAmCBCAEwQIQAmiBAAE0QIgAkiBMAEEQJggl+5y23hcPjUqVPq4Ndff00I+eqrr9QxgiAUFhZyqMw08Ct3ue3o0aPDhw/v7OzsboIZM2Zs3rw5kyWZDTpyuW3o0KHTpk2zWGK/joIgLFiwIMMlmQ0ilPOqqqq6+7lyi8Uyd+7cDNdjNohQzps7d25eXl70+Ly8vFmzZn3nO9/JfEmmggjlvAEDBsyaNSs/P/LOkKIoCxcu5FKSqSBCRrBw4cLoOwoFBQV33XUXl3pMBREyAlEUL7nkEu2Y/Pz8e+65p1+/frxKMg9EyAh69+49Z84cq9Wqjuno6Lj33ns5lmQeiJBBVFZWhsNhdXDAgAG33347x3rMAxEyiJKSkksvvZQ+tlqt8+fPLygo4FuSSSBCBpGfnz9//nzalwuHw5WVlbwrMgu8wcc4Pvjgg6lTpxJChgwZcuTIke7esgCphb1sHLfccsuwYcMIIVVVVchPxuCd2sTr9T777LO8q0iN/v37E0J27NhRXl7Ou5bU2LhxI+8SeoBzFTl06NCmTZt4V5EaI0eO7N+//6BBg3gXkgKHDx/OidcFrdA3sv9sp1N9ff28efN4V5EC9fX1FRUVvKvoGVohozFGfnIIIgTABBECYIIIATBBhACYIEIATBAhACaIEAATRAiACSIEwAQRAmCCCAEwQYQAmCBCAEwQoSQFAoHa2trS0lLehQBniFCSli9fvmDBAo/Hw7eM9vb26upqQRCqq6ubmpp0ziXEUlNT4/F4QqFQWgs2HkQoSatWreJdAgmFQjt37ly1alUwGJw+ffptt92mM9KKovj9fvo4GAwqiqIoSklJicvlqqqqCgQC6azaaBChHNbc3CyKIiFk4MCB8+fPJ4To71gWFRXRBwMHDqQPJkyYsHbtWkLIfffdh7ZIP0QoAaFQqLa2VhCE0tLSvXv3ap8KBAI1NTX0Kdqh0l4seTwe+lR7e7s6C53e5XIFAgH1B4KilxMHzY+WzWZTH9vtdrvdntAGFhUVLVu2zOPxNDc38920XKKYXl1dnc79IIqizWajPR+3263uQL/fL4qi2+1WFKWxsZEQ4vP51OPb6/UqiiLLMiHEZrPRRTkcDlmWFUUJBoOSJMVZjs6tCAaDhJCGhgZ1jCRJkiTFmSXmAUCXo9bJcdP0vy585UCJ6abzpWpoaCCEtLW10UF6qNEZaZzUKQkh9NiNOEa1g4QQv99PH9PLkjjL0aOxsVEURfXCRo/uzqHa8Rw3DRHKGTpfKtpH0o5Rj5voDhUdH+c4o0tzu93ag7675eghiiJtE/TTEyGOm4YI5QydL1X0q97d8dTdLNrBtrY29ahyOBzdrUInt9vtdDoTnSvm6mjrqjYRHDcNEcoZqYqQ2sfrbpboJfh8PnrOpodad8uJz+fz6e/vxSmPohcqjY2N2mm4bBoilDN0vlROp5N8+yJYPW7oU5Ik0a6L3+/XHjfR09PHaj/H5/PFX04cEdPQA1fXZsc67ulFvyiKEVvNZdMQoZyh86Wi951EUaS3m+jZmhBis9nUf1OqZFmO+N+levuBXmrT44kuSpZlejzFXE6ckugRHzGLelMu/h05tR7t4U7zo94M6K6kDGyaggjlEP0vlSzLtHNCY0Pv0tLjRpZlegPXZrPRg0N7uMQcpGdiorlgiLmcOLT/BVKpnaU4EYqei5YR84YEl01TcidC+H2hb767Gfsh2+TK64J3JwAwQYQAmODHUbKd+h6zmLK/n2N4iFC2Q0iyHDpyAEwQIQAmiBAAE0QIgAkiBMAEEQJggggBMEGEAJggQgBMECEAJogQABNECIAJIgTABO/U/kZ5eTnvEuBbDh8+zLsEXdAKkREjRpSVlfGuAiINHz48J14XfHcCABO0QgBMECEAJogQABNECIDJ/weC8lD0grOR1gAAAABJRU5ErkJggg==\n",
      "text/plain": "<IPython.core.display.Image object>"
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "plot_model(model, to_file='model.png')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "outputs": [
    {
     "data": {
      "text/plain": "251"
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_cols = [x for x in data.columns if x != '燃料消耗量']\n",
    "len(feature_cols)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "outputs": [],
   "source": [
    "train_data, valid = train_test_split(data, test_size=0.2, shuffle=True, random_state=666)\n",
    "valid_data, test_data = train_test_split(valid, test_size=0.5, shuffle=True, random_state=666)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "outputs": [],
   "source": [
    "X_train, Y_train = train_data[feature_cols], train_data['燃料消耗量']\n",
    "X_valid, Y_valid = valid_data[feature_cols], valid_data['燃料消耗量']\n",
    "X_test, Y_test = test_data[feature_cols], test_data['燃料消耗量']"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "outputs": [
    {
     "data": {
      "text/plain": "((922, 1, 251), (922, 1))"
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train = np.expand_dims(X_train.values, axis=1)\n",
    "y_train = Y_train.values.reshape(-1, 1)\n",
    "x_train.shape, y_train.shape"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "outputs": [],
   "source": [
    "x_valid = np.expand_dims(X_valid.values, axis=1)\n",
    "y_valid = Y_valid.values.reshape(-1, 1)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "outputs": [],
   "source": [
    "x_test = np.expand_dims(X_test.values, axis=1)\n",
    "y_test = Y_test.values.reshape(-1, 1)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "outputs": [],
   "source": [
    "callbacks = [keras.callbacks.EarlyStopping(monitor='val_loss', patience=int(10)),\n",
    "             keras.callbacks.ModelCheckpoint('./best_model.h5', monitor='val_loss', verbose=0, save_best_only=False, save_weights_only=False, mode='auto', period=1)]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 922 samples, validate on 116 samples\n",
      "Epoch 1/100\n",
      "922/922 [==============================] - 1s 1ms/step - loss: 0.0396 - val_loss: 0.0128\n",
      "Epoch 2/100\n",
      "922/922 [==============================] - 0s 69us/step - loss: 0.0074 - val_loss: 0.0057\n",
      "Epoch 3/100\n",
      "922/922 [==============================] - 0s 78us/step - loss: 0.0048 - val_loss: 0.0040\n",
      "Epoch 4/100\n",
      "922/922 [==============================] - 0s 61us/step - loss: 0.0035 - val_loss: 0.0028\n",
      "Epoch 5/100\n",
      "922/922 [==============================] - 0s 77us/step - loss: 0.0030 - val_loss: 0.0023\n",
      "Epoch 6/100\n",
      "922/922 [==============================] - 0s 69us/step - loss: 0.0025 - val_loss: 0.0020\n",
      "Epoch 7/100\n",
      "922/922 [==============================] - 0s 86us/step - loss: 0.0023 - val_loss: 0.0020\n",
      "Epoch 8/100\n",
      "922/922 [==============================] - 0s 78us/step - loss: 0.0023 - val_loss: 0.0018\n",
      "Epoch 9/100\n",
      "922/922 [==============================] - 0s 67us/step - loss: 0.0022 - val_loss: 0.0017\n",
      "Epoch 10/100\n",
      "922/922 [==============================] - 0s 61us/step - loss: 0.0019 - val_loss: 0.0016\n",
      "Epoch 11/100\n",
      "922/922 [==============================] - 0s 69us/step - loss: 0.0019 - val_loss: 0.0016\n",
      "Epoch 12/100\n",
      "922/922 [==============================] - 0s 69us/step - loss: 0.0019 - val_loss: 0.0017\n",
      "Epoch 13/100\n",
      "922/922 [==============================] - 0s 52us/step - loss: 0.0019 - val_loss: 0.0017\n",
      "Epoch 14/100\n",
      "922/922 [==============================] - 0s 69us/step - loss: 0.0018 - val_loss: 0.0015\n",
      "Epoch 15/100\n",
      "922/922 [==============================] - 0s 87us/step - loss: 0.0018 - val_loss: 0.0015\n",
      "Epoch 16/100\n",
      "922/922 [==============================] - 0s 52us/step - loss: 0.0016 - val_loss: 0.0013\n",
      "Epoch 17/100\n",
      "922/922 [==============================] - 0s 68us/step - loss: 0.0015 - val_loss: 0.0014\n",
      "Epoch 18/100\n",
      "922/922 [==============================] - 0s 73us/step - loss: 0.0014 - val_loss: 0.0013\n",
      "Epoch 19/100\n",
      "922/922 [==============================] - 0s 52us/step - loss: 0.0017 - val_loss: 0.0014\n",
      "Epoch 20/100\n",
      "922/922 [==============================] - 0s 70us/step - loss: 0.0017 - val_loss: 0.0013\n",
      "Epoch 21/100\n",
      "922/922 [==============================] - 0s 73us/step - loss: 0.0015 - val_loss: 0.0013\n",
      "Epoch 22/100\n",
      "922/922 [==============================] - 0s 69us/step - loss: 0.0014 - val_loss: 0.0013\n",
      "Epoch 23/100\n",
      "922/922 [==============================] - 0s 69us/step - loss: 0.0014 - val_loss: 0.0012\n",
      "Epoch 24/100\n",
      "922/922 [==============================] - 0s 69us/step - loss: 0.0013 - val_loss: 0.0014\n",
      "Epoch 25/100\n",
      "922/922 [==============================] - 0s 69us/step - loss: 0.0013 - val_loss: 0.0012\n",
      "Epoch 26/100\n",
      "922/922 [==============================] - 0s 69us/step - loss: 0.0013 - val_loss: 0.0011\n",
      "Epoch 27/100\n",
      "922/922 [==============================] - 0s 65us/step - loss: 0.0013 - val_loss: 0.0012\n",
      "Epoch 28/100\n",
      "922/922 [==============================] - 0s 89us/step - loss: 0.0013 - val_loss: 0.0013\n",
      "Epoch 29/100\n",
      "922/922 [==============================] - 0s 52us/step - loss: 0.0013 - val_loss: 0.0011\n",
      "Epoch 30/100\n",
      "922/922 [==============================] - 0s 70us/step - loss: 0.0013 - val_loss: 0.0012\n",
      "Epoch 31/100\n",
      "922/922 [==============================] - 0s 60us/step - loss: 0.0014 - val_loss: 0.0013\n",
      "Epoch 32/100\n",
      "922/922 [==============================] - 0s 70us/step - loss: 0.0012 - val_loss: 0.0012\n",
      "Epoch 33/100\n",
      "922/922 [==============================] - 0s 51us/step - loss: 0.0013 - val_loss: 0.0012\n",
      "Epoch 34/100\n",
      "922/922 [==============================] - 0s 69us/step - loss: 0.0012 - val_loss: 0.0012\n",
      "Epoch 35/100\n",
      "922/922 [==============================] - 0s 60us/step - loss: 0.0013 - val_loss: 0.0014\n",
      "Epoch 36/100\n",
      "922/922 [==============================] - 0s 69us/step - loss: 0.0012 - val_loss: 0.0011\n",
      "Epoch 37/100\n",
      "922/922 [==============================] - 0s 62us/step - loss: 0.0011 - val_loss: 0.0011\n",
      "Epoch 38/100\n",
      "922/922 [==============================] - 0s 77us/step - loss: 0.0012 - val_loss: 0.0012\n",
      "Epoch 39/100\n",
      "922/922 [==============================] - 0s 60us/step - loss: 0.0013 - val_loss: 0.0011\n",
      "Epoch 40/100\n",
      "922/922 [==============================] - 0s 69us/step - loss: 0.0012 - val_loss: 0.0011\n",
      "Epoch 41/100\n",
      "922/922 [==============================] - 0s 69us/step - loss: 0.0012 - val_loss: 0.0014\n",
      "Epoch 42/100\n",
      "922/922 [==============================] - 0s 69us/step - loss: 0.0013 - val_loss: 0.0014\n",
      "Epoch 43/100\n",
      "922/922 [==============================] - 0s 69us/step - loss: 0.0013 - val_loss: 0.0011\n",
      "Epoch 44/100\n",
      "922/922 [==============================] - 0s 51us/step - loss: 0.0012 - val_loss: 0.0011\n",
      "Epoch 45/100\n",
      "922/922 [==============================] - 0s 95us/step - loss: 0.0011 - val_loss: 0.0011\n",
      "Epoch 46/100\n",
      "922/922 [==============================] - 0s 68us/step - loss: 0.0011 - val_loss: 0.0011\n",
      "Epoch 47/100\n",
      "922/922 [==============================] - 0s 69us/step - loss: 0.0011 - val_loss: 0.0012\n",
      "Epoch 48/100\n",
      "922/922 [==============================] - 0s 66us/step - loss: 0.0011 - val_loss: 0.0010\n",
      "Epoch 49/100\n",
      "922/922 [==============================] - 0s 69us/step - loss: 9.9569e-04 - val_loss: 9.4151e-04\n",
      "Epoch 50/100\n",
      "922/922 [==============================] - 0s 69us/step - loss: 0.0010 - val_loss: 0.0011\n",
      "Epoch 51/100\n",
      "922/922 [==============================] - 0s 69us/step - loss: 0.0010 - val_loss: 0.0011\n",
      "Epoch 52/100\n",
      "922/922 [==============================] - 0s 77us/step - loss: 0.0010 - val_loss: 9.9602e-04\n",
      "Epoch 53/100\n",
      "922/922 [==============================] - 0s 52us/step - loss: 9.3358e-04 - val_loss: 0.0012\n",
      "Epoch 54/100\n",
      "922/922 [==============================] - 0s 69us/step - loss: 0.0010 - val_loss: 9.6337e-04\n",
      "Epoch 55/100\n",
      "922/922 [==============================] - 0s 69us/step - loss: 9.9897e-04 - val_loss: 9.9428e-04\n",
      "Epoch 56/100\n",
      "922/922 [==============================] - 0s 69us/step - loss: 0.0010 - val_loss: 9.1976e-04\n",
      "Epoch 57/100\n",
      "922/922 [==============================] - 0s 60us/step - loss: 9.8333e-04 - val_loss: 0.0011\n",
      "Epoch 58/100\n",
      "922/922 [==============================] - 0s 86us/step - loss: 0.0011 - val_loss: 0.0010\n",
      "Epoch 59/100\n",
      "922/922 [==============================] - 0s 69us/step - loss: 0.0010 - val_loss: 0.0010\n",
      "Epoch 60/100\n",
      "922/922 [==============================] - 0s 51us/step - loss: 9.6106e-04 - val_loss: 9.5494e-04\n",
      "Epoch 61/100\n",
      "922/922 [==============================] - 0s 87us/step - loss: 9.1071e-04 - val_loss: 8.9771e-04\n",
      "Epoch 62/100\n",
      "922/922 [==============================] - 0s 69us/step - loss: 9.1379e-04 - val_loss: 9.4967e-04\n",
      "Epoch 63/100\n",
      "922/922 [==============================] - 0s 69us/step - loss: 9.3075e-04 - val_loss: 9.1627e-04\n",
      "Epoch 64/100\n",
      "922/922 [==============================] - 0s 78us/step - loss: 8.8605e-04 - val_loss: 9.3663e-04\n",
      "Epoch 65/100\n",
      "922/922 [==============================] - 0s 69us/step - loss: 9.5708e-04 - val_loss: 0.0011\n",
      "Epoch 66/100\n",
      "922/922 [==============================] - 0s 68us/step - loss: 9.5701e-04 - val_loss: 8.9826e-04\n",
      "Epoch 67/100\n",
      "922/922 [==============================] - 0s 60us/step - loss: 9.4454e-04 - val_loss: 0.0011\n",
      "Epoch 68/100\n",
      "922/922 [==============================] - 0s 74us/step - loss: 9.5393e-04 - val_loss: 9.7981e-04\n",
      "Epoch 69/100\n",
      "922/922 [==============================] - 0s 104us/step - loss: 9.5125e-04 - val_loss: 0.0010\n",
      "Epoch 70/100\n",
      "922/922 [==============================] - 0s 78us/step - loss: 9.5720e-04 - val_loss: 9.7615e-04\n",
      "Epoch 71/100\n",
      "922/922 [==============================] - 0s 64us/step - loss: 9.2241e-04 - val_loss: 0.0010\n"
     ]
    },
    {
     "data": {
      "text/plain": "<keras.callbacks.callbacks.History at 0x2a12af4d9c8>"
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(x_train, y_train, epochs=100, batch_size=32, validation_data=(x_test, y_test), shuffle=True,\n",
    "          callbacks=callbacks)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "outputs": [],
   "source": [
    "y_pred = model.predict(x_test)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "outputs": [
    {
     "data": {
      "text/plain": "array([0.30161506, 0.12067786, 0.43900865, 0.4143401 , 0.11434203,\n       0.87028706, 0.15387392, 0.86223227, 0.8570186 , 0.4433931 ,\n       0.7649788 , 0.36369222, 0.33063045, 0.7437426 , 0.3493362 ,\n       0.7671248 , 0.7743846 , 0.12363896, 0.4528606 , 0.75037146,\n       0.45131576, 0.34990048, 0.7552419 , 0.7508755 , 0.7558205 ,\n       0.3391131 , 0.76618046, 0.38217723, 0.30887872, 0.36930698,\n       0.3591324 , 0.83817935, 0.9321221 , 0.35430533, 0.11836711,\n       0.764429  , 0.7478696 , 0.74976325, 0.3656214 , 0.3482211 ,\n       0.3658831 , 0.35415024, 0.29030812, 0.7965492 , 0.9372817 ,\n       0.11179626, 0.33758143, 0.305908  , 0.12149343, 0.41378874,\n       0.09611899, 0.36266702, 0.76215094, 0.41939664, 0.7642038 ,\n       0.36630815, 0.36369124, 0.775969  , 0.7431689 , 0.49149197,\n       0.35072863, 0.7608663 , 0.88904417, 0.11546668, 0.32508087,\n       0.78478754, 0.2949888 , 0.9328996 , 0.26087016, 0.15387377,\n       0.7867287 , 0.35840425, 0.8485855 , 0.36474293, 0.86086893,\n       0.85082245, 0.37929475, 0.88898706, 0.44798538, 0.74498856,\n       0.7642088 , 0.9374167 , 0.24628928, 0.1150094 , 0.35409844,\n       0.34573317, 0.1182591 , 0.35334843, 0.8806509 , 0.3744196 ,\n       0.12240422, 0.7410463 , 0.3571657 , 0.44970232, 0.8927134 ,\n       0.76465344, 0.7640152 , 0.33744502, 0.7715051 , 0.44094718,\n       0.33831298, 0.93699497, 0.30656263, 0.10126469, 0.8242742 ,\n       0.85100025, 0.42451733, 0.362445  , 0.77677643, 0.40487826,\n       0.78558755, 0.339495  , 0.8240729 , 0.7534524 , 0.93057597,\n       0.3128613 ], dtype=float32)"
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = np.squeeze(y_pred)\n",
    "y_pred"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "outputs": [],
   "source": [
    "from sklearn.metrics import mean_absolute_error, mean_squared_error, mean_absolute_percentage_error, r2_score"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "outputs": [],
   "source": [
    "y_true = np.squeeze(y_test)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MSE: 1.02E-03\n",
      "RMSE: 0.032\n",
      "MAE: 0.0236\n",
      "MAPE: 7.07%\n",
      "R_2: 0.9858\n"
     ]
    }
   ],
   "source": [
    "MSE = mean_squared_error(y_true, y_pred)\n",
    "RMSE = np.sqrt(mean_squared_error(y_true, y_pred))\n",
    "MAE = mean_absolute_error(y_true, y_pred)\n",
    "MAPE = mean_absolute_percentage_error(y_true, y_pred)\n",
    "R_2 = r2_score(y_true, y_pred)\n",
    "print(f\"MSE: {format(MSE, '.2E')}\")\n",
    "print(f'RMSE: {round(RMSE, 4)}')\n",
    "print(f'MAE: {round(MAE, 4)}')\n",
    "print(f'MAPE: {round(MAPE * 100, 2)}%')\n",
    "print(f'R_2: {round(R_2, 4)}')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "outputs": [],
   "source": [
    "def recover(x, col='燃料消耗量'):\n",
    "    return np.expm1(x * (maxs[col] - mins[col]) + mins[col])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "outputs": [],
   "source": [
    "y_true_recover = recover(y_true)\n",
    "y_pred_recover = recover(y_pred)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MSE: 7.76E+04\n",
      "RMSE: 278.5442\n",
      "MAE: 166.5543\n",
      "MAPE: 9.54%\n",
      "R_2: 0.9717\n"
     ]
    }
   ],
   "source": [
    "MSE = mean_squared_error(y_true_recover, y_pred_recover)\n",
    "RMSE = np.sqrt(mean_squared_error(y_true_recover, y_pred_recover))\n",
    "MAE = mean_absolute_error(y_true_recover, y_pred_recover)\n",
    "MAPE = mean_absolute_percentage_error(y_true_recover, y_pred_recover)\n",
    "R_2 = r2_score(y_true_recover, y_pred_recover)\n",
    "print(f\"MSE: {format(MSE, '.2E')}\")\n",
    "print(f'RMSE: {round(RMSE, 4)}')\n",
    "print(f'MAE: {round(MAE, 4)}')\n",
    "print(f'MAPE: {round(MAPE * 100, 2)}%')\n",
    "print(f'R_2: {round(R_2, 4)}')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}