predict_future_sales_with_text_features.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import gc\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.metrics import r2_score\n",
    "import lightgbm as lgb\n",
    "from catboost import CatBoostRegressor\n",
    "from tqdm import tqdm_notebook\n",
    "from itertools import product"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "input_folder = \"../predict_future_sales\" # or input_folder = os.getcwd()\n",
    "file_names = (os.listdir(input_folder))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['.ipynb_checkpoints',\n",
       " '1.feather',\n",
       " 'benchmark.ipynb',\n",
       " 'city_name_eng.csv',\n",
       " 'city_name_rus.csv',\n",
       " 'coordinates.csv',\n",
       " 'EDA.ipynb',\n",
       " 'feature generation.ipynb',\n",
       " 'geocoding_and_ translation.ipynb',\n",
       " 'GoogleNews-vectors-negative300.bin.gz',\n",
       " 'hol.feather',\n",
       " 'itc_en.feather',\n",
       " 'itc_en_with_word_embed.feather',\n",
       " 'items.csv',\n",
       " 'item_categories.csv',\n",
       " 'it_en.feather',\n",
       " 'it_en_with_word_embed.feather',\n",
       " 'learn_error.tsv',\n",
       " 'meta.tsv',\n",
       " 'predict_future_sales.ipynb',\n",
       " 'predict_future_sales_with_text_features.ipynb',\n",
       " 'Programming_assignment_week_4 (1).ipynb',\n",
       " 'references.txt',\n",
       " 'sales_train.csv.gz',\n",
       " 'sales_train_v2.csv',\n",
       " 'sample_submission.csv',\n",
       " 'sample_submission.csv.gz',\n",
       " 'sample_submission1.csv',\n",
       " 'shops.csv',\n",
       " 'shops_city_longitude_latitude_en.csv',\n",
       " 'sh_en.feather',\n",
       " 'test.csv',\n",
       " 'test.csv.gz',\n",
       " 'time_left.tsv',\n",
       " 'train']"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "sales_train = pd.read_csv('sales_train.csv.gz')\n",
    "shops = pd.read_csv('shops.csv')\n",
    "items = pd.read_csv('items.csv')\n",
    "item_cats = pd.read_csv('item_categories.csv')\n",
    "sample_sub = pd.read_csv('sample_submission.csv.gz')\n",
    "test = pd.read_csv('test.csv.gz')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "import feather\n",
    "import pyarrow\n",
    "hol = pyarrow.feather.read_feather('hol.feather')\n",
    "it_en = pyarrow.feather.read_feather('it_en.feather')\n",
    "itc_en = pyarrow.feather.read_feather('itc_en.feather')\n",
    "sh_en = pyarrow.feather.read_feather('sh_en.feather')\n",
    "coordinates = pd.read_csv('coordinates.csv')\n",
    "city_name_eng = pd.read_csv('city_name_eng.csv')\n",
    "it_en_with_word_embed = pyarrow.feather.read_feather('it_en_with_word_embed.feather')\n",
    "itc_en_with_word_embed = pyarrow.feather.read_feather('itc_en_with_word_embed.feather')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "it_en.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "itc_en.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "items.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "coordinates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sh_en.head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "shops.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "sample_sub.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "test['item_cnt_day'] = -1\n",
    "test['date_block_num'] = 34\n",
    "sales_train['ID'] = 999"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "sales_train.head()"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# concat with item and item_cat text features\n",
    "sales_train = sales_train.merge(it_en_with_word_embed[['it_id','itc_id']], left_on='item_id', right_on='it_id', how='left').drop(labels =['it_id'],axis=1) # add itc_id\n",
    "sales_train = sales_train.merge(it_en_with_word_embed.drop(labels=['itc_id','en_it_name'],axis=1).iloc[:,1:], left_on='item_id', right_on='it_id', how='left').drop(labels =['it_id'],axis=1) # add item word embedding features \n",
    "sales_train = sales_train.merge(itc_en_with_word_embed.drop(labels=['en_itc_name'],axis=1).iloc[:,1:], left_on='itc_id', right_on='itc_id', how='left').head() # add item category word embedding features "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>date_block_num</th>\n",
       "      <th>shop_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>item_price</th>\n",
       "      <th>item_cnt_day</th>\n",
       "      <th>ID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>02.01.2013</td>\n",
       "      <td>0</td>\n",
       "      <td>59</td>\n",
       "      <td>22154</td>\n",
       "      <td>999.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>03.01.2013</td>\n",
       "      <td>0</td>\n",
       "      <td>25</td>\n",
       "      <td>2552</td>\n",
       "      <td>899.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>05.01.2013</td>\n",
       "      <td>0</td>\n",
       "      <td>25</td>\n",
       "      <td>2552</td>\n",
       "      <td>899.00</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>06.01.2013</td>\n",
       "      <td>0</td>\n",
       "      <td>25</td>\n",
       "      <td>2554</td>\n",
       "      <td>1709.05</td>\n",
       "      <td>1.0</td>\n",
       "      <td>999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>15.01.2013</td>\n",
       "      <td>0</td>\n",
       "      <td>25</td>\n",
       "      <td>2555</td>\n",
       "      <td>1099.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>999</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         date  date_block_num  shop_id  item_id  item_price  item_cnt_day   ID\n",
       "0  02.01.2013               0       59    22154      999.00           1.0  999\n",
       "1  03.01.2013               0       25     2552      899.00           1.0  999\n",
       "2  05.01.2013               0       25     2552      899.00          -1.0  999\n",
       "3  06.01.2013               0       25     2554     1709.05           1.0  999\n",
       "4  15.01.2013               0       25     2555     1099.00           1.0  999"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sales_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def downcast_dtypes(df):\n",
    "    '''\n",
    "        Changes column types in the dataframe: \n",
    "                `float64` type to `float32`\n",
    "                `int64`   type to `int32`\n",
    "    '''\n",
    "\n",
    "    # Select columns to downcast\n",
    "    float_cols = [c for c in df if df[c].dtype == \"float64\"]\n",
    "    int_cols =   [c for c in df if df[c].dtype == \"int64\"]\n",
    " \n",
    "    # Downcast\n",
    "    df[float_cols] = df[float_cols].astype(np.float32)\n",
    "    df[int_cols]   = df[int_cols].astype(np.int32)\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "sales = sales_train[['ID','date_block_num','shop_id','item_id','item_cnt_day']]\n",
    "test = test[['ID','date_block_num','shop_id','item_id','item_cnt_day']]\n",
    "sales = pd.concat([sales, test])\n",
    "# sales = sales[sales['shop_id'].isin([26, 27, 28])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\groupby\\groupby.py:4658: FutureWarning: using a dict with renaming is deprecated and will be removed in a future version\n",
      "  return super(DataFrameGroupBy, self).aggregate(arg, *args, **kwargs)\n"
     ]
    }
   ],
   "source": [
    "# Create \"grid\" with columns\n",
    "index_cols = ['shop_id', 'item_id', 'date_block_num']\n",
    "\n",
    "# For every month we create a grid from all shops/items combinations from that month\n",
    "grid = [] \n",
    "for block_num in sales['date_block_num'].unique():\n",
    "    cur_shops = sales.loc[sales['date_block_num'] == block_num, 'shop_id'].unique()\n",
    "    cur_items = sales.loc[sales['date_block_num'] == block_num, 'item_id'].unique()\n",
    "    grid.append(np.array(list(product(*[cur_shops, cur_items, [block_num]])),dtype='int32'))\n",
    "\n",
    "# Turn the grid into a dataframe\n",
    "grid = pd.DataFrame(np.vstack(grid), columns = index_cols,dtype=np.int32)\n",
    "\n",
    "# Groupby data to get shop-item-month aggregates\n",
    "gb = sales.groupby(index_cols,as_index=False).agg({'item_cnt_day':{'target':'sum'}})\n",
    "# Fix column names\n",
    "gb.columns = [col[0] if col[-1]=='' else col[-1] for col in gb.columns.values] \n",
    "# Join it to the grid\n",
    "all_data = pd.merge(grid, gb, how='left', on=index_cols).fillna(0)\n",
    "\n",
    "# Same as above but with shop-month aggregates\n",
    "gb = sales.groupby(['shop_id', 'date_block_num'],as_index=False).agg({'item_cnt_day':{'target_shop':'sum'}})\n",
    "gb.columns = [col[0] if col[-1]=='' else col[-1] for col in gb.columns.values]\n",
    "all_data = pd.merge(all_data, gb, how='left', on=['shop_id', 'date_block_num']).fillna(0)\n",
    "\n",
    "# Same as above but with item-month aggregates\n",
    "gb = sales.groupby(['item_id', 'date_block_num'],as_index=False).agg({'item_cnt_day':{'target_item':'sum'}})\n",
    "gb.columns = [col[0] if col[-1] == '' else col[-1] for col in gb.columns.values]\n",
    "all_data = pd.merge(all_data, gb, how='left', on=['item_id', 'date_block_num']).fillna(0)\n",
    "\n",
    "# Downcast dtypes from 64 to 32 bit to save memory\n",
    "all_data = downcast_dtypes(all_data)\n",
    "del grid, gb \n",
    "gc.collect();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>shop_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>date_block_num</th>\n",
       "      <th>target</th>\n",
       "      <th>target_shop</th>\n",
       "      <th>target_item</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>11128045</th>\n",
       "      <td>45</td>\n",
       "      <td>18454</td>\n",
       "      <td>34</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-5100.0</td>\n",
       "      <td>-42.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11128046</th>\n",
       "      <td>45</td>\n",
       "      <td>16188</td>\n",
       "      <td>34</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-5100.0</td>\n",
       "      <td>-42.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11128047</th>\n",
       "      <td>45</td>\n",
       "      <td>15757</td>\n",
       "      <td>34</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-5100.0</td>\n",
       "      <td>-42.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11128048</th>\n",
       "      <td>45</td>\n",
       "      <td>19648</td>\n",
       "      <td>34</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-5100.0</td>\n",
       "      <td>-42.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11128049</th>\n",
       "      <td>45</td>\n",
       "      <td>969</td>\n",
       "      <td>34</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-5100.0</td>\n",
       "      <td>-42.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          shop_id  item_id  date_block_num  target  target_shop  target_item\n",
       "11128045       45    18454              34    -1.0      -5100.0        -42.0\n",
       "11128046       45    16188              34    -1.0      -5100.0        -42.0\n",
       "11128047       45    15757              34    -1.0      -5100.0        -42.0\n",
       "11128048       45    19648              34    -1.0      -5100.0        -42.0\n",
       "11128049       45      969              34    -1.0      -5100.0        -42.0"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_data.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2d905adb9ee24c7daf79b1f3d9ab76c8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=6), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# List of columns that we will use to create lags\n",
    "cols_to_rename = list(all_data.columns.difference(index_cols)) \n",
    "\n",
    "shift_range = [1, 2, 3, 4, 5, 12]\n",
    "\n",
    "for month_shift in tqdm_notebook(shift_range):\n",
    "    train_shift = all_data[index_cols + cols_to_rename].copy()\n",
    "    \n",
    "    train_shift['date_block_num'] = train_shift['date_block_num'] + month_shift\n",
    "    \n",
    "    foo = lambda x: '{}_lag_{}'.format(x, month_shift) if x in cols_to_rename else x\n",
    "    train_shift = train_shift.rename(columns=foo)\n",
    "\n",
    "    all_data = pd.merge(all_data, train_shift, on=index_cols, how='left').fillna(0)\n",
    "\n",
    "del train_shift\n",
    "\n",
    "# Don't use old data from year 2013\n",
    "all_data = all_data[all_data['date_block_num'] >= 12] \n",
    "\n",
    "# List of all lagged features\n",
    "fit_cols = [col for col in all_data.columns if col[-1] in [str(item) for item in shift_range]] \n",
    "# We will drop these at fitting stage\n",
    "to_drop_cols = list(set(list(all_data.columns)) - (set(fit_cols)|set(index_cols))) + ['date_block_num'] \n",
    "\n",
    "# Category for each item\n",
    "item_category_mapping = items[['item_id','item_category_id']].drop_duplicates()\n",
    "\n",
    "all_data = pd.merge(all_data, item_category_mapping, how='left', on='item_id')\n",
    "all_data = downcast_dtypes(all_data)\n",
    "gc.collect();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['target_shop', 'target_item', 'target', 'date_block_num']"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "to_drop_cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>shop_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>date_block_num</th>\n",
       "      <th>target</th>\n",
       "      <th>target_shop</th>\n",
       "      <th>target_item</th>\n",
       "      <th>target_lag_1</th>\n",
       "      <th>target_item_lag_1</th>\n",
       "      <th>target_shop_lag_1</th>\n",
       "      <th>target_lag_2</th>\n",
       "      <th>...</th>\n",
       "      <th>target_lag_4</th>\n",
       "      <th>target_item_lag_4</th>\n",
       "      <th>target_shop_lag_4</th>\n",
       "      <th>target_lag_5</th>\n",
       "      <th>target_item_lag_5</th>\n",
       "      <th>target_shop_lag_5</th>\n",
       "      <th>target_lag_12</th>\n",
       "      <th>target_item_lag_12</th>\n",
       "      <th>target_shop_lag_12</th>\n",
       "      <th>item_category_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6639289</th>\n",
       "      <td>45</td>\n",
       "      <td>18454</td>\n",
       "      <td>34</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-5100.0</td>\n",
       "      <td>-42.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>702.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>675.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>622.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6639290</th>\n",
       "      <td>45</td>\n",
       "      <td>16188</td>\n",
       "      <td>34</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-5100.0</td>\n",
       "      <td>-42.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>702.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6639291</th>\n",
       "      <td>45</td>\n",
       "      <td>15757</td>\n",
       "      <td>34</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-5100.0</td>\n",
       "      <td>-42.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>702.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>675.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>622.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1251.0</td>\n",
       "      <td>55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6639292</th>\n",
       "      <td>45</td>\n",
       "      <td>19648</td>\n",
       "      <td>34</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-5100.0</td>\n",
       "      <td>-42.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>702.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>675.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>622.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6639293</th>\n",
       "      <td>45</td>\n",
       "      <td>969</td>\n",
       "      <td>34</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-5100.0</td>\n",
       "      <td>-42.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>702.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>675.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>622.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>1251.0</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         shop_id  item_id  date_block_num  target  target_shop  target_item  \\\n",
       "6639289       45    18454              34    -1.0      -5100.0        -42.0   \n",
       "6639290       45    16188              34    -1.0      -5100.0        -42.0   \n",
       "6639291       45    15757              34    -1.0      -5100.0        -42.0   \n",
       "6639292       45    19648              34    -1.0      -5100.0        -42.0   \n",
       "6639293       45      969              34    -1.0      -5100.0        -42.0   \n",
       "\n",
       "         target_lag_1  target_item_lag_1  target_shop_lag_1  target_lag_2  \\\n",
       "6639289           1.0                2.0              702.0           0.0   \n",
       "6639290           0.0                1.0              702.0           0.0   \n",
       "6639291           0.0                5.0              702.0           0.0   \n",
       "6639292           0.0                2.0              702.0           0.0   \n",
       "6639293           0.0                3.0              702.0           0.0   \n",
       "\n",
       "               ...         target_lag_4  target_item_lag_4  target_shop_lag_4  \\\n",
       "6639289        ...                  0.0               12.0              675.0   \n",
       "6639290        ...                  0.0                0.0                0.0   \n",
       "6639291        ...                  0.0                4.0              675.0   \n",
       "6639292        ...                  0.0                2.0              675.0   \n",
       "6639293        ...                  0.0                2.0              675.0   \n",
       "\n",
       "         target_lag_5  target_item_lag_5  target_shop_lag_5  target_lag_12  \\\n",
       "6639289           0.0               19.0              622.0            0.0   \n",
       "6639290           0.0                0.0                0.0            0.0   \n",
       "6639291           0.0                8.0              622.0            0.0   \n",
       "6639292           0.0                4.0              622.0            0.0   \n",
       "6639293           0.0                2.0              622.0            0.0   \n",
       "\n",
       "         target_item_lag_12  target_shop_lag_12  item_category_id  \n",
       "6639289                 0.0                 0.0                55  \n",
       "6639290                 0.0                 0.0                64  \n",
       "6639291                 9.0              1251.0                55  \n",
       "6639292                 0.0                 0.0                40  \n",
       "6639293                 6.0              1251.0                37  \n",
       "\n",
       "[5 rows x 25 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_data.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "it_en_with_word_embed.iloc[:,4:] = it_en_with_word_embed.iloc[:,4:].astype(np.float16)\n",
    "itc_en_with_word_embed.iloc[:,3:] = itc_en_with_word_embed.iloc[:,3:].astype(np.float16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add text features\n",
    "all_data = all_data.merge(it_en_with_word_embed[['it_id','itc_id']], left_on='item_id', right_on='it_id', how='left').drop(labels =['it_id'],axis=1) # add itc_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "42"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_data = all_data.merge(it_en_with_word_embed.drop(labels=['itc_id','en_it_name'],axis=1).iloc[:,1:], left_on='item_id', right_on='it_id', how='left').drop(labels =['it_id'],axis=1) # add item word embedding features "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "28"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(6639294, 626)"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_data = all_data.merge(itc_en_with_word_embed.drop(labels=['en_itc_name'],axis=1).iloc[:,1:], left_on='itc_id', right_on='itc_id', how='left') # add item category word embedding features "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "28"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub_test_data = all_data[all_data['date_block_num']==34].drop(to_drop_cols, axis=1)\n",
    "all_data = all_data[all_data['date_block_num']!=34]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test `date_block_num` is 33\n"
     ]
    }
   ],
   "source": [
    "# Save `date_block_num`, as we can't use them as features, but will need them to split the dataset into parts \n",
    "dates = all_data['date_block_num']\n",
    "\n",
    "last_block = dates.max()\n",
    "print('Test `date_block_num` is %d' % last_block)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pyarrow.feather.write_feather(all_data,'all_data_no_sub_test.feather')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "dates_train = dates[dates <  last_block]\n",
    "dates_test  = dates[dates == last_block]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_data['target'] = all_data['target'].clip(0,20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-238172"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "6186922-6425094"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(6425094,)"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_data['date_block_num'].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = all_data.loc[dates <  last_block].drop(to_drop_cols, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test =  all_data.loc[dates == last_block].drop(to_drop_cols, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "ename": "MemoryError",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mMemoryError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-58-c9debfd7da29>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0mall_data\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'target'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mall_data\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'target'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclip\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m20\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 5\u001b[1;33m \u001b[0mX_train\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mall_data\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloc\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mdates\u001b[0m \u001b[1;33m<\u001b[0m  \u001b[0mlast_block\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdrop\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mto_drop_cols\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      6\u001b[0m \u001b[0mX_test\u001b[0m \u001b[1;33m=\u001b[0m  \u001b[0mall_data\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloc\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mdates\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0mlast_block\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdrop\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mto_drop_cols\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      7\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\frame.py\u001b[0m in \u001b[0;36mdrop\u001b[1;34m(self, labels, axis, index, columns, level, inplace, errors)\u001b[0m\n\u001b[0;32m   3692\u001b[0m                                            \u001b[0mindex\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mindex\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcolumns\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3693\u001b[0m                                            \u001b[0mlevel\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mlevel\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minplace\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0minplace\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3694\u001b[1;33m                                            errors=errors)\n\u001b[0m\u001b[0;32m   3695\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3696\u001b[0m     @rewrite_axis_style_signature('mapper', [('copy', True),\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\generic.py\u001b[0m in \u001b[0;36mdrop\u001b[1;34m(self, labels, axis, index, columns, level, inplace, errors)\u001b[0m\n\u001b[0;32m   3106\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0maxis\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabels\u001b[0m \u001b[1;32min\u001b[0m \u001b[0maxes\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3107\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mlabels\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3108\u001b[1;33m                 \u001b[0mobj\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_drop_axis\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlabels\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlevel\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mlevel\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0merrors\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   3109\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3110\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0minplace\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\generic.py\u001b[0m in \u001b[0;36m_drop_axis\u001b[1;34m(self, labels, axis, level, errors)\u001b[0m\n\u001b[0;32m   3139\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3140\u001b[0m                 \u001b[0mnew_axis\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0maxis\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdrop\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlabels\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0merrors\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3141\u001b[1;33m             \u001b[0mdropped\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreindex\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m**\u001b[0m\u001b[1;33m{\u001b[0m\u001b[0maxis_name\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mnew_axis\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   3142\u001b[0m             \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3143\u001b[0m                 \u001b[0mdropped\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0maxes\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0maxis_\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mset_names\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0maxis\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnames\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minplace\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\util\\_decorators.py\u001b[0m in \u001b[0;36mwrapper\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m    184\u001b[0m         \u001b[1;33m@\u001b[0m\u001b[0mwraps\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    185\u001b[0m         \u001b[1;32mdef\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 186\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    187\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    188\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mPY2\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\frame.py\u001b[0m in \u001b[0;36mreindex\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   3561\u001b[0m         \u001b[0mkwargs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpop\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'axis'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3562\u001b[0m         \u001b[0mkwargs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpop\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'labels'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3563\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0msuper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreindex\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   3564\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3565\u001b[0m     \u001b[1;33m@\u001b[0m\u001b[0mAppender\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m_shared_docs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'reindex_axis'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m%\u001b[0m \u001b[0m_shared_doc_kwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\generic.py\u001b[0m in \u001b[0;36mreindex\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   3683\u001b[0m         \u001b[1;31m# perform the reindex on the axes\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3684\u001b[0m         return self._reindex_axes(axes, level, limit, tolerance, method,\n\u001b[1;32m-> 3685\u001b[1;33m                                   fill_value, copy).__finalize__(self)\n\u001b[0m\u001b[0;32m   3686\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3687\u001b[0m     def _reindex_axes(self, axes, level, limit, tolerance, method, fill_value,\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\frame.py\u001b[0m in \u001b[0;36m_reindex_axes\u001b[1;34m(self, axes, level, limit, tolerance, method, fill_value, copy)\u001b[0m\n\u001b[0;32m   3491\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mcolumns\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3492\u001b[0m             frame = frame._reindex_columns(columns, method, copy, level,\n\u001b[1;32m-> 3493\u001b[1;33m                                            fill_value, limit, tolerance)\n\u001b[0m\u001b[0;32m   3494\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3495\u001b[0m         \u001b[0mindex\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0maxes\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'index'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\frame.py\u001b[0m in \u001b[0;36m_reindex_columns\u001b[1;34m(self, new_columns, method, copy, level, fill_value, limit, tolerance)\u001b[0m\n\u001b[0;32m   3516\u001b[0m         return self._reindex_with_indexers({1: [new_columns, indexer]},\n\u001b[0;32m   3517\u001b[0m                                            \u001b[0mcopy\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcopy\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfill_value\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mfill_value\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3518\u001b[1;33m                                            allow_dups=False)\n\u001b[0m\u001b[0;32m   3519\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3520\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_reindex_multi\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maxes\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfill_value\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\generic.py\u001b[0m in \u001b[0;36m_reindex_with_indexers\u001b[1;34m(self, reindexers, fill_value, copy, allow_dups)\u001b[0m\n\u001b[0;32m   3804\u001b[0m                                                 \u001b[0mfill_value\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mfill_value\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3805\u001b[0m                                                 \u001b[0mallow_dups\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mallow_dups\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3806\u001b[1;33m                                                 copy=copy)\n\u001b[0m\u001b[0;32m   3807\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3808\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mcopy\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mnew_data\u001b[0m \u001b[1;32mis\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_data\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\internals.py\u001b[0m in \u001b[0;36mreindex_indexer\u001b[1;34m(self, new_axis, indexer, axis, fill_value, allow_dups, copy)\u001b[0m\n\u001b[0;32m   4419\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0maxis\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   4420\u001b[0m             new_blocks = self._slice_take_blocks_ax0(indexer,\n\u001b[1;32m-> 4421\u001b[1;33m                                                      fill_tuple=(fill_value,))\n\u001b[0m\u001b[0;32m   4422\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   4423\u001b[0m             new_blocks = [blk.take_nd(indexer, axis=axis, fill_tuple=(\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\internals.py\u001b[0m in \u001b[0;36m_slice_take_blocks_ax0\u001b[1;34m(self, slice_or_indexer, fill_tuple)\u001b[0m\n\u001b[0;32m   4499\u001b[0m                     blocks.append(blk.take_nd(blklocs[mgr_locs.indexer],\n\u001b[0;32m   4500\u001b[0m                                               \u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnew_mgr_locs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mmgr_locs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 4501\u001b[1;33m                                               fill_tuple=None))\n\u001b[0m\u001b[0;32m   4502\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   4503\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mblocks\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\internals.py\u001b[0m in \u001b[0;36mtake_nd\u001b[1;34m(self, indexer, axis, new_mgr_locs, fill_tuple)\u001b[0m\n\u001b[0;32m   1252\u001b[0m             \u001b[0mfill_value\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfill_value\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1253\u001b[0m             new_values = algos.take_nd(values, indexer, axis=axis,\n\u001b[1;32m-> 1254\u001b[1;33m                                        allow_fill=False)\n\u001b[0m\u001b[0;32m   1255\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1256\u001b[0m             \u001b[0mfill_value\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfill_tuple\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\algorithms.py\u001b[0m in \u001b[0;36mtake_nd\u001b[1;34m(arr, indexer, axis, out, fill_value, mask_info, allow_fill)\u001b[0m\n\u001b[0;32m   1654\u001b[0m     func = _get_take_nd_function(arr.ndim, arr.dtype, out.dtype, axis=axis,\n\u001b[0;32m   1655\u001b[0m                                  mask_info=mask_info)\n\u001b[1;32m-> 1656\u001b[1;33m     \u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0marr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mindexer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mout\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfill_value\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1657\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1658\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mflip_order\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\algorithms.py\u001b[0m in \u001b[0;36mfunc\u001b[1;34m(arr, indexer, out, fill_value)\u001b[0m\n\u001b[0;32m   1444\u001b[0m         \u001b[0mindexer\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_ensure_int64\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1445\u001b[0m         _take_nd_object(arr, indexer, out, axis=axis, fill_value=fill_value,\n\u001b[1;32m-> 1446\u001b[1;33m                         mask_info=mask_info)\n\u001b[0m\u001b[0;32m   1447\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1448\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\algorithms.py\u001b[0m in \u001b[0;36m_take_nd_object\u001b[1;34m(arr, indexer, out, axis, fill_value, mask_info)\u001b[0m\n\u001b[0;32m   1301\u001b[0m         \u001b[0marr\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0marr\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mout\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1302\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0marr\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0maxis\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m>\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1303\u001b[1;33m         \u001b[0marr\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtake\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m_ensure_platform_int\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0maxis\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mout\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1304\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mneeds_masking\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1305\u001b[0m         \u001b[0moutindexer\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mslice\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m*\u001b[0m \u001b[0marr\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mndim\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mMemoryError\u001b[0m: "
     ]
    }
   ],
   "source": [
    "y_train = all_data.loc[dates <  last_block, 'target'].values\n",
    "y_test =  all_data.loc[dates == last_block, 'target'].values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Linear Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test R-squared for linreg is 0.146042\n"
     ]
    }
   ],
   "source": [
    "lr = LinearRegression()\n",
    "lr.fit(X_train.values, y_train)\n",
    "pred_lr = lr.predict(X_test.values)\n",
    "\n",
    "print('Test R-squared for linreg is %f' % r2_score(y_test.clip(0,20), pred_lr.clip(0,20)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.049918661908493\n",
      "1.0493949987899915\n"
     ]
    }
   ],
   "source": [
    "print(np.sqrt((sum((y_test.clip(0,20)-pred_lr.clip(0,20))**2))/(y_test.shape[0])))\n",
    "print(np.sqrt((sum((y_train.clip(0,20)-lr.predict(X_train).clip(0,20))**2))/(y_train.shape[0])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Found array with 0 sample(s) (shape=(0, 21)) while a minimum of 1 is required.",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-22-d5a45d952cb5>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0msub_pred_lr\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlr\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msub_test_data\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\base.py\u001b[0m in \u001b[0;36mpredict\u001b[1;34m(self, X)\u001b[0m\n\u001b[0;32m    254\u001b[0m             \u001b[0mReturns\u001b[0m \u001b[0mpredicted\u001b[0m \u001b[0mvalues\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    255\u001b[0m         \"\"\"\n\u001b[1;32m--> 256\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_decision_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    257\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    258\u001b[0m     \u001b[0m_preprocess_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mstaticmethod\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m_preprocess_data\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\base.py\u001b[0m in \u001b[0;36m_decision_function\u001b[1;34m(self, X)\u001b[0m\n\u001b[0;32m    237\u001b[0m         \u001b[0mcheck_is_fitted\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"coef_\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    238\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 239\u001b[1;33m         \u001b[0mX\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcheck_array\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maccept_sparse\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'csr'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'csc'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'coo'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    240\u001b[0m         return safe_sparse_dot(X, self.coef_.T,\n\u001b[0;32m    241\u001b[0m                                dense_output=True) + self.intercept_\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\validation.py\u001b[0m in \u001b[0;36mcheck_array\u001b[1;34m(array, accept_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, warn_on_dtype, estimator)\u001b[0m\n\u001b[0;32m    460\u001b[0m                              \u001b[1;34m\" minimum of %d is required%s.\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    461\u001b[0m                              % (n_samples, shape_repr, ensure_min_samples,\n\u001b[1;32m--> 462\u001b[1;33m                                 context))\n\u001b[0m\u001b[0;32m    463\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    464\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mensure_min_features\u001b[0m \u001b[1;33m>\u001b[0m \u001b[1;36m0\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0marray\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mndim\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m2\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: Found array with 0 sample(s) (shape=(0, 21)) while a minimum of 1 is required."
     ]
    }
   ],
   "source": [
    "sub_pred_lr = lr.predict(sub_test_data.values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=4,\n",
       "           max_features='auto', max_leaf_nodes=None,\n",
       "           min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "           min_samples_leaf=1, min_samples_split=2,\n",
       "           min_weight_fraction_leaf=0.0, n_estimators=100, n_jobs=-1,\n",
       "           oob_score=False, random_state=0, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestRegressor\n",
    "clf = RandomForestRegressor(n_estimators=100, max_depth=4, random_state=0, n_jobs = -1, verbose=1)\n",
    "clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pred_rf = clf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test R-squared for rf is 0.254873\n",
      "0.9807371321492697\n",
      "0.9381757394915309\n"
     ]
    }
   ],
   "source": [
    "print('Test R-squared for rf is %f' % r2_score(y_test.clip(0,20), pred_rf.clip(0,20)))\n",
    "print(np.sqrt((sum((y_test.clip(0,20)-pred_rf.clip(0,20))**2))/(y_test.shape[0])))\n",
    "print(np.sqrt((sum((y_train.clip(0,20)-clf.predict(X_train).clip(0,20))**2))/(y_train.shape[0])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sub_pred_rf = clf.predict(sub_test_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Benchmark: Mean "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test R-squared for rf is -0.000711\n",
      "1.136558396230848\n"
     ]
    }
   ],
   "source": [
    "pred_mean =  np.full((238172,), np.mean(y_train))\n",
    "print('Test R-squared for rf is %f' % r2_score(y_test.clip(0,20), pred_mean.clip(0,20)))\n",
    "print(np.sqrt((sum((y_test.clip(0,20)-pred_mean.clip(0,20))**2))/(y_test.shape[0])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Light gbm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test R-squared for LightGBM is 0.352241\n"
     ]
    }
   ],
   "source": [
    "lgb_params = {\n",
    "               'feature_fraction': 0.45,\n",
    "               'metric': 'rmse',\n",
    "               'nthread':11, \n",
    "               'min_data_in_leaf': 2**4, \n",
    "               'bagging_fraction': 0.84, \n",
    "               'learning_rate': 0.122, \n",
    "               'objective': 'mse', \n",
    "               'bagging_seed': 2**7, \n",
    "               'num_leaves': 2**6,\n",
    "               'bagging_freq':1,\n",
    "               'verbose':0 \n",
    "              }\n",
    "\n",
    "model = lgb.train(lgb_params, lgb.Dataset(X_train, label=y_train), 100)\n",
    "pred_lgb = model.predict(X_test)\n",
    "\n",
    "print('Test R-squared for LightGBM is %f' % r2_score(y_test.clip(0,20), pred_lgb.clip(0,20)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9144167633545289\n",
      "0.8168128467909939\n"
     ]
    }
   ],
   "source": [
    "print(np.sqrt((sum((y_test.clip(0,20)-pred_lgb.clip(0,20))**2))/(y_test.shape[0])))\n",
    "print(np.sqrt((sum((y_train.clip(0,20)-model.predict(X_train).clip(0,20))**2))/(y_train.shape[0])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "0.352241\n",
    "0.914416763355\n",
    "0.816812846791"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sub_pred_lgb = model.predict(sub_test_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Catboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:\tlearn: 1.1029154\ttotal: 2.16s\tremaining: 41s\n",
      "1:\tlearn: 1.0294724\ttotal: 3.9s\tremaining: 35.1s\n",
      "2:\tlearn: 0.9857883\ttotal: 5.57s\tremaining: 31.6s\n",
      "3:\tlearn: 0.9552109\ttotal: 7.28s\tremaining: 29.1s\n",
      "4:\tlearn: 0.9371892\ttotal: 9.27s\tremaining: 27.8s\n",
      "5:\tlearn: 0.9186031\ttotal: 11s\tremaining: 25.8s\n",
      "6:\tlearn: 0.9091894\ttotal: 13.1s\tremaining: 24.4s\n",
      "7:\tlearn: 0.9019890\ttotal: 15s\tremaining: 22.5s\n",
      "8:\tlearn: 0.8967270\ttotal: 16.8s\tremaining: 20.5s\n",
      "9:\tlearn: 0.8853466\ttotal: 18.2s\tremaining: 18.2s\n",
      "10:\tlearn: 0.8794866\ttotal: 20s\tremaining: 16.3s\n",
      "11:\tlearn: 0.8753645\ttotal: 22.2s\tremaining: 14.8s\n",
      "12:\tlearn: 0.8714032\ttotal: 23.6s\tremaining: 12.7s\n",
      "13:\tlearn: 0.8694466\ttotal: 25.2s\tremaining: 10.8s\n",
      "14:\tlearn: 0.8680499\ttotal: 26.8s\tremaining: 8.93s\n",
      "15:\tlearn: 0.8651640\ttotal: 28.3s\tremaining: 7.07s\n",
      "16:\tlearn: 0.8641188\ttotal: 30.4s\tremaining: 5.36s\n",
      "17:\tlearn: 0.8623801\ttotal: 31.6s\tremaining: 3.52s\n",
      "18:\tlearn: 0.8524468\ttotal: 33.4s\tremaining: 1.76s\n",
      "19:\tlearn: 0.8503367\ttotal: 34.8s\tremaining: 0us\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<catboost.core.CatBoostRegressor at 0x17b137964e0>"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from catboost import CatBoostRegressor\n",
    "cat_features = [0,1,20]\n",
    "cb = CatBoostRegressor(iterations=20, learning_rate=0.3, depth=3, thread_count=11)\n",
    "cb.fit(X_train, y_train, cat_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test R-squared for catboost is 0.172418\n",
      "1.0335773814558504\n"
     ]
    }
   ],
   "source": [
    "pred_cb = cb.predict(X_test)\n",
    "print('Test R-squared for catboost is %f' % r2_score(y_test.clip(0,20), pred_cb.clip(0,20)))\n",
    "print(np.sqrt((sum((y_test.clip(0,20)-pred_cb.clip(0,20))**2))/(y_test.shape[0])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8364470663085795\n"
     ]
    }
   ],
   "source": [
    "print(np.sqrt((sum((y_train.clip(0,20)-cb.predict(X_train).clip(0,20))**2))/(y_train.shape[0])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_test_level2 = np.c_[sub_pred_lr, sub_pred_lgb] "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train on whole traning data Lightgbm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "all_data['target'] = all_data['target'].clip(0,20)\n",
    "X_train = all_data.drop(to_drop_cols, axis=1)\n",
    "\n",
    "\n",
    "y_train = all_data.loc[:, 'target'].values\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_preds = model.predict(sub_test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lgb_params = {\n",
    "               'feature_fraction': 0.45,\n",
    "               'metric': 'rmse',\n",
    "               'nthread':11, \n",
    "               'min_data_in_leaf': 2**4, \n",
    "               'bagging_fraction': 0.84, \n",
    "               'learning_rate': 0.122, \n",
    "               'objective': 'mse', \n",
    "               'bagging_seed': 2**7, \n",
    "               'num_leaves': 2**6,\n",
    "               'bagging_freq':1,\n",
    "               'verbose':0 \n",
    "              }\n",
    "\n",
    "model = lgb.train(lgb_params, lgb.Dataset(X_train, label=y_train), 100)\n",
    "sub_pred_lgb = model.predict(sub_test_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train on whole traning data Linear Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lr = LinearRegression()\n",
    "lr.fit(X_train.values, y_train)\n",
    "pred_lr = lr.predict(X_test.values)\n",
    "sub_pred_lr = lr.predict(sub_test_data.values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Second level training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Simple convex mix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dates_train_level2 = dates_train[dates_train.isin([27, 28, 29, 30, 31, 32])]\n",
    "\n",
    "# That is how we get target for the 2nd level dataset\n",
    "y_train_level2 = y_train[dates_train.isin([27, 28, 29, 30, 31, 32])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n"
     ]
    }
   ],
   "source": [
    "# And here we create 2nd level feeature matrix, init it with zeros first\n",
    "X_train_level2 = np.zeros([y_train_level2.shape[0], 2])\n",
    "\n",
    "# Now fill `X_train_level2` with metafeatures\n",
    "for cur_block_num in [27, 28, 29, 30, 31, 32]:\n",
    "    \n",
    "    print(cur_block_num)\n",
    "    \n",
    "    '''\n",
    "        1. Split `X_train` into parts\n",
    "           Remember, that corresponding dates are stored in `dates_train` \n",
    "        2. Fit linear regression \n",
    "        3. Fit LightGBM and put predictions          \n",
    "        4. Store predictions from 2. and 3. in the right place of `X_train_level2`. \n",
    "           You can use `dates_train_level2` for it\n",
    "           Make sure the order of the meta-features is the same as in `X_test_level2`\n",
    "    '''  \n",
    "    M=15\n",
    "    X_train_m = X_train[dates_train.isin(list(range(27-M,cur_block_num)))]\n",
    "    y_train_m = y_train[dates_train.isin(list(range(27-M,cur_block_num)))]\n",
    "    X_test_m = X_train[dates_train==cur_block_num]\n",
    "    lr = LinearRegression()\n",
    "    lr.fit(X_train_m.values, y_train_m)\n",
    "    pred_lr_train = lr.predict(X_test_m.values)\n",
    "    \n",
    "    lgb_params = {\n",
    "               'feature_fraction': 0.45,\n",
    "               'metric': 'rmse',\n",
    "               'nthread':11, \n",
    "               'min_data_in_leaf': 2**4, \n",
    "               'bagging_fraction': 0.84, \n",
    "               'learning_rate': 0.122, \n",
    "               'objective': 'mse', \n",
    "               'bagging_seed': 2**7, \n",
    "               'num_leaves': 2**4,\n",
    "               'bagging_freq':1,\n",
    "               'verbose':0 \n",
    "              }\n",
    "\n",
    "    model = lgb.train(lgb_params, lgb.Dataset(X_train_m, label=y_train_m), 100)\n",
    "    pred_lgb_train = model.predict(X_test_m)\n",
    "    \n",
    "    index1 = dates_train.isin(list(range(27,cur_block_num+1))).sum()- (dates_train==cur_block_num).sum()\n",
    "    index2 = dates_train.isin(list(range(27,cur_block_num+1))).sum()\n",
    "    X_train_level2[index1:index2,0] = pred_lr_train\n",
    "    X_train_level2[index1:index2,1] = pred_lgb_train\n",
    "    #  YOUR CODE GOES HERE\n",
    "    \n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train_level2 = X_train_level2.clip(0,20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAD8CAYAAAB0IB+mAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzs3XlwXNd96Pnv797b+4rGvhAAN3EVF4mSKFm7ZMmyHW/ynsRL8iJPKk5eal5NtqmaZOY912Sq5iXjqbw4I5ft2Ilt2bHiWLblRZas3aTETdxJgSv2Heh9ufee+aObC0iIAEiAAIjzqUIROH379q9B9K9P/86554hSCk3TNO3GZ8x3AJqmadr1oRO+pmnaEqETvqZp2hKhE76madoSoRO+pmnaEqETvqZp2hKhE76madoSoRO+pmnaEqETvqZp2hJhzXcAADU1Naq9vX2+w9A0TVtUdu/ePaSUqp3u8Qsi4be3t7Nr1675DkPTNG1REZEzMzlel3Q0TdOWCJ3wNU3Tlgid8DVN05YInfA1TdOWCJ3wNe06UaqE4/TgOiOzfm5XFSjZ3TjO6Kycr+iWGMyPkLazs3K+6yFrZxguDFN0i/MdyoK1IGbpaNqNrlTcTyH7fVA5FC6WZw2+4KcxjOg1nzuT30ky+x+4qgi4BLybiIc+iWEEZnwupRT7x4/z2uAebOWgUKyJLOfB+jvwGp5rjnUulNwSO4dfpyN9HEEQMbm1ahvroxsRkfkOb0HRPXxNm2OO00Mh88+I+DDMJgyjGbvUQT7zr1zrjnOF0gnGMt9BjDAeqwnLbCJXfIuxzA+u6nxns7280L+DsBWg1ldFjTfO0eRJXhncfU1xzqXdo29yPHWMuCdBlbeasBlmx/DrdOXOzndoC45O+Jo2x+zCmyAGIkEARATDaMCx30a5g9d07kz+NUQCGOKvnNvAMpvJFffiuKkZn++tsaMETD+eSm/eEIMaXxWHkycoOAuvVFJySxxPHaXKm8CQcjqzDIuAGeTQ+MF5jm7hmbKkIyJ+4GXAVzn+B0qpvxaRfwbuA8Yrh35OKbVPyp+hvgy8F8hW2vfMRfCaNl+G+sZ488UjdJ0YJFEX5fYH17FsZf2kxyp3FPBOaCu/TAyUyl1THI4aQ8R3ybkNECrnjszofGk7h9eYmBZMMVBKUXRL+EzvO9xzfpTcEq5yMS7pu3rEQ9ZZPOMP18t0evgF4EGl1GZgC/AeEdleue1/UUptqXztq7Q9BqyufD0BfGW2g9a0+TTUN8Z3/t/neHt/J6Yl9HUO8b1/fJ63D3ZOerzhWY9SE5OPUnlEPBjm5G8S0xXwbMR1xye0uW4GUyKYRvWMz7ci1ELqkoHajJ0j6gkTsmY+JjDXAmaAmCdG7pLknnFStAbb5imqhWvKhK/K0pUfPZWvKxUePwh8q3K/HUBcRBqvPVRNWxjeeOEwSrkk6qJ4fR5iiTDRqiAv/3gfrutedrzHuxnTXI7jnMV1R3Gdflx3EG/gw0ilFHO1gr7tWGYjJfssjjuK7fThqjFiwY8hYs74fJviNxHzROjPj5AqZRgujpFz8jxQd/v5kslCIiLcWXM3RVVgrDRCxs4wXBwkZIVZH90w3+EtONOapSPlv5zdwCrgfyildorIHwJfEpH/DXge+AulVAFoBi7u6nRV2npnNXJNmyddJweJxIMT2oJhP/3doxTzJfzBS0ssPgKRL1Aq7sMpHUCMCB7vdkyr/ZpjMYwgtdE/JlvYTaF0BNNMEPLdicdqvqrzBa0An2h9jKPJk5zN9BL3RtgQW0WNr+qaY50rDf5GPtD0EY6njjJeGqfR38SqyGr85sL7RDLfppXwlVIOsEVE4sAPRWQj8JdAH+Xi5JPAnwP/BzDZPKjLPhGIyBOUSz60trZeVfCaNh8StVEGekbw+i5MUywWSvj9Xjy+yacuivjx+raDb/ukt18LwwgSDtxDOHDPrJwvYPrYWrWOrVXrZuV810PcW8Xt1XfOdxgL3ow+oymlxoAXgfcopXorZZsC8A3g9sphXcCyi+7WAvRMcq4nlVLblFLbamunvbqnps272x9aTzZVIJcpAOVkP9yX5I6HN2CaC6/soWnnTPnXKSK1lZ49IhIAHgaOnqvLV2blfAg4NwfqGeAzUrYdGFdK6XKOdsNoXVXPBz9/D2IIA92j5NJFHvjQrdxyz03zHZqmXdF0SjqNwDcrdXwD+L5S6ici8oKI1FIu4ewD/qfK8c9SnpLZQXla5udnP2xNm1+rb17Gyg3NFPMlPD6P7tlri8KUCV8ptR/YOkn7g+9wvAL+6NpD07SFzTCMywZoNW0h090STdO0JUInfE3TtCVCJ3xN07QlQid8TdO0JUInfE3TtCVCJ3xN07QlQid8TdO0JUInfE3TtCVCJ3xN07QlQid8TdO0JUInfE3TtCVCJ3xN07QlQid8TdO0JUInfE3TtCVCJ3xN07QlQid8TdO0JUInfE3TtCVCJ3xN07QlQid8TdO0JWLKhC8ifhF5Q0TeEpFDIvK/V9qXi8hOEXlbRL4nIt5Ku6/yc0fl9va5fQqapmnadEy5iTlQAB5USqVFxAO8KiI/A/5n4O+VUk+JyD8Bvw98pfLvqFJqlYh8Evi/gE/MUfyaNudcV3Gqb5j9e08yemqQluo4m+5cTUN73XyHpmkzMmXCV0opIF350VP5UsCDwKcr7d8E/oZywv9g5XuAHwD/ICJSOY+mLSqFks23f72Hl14+yPhwCtNjEnSEtc8f4Lc+ex+b7lk33yFq2rRNq4YvIqaI7AMGgOeAE8CYUsquHNIFNFe+bwY6ASq3jwPVsxm0pl0vvzl6hn1Hz2IPp2mIhakNBnFDHoYSXp7/7uvkM/l3vG/BKXE82cP+0bMMF1LXMWpNm9x0SjoopRxgi4jEgR8Ck3VrzvXg5Qq3nSciTwBPALS2tk4rWE273nZ1dCHpEpZlce5POyQmA8qmyYGBzmGWrWniTHKMff29OEqxqbaesN/k26dfJeMUzv/131u3lvvrNyAy2UtkdrjKZrxwjGTxbSwjRMK/iYBVP2ePdymlFN25Xk5mzmBgsCqynHpf7Zw+Z236ppXwz1FKjYnIi8B2IC4iVqUX3wL0VA7rApYBXSJiATFgZJJzPQk8CbBt2zZd7tEWLI/XRLnuZe1KKbx+D8+fOcnPThzDMgxEhFc7T2OHRmmNRWn0xwFwlMtLA0doD9exPDw3tX9XlTg5/hTjxWOY4kdh0599leXRj1Pl3zAnj3kxpRSvDu3kwPhhPGKhUOwfP8z26lu5pWrTnD++NrXpzNKprfTsEZEA8DBwBPg18NHKYZ8FflT5/pnKz1Ruf0HX77XFatuqFoj6cZXCLpUrmGnXIZZzqGuuxqoN8YuTb9MQjtAQjlAfClMV8nIqOYS45vnzmGLgNSwOjXXOWaxjhWOMF48SNJvwm9UEzHq8RoyzqR/hqtKcPe45g4VhDo4focaboMobJ+GtotobZ+fIbpIlXdJaCKbTw28EvikiJuU3iO8rpX4iIoeBp0TkvwF7ga9Vjv8a8C8i0kG5Z//JOYhb096Ro4oM5nYwktsNCDWBbVQHbscszxyekTvXtnF6YJRSrkj3iT6cXIGgErY2NvChP3yEU+kkCoVlXOg7iYAgjOZzRH2+C+2AM4d9n/HCESwJTiifWEaAkj1Ozu4n5GmZs8cG6Mv3A2DIhd+FKSYoGCgMEfVE5vTxtalNZ5bOfmDrJO0ngdsnac8DH5uV6DRthpRyOTX+HVLF43iNakDRlX6WVPEUK2K/M+Nass9j8ZkHbuX+jSvoH0lhp/KsbKimflk1IoJv6PJB26DhxxST0kW9ale55J0SG+Jzl3QtI4iLPaFNKYVCYVzFm91MeYx3fgyPzKh6rM0R/b+g3VAypTOkim8TMJvPJ3dTAiSLx8jYZwl72mZ8TsMQ2usTtNbFSZbGJ/RgV1YlCHu9jBfyxHx+ALIlmzarhbDPoTc3xrlR29trVrFijur3AAn/FgZzO3BUAVN8KKUouMOErGb8Zu2EY5VSjGfzOK5LVSiIYVz7oGprsBlLTHJOnoBZ/l2k7QwBM0BToOGaz69dO53wtRtK3hkCmNCTP/d9wR66qoQPMFjo55XBX5OykwDUeGu5p/ZBop4Yv79pG988sIfuVBIDIeDx8MUtd9MUjXAi1U/eKdASrKExEJ/T2SohTzOtkQ/Rlf4pReWgUAStZpbHPjHhcUfTOX644yCnB0ZAoDoc5PHtG6mLnqJUfAnlZrA8W/D678UwotN/fCvIY40P86v+lxgqjoCCsCfEo/UP4DE8c/GUtRmShTCeum3bNrVr1675DkO7AYwXjnFy/F8JWo0T2rN2DyvjnyHqvWnG58w5OX7Y9T1MMQlaIZRSpO0kPjPAh5o/hikmjuvSnU7iKkVzOIrHNKc+8Ryx3Rw5ux/T8BEwGyYke8d1+cef/YbRdI6aSLnen8zmaY6/wWObz+K1qhHx4rrDGEYtocifIkZwho9vM1QcQRBqfdUTPhFps0tEdiultk33eP0/oU2Lq1wG8gOczZ4lbaenvsM8iXhX4DdryDv9KOWilEPOGcBv1RP2rLiqc3Zmz1B0iwStEFD+xBDxxEiXkvTn+wAwDYPWaJz2WNW8JnsoD9RGvO0ErcbLPlF0Do0zMJ6mNho6f1si7NIS28NwOoZhxBAJYJotuO4gpeK+q3h8iwZ/HfX+Wp3sFxhd0tGmlLEzPNf3HEPFofNJYnNsM7dW3brgLqgxxMPK+OfoTj/LeOEwAHHfRprCj2Fc5cBh3slN+jwVUHQL1xLudZcrFJFLro30W+OICMXSxHaRAI5zArjrOkaozSWd8LUpvTL4CqOlUWp8NUC5t79ndA+1vlraQldXE59LXjPG8tincFQR4KqmY16szl+Pq1yUUucTv6tcRCDhXVyrhjRURQGF47qYlamkBTuAwiUW8k04Vqk8hqEXiLuR6M9b2hVl7Szd+W7invj5NkMMAmaAo6mj8xjZ1EzxXnOyB6jzNbAitJLh4iBpO0WqlGSkOMTG6GaintgsRHr9VIUD3L1+Ob2jKUbSWcazeU4NCmJsJOofRikbpRSuO4KIB4/3tvkOWZtFuoevXZFdSQCXMsSg5M791ZsLgSEGd9c+QGuwnZOZE1hisjJ8E82BZfMd2lV59+bVtNVWsetEFyXb4ZEtN7Gx7R6c4s8oFXYADqbZij/4OIaZmO9wtVmkE752RRErQtwTJ+tkCVUGLZVSZOwMt1TdMs/RXT+mmCwPr2J5eNV8h3LNRIQ1zbWsaZ44Nx/ro/gDHwBVgkuu2NVuDLqko12RiHBv7b04ymG4OMxocZSh4hAtwRZW3QDJ70ailMKxz1Is7MQuHURVxjBmQsSLGCGd7G9QuoevTanOX8dHWz7KqcwpMnaGhkADzYHm8jop2oKglE0++11Kxd2U5w8JhpEgGP5DjEuustWWLp3wtWkJWkE2xOZ+iV3t6pSKeykV3sAw2y7MJHIHyGW/RyjyxXmOTlsodElH0+aA47gMdo0w1DM66aD3TChVQrnpK56nVNyJGFWXLClRi2OfwHXHrunxtRuH7uFr2izrOTXAT7/+IsmRDADVjXHe/3v3U9NUNaPzKGVTyj+HXXgJpUoYRg2ewIexvGsnO/raA9dueLqHr2mzKJvK8/Q//BLbdqhblqC2pYr0WIan/8cvKRXtqU9wkVLuZ5TyPwepwjCbUapIMfNVHPvyTVQ83jtQauKnCaWGMK3liCyuawW0uaMTvqbNolOHOinmCoRj5QXHRIRYTYT0WIbO473TPo9SOUrFVxCjGZHySpNiRFBY2IVXLjve470Fj3cbrtuF43TiOJ2IBAgEP6ln3Gjn6ZKOps2iXKZQ3vJqEsX89C9UU24GlIsYE1+iIiFcd+Cy40Us/MHP4PXdh+P0IhLG8tyEiO+yYxej8tW/fSg3i2E2YsxwBU+tTCd8TZtFTctrUa7CddX5TUUcp7wBen3r9NfdESOGiB+l8oj4L9ygxjHNyS94ExFMqx3Tar/q+Bci102SSX8T2z4BYiAY+AMfwO+/d75DW3R0wte0WdS4vI4Nd67mwGvHCYT9KKXIZwtsf88WquqmX0sX8eAJfIBi9jsoiZSTvjsGEsLyv2sOn8HCk808hW2fxqjsYqZUkVz2B5hmIx7Palx3lGJxL647hmWtwuNZd74Mpk00ZcIXkWXAt4AGwAWeVEp9WUT+BvgDYLBy6F8ppZ6t3Ocvgd8HHOBPlFK/mIPYNW3BEREe+e27WbW5jaNvnsS0DNbfvoq2dU0zPpfHdzuGEaGUfwnljmD67sTy3YdhzGy2z2LmuqOUSofPJ3soXw2MBCgWdiJikk79E4oSYFHIv4jlWU04/MQNU86aTdPp4dvAf1FK7RGRCLBbRJ6r3Pb3Sqn/++KDRWQ98ElgA9AE/EpEblJKObMZuKYtVKZpsHpzG6s3X/vS0aZnHaZn3SxEtTgpVR4TuXTgWcSD46bIZL4D4sOsLOOslMIuvU2hsAv/EvskNB1TztJRSvUqpfZUvk8BR4DmK9zlg8BTSqmCUuoU0AHcPhvBapq2tBhGDYZEcN0Lu6wppVBuEo/VXtmK8UKpTEQQI0aptGc+wl3wZjQtU0Taga3AzkrTF0Vkv4h8XUTOfc5sBi6eKNzFJG8QIvKEiOwSkV2Dg4OX3qxpmoaIRSD4KZRK4di9OM4QrtOFaa3A67sFFJdfgaxsRALzE/ACN+2ELyJh4GngT5VSSeArwEpgC9AL/Pdzh05y98suA1RKPamU2qaU2lZbqxd30jRtcl7vOqLRP8PnfxCv52aCoc8QifwhplmPx7MW1+k7n/SVsnFVGp9Pb8s4mWnN0pHykPfTwLeVUv8OoJTqv+j2rwI/qfzYBVy8M0QL0DMr0WraEqOUYmg0Q6FYoqYqjN+3NGefmGY9weD7LmsPhj5JOv11HOdsZZFQIRB4H5a1dMc9rmQ6s3QE+BpwRCn1dxe1Nyqlzl06+GHgYOX7Z4DviMjfUR60XQ28MatRa9oSkM4W+OEv3+JM9whigGkYPPKutdyysXW+Q1swDCNGJPKnOE4XSmUwzcYJNX1toun08N8F/C5wQET2Vdr+CviUiGyh/L56GvgCgFLqkIh8HzhMeYbPH+kZOpo2cz9+4QCdvSPU10QQEYolm5++dIiaRJjWJr314DkigmUtzu0mr7cpE75S6lUmr8s/e4X7fAn40jXEpWlL2ngqx4mzQ9RXR85PSfR6LLwei72Hu3TC166KXjxN0xagQtFGhMvmn3ssg0xu5lsXahrohK9pC1IiHiLo95LNX0juSinS2QJrV9TNY2TaYqYTvqYtQJZp8P4HNpLKFBgYTjE6nqV3IElrU4INq2e+TIOmgV48bclwbIdTHQN0nh4kHPGzZkML0bheYnYhW91exxOfeBf7j3WTTOdZ2VrD2hX1eD36ZatdHf2XswSUSjb/8d0dnD4xgMdj4Tgur714lMd/+y6WtdfMd3jaFdQmwjx055r5DkO7QeiSzhJw9GAXpzoGqG+MU10boa4hht/v4ec/2oPruvMdnqZp14lO+EvAsYPdhMP+CTM+QmE/ybEso5WNthcyV7kUXfvyNVM0TZsRXdJZArw+z/ldl845lzw9ljkfIU2Lq1zeGH6b1wePkHWK1PtjvLthC+3h+vkOTdMWJd3DXwI23dJGLlfEti9c8Dw8lGLZ8poFPXD7+uBRftG7F7/ppcEfJ23n+c6Zl+nNjcx3aJq2KOmEvwS0razj3oc3MDKUZqBvnP7eMWpqo7zng5PvjboQFF2b14eOUu+P4TM9iAhRTxBLDHYOH5/v8DRtUdIlnSVARNh+7xo2bm1lsD9JIOClrjGGYSzc9/ucXcBWDh5j4p9o0PQxkB+fp6g0bXHTCX8JCUcChCOLY2OIkOXHb3opOCV85oUlgVN2ni2RxiveN2vn6MsPYYlJY6AWj7E0lxS+Etd1yWUKeH0ePF6dBpYK/T+tLUiWYfJA3c38uPsNIp4AAdPLeCmLJQZ3VN/0jvc7nHybFwd24qryILXf9PP+pgdp8OvrDc45dbiLF/7tDcaHU1iWyS0PrOfOxzZjLuABfG12LNzP9NqSt6VqOZ9ovZu4J0TOKXJTpInPrXiYhC8y6fEjxTGe7/8NUStErS9BrS+BgfDTnhcoufZ1jn5h6jszxA//6Xkc26auJUE0EeI3P3uLV3+8d75D064D3cPXFiwRYU2shTWxlmkdfzJ9FgOZUMIJWQEGCyP05wdpCV65FLQU7H3pCJbHJFgp7Vlei7qWKva+fITt79mEL+Cd5wi1uaR7+NoNw3YdZLKdGwBH6SuKAYb7xvCHfBPaTMvEdVxy6fw8RaVdLzrhazeMtlAztnLP1+8Bim4JU0wa/LXzGNnC0bamicx4dkJbIVfEH/QRjofmKSrtetElHW3WuapI0T6Ncm3GBiJkUwaJ2ihVtZPX3mdLg7+WW+Ib2Dt2CEMMFApBeKT+HnzmwitV5OwSp1OjmCK0RarwmXP/ctx8zxoO7uhgsGeUSDxEMV8kk8zz3s/eg+XRg7Y3OplqfRIRWQZ8C2gAXOBJpdSXRSQBfA9op7yn7ceVUqOVTc+/DLwXyAKfU0rtudJjbNu2Te3atesan4q2EBTs0wylvkaxmKSzo5/0eInuQ7eT7G9l852reOBDt2KaFz5YOrbD2eO9DHSNEE2EWbGh5ZrqyEop+gvDdGV7sAyL5cFlxLxz+0ZzNfYP9fLdjn3YjosSCJoePr9uG8ujc7914fhwmt2/PsTpwz3EqsNse2gDbWv1GvuLkYjsVkptm/bx00j4jUCjUmqPiESA3cCHgM8BI0qpvxWRvwCqlFJ/LiLvBf6YcsK/A/iyUuqOKz2GTvg3BlcV6B39r4iYnDqcZLB3jHDMwPKn6XnrI3SfsHn0k9vZdMdKoFxK+I8nn6fzeB9iCEopIvEQH/vjR0nUx+b52cydkXyWv93zInGfn4BVHmBOFQuUXJf/dduD+K9DT1+7Mcw04U9Zw1dK9Z7roSulUsARoBn4IPDNymHfpPwmQKX9W6psBxCvvGloN7hC6QSuyoAbZqh3jGDYh3K9oBTBRBfRRIj9v3n7/PH7Xj3K2eO91C1LUNeSoH5ZNYVckef/bcc8Pou5d2R0AFe555M9QMTrI2cXOTk+PI+RaTe6GQ3aikg7sBXYCdQrpXqh/KYAnNtosxnovOhuXZU27YbnAOWeuuKiDbhFELExTaFYuDAf/vDOE8QS4QnLNsdrI5w91kM+W7i+oV9HtuvCZLOJBGw9m0ibQ9NO+CISBp4G/lQplbzSoZO0XVY3EpEnRGSXiOwaHBycbhjaAua12hExMCybaFWIfLYIuAguhVQT4yMZ1t/afv540zIvW+NeueWB1oW8zs90FJwC/flBkqXUZbetjFWjFDgXbT5TdGwMyoO3mjZXplUsFBEP5WT/baXUv1ea+0WkUSnVWynZDFTau4BlF929Bei59JxKqSeBJ6Fcw7/K+LUFxDQiVAU/xkjmKZatgdNHx3GcEt1H19NzzKCxtZotd60+f/zmd93EL779Gv6QH8Mo9xOG+8a56ZZ2vP7Fuf6NUoq3xg7yxsheFApXuawML+f+urvwGuXB6OZQlIdaVvF8VwdmZTaRUorHV95MzOu/6sdOjmdxbJdYVXDRv2Fqc2PKhF+ZdfM14IhS6u8uuukZ4LPA31b+/dFF7V8UkacoD9qOnyv9aDe+kP8OvJ42csGDxG/N0dMRJWhH2XZHNcvXNk1YqGvjnavpPjnA4TdPlMs6SlHfWsMDj98+j8/g2pzOdPLa8JtUe+JYhoVSihPpU/gML/fV3QWUS12Pta5hY6KBo2MDGAgbqxtoCF7dbKLkeJZf/GgPZ04MgkC8KsRjH7mV5mXVs/nUtBvAdGbp3A28AhygPC0T4K8o1/G/D7QCZ4GPKaVGKm8Q/wC8h/K0zM8rpa44BUfP0pkfju3gKoXHM3+zQpRSDPWOMdI3RigapGlF7aLunT7T8wuG88NEPOHzbY5yGCsl+fzyT53v5c8W13X51ydfZHgwRXVtBBEhncpRLDj83h8/TCS6OFZHXQwKRZsTnYOMJXPUVkdY3lyNZc7v3+pMZ+lM+UpXSr3K5HV5gIcmOV4BfzTdALTrr5Av8fqLR3hr12kc26F9VT33P7qR6trodY9FRKhtqqK26caoXefs3GVr+BsYKKUoufasJ/y+7jEGesepb4qfbwtHAgykxzh+qJtb71w1q4+3VI2lsvzLj99kNJlFpDwxYVlDFZ967Fb8vsVTfly8XSntqiilePaHu9n1egexqiA19TG6zgzxvW+8SuYKa6mMj6Q5/tZZzhzrxS4573jcYpIsZTid6WW4MD5rG6SvCLeRciZuDJ91csS9MYLm7Pe28/nipOsHGaZBKpmb9cdbqn75+lFSmTyNtTEaaqI01ETp7B1l54HT8x3ajOgrPJaYkaEUHUd7qW+Kn58OmaiJ0N9b7hFurVwUdY5Sih2/PMDrv9iPVKZcRqtCfOSJB6luWJwXRznK5aWBPewZPY6B4KJYEW7ifY13XfMSDBuia+lIn2KwMIzf8FFUJQTh3Q33T5h+Oltq62MowHHc81cwK6WwSw7Lluv1g2ZDyXY4dnqAusSFMRYRoSoW5K1j3dy3bfUV7r2w6B7+EpNK5jEMuSz5eDwmw0OXTyHs7Ojn1WffoqY+Rl1zFfUtCYr5Ej/51iuz1iu+3g6Nn+LNkSPU+mLU+uPU+eKcSHfzyuBb13zuoBXgw83v457a7TQFG9kc38jHl32QpkDDLER+uUg0wJ33rqG/Z4yx0QypZI7e7lHaV9XTvkIn/NkggCGCumR2uVIKc5GNN+ke/hJTlQihXIXruhMGR0tFh4ZJ6uhHdp/G5/dM2A0pmggx2DPGUO/Yoqy97x09RswTxpDy8xcRan0xDoyf4P66W7CMa1tEzG/62Bhbx8bYutkId0p3PbCOxmUJDuw+Q6FQ4u4H17NuU4vewWqWWJbJxtWNHDjeQ31NeZxLKcVIMst77ro+/8ezRSf8JSZWFWLLbcvZvfME8UQIyzQYG8mQqAmzet3lC2g5tnONWMi5AAAgAElEQVR+jvw55z4duO7i7OEX3CKWTOyZGRg4ysFRDhaLK1GKCCtWN7Bi9dx8ipiKUnncwpuo0kEwwhje7Yi1ak5KWPPl4TvWMDSWoad/DCqDthtWNrJtQ+t8hzYjOuEvQQ88toma+ih7dp6gkLe59c5V3HbXanyTXOx005ZWDr15gqgbPp/4M6kcoWiAmkVaw18baeONkcPUmxdWphwrpWkNNizIZZQXMqUKOOknUfZZMKLg9OAU92EEPoTpv2e+w5s1oaCPz39wO139oyQzBapjIRpqIovuTU0n/CXINA223LaCLbetmPLY5eua2Xj7Kg69eQLDKE8v9PgsPvLEg4u2ZHBrYh0n0t305UbwmRYl18Fvenmg/tb5Dm3RcYsHUPYZxLpwcb1SEdz8sxjeWxEjOI/RzS7DEFob53756rmkE752RaZp8OintnPznavoPjlAIORjxfpmwrHF+0IOWX4+3fYIHekuenJDJLwx1kZbCVn6IqWZUvYxkIl/CyJeFC7K7UeM5fMUmTYZnfC1KRmGQcuKOlpW1E198CLhM71siK1gQ2zqTznaOxMjgaI4oa08e8tFRG+ZuNDohK9NW/fgOHuOdZFM51jdWsemlY34fR6UKpErHaVQ6sA04gS9W7DMxTd7R5s5w3srbuEllJtGjDBKueD0IZ6bwNDTQhcanfC1aTl4spenX3gLj2ni9Zq83TnE3mNd/O5jm0gXvkW+dBwRH4oS49mfUxf9Aj6P7j3f6MSswwh+Hjf3A5RTXhRXPBswg48vugHNpUAnfG1Ktu3w09cOUxUJnF83JBry0zOY5NDp56irOYbXbD3/AnfcJMPp79IY/0tEFteFKdrMmd41GJ6/AHcExIcY139NJm169KtRm9JwMkuhaF+2SFQ46GU4uQvTiE3ozZlGFNsZxnb1dn1LhYiJmLU62S9wOuFrU/J7LRTgXrKUQqFk4/OGQNkT2s8tuSD6A6SmLSj6FalNKRYOsLatjmNnBqhLhDFEyBdLlGyX5Q0Pky98k4HeIuOjBfwBi4Zmh+qqDXrgVtMWGJ3wtWn5rbs3gIKjZwcQwOe1ePyBTcS8UZ79WStVjbvx+EwKJYdD+6rYsPpe6q7hQtwzmX7eGD7CWClNW6ie2xJrqfJe3Y5Qsy1VSnIoeZC+XA9Rb4wN0Zup98/PsgaaNhM64WvTEvR7+cS7tzKezpEr2FTHgngsk+d/cYC+0+tRpU14/cM4tp/kaJQXT5/lprVrzi/ZOxOHx0/zTPdrBEwfftPL/tETHE2e5TPtjxL3hqc+wRxKlpL8tOc/KLpFgmaInmw3p9OneKj+EdpC7fMam6ZNRdfwtRmJhQM0VEfwVJZVOHNqkEjUj2sHyaeXUcrXEgj4yKQLZDOFGZ/fUS6/HthHlTdC3BvGb3qp9ccpOCV2jxyf7aczY4fGD1BSJaq81fhMP1FPjLAV5o2R3+Aqd+oTaNo80glfuyaJRJh8rjShzS45WJYx6WJsU8nYObJ2Dv8li5hFPAHOZPuuKdbZ0JPrJmhOvILUbwbI2BnyzjvvGKZpC8GUCV9Evi4iAyJy8KK2vxGRbhHZV/l670W3/aWIdIjIMRF5dK4C1+aPq1yOJk/z/c5f0dfaS29ymEymnOzsksPgQJJb71iJ1zvziqHf9GGKie1O3EYx7xRJeOd/yl/UE6XgTFxKwHZtLLFmfb9aTZtt0+nh/zPwnkna/14ptaXy9SyAiKwHPglsqNznH0VkcS6pqL2jF/p38eOeVxjOj0ONg/muAifSXfT3jZFM5njXfWu56541V3Vur2GxLbGGwcIYJbc83TNr5yk4JW5LXN05Z9PG2CYKbp6CUy5X2a7NWGmE9dGNWIYeEtMWtin/QpVSL4tI+zTP90HgKaVUATglIh3A7cBvrjpC7TJKKc4e7eHga8fIZfKsvmU5625fhdc38xLKTA0Xxnlr7G3qfYnzO0atWdtEX9sw76/eyqpEMx5P+T1+rJDDVi7VvuCMLrN/V81GDDF4c/goJWUT9YT4yLJ7aQ7O/9osjYEmHqh7iDdGdjBSHMYUk83xW9hctXW+Q9O0KV1Ll+SLIvIZYBfwX5RSo0AzsOOiY7oqbZcRkSeAJwBaWxfXrjHzbffzB3nhqdfxh7xYHouTBzo58kYHH/2Tx7A8c9vLHCqMgXA+2UN5xyXTMElZaTwek7FCju+f3EfH+DAI1PpDfGLlFlrD05uXbxomd9fezB3V6yi6NgHTO+Hx5tvy8EraQsvJOVm8hg+PMfdvtJo2G672VfQVYCWwBegF/nulfbJu3KT74CmlnlRKbVNKbautnf+e22KRTeV4+emd1DZXUVUXI1IVoqG9hs6jPZzYf3bOHz9g+SZtd1GELT+uUnzj2BucTo3QFIzQFIiQtYt89ehOUsWZDWp6DIuQ5V9Qyf4cQwxCVlgne21RuapXklKqXynlKKVc4KuUyzZQ7tEvu+jQFqDn2kK8cSmlKNg9jOV+zXjuFUrO0JT3GeoeQSmwLhoQFRF8AS9njnTPZbgANAfqqPbGGC6Oo5RCKcV4KU3I8rMi3MyZ9Ci92ST1gfL2byJC3BugYJc4ODL/s2w0bSm7qs//ItKolOqt/Phh4NwMnmeA74jI3wFNwGrgjWuO8gaklGIs9ytGsz+HSn17JPsMNaFPEfHf8o7384d8KOWilJpQF7dLDpGq2d9wQilF2i5gihC0fJhi8JGWB/hl3w7OZPsQoM5fzaMN2/GbPrL26KT1elMMxoq5CW29A+PsOdzJWDLPqrYaNq1pJnAVUzk1TZueKRO+iHwXuB+oEZEu4K+B+0VkC+VyzWngCwBKqUMi8n3gMGADf6SUciY771JXdHoYzf4cr9mASPm/wVUFhjLfI+i9CdOY/IrS2pZqmlbU0392iOrGOCJCNpVDRFh326pZjbE/l+SHZ/dwsLOT0cEUNfkgj6+4le3b1/LRZQ+RsXO4yiVsXRiUbQxGUYDjuphG+QOkUoqSclgerT5/7mMn+/m3n+/FMg28HouTnYPsOdTJZz9yB0G/nt6oaXNhOrN0PjVJ89eucPyXgC9dS1BLQa50HETOJ3sAQ3wo5ZC3TxPybpz0fiLCB77wMD//5kucPtyNCIRjIT78xUeJ183ePPWsXeQbHa9y6mQ/qbMpfH4PfZ4U/9+Bl+g+MsjH/9N9k+4Bm/AFeaBxJc91v03E48UUg/FinnVVdayO1gBgOy4/e/kwsbCfQCW5R8N+egfH2Xeki7u26o1TNG0u6InD8+Qdlw4WQaYYWgnHQ3z0P7+X5HCKUtEmXhe74po1BafEiXQvo8U0Nd4YYTuEgUFdVfh8L/xSx8f7GE6nyXRliEQD5XECPIxXFTj8djdnOvpZsbZp0vu+Z9la2iIJdg6coeg4PNqyhq01LecfazyVI5srUlc9cTG0SMjP8VMDOuFr2hzRCX+eBL3rGMk+g6sKGFKe+eK4aQzx4beml/Ci1VOvHjlWzPDtMy8yVkyTT7t0vJXCLHhYHm6gKhTk8fs20VZ/+XTJsVKOUr6ECBNr8gKuV9F7duQdE76IsL6qnvVV9ZPefn59fVdhGBfOXSzaRMP+KZ+TpmlXZ+HNd1siPGYNNaFPYLujFOxuCk43LgXqwp/DMGYv6T3ft490KUetJ073/hIex4sK2Nj+Akopvv3cblLZyxc5aw7GEcuYsOmJqsywDeRMolXBq44pFPSxcXUjA8PJ8+cvFG3yRZttG/U1GZo2V3QPfx5F/NsIeteQt08jGPitlbOW7EuuzSsDx/hB5w68YhHOBmkInWJj6ykso0B3soWo837ODrgc7xzk1jUtE+6/IlLLzQ0t/LJzhNFUFn/QQ96wqRsKUO0Ns3LdpNfTTdt77l2H67oc7uhDRPB4TD748CZamxLXdF5N096ZTvjzzDQihLw3z/p5f9S1h70jp7HEwGd6aPK9wfJlJ0GqKdhe2qpOEjefpnf4veQKxcvub4rB76y4k+WeGn70xpuM9qdoGw+zObGMR/7gNoLhyS/Ami6f18OHH9nCQ3flyRdKVFXW19c0be7ohH8DGswnOTB6lmXBBEU3R7LUw/rqLs6eiRL0m4hA0GnE8iVJRE7SWn/vpOfxmhYPtK/jgfZ15LNFHMclGPbNaF2cqUTDfl2317TrRNfw51nRdija5UsVXKUYL+TJ2/YU9yqz3SwFZxR1ycYbY8UsRuUq17ZQA3UeF8dwMaNCLl/Ccj1YrkUqC+vbbZbVxad8LH/QSyjin9Vkr2na9aV7+PNkLJvn2YNHOdgzAEBdVZAxT560U8QQ4V2NbTzWvgaveXmZI10aYt/wVxnNH8AQk4SvnbWxTwODpAp7cGyXiOmi1CY8hsXy8FoS1m6KCT/NdbV48l6UUtTFFVVVmziW7McUoS1UjdfUfxKadqPSr+7r7O3hYV46dYpfHezAFKiOBsi5Nq8fPkN1OMhd61txleLFrpPYyuXxVRcuwCq6BTozR9kz+P8Ao/iNGlyE3txxisU/pc6/Cr/ViFccbg510JEdpmTejyNR+ortNPlPszZegyU+HHeQkYKPr5wYI+/uQClFwPLyuyu30x6ufucnoGnaoqUT/nX0ZncXTx04QLFgM5hNkTRymHkDj2GSw0alFSPJHDJukzmZ4unX3qT2IS93bF1BQcZ5ZfCHDOU7EKcTJISjUkQ9CWzXIFkapjcVxWsGWRFLsDKyGZ95lDdSg2TtIA3xx7kp0k028xpQxBNYx9P9UQJWFVVmef2adKnAv5zYwZ9tfASfqde00bQbjU7410nRcXjm6DFqQyH6SylSboGAx4PtuJQcF59pkckX6TjYh9Vr4wlY2LbLcy8d4czpYRrvOYOrHExcRDyIeCi6eUYLGdKZNLV+wWMWSRcL7Orr4tb6ZhoDCb5Qt5mwdyNnuof53tM5cvlHEIGRSJ7SCptV1RcSe9jjoyc3xqn0MGtjDfP425qeouNwZGSAvkyK2kCI9dX1+C39J61p70S/Oq4DpRSHBwfoSSdpikRwLRflKiiVrzR1UdiuC65irDdFa3UcB4XXY9HaUMXprgGyZ/pYtbyenFlF0QERhSEmI7k0FhaWuBTdKH7LgyB0jA2zpc7AY0TJ5Uv820/24PNZ1NeWB2gHCz2c7BymPV6DddGyDILgXDIIvBCligX+af9O+jJpTKMcc7U/xB9uuoMq/+Vr/GiapmfpzLmS4/CtA/v4xv49nBgf4c3+bjo6ejG7iuTOFCicyOMZVagi4CosBXnXJlMqsqaqnIwt02BsoHxFatTbSEHqQWWAArZbwsXFUSFMPIDCZwpFpxeftQyftYyTZ4colBxCwQtz55f5qnAch+Hx9Pm2omNjILSFFv7FT8+f7aAvk6YlEqUxFKElHGO8kOOnp47Od2iatmDpHv4c29Hdyb7+XtpicbLFEp1nhijuTROIuDhBg7whGOMuq2JRCvUu5qBLzBegPRKnOlBZvkCZNCbqSdujRDwJqow7ONm5DzF6GU0GSJdWYjdtoiG0G585gO24uGoVTdHPIWJQLNqXbUUWwU9LvoqhQho3p0CBEni8dSthz8KfF79noIe64MT1/2uDId4a7OXTa7dg6OmjmnYZnfDn2I6eLqoD5fXi19TUkHxzmFGPoDAJuUKDz4vlCupklv/8W+/mpDNA/2CSKl+gvJtUMkfA7+GxzY+wN/tzRnL9dJzsY7wvxvi+lST7AvSTp6t9nDvf91sUSTGcK/I7G7djGeXlkluaqspr1Dvu+VU1HUexLJfgw+s3M6DSmGKwLt5ArX/qBdkWAsswK6WnC9NWXaUwDXPSfTY1TdMJf865Sp3f6dc0DOLKQ7g6ThaHrcFq3j7RTbFYgixkXx/iQ+/dyvMvH+HtE/0ANNbHeezdG6mNRnko8inePLyXg/v3MHrIwcJLXdTC53jp7Uix99cnWP/Acn57/VY2114YdK1NhLnr1uW8tuskHssAEWzb4f47bmJDQzMb5uMXc43e1dTKj08epTUSQ0RQStGXyXBfc7u+OEzT3oFO+HPs9sZmnuk4StDyICKEG8J0HxmgOhTg5L5OggEvQcuDG1bsf/UYXsvk8cdvJ5sr4rqKUNB7PoGZYuKMREid8oOTwxuyEIS4FcD0Qe2Inz/ZeAfhyMRBSxHhgTvXsKq9jmMdfRRsm4Dfiy/goW8oSX115B2TZF/3KDtfPkZf9yj1TXFuv2cNTcvmv8Z/T/NyulJJ9g/1YYjgKpc1VTU82r56vkPTtAVLJ/w5dmdLK8dGhnh7ZBgRwW724D/lQfoLKMB2FY7tsG5FLfGIn/2vHeeux7YQDF6+zZ/tupzydtEZPgGmIu/zEi4k8JZCCELQ7yWbKV6W8KGc9FubEijgqWd3U7RtQEAp7tjczrvvXHtZ0u85O8xTX3sZy2MSCvvpPDXIiaO9fPz37mVZe83c/MKmyWuafGb9VnozKYbzWeK+AC3hqO7da9oVTGdP268D7wcGlFIbK20J4HtAO+U9bT+ulBqV8qvty8B7gSzwOaXUnrkJfXHwWxZ/sGUbJ8dG6U2nqPIHqL87wJP/9Rn6iw6xaICm+sj5BcSUUuSzBfyXJHzHdXnywK/YX3qZ0WaTkVEvph8SuUEauvPUxesIBL1E4++8Tr1tOzz93F78PovqeHnA03Vddrx1mpva6mhvnniF7Su/OozXZxGrbI7u9YVJjmd55bmDfPoP7p/F39LVERGawlGawrO3taOm3cimMy3zn4H3XNL2F8DzSqnVwPOVnwEeA1ZXvp4AvjI7YS5upmGwOlHNva3t3FxXT111lHfdexN1XqG1NkQkVJ4uWcgV8QV9RCZJ2keGBziaeZmupEO+QWH5XdwC9IVMetpKxGIB7r5vLT6fRd/pQd58bj9vvXKE1MiFaZe9Q0my+RKhwIXpmYZh4LVMjpzsv+wxezqHiUQnflqIRAP0do6gLtoYRdO0xWE6m5i/LCLtlzR/ELi/8v03gReBP6+0f0uVs8EOEYmLSKNSqne2Al6ojo0M8lLnacbyOdZW13JPS/s7XgB06PVj7PrZXjoPnuHknpPE62MsW78Mx1G87zP3YE6yLvwbgy+RdkdJFmoI+13UqgKFIYv8mI9sXLjzjnVs27qSX//bDva8cAgxBKXAsgw+8IWHWbFxWbncMUmeVkphGpeXQhI1YXLZIqGLli/OZYvEr1Dz1zRt4braC6/qzyXxyr91lfZmoPOi47oqbTe0N3q7+Kd9b9KVGqfkurzSdZr/c8dLvNp1mjPJ0Qm94b5TAzz79V8Tr3G55cEwbet8FFJpRruG+fgXH2H9bZfvZ1t08+Slg0KhXFoxRDC9JoGmEpG1eWra/ERbonR39LP7+UPUtiSoW1ZNfWs1oViQZ7/xIsVCiYaaKJGQj1Qmf/7c55Z2WLfi8qUUtt+3luRolnyuvEFKPldkfDTDnfevme1foaZp18FsD9pO1u2b9LO/iDxBuexDa+vi3ce06Dg803GUumAIv2XhuIrBbIZT46N0psZpjkRYHkvwuY23EPJ4OfSb44icRRkDGIZB0wpF4woY74sQjl7oSTvuOAX7LAZe8kRJBAIEzSIKhYtbWQJBYRou9YF64r4Ab+97G4/PxLhoqQR/yMfQ4Di/eHM/0bYEd9+9ml+/eJTeoSSgEIT7b1tFS8Pla+LftKGZ933sNl751WEGescJhn089pFtrNu07Hr8ajVNm2VXm/D7z5VqRKQRGKi0dwEXZ4MWoGeyEyilngSeBNi2bduiLQiPFfIUHJvqQLl805kaoy+TJu7zg4KmUJRTYyP87ORxPrpmI6mxs2D0IRJHkPJEGUo47klGhoc4dGaQY6d34w8eYsPaAjUJByVBvOLlrsY6upKjZIslfFYJQwziZhNrq5pZEUvQ4zG5dBmcoWyG/UN9dHV24HciKOBdd7aywVtDyXZpro+TiE0+0CsibNjaxrrNyygWbLw+C8PQq3Fo2mJ1ta/eZ4DPVr7/LPCji9o/I2XbgfEbvX4f9ngQylMmATpT4wQtD45ShLzluff1oQhv9HXhuC5tG/LkMxNr6cWciYPiqz94kW8+9Twdp/Zy+FiQH/60noMn4uw5201pZD+nRk7y0DIft9TWEjETxK1qHm29hf+04TYMEW7a2o5jO9il8o5Ztuuy+/RZAhE/K1Y20RyK0hgI81rfGbzVXm6+qekdk/3FDMPAH/DqZK9pi9x0pmV+l/IAbY2IdAF/Dfwt8H0R+X3gLPCxyuHPUp6S2UF5Wubn5yDm685xhlDuCIZRjWFOnLoY9Hi5q7mVlzpP0xgO4yqF4ypcpVgWKZdJRMoDowpYuaWeZWstuo4V8QcNbFtRKrrQBv2DWaoTGXAL5HOjOAEvz7xS4O4HLaq9fmpzbfSlBlmVKPD4TZtZF9tOta/2fCwN7bXc/7HtvPTvb4BSJAsFXEfR9oF19L5wirFDAyAgq6Psq+5ixc3zfwGVpmnXz3Rm6XzqHW56aJJjFfBH1xrUQqFUkVz2+xQLuxAMFC5e3x0Egh9F5MI68u9bsQYD4WfHjpEaL5BxC9zW0ELUV55LP5TNcnNtPZZhEApv5b1/8EtOHwpzcKeDjUGibZzfHKsiP+IyXhoh2FCg4C/hljIUS2GGChmOnw1x9pQf213HUNTPpz75EIWxEmOBLPHohV76toc2smZrOz2nBukvZkgmTzDw7FnyQ1m8iQAoGN0/yIHiPj60YeM79trHxrP0DSTx+z20NFVNWEJZ07TFSV9pewWF/AsUCzsxzGWIGCjlUiy8jmHU4g88fP44yzCQMyXqT5jU+eo5xDBHTvcxlslRUx3Gb1nc3dIGgGk2EQh9gmP9P+Ws8uGJJRk2BhmmHhLdZHIunqCLKgiuDW6pxOG9taTzfixDIcDpI/382ZeeZmVTDSjFmlUNvPfhm/H7ym9CkUSYNYkwK1yHnzx1jKG+JNHGGACucrFq/MhAnu4TA/9/e3ceJMd1H3j++97Lq+6u6q6+0QAaN0DwBE+RuiXLuqiRLEsa2+MJa0OxsePdmY2NWNt7xe7OP54/dmfWGw6N5V3POjwztix7PJZliaIkiiJtkSLBEzcI4uhu9N11H3m//aOKECCCFkk0CKI7PxGI6szOyn6/V4Vfvcp8B1t2Xdk7R2vNk0+9wo+febW3DRQLKX75M3dTKl45O2Uikbi5JM22N6C1xveeQKpRhOhVkxASqYbxvSeuOHZ+qc5Lx+eYKBfZMlDgfcMF7i07BKsVGm4LX/l89fhT/LuTh2kHPidPj3F+9hATOyYY3Fen66ZxVJcgNjGHAsJQopXGDRxyw3VadYXjGASiw0qlRVD1qbse6bRFeSjPqTOLfO9Hx18XgykVHyptJdaapu/R8F3aYcC+wWEypkV9rfW655ybWeWJp19haDDLyHCe0eE8na7PNx95MRlslUjc5JIW/s9wwy5HVk5ztlJjl5yluWgQRHXsnGR0ZJzywAhCd694zux8tTf/uhFQKT1HYNYx4oii1SSfGWM0dx9og+PVRf7yLPgvdygW8qjB4zQaJhJFymyQUzWaF1P4kaKhHPYdmCOf7dC4kCUt12h50FoVOIai2nU5tbTKoe0TlIdyHDs5z4ce2kc6deWUDHu2jnFbeQyjnCLWmgHbwVaK5VqFgfLrp0I+cvwiace8NI0ywEAhzcJSnUqtzWAxe9V601oTxb3OWkoOJwOzEol3oU2b8JtNl9m5NWqey5Ld4WJrjtbas1Tt87gIlBK0nHnGh5roIEcUCF5dWsPVrzA19IUrzpVyDLTWNHNnCMwGbtdmvtPA7SrU4DLV8DQl4wBj6TxHKotsDVO9DwirTuiZdJoGr740hhsaSOFjz3eY2LZGaqLDs5VJ1rIFlIoZyXfJ+g1cN4WVsllptWl5PlnHRgN+EL4u4W/ZPcbE9jIL51cZHC2gI83SxQpTe0YZ317mZ4VhjPiZUbdCiF6//+jqLfwwnKfR/hPCqNchS6kRCplfwzAm3/4L9HM0ah1eePYsF15dpjSU5c57d67rLJ5aa7woxJAKI+mdlNggNmXCP3J0lu989wi1rsurnQWG9rzC2LZzmKM+qutgdi0iV7LgZBg3a0jdJkMaoQMqDbCLoxQuO9/ObcNYtkHTnMHrGKx5LUI/QCpNhxjXO858M0+gY6JIcP+uMU4+s4DTLeGKDi+8UCAyBNIKcXWKTstGHWtyrlxiYnuTrkzTbZrMNFMU2i2oQkCHQMfUJruIGAq5FPns66dyUEry2f/ywzzz6BGO/uQMCMG9v3CQez5y8Ko3bPfvGeXkKwsUcqlLrfRW2yWfTzFUen3rXmuPWvOraB2iZG9QdRxXqTW/Smngf0KK9V9ftlHr8O//8HG6HY9sPsXZ00ucPHKRz3zpXnbuHb/m85+vVfnm6ZPM1OvYhuKhLdv40PZpTPX6KS8SiZvJpkv4tVqHb3/3CAOFFCeDVaYOncPJzpPJdGhFNobSWFmXZiOFLwTPNqdIBTHbYpPYHWBuzeGu0QUmXZ8ojsmmbDJpmy988hD/7uSTrDZboCOUERCXNIgIPw6Yb58lY4zgRyHHS3OMTAxw+KksNdMh8jW2FaC9kGABYt8gsDIYf9VlYTpD5sNtvPMp8BXtnYrsMUmEpLPcZnm2ijWh+MLDdyGvMh8OQCrr8L7P3s37Pnv3z62fXTtGObBnnGOnF3rz62gwLcUXPnHnVc/vB6eJ4waG8dPxdkqUCKMZ/OAUjnX723+x3sBzT5+h2/YZHut1e02nbTptj8ceOcL07tFrGi+w3G7xb597BtswmMjlCOKY7559hW4Y8I/27l+vEN512oHPhUYNQ0q2F4qYMvlwu968qEonmEdJh6w5hbys59/1sukS/rnzK+hYE0nQdhUn28LRPlrL3gCqSGIZIcrorQMbIFnuFoncEnbLIrYifvxMxOPVH/ZG0g4V+PT9B5iaKPE+9R7+5rWwqYgAACAASURBVMKjyNhFSIOlBUGgQmp2Di/ySMuAe0YmCWVEdZuPbI5ROx2TVstkHJe5So5wwEBLQWxJZJQie6FG8HSImQ9wcj6xrbBNh24QIaWB3fR56NAOTCl54cWzlEZccoUaQphkrT2Y6sqpg4M44my9QjsImMzmGU5f2Wo3lOTjH72V8S1FlpYaTIwU2L1jlGzG5mq07oC42qUegY476/Oi/YzzZ5bJD1z5zSGdsVlerNNpe1ddD+DNevriHMClie8spZjM5XlqbpaPTO8ka71+nYKb3bOLc/zFmaO9JSM15Cyb3zhwiC25ws9/cuIt01qz0H6c+c7jCEBrsFSeXQO/RsoY/rnPvxabLuG/1tHEkAJpBmgNUmpkpDHsGILe76UShIFEmhqhBaYP3a7LxfYQWVlg62AOAaw12vzxo8/ym595kLsGH+TbF39Ee6ZD5cc2Wmq6XYuumWLkQZf7txqk7SxeFPD9p85QmfXQkcuY4zHfKuCbBsoLEEpCoIkdSWc4R/Z0FXVXRJxSePOSoOMhQnAaLg1V4fe/9j1Wmm123zHPgYOvUipk2b9nAtOwmcz9Kjl7LwAr3TZ/eOQZKt5Pbzo/OL6VT0/vv7To9/nFCn/xw5do9ydMO1utMT5RfMOEb6gt/XqNL/Vm0joG9BWt/vVUKGa4OLuGc9n9ijCMUEpi2dfWSlpsNUmZV55D9b8xNH1vwyX8pU6Tr59+maFUGlv10kHdc/mjY4f5H+55f9LSvw6awTkutn9AyhhFil6de1GVs/VvsL/0X13XDg+b7m7Utm1DvZuQsaAoy0SxphvkMHSMEWmydhchNZEWdAOHer3AoPQRZkBuuogZTbJ9cAIpBEIIirk0HS/g1MwyOTPL7uBuZl/ME97iER/0se51yY+1GXk5Imf2ksWZl5aZXXDp5H3EsM9KKUc4oBBRDFKgARWGoGNC20QbinbWplZJEb5iIiKw673yXpA+M34blfNxGxFr3RznLgiOnQgwRI755p8Rax+tNX966iXaYcBENs9ENs9oJsvjc+c4vtabC7/V8fiPjz6HEDA6mGN0MIcXBPz7R5/D70/X8LOUGiNlv5comiGK1oiiNcJoBse+H0Ndn5u2hx7YSbfj47q9D6UwjFhZbHDnvdNY1rW1YaYHirR8/4p9fhShpKDoOG/wrJvXkdUlEOJSsgco2A7NwONCo3YDS7ZxVdyXUcK5lOwBLDmAGy7TjZb/gWdeu03Xwi8VM3z4g/v5/mPHGIiKrF0cJR6dpR3nSMctLAmxjiiYglT+Ng4OP8i0vR/D0lxc1CyfP8FMVGMhbmCimFJFhIRqu0sYxbx8ZIXUdBe/axDHEkSMNebROG9w+tkKbupVLq61KY6kkbkIrTXxMQjX7N4ne6QReYFWoIIIHQvaKk2rmSX3TBXD1TihQPgxwWieVsrAFoKBfJfQM2i2bUrpgIuLVe68ZRuRqNENZnHjMWaaNcYzP+2KqYQkZ1k8uzTHLUOjvDK3gh9EDBZ+OsCqkE2xsNbgwmKVXVte36tHCEE2/Vkscw+u/yxojW0dwrYOXreWytT2Mp/6/N388JEjNGo1pJTc+9AuHnj/vms+9z0Tk/x4bpaFVpOSk8KLQqquy8N79uIY1/8a6zvNj6Ort/r0T+eHSqyvWIeXvg2/RojeRIro6Lr+7U2X8AHuunMb09vLXJhdQ4jb8ItzrHCcOKqQjU12DdzKQPZWTFXAkj+9lBEONDgmFvGDkJQwidAsRHW2WjEyZfPHf7fArH4VoyOIQiDWgMC7qFCyy6o+TVyzMIOQiXKLlmlT/2tBZqWDTuUIizlohei2AkegpUIYMcGURbplIm8bQoUu/oIAbRAULAg0QvQWO0GAAjwF0ouJ4ri3QwjiNxg0JURvmmWArnf1VjwavDdo4ffOIbGtg9jWwbf1erwd+w5uYfe+cVpNFydlYTvrk4zztsNv3n0vPzx/luMrKxScFA/v3setI69fL2Aj2Fss8/0LZ4i1vnRZzw1DDCnZmn/9lNmJa1dybmHNfRFLDlxK/EHcwpQ5UsbIdf3bmyLhd0OPI6sLtDoBW9JFVipNOkFI2bCpzbdQKyXu2vd5RiZ7b3DXDzg3V8H1VxgvFxguZhFCsBLMc+/WM5RZpBUWeKU5TiZ7GtOu8sj8k3hRzOCEwcrpIloILn2Iz4F5R0TomhAZRBrEWky61MX1DeKSgaU9YtsiGnOIO0DWAEcT7/JRNYvM2QA3IxElSaasaZ8xQYPQmiiOadaz5MYapPMuQSTIZtIo1QWRIWVsIW0YjKSz1Dz30g1J3R+Be+dwryvjVH9O/DjWl3rkhFEMQjBZfvf951eGurTe7noqpdJ8bt8tfO7avzC8623PF3lochtPzp1H9S8nCuBLe24jtQG/0bwbFKxdDDl3sOa+CEKC1khhsWvg1xDi+t4z2dAJP9Yxfzv7Iv/33/+EypLfW93Jg5QwSQcK3w25xcizzyzw9BOn+NDHb2V0eoj/8J3DtF0frXtv/ntv2cpH7hnFcL/KB8ZXqbcV9e5ZRkovcLJV5IX2MNpXgMbO+Fhbu3ivZqD3WmIQEZmCbkshRYA2JbGWqFaMMaRxV03CQCHdNlJ4RL4kuMXBmAoJVyxENoZIYFZiusLEz4ExFOGsWoAg8iLakcLP2zjpFgUtOLirCCJkS/6fXuru9aU9t/G1I89wsdkAoYk13FEe59ahMQAmhgoc2ruFZ47P4NgGsdb4QcSHDu1i4Bp6viTevYQQfGZ6P7cPjXG6uoqpFLcMjryu91Zi/Qih2Jb/LEOpQ7SCCxgizYC9B1O9fuT7uv/td8P8KIcOHdKHDx9e9/M+u3qG//nx71GdDXEcSaXpE8YxIpKkXUVBmHRFxCeHtjEmU1TWmojtBSJiCtkUXhxy0l3ldHONz25/hd0DJzgRODQESCXRTofTzUHW2ilsT/Q+qe0Q1zQIVixUS4AdI17QOLf6+A0LqTXKMIiiCNtos/psimYjhyV8pONCDGJN0/mlAkHWwqpDuqFJv+LQ7QooSMjBxw4tELTfw1MvVRAKzLxByw/YNqT4Z5/awfbJMdLmNEpeeaOxE/icqCzTDHymcgNsyxcvfZWHXuv+7Pwax84tYijBgekxto4Uk6kSEol3ISHEc1rrQ2/2+A3dwn90/hjNSohtSsI4Jop7NyojpfFUhIGNjGOOd6psKWbpBBH1pSq7d4wSxBGPd8/T8FoMhWeQznFORFCzJKJrIHSMGyq6mDhGhO5IkCACiAJFo5nCNsDzJR0MJis1jEJI3JKIMMJyYrxUnposEJckkZFChimcxSp6RKFtA+WbBOmI7BmYTCsqoYcVQUlGOMsRrxyrkO1qSvk0ZZnjk796iDt2T17qRng1adPirpE37j0jpWDn5BA7J4eux0uSSCRuoA2d8BuBC5FAKImOLutxIED3GuRILQguWxdQSsli1OTv3HMstKocnDjPIE1CrcHSOMInSkdErk2sX5slXwAatMAwY9o1m/icyYrIEKQk4aDg3GqKtOtRzDfJWR2aUZrm3DhGqYGeD4gFhJaJN5SHhzTagriridcscoGPEmAJhWW6BJ2QU8cG8aRJsWxx545J2g2XY0+e5a49yXqziUTi6jZ0wr+1NMFPsiv4lQjTlggBMf2BVLEgjmNCGbPNydFqdCnlUzCieGTlOJ3FFsVMg+J4k2bX5pQeYnd+FRlplAiJPUUmjnglBi3BsCOkhJZr49UspIyRVoyMJdLRRBlJt5qm00hjuiEhJnauSXo0ws3aGFGvq5ZvFMgELeSKpF03ELaBrxXNto8UMJgHgcVafZztYyUmBwuYSlIYSLG83GBttcXQVWbBTCQSiQ2d8D8ydpBndlzg6cYqQVdgKnB9jaElecuh1Q0oa5uBukBkBZ/70r38yekf0T3XQAqFkw0xzBjZ0DSkRaWSIlvyUCLGSvksz+eQdUV32KTrGbieTegqzCZEzYB4KEYOeNiBQFVMhB0RCUlgKkw3Qvkm2gwQNkShgTA0ItDEniJIS2LHYHsjQ2R5ZDMjHNie570P7eTMcY8L51YYuKyHymszWsZR0nc6kUhc3YZO+MNOgf/10MM8Uj7K90+/ituCXfkyqchCo7l9yzg70nlSlsXwSIeFxr/k3GybAbNAKh3jRQohez2niGF1KY0vBHYqpDafoVJL42rF3JlBIlOws1Th/i0XGB1r4O42eezCHmbW0kyPtamm8wgrBkeDGTGWr3BuZoyOa2Blu+iugdu1SOVcOmmJZzsMD0g++YEdrAV1HirfwoPlAwAYYp4Tx+cpDOhLN1PbLY9M1mbwKq37WMfEOkYJldx8TSQ2sWtK+EKI80ATiIBQa31ICFECvg5sA84Dv6y1rl5bMd++opXlS7vu40u77rtif6w1zyzP8J8WTtCtNbm/+ycsuavU2rupmxa2aOH4mnbXJF10aTVtjE4MNUmnZuMZiotrQ8zkC8SB5I6BBT49eoSVbobFuIDT8Xho8Cw/rm3FvajwERRGXVwMyukG28prZAdczi2VWayXUEQMiDqjW1ZYUqMEaYu7h0d6a8kGYFzWP3fn7lEOHJzgxNGLSCnRWmOais998b4rFi6JdcyLtSO8XDuCH/uU7SEeGLqXEef6Du5IJBLvTuvRwv+A1nr1su3fBn6gtf5dIcRv97d/ax3+zrp6dO4kj86dZtDOUIxO8J1vK049dRtBU6InBOd2DZGJuiy9mmHLeJUtZoXs9oDQM4i04sTf76Dt5VBOhAo07504y6qbxsPEbZiceWy4N+eNhqWlHOFITHvEREnNRKYKnoQu7N47hzcjMKKA4W4dbMWBoRWUtZVdhTJB3BvhujM3dqnsSkk+8fBd3HbHNuZm1nDSFrt2j5LLX9lX/tnKczxffYmSNUDOyNIImvzN/CN8bvJhita7byBVIpG4vq7HJZ2Hgff3f/5j4HHeZQm/Hfg8Pn+WiXSBlj/H818/x9IP80S1BiKC0pE6mVN51j5aJHeyzeqLOdLvCUnlW2g3zXNntyIqGnuqS802MIOYnOOx0MoSa8HcE0VErDHzEUWjQ6Vu4lctSs0aw1MddCAJY4Vd9lB2yOBoHeY1hYLmvq0lzmhBLFusunWEgF8cPcSQfeVUtVIKprYNMbXt6t0nvcjjSP0YQ3YJ1f92kDOzVPwax+oneLB8/3Wv50Qi8e5yrQlfA48KITTwB1rrrwEjWusFAK31ghDiqhM8CyG+AnwFYGpq6hqL8eZ1ozZPLD/JufYJqqHEnauw/ERM3A3AUmAIRKhxztUpvirJ6w4f+/RZssMR0tKksprthRkaOww6OYOnWtPU2ynm3RwFx2VxIUfYkaQLAZYR0WwaFHUb3/AodZqUh1s4ToST9TjSGianNaIJ2x3Be3aXOTQ+xtZzLV5dMxga28vde3ZQTL31UY/dqNubdfNnhmo70qbiV9apNhOJxM3kWhP+e7TW8/2k/j0hxMk3+8T+h8PXoDfS9hrL8aYEsc/jS/+ZFb+GIUz8qEF9pkFYCyGnoCt6PeoNARKsix32f2qFrOXSVik6HZPu8w7zLw+w0kjRNSQ8EGNsiXissZVf2fIyedNEKU3KDBBhzMxCCVuH2EGM1DA8XMfH5MzaMGkFeWI+dYvFdG4KFdn82TdiKlWBaeaZOTrH3EsNvvj5e0inrz4f/RvJGBmUUARxiCl/+jJ3Y5ddzs51rtlEInEzuKaEr7We7z8uCyH+CrgHWBJCjPVb92PA9Z3g+S1Y6F6gEdYYTZXZmjvFkbU2iAhiCEMDEWkU/RkmlSCKBNsHVlkhQ2clzZlHt9A6b2PaEUiJDhTxd1zCA4qFAzn+1LuNuwtzGFZEdc5goZWn6xt4UiHbYIQtTreH0UIzku8wYBe5Y+gWdltLKOXy3R9Are4zMmxhWtsQwmZxqc5TT5/hQx888JZiNaXJoeId/P3q0+SMDKa0aIYtLGGyP7/3+lRwIpF4V3vbCV8IkQGk1rrZ//mjwP8OfBP4deB3+49/vR4FXQ9Vb5W5ExGHjy+y1nRxxqCGBX4XIwwJUgZIQSoTgBUixwVPNfbT9FK0FrJYkU8x28YwNd3QIvIcRC4kc8HDnciwODvIt0UBsxyQvhAiqxFKaWKt8EYiXprexrGTDu/dY/HpHQ+yJbebtCoRBkfwuo9z6pUWQ4OTmNZ2hOi16AdLGY4cu/iWEz7AwcIB0irNS7UjtKM2O7PbuX3gNnJmMjFWIrEZXUsLfwT4q36/bgP4j1rrR4QQzwJ/LoT4MjADfP7ai7k+Th92ef4Jl8paG3fJQ33Xw/QV1h0K92iA44dkp1wINMaEpvCZGMfpUKh0OefbVNNpYi0Y7TQJdW8qWSKTbj6NWRcEviTCIVIWwX1dSn4bIwqJ0hJj2OSOoXHGtuZ4cHQve4s/ne/ItG7FMA9i2Y+gjPQVU6TGMVd0tXwrhBDszE2zMzd9jTWXSCQ2gred8LXWZ4HbrrJ/DfjQtRTqeuh0PL733AKn2x4ajZxMoSfTOCd9smsR4nabwXIF2QU5aCD3WXhCkAlbpPIhWkjS0qeVdui2unjaBKXxTIvANjClj235xJ4ijgRWHGEXOjipEv6OiHQhx87SEA+U9/JQef/ryieE4Pbbpjj83HlGhvP9RU00lUqLhx7cfQNqLJFIbDQbeqTt5WaWqxxtLGNVOwhHI0JN5AnaU2msYy5mJqb0YejUMohYENoK/BhfG5CL8CxB7Bv4gUGtk8KQMSKAIK8Q5ZhSvkOoJH5gEnQEYcvEewxEscnA4QxyzGLiV8f4wMhBtNa8sDbHE4tnaIYe+wujvH9sFw8+sJuVlSYXZtYuJfxdu0a45+6khZ5IJK7dpkn4ldAlDLooCaEwCT2JJAZTE6ZNZCdGoxA6JsDEVRLDk4Smibfm0Fky0cpAGhppC7SriIYDjB2wp7xKSvi9m78pyVqYYu5cEVXVyH1pyOcx1zR//5fP849vuY+/Wz7Lt+eOU7RSONLg8NoMJ+qL/Nf73scXf/le5hdqNBpdigMZRkbyyXQIiURiXWyahJ9J2eh0xJBuk84EVLppVltpIkuAEeEaOSqzHRpli9Ugiwg0WkvGnSr+rEGUFwg/YmCtTfkDVdR4yHljgPvjs8yeGEc7GseK0L4mZUeMpVcoT9XJl0LOuwb50jC1sw1Wqg1+MH+KsVQeU/au1Y+m8sx36jy3OsMHxnczMV5kYrx4g2sskUhsNJsm4dNc4wv3HmZU1cDTCANOLJX5/sldmBmXbrbEqeUJjJxHzuz2+uMLzfnKEGYjIhd2MXWIo0Lqr6SpjFtklMstk3MErsPC/BCx5yNiKOUaHLh9gdZ8EVM1GDZNOtEAOdOh7XvEWl9K9q/JGBbn28mAqEQicf1s+ITfjVz+9syPmFv8A6bGVlnp5ggihdAR2VLMlu1tKnYRtRrRGbCxFgSRYSDQBKFJrGIsJ6I42yaOwLdNui0THUGhGWKMaaanFhgfW6UQhxTTbWyjy8wTg1SEIooiSmad+UrEHbt3MTo0APMQ6Rglftr7phv6jKbyN7CmEonERrdhEn7La7FQnyPnFBjN9yYa01rze0f/gifPPM+vbl1ltZvBjAOMfMCPj+zFDwxsI6ZVyyDTEZYKSA/4RK6i27UglNjC692MNW0MK2SYJu6kYml1gGXytAOHnO1iGhEyjrGzHoOqy3IQES9HaENgSIfbBid5+FfeSyvw2ZopcaK+xJbMAKZU1PwuSkruHnrnpphIJBKbz02f8OM45psnvsH3z/2I1TVFKhVx69Qo/8Xt/y3HKos8PnMUfVZgpiNYEbi2w0xjkM6azUCxTWxJlBUzMNRCRGnsoosjQwLfoDmfpi1s7IamGSl0KHBFivSWKmFTYgjN0+d28r7pExTNDkJDMe6QM+DuT66x9UCHxpzNmj7I8I5p/sUjX2c+bpPOKXJDNg2vTTmdZ1t2kE9N3cKQkwyISiQS189Nn/CfOP8j/vAbz9E9OtybcljFzA57VD7/fxBaBwl/EiNPK87nhxga7tCom1TrGQwzwgw1F1czGG5IuKKwsjFxoKhKk0AbhGMCGpBe8lGxwCwEiILGVJLdu+dZXBzgxPwYS+08A/kWU/U2xtQsA9uWwfCZ3O3R2OHwey9nac98mzAlyGifThuWqimmttb5yp6Pc//oHTe6GhOJxCZw0yf8P3z0UbqHM0QpiMoxwgdjweYn/9klOngUYyZmcEeTo5UxPjV9hFy6w2y7iN8p0uykWFooEGYFddtC+gKn4xOkFbEAW4V0TAO9O6agur3r+r6iMZfHHAgYG6/SjBzmvTwr5/NUl33u2TtHO3ZIEXDcLfI3a7too4jyYHQiRjNN7p2cwSLiqdWdfOvcd7lv5DaEeHujaROJROLNuukT/uKzBq0xQWu7Qvdzpl2NGDhloYcb7N69zAPvOQORYnElxdzLOWbPOJxzBvEMk7BkgoqRgQbLpx3aZAouhcglDhXdIEVNpRnQHloLpNDoSKKXDaJMTDls0z2exxYBzSGDb1R2MqYb7HEWeaExTiXIgIQYwcHBJX6xfAovMsgIn89ln2c5XqUTLJCxJm5sRSYSiQ3vpk/4HSVpbDfQujc9AZGmWzIQW2NKTsxnPvw8thnz+LE9HPlWGde16Gobaj4qLdGGJMoZxCpGGxqrERFoRYjE9WyIJVppPBQpHeG7BrbjIlyNRqDakJlzkfsCwlHNWrPAkG7zwplddCspaqKE2KpxSgGD6TbLbhaNINISD4MDQ/Po+CKQJPxEInF93fTXERpbJLEhMFWME4TYIgIh6IwLPnH3CxQyHY6sjnHx5TRGPURLRRwqzDjCbrYozSwznl+iVGxiql7/fCE0wtSkMi7CiIhjQbdj4bdNlBWRtjtoCWYmonnBwk65yFdicMGpR6w+OUJzOYs2NHHTxH7epFD1mbAaIOhNwSw1CMVIxsDQnRtdjYlEYhO46Vv4Ih1juSGhI3AzBjLS2K0QHM13/u0+Hhu+nUYrQ9iWoDTWxRYyFSEymlQcE7csxFFFJhWSHveIdvssBjmC2KDgdMmWWzTbKWQqoJCvkHVcRFsiFDTPWnTPWKQzAaYXw7KJKSVxAKlcl2qURqdgPNVm21KDDz5wnjBWLHXSLLo5pgcEo+ltSJW70dWYSCQ2gZs+4dudmOqQiRYCtCZSkmBQ4bR96qUy1BTajcAx0LbEyztkT1cwTYh8uzdzZhYiLVEzgun9F1mNM4S+QS3MULC7EGu0jFk4nWdQCbIrHu1KirVaEa0FnZZBSkYMFluoGUU64xMJQTvKMZ3y2JttU1kbxJYWKcOjZLc4YI0ymN6LlBLD2HejqzGRSGwCN3XC972ATsYiRqDaAhkIUBCmYry0yY70MtpKURc5YimAmMA1ae8skb3QQYoYWRQgBLGQZAotOG1z363nCRwTEcYM5ho8eWE3ONAdNKlXc4SVSVTJpZwNaVdBdiwGRocYnGjTbWmCVpa0HGInJiX5MmldplBOMV3+DEHwEkR1EHWkzJLO/jpSpm90VSYSiU3gpk343Y7Hn//RkwSGwqxJpA/Qm1XS7ArCQkzgg9vOEmgTGUPsgMpHxEIQDNgIM8TMgPA1A6pFLurgL1qkxgMy21wypsfc0iCUYuIy4EHYKXDrx/ZROB6i4xhVkExuH2bwrjGeOHOB0W2S04fnCIXEtARxGLBWlXz2EwIpc9j2e4jjFjpeIJ3/LaQ0b2Q1JhKJTeSmTfgvPv0qK4s1jIwgDgXKjVCeJlaCIC2RHcVsZpi0qUiFPqFrIFxBqEycchdfKqaH5tk7uYgpQtbms6xdKOB5BtZFSWlnl6af5tWwjJ6M0Z7CNAUHRif5l7/yWQIvZG2pju1YlIZznJhbRp+5wEA5y647J5g9vUK74WJmczz8oTr796b6JReAj2HemiT7RCLxjropE77WmieePMpLu5/DfGEA5bmkF33QgNYEBYP2qEO55LJ3aBkRac6tlFlaLKJcjS8ko7evMFqoYgY+jgzZessK2bzLU3++Ay+WXCiXWKmWMA60MVwDQoUyBf/NL3wUKQS2YzK+dehSmXaODVLKplmutxgqZ8kNpVmutdgxWuae238M8QJaZIE2oDBTn7pR1ZdIJDap65bwhRAfA/4vQAH/j9b6d9fr3MeOX+T58hHc8TbW4xkyhVfZMWET1xUyFVPPCy4+PcCezyySyoUYluYD245xfq7M0+d30s0YuGmD5ztTqDYc8ObJ45ErthgcabDQGqLSymOEkDYV0rCxHIP3j+9n3+j4VctkGQa/8cFDPPLCKY7NLSER3Lljkl+4fTcp835C7ymi6AJK3Y5hP4BUw+tVHYlEIvGmXJeEL3qrcP8+8BFgDnhWCPFNrfXxaz13HGse+7ujhDu65KRHeXSO0qk8zZaFNGLiSKCEZsfdVc797Raa90jE9phsymWqWGGw1qRlGGSdLlGkCGzFyWCcu/QF4kAwOlXlrFUitbWNM2sylIOxwih7CuP8kx3/8FK9A5kUX3zwdoIoQiAwLi0+PoiV/uS1hp5IJBLX5Hq18O8BzvQXOkcI8WfAw8A1J/wwjKh2OkhT4K21GG/kaTRN7IIP/aUAw65EnLPxtr1E/titVIcNDEOz7OdwMh7aCZCRJhYaK4poYdHWFkUVsTSeQw+aZCqSXSN5xvxpvnzwIcac0pteatBU6ucflEgkEu+w6zXSdgKYvWx7jnWaO8A0FUOpHKaX4uSjLu6ShZGKLiV7AMOJ8FoG+dFJhCsxGho3MknJgBopcuk2gW9gmRG6K9ACjJJPZ8rkYrFMrhahGlNkjd3sH9jBeGowWVc2kUjc9K5Xwr9adtRXHCDEV4QQh4UQh1dWVt78iYXg/e/dz9iFKQIvwrB7l3Gu+ENxrwSujPpTGIAhI8JI0okNQqVYXCvQXHLQJmQHXFQq4tyLE0zOuoxYFrlCCkeZ3L9vP/F4HQAABj9JREFU69uJP5FIJN51rtclnTlgy2Xbk8D85Qdorb8GfA3g0KFDV3wY/Dz7943zFfUJMhJmZ39C9BMHZUVIQ4AGv2WSm+oQX4B2PoKiIOu4nG0MsXtwjrOzo3RWHDoNG0tFlITPuXAbzkDI+O4VTi4d5IHpSX7p/oOMl5JlBxOJxMYgtH5LufbNnVQIAzgNfAi4CDwL/GOt9bGrHX/o0CF9+PDht/W37vutX+aWoED9ZBYd9xJ+dqzL4k7QzxWZP5QiLgoMI2QitUrqTMTqMwW0hvHpNYp3dcG12D7YZrjssGfqy9w2dR8py0wu4yQSiXc1IcRzWutDb/b469LC11qHQojfBL5Lr1vmH71Rsr9WT/+rP2f6wBgHf+MzFLwI15I8ffIsW+p7WRvzCOomhU6VHTYYjRTzZprtn2+xdauFq/ezb3gfn9o5hYEkl5pGCut6FDORSCRuuOvWD19r/W3g29fr/Jc7e2yh/zcjQL6uZd5uuSzMVTEMycTWQaqBR7XboZRKU05n3okiJhKJxA13U460fSO97v+vl8k67Nw7dml72DQYThJ9IpHYZG76BVASiUQi8eYkCT+RSCQ2iSThJxKJxCaRJPxEIpHYJJKEn0gkEpvEdRl49ZYLIcQKcGGdTjcErK7TuW4mSdybz2aNfbPGDa+PfavWuvxmn/yuSPjrSQhx+K2MPNsokrg3n80a+2aNG6499uSSTiKRSGwSScJPJBKJTWIjJvyv3egC3CBJ3JvPZo19s8YN1xj7hruGn0gkEomr24gt/EQikUhcxYZJ+EKIjwkhTgkhzgghfvtGl2e9CSH+SAixLIQ4etm+khDie0KIV/qPxf5+IYT4vX5dvCyEuPPGlfzaCCG2CCF+KIQ4IYQ4JoT45/39Gzp2IYQjhHhGCPFSP+7/rb9/uxDiJ/24vy5Ebz5vIYTd3z7T//22G1n+ayWEUEKIF4QQ3+pvb5a4zwshjgghXhRCHO7vW7f3+oZI+KI3TebvA78I7Ae+JITYf2NLte7+P+BjP7Pvt4EfaK13AT/ob0OvHnb1/30F+Oo7VMbrIQT+O631PuA+4J/1X9uNHrsHfFBrfRtwO/AxIcR9wL8C/nU/7irw5f7xXwaqWuudwL/uH3cz++fAicu2N0vcAB/QWt9+WffL9Xuva61v+n/A/cB3L9v+HeB3bnS5rkOc24Cjl22fAsb6P48Bp/o//wHwpasdd7P/A/4a+Mhmih1IA88D99IbdGP0919639NbbOj+/s9G/zhxo8v+NuOd7Ce2DwLfordG9oaPux/DeWDoZ/at23t9Q7TwgQlg9rLtuf6+jW5Ea70A0H8c7u/fkPXR/7p+B/ATNkHs/csaLwLLwPeAV4Ga1jrsH3J5bJfi7v++Dgy+syVeN/8G+O+BuL89yOaIG0ADjwohnhNCfKW/b93e6xtlAZSrLT67mbsfbbj6EEJkgb8E/oXWuvEPrDe8YWLXvSXcbhdCDAB/Bey72mH9xw0RtxDik8Cy1vo5IcT7X9t9lUM3VNyXeY/Wel4IMQx8Twhx8h849i3HvlFa+HPAlsu2J4H5G1SWd9KSEGIMoP+43N+/oepDCGHSS/b/QWv9n/q7N0XsAFrrGvA4vXsYA0KI1xpql8d2Ke7+7wtA5Z0t6bp4D/BpIcR54M/oXdb5N2z8uAHQWs/3H5fpfcjfwzq+1zdKwn8W2NW/k28BXwS+eYPL9E74JvDr/Z9/nd717df2/5P+Xfz7gPprXwlvNqLXlP9/gRNa6//zsl9t6NiFEOV+yx4hRAr4ML2bmD8Efql/2M/G/Vp9/BLwmO5f2L2ZaK1/R2s9qbXeRu//8WNa619hg8cNIITICCFyr/0MfBQ4ynq+12/0TYp1vNnxceA0veuc/+ONLs91iO9PgQUgoPfJ/mV61yp/ALzSfyz1jxX0ei29ChwBDt3o8l9D3A/S+5r6MvBi/9/HN3rswK3AC/24jwL/S3//NPAMcAb4BmD39zv97TP930/f6BjWoQ7eD3xrs8Tdj/Gl/r9jr+Wx9XyvJyNtE4lEYpPYKJd0EolEIvFzJAk/kUgkNokk4ScSicQmkST8RCKR2CSShJ9IJBKbRJLwE4lEYpNIEn4ikUhsEknCTyQSiU3i/wcCpA+WGwlUMwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x295017b83c8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# YOUR CODE GOES HERE\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "colors = np.random.rand(X_train_level2.shape[0])\n",
    "plt.scatter(X_train_level2[:,0], X_train_level2[:,1], c=colors, alpha=0.5)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best alpha: 0.000000; Corresponding r2 score on train: 0.409477\n"
     ]
    }
   ],
   "source": [
    "alphas_to_try = np.linspace(0, 1, 1001)\n",
    "r2_train = np.zeros([np.shape(alphas_to_try)[0],])\n",
    "# YOUR CODE GOES HERE\n",
    "index=0\n",
    "for alpha in alphas_to_try:\n",
    "    r2_train[index] = r2_score(y_train_level2.clip(0,20),(X_train_level2[:,0].clip(0,20)*alpha + X_train_level2[:,1].clip(0,20)*(1-alpha)))\n",
    "    index += 1\n",
    "best_alpha = alphas_to_try[np.argmax(r2_train)] # YOUR CODE GOES HERE\n",
    "r2_train_simple_mix = r2_score(y_train_level2.clip(0,20),(X_train_level2[:,0].clip(0,20)*best_alpha + X_train_level2[:,1].clip(0,20)*(1-best_alpha))) # YOUR CODE GOES HERE\n",
    "\n",
    "print('Best alpha: %f; Corresponding r2 score on train: %f' % (best_alpha, r2_train_simple_mix))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best alpha: 0.000000; Corresponding rmse score on train: 0.820599\n"
     ]
    }
   ],
   "source": [
    "alphas_to_try = np.linspace(0, 1, 1001)\n",
    "rmse_train = np.zeros([np.shape(alphas_to_try)[0],])\n",
    "# YOUR CODE GOES HERE\n",
    "index=0\n",
    "for alpha in alphas_to_try:\n",
    "    rmse_train[index] = np.sqrt(np.sum((y_train_level2.clip(0,20)-(X_train_level2[:,0].clip(0,20)*alpha + X_train_level2[:,1].clip(0,20)*(1-alpha)))**2)/y_train_level2.shape[0])\n",
    "    index += 1\n",
    "best_alpha = alphas_to_try[np.argmin(rmse_train)] # YOUR CODE GOES HERE\n",
    "rmse_train_simple_mix = np.sqrt(np.sum((y_train_level2.clip(0,20)-(X_train_level2[:,0].clip(0,20)*best_alpha + X_train_level2[:,1].clip(0,20)*(1-best_alpha)))**2)/y_train_level2.shape[0]) # YOUR CODE GOES HERE\n",
    "\n",
    "print('Best alpha: %f; Corresponding rmse score on train: %f' % (best_alpha, rmse_train_simple_mix))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.82059863, 0.82059887, 0.82059944, ..., 0.96866464, 0.96893872,\n",
       "       0.96921299])"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rmse_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test R-squared for simple mix is 0.352241\n",
      "Test R-squared for single LGB is 0.352241\n"
     ]
    }
   ],
   "source": [
    "test_preds = pred_lr.clip(0,20)*best_alpha+pred_lgb.clip(0,20)*(1-best_alpha)# YOUR CODE GOES HERE\n",
    "r2_test_simple_mix = r2_score(y_test.clip(0,20),test_preds.clip(0,20)) # YOUR CODE GOES HERE\n",
    "\n",
    "print('Test R-squared for simple mix is %f' % r2_test_simple_mix)\n",
    "print('Test R-squared for single LGB is %f' % r2_score(y_test.clip(0,20), pred_lgb.clip(0,20)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9144167633544937"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sqrt(np.sum((y_test.clip(0,20)-test_preds.clip(0,20))**2)/y_test.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.sqrt(np.sum((y_train_level2.clip(0,20)-(X_train_level2[:,0].clip(0,20)*alpha + X_train_level2[:,1].clip(0,20)*(1-alpha)))**2)/y_train_level2.shape[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stacking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearRegression(copy_X=True, fit_intercept=True, n_jobs=1, normalize=False)"
      ]
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# YOUR CODE GOES HERE\n",
    "lr = LinearRegression()\n",
    "lr.fit(X_train_level2, y_train_level2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train R-squared for stacking is 0.276578\n",
      "Test  R-squared for stacking is 0.349031\n"
     ]
    }
   ],
   "source": [
    "train_preds = lr.predict(X_train_level2) # YOUR CODE GOES HERE\n",
    "r2_train_stacking = r2_score(y_train_level2,train_preds)# YOUR CODE GOES HERE\n",
    "\n",
    "X_test_level2 = np.c_[pred_lr, pred_lgb] \n",
    "test_preds = lr.predict(X_test_level2)# YOUR CODE GOES HERE\n",
    "r2_test_stacking = r2_score(y_test,test_preds)# YOUR CODE GOES HERE\n",
    "\n",
    "print('Train R-squared for stacking is %f' % r2_train_stacking)\n",
    "print('Test  R-squared for stacking is %f' % r2_test_stacking)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Stacking: whole model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dates_train = dates\n",
    "\n",
    "\n",
    "X_train = all_data.drop(to_drop_cols, axis=1)\n",
    "X_test =  sub_test_data\n",
    "\n",
    "y_train = all_data.loc[:, 'target'].values\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lr = LinearRegression()\n",
    "lr.fit(X_train.values, y_train)\n",
    "sub_pred_lr = lr.predict(sub_test_data.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lgb_params = {\n",
    "               'feature_fraction': 0.45,\n",
    "               'metric': 'rmse',\n",
    "               'nthread':11, \n",
    "               'min_data_in_leaf': 2**4, \n",
    "               'bagging_fraction': 0.84, \n",
    "               'learning_rate': 0.122, \n",
    "               'objective': 'mse', \n",
    "               'bagging_seed': 2**7, \n",
    "               'num_leaves': 2**4,\n",
    "               'bagging_freq':1,\n",
    "               'verbose':0 \n",
    "              }\n",
    "\n",
    "model = lgb.train(lgb_params, lgb.Dataset(X_train, label=y_train), 100)\n",
    "sub_pred_lgb = model.predict(sub_test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dates_train_level2 = dates_train[dates_train.isin([27, 28, 29, 30, 31, 32, 33])]\n",
    "\n",
    "# That is how we get target for the 2nd level dataset\n",
    "y_train_level2 = y_train[dates_train.isin([27, 28, 29, 30, 31, 32, 33])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n"
     ]
    }
   ],
   "source": [
    "# And here we create 2nd level feeature matrix, init it with zeros first\n",
    "X_train_level2 = np.zeros([y_train_level2.shape[0], 2])\n",
    "\n",
    "# Now fill `X_train_level2` with metafeatures\n",
    "for cur_block_num in [27, 28, 29, 30, 31, 32, 33]:\n",
    "    \n",
    "    print(cur_block_num)\n",
    "    \n",
    "    '''\n",
    "        1. Split `X_train` into parts\n",
    "           Remember, that corresponding dates are stored in `dates_train` \n",
    "        2. Fit linear regression \n",
    "        3. Fit LightGBM and put predictions          \n",
    "        4. Store predictions from 2. and 3. in the right place of `X_train_level2`. \n",
    "           You can use `dates_train_level2` for it\n",
    "           Make sure the order of the meta-features is the same as in `X_test_level2`\n",
    "    '''  \n",
    "    M=15\n",
    "    X_train_m = X_train[dates_train.isin(list(range(27-M,cur_block_num)))]\n",
    "    y_train_m = y_train[dates_train.isin(list(range(27-M,cur_block_num)))]\n",
    "    X_test_m = X_train[dates_train==cur_block_num]\n",
    "    lr = LinearRegression()\n",
    "    lr.fit(X_train_m.values, y_train_m)\n",
    "    pred_lr_train = lr.predict(X_test_m.values)\n",
    "    \n",
    "    lgb_params = {\n",
    "               'feature_fraction': 0.45,\n",
    "               'metric': 'rmse',\n",
    "               'nthread':11, \n",
    "               'min_data_in_leaf': 2**4, \n",
    "               'bagging_fraction': 0.84, \n",
    "               'learning_rate': 0.122, \n",
    "               'objective': 'mse', \n",
    "               'bagging_seed': 2**7, \n",
    "               'num_leaves': 2**4,\n",
    "               'bagging_freq':1,\n",
    "               'verbose':0 \n",
    "              }\n",
    "\n",
    "    model = lgb.train(lgb_params, lgb.Dataset(X_train_m, label=y_train_m), 100)\n",
    "    pred_lgb_train = model.predict(X_test_m)\n",
    "    \n",
    "    index1 = dates_train.isin(list(range(27,cur_block_num+1))).sum()- (dates_train==cur_block_num).sum()\n",
    "    index2 = dates_train.isin(list(range(27,cur_block_num+1))).sum()\n",
    "    X_train_level2[index1:index2,0] = pred_lr_train\n",
    "    X_train_level2[index1:index2,1] = pred_lgb_train\n",
    "    #  YOUR CODE GOES HERE\n",
    "    \n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearRegression(copy_X=True, fit_intercept=True, n_jobs=1, normalize=False)"
      ]
     },
     "execution_count": 158,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# YOUR CODE GOES HERE\n",
    "lr = LinearRegression()\n",
    "lr.fit(X_train_level2, y_train_level2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_test_level2 = np.c_[sub_pred_lr, sub_pred_lgb] \n",
    "sub_pred_stack = lr.predict(X_test_level2)# YOUR CODE GOES HERE\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Submission LGB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sub_test_data['item_cnt_month'] = sub_pred_lgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>shop_id</th>\n",
       "      <th>item_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>5037</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>5320</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>5233</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>5232</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>5268</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ID  shop_id  item_id\n",
       "0   0        5     5037\n",
       "1   1        5     5320\n",
       "2   2        5     5233\n",
       "3   3        5     5232\n",
       "4   4        5     5268"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test[['ID','shop_id','item_id']].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>shop_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>target_lag_1</th>\n",
       "      <th>target_item_lag_1</th>\n",
       "      <th>target_shop_lag_1</th>\n",
       "      <th>target_lag_2</th>\n",
       "      <th>target_item_lag_2</th>\n",
       "      <th>target_shop_lag_2</th>\n",
       "      <th>target_lag_3</th>\n",
       "      <th>target_item_lag_3</th>\n",
       "      <th>...</th>\n",
       "      <th>target_item_lag_4</th>\n",
       "      <th>target_shop_lag_4</th>\n",
       "      <th>target_lag_5</th>\n",
       "      <th>target_item_lag_5</th>\n",
       "      <th>target_shop_lag_5</th>\n",
       "      <th>target_lag_12</th>\n",
       "      <th>target_item_lag_12</th>\n",
       "      <th>target_shop_lag_12</th>\n",
       "      <th>item_category_id</th>\n",
       "      <th>item_cnt_month</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6425094</th>\n",
       "      <td>5</td>\n",
       "      <td>5037</td>\n",
       "      <td>0.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>1052.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>110.0</td>\n",
       "      <td>1092.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>119.0</td>\n",
       "      <td>...</td>\n",
       "      <td>54.0</td>\n",
       "      <td>991.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>105.0</td>\n",
       "      <td>954.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>65.0</td>\n",
       "      <td>1445.0</td>\n",
       "      <td>19</td>\n",
       "      <td>0.383015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6425095</th>\n",
       "      <td>5</td>\n",
       "      <td>5320</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>55</td>\n",
       "      <td>0.238589</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6425096</th>\n",
       "      <td>5</td>\n",
       "      <td>5233</td>\n",
       "      <td>1.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>1052.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>80.0</td>\n",
       "      <td>1092.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>...</td>\n",
       "      <td>37.0</td>\n",
       "      <td>991.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>119.0</td>\n",
       "      <td>954.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>19</td>\n",
       "      <td>0.866186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6425097</th>\n",
       "      <td>5</td>\n",
       "      <td>5232</td>\n",
       "      <td>0.0</td>\n",
       "      <td>28.0</td>\n",
       "      <td>1052.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>48.0</td>\n",
       "      <td>1092.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>65.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>23</td>\n",
       "      <td>0.224132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6425098</th>\n",
       "      <td>5</td>\n",
       "      <td>5268</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>20</td>\n",
       "      <td>2.459580</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         shop_id  item_id  target_lag_1  target_item_lag_1  target_shop_lag_1  \\\n",
       "6425094        5     5037           0.0               25.0             1052.0   \n",
       "6425095        5     5320           0.0                0.0                0.0   \n",
       "6425096        5     5233           1.0               42.0             1052.0   \n",
       "6425097        5     5232           0.0               28.0             1052.0   \n",
       "6425098        5     5268           0.0                0.0                0.0   \n",
       "\n",
       "         target_lag_2  target_item_lag_2  target_shop_lag_2  target_lag_3  \\\n",
       "6425094           1.0              110.0             1092.0           3.0   \n",
       "6425095           0.0                0.0                0.0           0.0   \n",
       "6425096           3.0               80.0             1092.0           1.0   \n",
       "6425097           0.0               48.0             1092.0           1.0   \n",
       "6425098           0.0                0.0                0.0           0.0   \n",
       "\n",
       "         target_item_lag_3       ...        target_item_lag_4  \\\n",
       "6425094              119.0       ...                     54.0   \n",
       "6425095                0.0       ...                      0.0   \n",
       "6425096              150.0       ...                     37.0   \n",
       "6425097               65.0       ...                      0.0   \n",
       "6425098                0.0       ...                      0.0   \n",
       "\n",
       "         target_shop_lag_4  target_lag_5  target_item_lag_5  \\\n",
       "6425094              991.0           1.0              105.0   \n",
       "6425095                0.0           0.0                0.0   \n",
       "6425096              991.0           2.0              119.0   \n",
       "6425097                0.0           0.0                0.0   \n",
       "6425098                0.0           0.0                0.0   \n",
       "\n",
       "         target_shop_lag_5  target_lag_12  target_item_lag_12  \\\n",
       "6425094              954.0            1.0                65.0   \n",
       "6425095                0.0            0.0                 0.0   \n",
       "6425096              954.0            0.0                 0.0   \n",
       "6425097                0.0            0.0                 0.0   \n",
       "6425098                0.0            0.0                 0.0   \n",
       "\n",
       "         target_shop_lag_12  item_category_id  item_cnt_month  \n",
       "6425094              1445.0                19        0.383015  \n",
       "6425095                 0.0                55        0.238589  \n",
       "6425096                 0.0                19        0.866186  \n",
       "6425097                 0.0                23        0.224132  \n",
       "6425098                 0.0                20        2.459580  \n",
       "\n",
       "[5 rows x 22 columns]"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub_test_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sub_file = sub_test_data.merge(test[['ID','shop_id','item_id']], on=['shop_id','item_id'], how='left')[['ID','item_cnt_month']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sub_file[\"item_cnt_month\"] = sub_file.item_cnt_month.fillna(0).clip(0, 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sub_file.to_csv('sample_submission1.csv',index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Submission: Stacking "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sub_test_data['item_cnt_month'] = sub_pred_stack\n",
    "sub_file = sub_test_data.merge(test[['ID','shop_id','item_id']], on=['shop_id','item_id'], how='left')[['ID','item_cnt_month']]\n",
    "sub_file[\"item_cnt_month\"] = sub_file.item_cnt_month.fillna(0).clip(0, 20)\n",
    "sub_file.to_csv('sample_submission1.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "widgets": {
   "state": {
    "2b965bdc846f493caf3bb07f15a55d4a": {
     "views": [
      {
       "cell_index": 21
      }
     ]
    }
   },
   "version": "1.2.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}