src/word_generation_demo.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from multiprocessing import cpu_count\n",
    "from typing import List, Iterable, Set, Optional, Dict, Tuple\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from tqdm import tqdm\n",
    "import numpy as np\n",
    "\n",
    "# from model import get_m1_model, get_m1_bigger_model, get_m1_smaller_model\n",
    "from model import MODEL_GETTERS_DICT\n",
    "from ns_tokenizers import CharLevelTokenizerv2, KeyboardTokenizerv1\n",
    "from ns_tokenizers import ALL_CYRILLIC_LETTERS_ALPHABET_ORD\n",
    "from dataset import CurveDataset, CurveDatasetSubset\n",
    "from word_generators_v2 import GreedyGenerator, BeamGenerator, WordGenerator\n",
    "from predict_v2 import Predictor\n",
    "from metrics import get_mmr\n",
    "from aggregate_predictions import (separate_out_vocab_all_crvs,\n",
    "                                   append_preds,\n",
    "                                   create_submission,\n",
    "                                   merge_default_and_extra_preds)\n",
    "from feature_extractors import weights_function_v1\n",
    "from feature_extractors import get_val_transform\n",
    "from grid_processing_utils import get_grid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "IN_KAGGLE = False\n",
    "\n",
    "if IN_KAGGLE:\n",
    "    DATA_ROOT = \"/kaggle/input/yandex-cup-playground\"\n",
    "    MODELS_DIR = \"\"\n",
    "else:\n",
    "    DATA_ROOT = \"../data/data_separated_grid\"\n",
    "    MODELS_ROOT = \"../data/trained_models_for_final_submit\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "VAL_PATH = os.path.join(DATA_ROOT, \"valid__in_train_format.jsonl\")\n",
    "TEST_PATH = os.path.join(DATA_ROOT, \"test.jsonl\")\n",
    "\n",
    "VOCAB_PATH = os.path.join(DATA_ROOT, \"voc.txt\")\n",
    "G_NAME_TO_GRID_PATH = os.path.join(DATA_ROOT, \"gridname_to_grid.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accumulating out-of-bounds coordinates...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "63a8338f3aa7485c81d6cf13f47dee72",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eceb66598a41489b9a586efdc2ce6a3a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "augmenting gname_to_out_of_bounds\n"
     ]
    }
   ],
   "source": [
    "kb_tokenizer = KeyboardTokenizerv1()\n",
    "char_tokenizer = CharLevelTokenizerv2(VOCAB_PATH)\n",
    "\n",
    "data_paths = [\n",
    "    VAL_PATH,\n",
    "    TEST_PATH\n",
    "]\n",
    "\n",
    "# takes almost no time\n",
    "dist_transform_v1 = get_val_transform(\n",
    "    gridname_to_grid_path=G_NAME_TO_GRID_PATH,\n",
    "    grid_names=('default', 'extra'),\n",
    "    transform_name=\"traj_feats_and_distances\",\n",
    "    char_tokenizer=char_tokenizer,\n",
    "    dist_weights_func=weights_function_v1,\n",
    "    include_time=False,\n",
    "    include_velocities=True,\n",
    "    include_accelerations=True\n",
    ")\n",
    "\n",
    "# takes a lot of time\n",
    "kb_transform = get_val_transform(\n",
    "    gridname_to_grid_path=G_NAME_TO_GRID_PATH,\n",
    "    grid_names=('default', 'extra'),\n",
    "    transform_name=\"traj_feats_and_nearest_key\",\n",
    "    char_tokenizer=char_tokenizer,\n",
    "    uniform_noise_range=0,\n",
    "    ds_paths_list=data_paths,\n",
    "    totals = [10_000, 10_000],\n",
    "    include_time=False,\n",
    "    include_velocities=True,\n",
    "    include_accelerations=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "10000it [00:00, 11101.07it/s]\n",
      "10000it [00:00, 13174.88it/s]\n"
     ]
    }
   ],
   "source": [
    "val_dataset = CurveDataset(\n",
    "    data_path=VAL_PATH,\n",
    "    store_gnames=True,\n",
    "    init_transform=None,\n",
    "    # get_item_transform=kb_transform,\n",
    "    get_item_transform=dist_transform_v1,\n",
    ")\n",
    "\n",
    "test_dataset = CurveDataset(\n",
    "    data_path=TEST_PATH,\n",
    "    store_gnames=True,\n",
    "    init_transform=None,\n",
    "    # get_item_transform=kb_transform,\n",
    "    get_item_transform=dist_transform_v1,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_default_dataset = CurveDatasetSubset(val_dataset, \"default\")\n",
    "val_extra_dataset = CurveDatasetSubset(val_dataset, \"extra\")\n",
    "\n",
    "test_default_dataset = CurveDatasetSubset(test_dataset, \"default\")\n",
    "test_extra_dataset = CurveDatasetSubset(test_dataset, \"extra\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_targets(dataset: CurveDataset, \n",
    "                char_tokenizer: CharLevelTokenizerv2) -> List[str]:\n",
    "    targets = []\n",
    "    for _, target_tokens in dataset:\n",
    "        target = char_tokenizer.decode(target_tokens[:-1])\n",
    "        targets.append(target)\n",
    "    return targets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_vocab_set(vocab_path: str):\n",
    "    with open(vocab_path, 'r', encoding = \"utf-8\") as f:\n",
    "        return set(f.read().splitlines())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab_set = get_vocab_set(os.path.join(DATA_ROOT, \"voc.txt\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['m1', 'm1_bigger', 'm1_smaller', 'transformer_m1_bigger', 'transformer_bb_model', 'weighted_transformer_bigger', 'v2_weighted_transformer_bigger', 'v2_nearest_transformer_bigger', 'v3_weighted_and_traj_transformer_bigger', 'v3_nearest_and_traj_transformer_bigger', 'v3_nearest_only_transformer_bigger', 'v3_trainable_gaussian_weights_and_traj_transformer_bigger'])"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "MODEL_GETTERS_DICT.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid_name = \"default\"\n",
    "model_getter = MODEL_GETTERS_DICT['v3_weighted_and_traj_transformer_bigger']\n",
    "weights_path = r\"..\\results\\models_for_debug\\weighted_transformer_bigger-default--epoch=60-val_loss=0.442-val_word_level_accuracy=0.875.pt\"\n",
    "\n",
    "# model_getter = MODEL_GETTERS_DICT['v3_nearest_and_traj_transformer_bigger']\n",
    "# weights_path = r\"..\\results\\models_for_debug\\my_features_1\\v3_nearest_and_traj_transformer_bigger-default--epoch=32-val_loss=0.441-val_word_level_accuracy=0.864.pt\"\n",
    "\n",
    "model = model_getter(device, weights_path)\n",
    "grid_name_to_greedy_generator = {grid_name: GreedyGenerator(model, char_tokenizer, device)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_suffix(s: str, suffix: str) -> str:\n",
    "    if s.endswith(suffix):\n",
    "        return s[:-len(suffix)]\n",
    "    return s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_classes = 35\n",
    "\n",
    "greedy_generator__no_vocab = GreedyGenerator(model, char_tokenizer, device)\n",
    "beam_generator__no_vocab = BeamGenerator(model, char_tokenizer, device)\n",
    "\n",
    "greedy_generator_with_vocab = GreedyGenerator(model, char_tokenizer, device, vocab_set, max_token_id=n_classes-1)\n",
    "beam_generator__with_vocab = BeamGenerator(model, char_tokenizer, device, vocab=vocab_set, max_token_id=n_classes-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "def denormalize_beamsearch(score, pred_len, normalization_factor):\n",
    "    return score * (pred_len + 1)**normalization_factor\n",
    "\n",
    "def beamsearch_score_to_prob(score, pred_len, normalization_factor):   \n",
    "    return np.exp(denormalize_beamsearch(-score, pred_len, normalization_factor))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import Callable\n",
    "\n",
    "MAX_WORD_LEN = 35\n",
    "\n",
    "def predict(word_generator: WordGenerator, dataset, n_hypotheses, scores_to_prob: Callable, \n",
    "            max_steps_n: int = MAX_WORD_LEN, verbose = True, \n",
    "            n_examples = None, generator_call_kwargs = None, ) -> List[List[Tuple[str, float]]]:\n",
    "    \n",
    "    curve_id_to_hypotheses = []\n",
    "\n",
    "    n_examples = n_examples or len(dataset)\n",
    "    generator_call_kwargs = {} if generator_call_kwargs is None else generator_call_kwargs\n",
    "    \n",
    "\n",
    "    if verbose:\n",
    "        print((\"{:<22}\" + \"{:<29}\" * n_hypotheses).format(\"target\", *[f\"pred{i}\" for i in range(1, n_hypotheses+1)]))\n",
    "        print(\"-\"*(15+30*n_hypotheses))\n",
    "\n",
    "    for i, data in enumerate(val_default_dataset):\n",
    "\n",
    "        (encoder_in, dec_in), target = data\n",
    "\n",
    "        scores_and_preds_full = word_generator(encoder_in, max_steps_n = max_steps_n, \n",
    "            **generator_call_kwargs)\n",
    "\n",
    "        preds_and_probs_full = [(pred, scores_to_prob(pred, score)) \n",
    "                                for score, pred in scores_and_preds_full]\n",
    "\n",
    "        curve_id_to_hypotheses.append(preds_and_probs_full)\n",
    "\n",
    "        true_label = char_tokenizer.decode(target[:-1])\n",
    "\n",
    "        flat_preds_and_scores = (item for pair in preds_and_probs_full[:n_hypotheses] for item in pair)\n",
    "        if verbose:\n",
    "            print((\"{:<15}   |   \" + \"{:<16}{:.4f}   |   \" *n_hypotheses ).format(true_label, *flat_preds_and_scores))\n",
    "\n",
    "        if i >= n_examples:\n",
    "            return curve_id_to_hypotheses\n",
    "    \n",
    "    return curve_id_to_hypotheses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "target                pred1                        \n",
      "---------------------------------------------\n",
      "на                |   на              0.8950   |   \n",
      "все               |   все             0.8705   |   \n",
      "добрый            |   добрый          0.8608   |   \n",
      "девочка           |   девочка         0.8550   |   \n",
      "сказала           |   сказала         0.8633   |   \n",
      "скинь             |   скинь           0.8786   |   \n",
      "геев              |   геев            0.8842   |   \n",
      "тобой             |   тобой           0.8880   |   \n",
      "была              |   баса            0.4085   |   \n",
      "да                |   да              0.8957   |   \n",
      "муж               |   муж             0.8350   |   \n",
      "щас               |   щас             0.9516   |   \n",
      "она               |   она             0.9011   |   \n",
      "проблема          |   проблема        0.8468   |   \n",
      "билайн            |   билайн          0.8468   |   \n",
      "уже               |   уже             0.9064   |   \n"
     ]
    }
   ],
   "source": [
    "curve_id_to_hypotheses = predict(\n",
    "    greedy_generator_with_vocab, val_dataset, n_hypotheses=1, \n",
    "    scores_to_prob=lambda pred, score: np.exp(-score), n_examples=15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "target                pred1                        \n",
      "---------------------------------------------\n",
      "на                |   на              0.8729   |   \n",
      "все               |   все             0.8297   |   \n",
      "добрый            |   добрый          0.7269   |   \n",
      "девочка           |   девочка         0.6857   |   \n",
      "сказала           |   сказала         0.7000   |   \n",
      "скинь             |   скинь           0.7596   |   \n",
      "геев              |   геев            0.7808   |   \n",
      "тобой             |   тобой           0.7647   |   \n",
      "была              |   баса            0.3738   |   \n",
      "да                |   да              0.8705   |   \n",
      "муж               |   муж             0.7908   |   \n",
      "щас               |   щас             0.8447   |   \n",
      "она               |   она             0.8367   |   \n",
      "проблема          |   проблема        0.6630   |   \n",
      "билайн            |   билайн          0.7044   |   \n",
      "уже               |   уже             0.8313   |   \n"
     ]
    }
   ],
   "source": [
    "curve_id_to_hypotheses = predict(\n",
    "    greedy_generator__no_vocab, val_dataset, n_hypotheses=1, \n",
    "    scores_to_prob=lambda pred, score: np.exp(-score), n_examples=15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "target                pred1                        pred2                        pred3                        pred4                        \n",
      "---------------------------------------------------------------------------------------------------------------------------------------\n",
      "на                |   на              0.8950   |   нан             0.0014   |   нам             0.0013   |   нас             0.0013   |   \n",
      "все               |   все             0.8705   |   всенародная     0.0002   |   всенародно      0.0001   |   всенародную     0.0001   |   \n",
      "добрый            |   добрый          0.8608   |   добрый-добрый   0.0011   |   добрым          0.0039   |   добрые          0.0019   |   \n",
      "девочка           |   девочка         0.8550   |   девочка-волшебница0.0002   |   девочки         0.0029   |   девочку         0.0025   |   \n",
      "сказала           |   сказала         0.8633   |   сказал-сделал   0.0008   |   сказал          0.0037   |   сказали         0.0026   |   \n",
      "скинь             |   скинь           0.8786   |   скиньте         0.0025   |   скинь-ка        0.0012   |   скинься         0.0011   |   \n",
      "геев              |   геев            0.8842   |   генеалог        0.0017   |   генетическая    0.0004   |   генетическую    0.0002   |   \n",
      "тобой             |   тобой           0.8880   |   тобой-то        0.0011   |   тобоган         0.0012   |   тобрекс         0.0006   |   \n",
      "была              |   баса            0.4085   |   бычатся         0.1309   |   бычачий         0.0969   |   бычка           0.0666   |   \n",
      "да                |   да              0.8957   |   даяна           0.0011   |   дак             0.0013   |   дар             0.0013   |   \n",
      "муж               |   муж             0.8350   |   мудехар         0.0059   |   мудр            0.0120   |   мал             0.0106   |   \n",
      "щас               |   щас             0.9516   |   щаденко         0.0004   |   щаного          0.0005   |   щавель          0.0004   |   \n",
      "она               |   она             0.9011   |   онассис         0.0013   |   оная            0.0013   |   онам            0.0012   |   \n",
      "проблема          |   проблема        0.8468   |   проблемы        0.0031   |   проблема-то     0.0011   |   проблемам       0.0014   |   \n",
      "билайн            |   билайн          0.8468   |   биоразнообразие 0.0006   |   библа           0.0072   |   биоразнообразия 0.0003   |   \n",
      "уже               |   уже             0.9064   |   ужеобразные     0.0014   |   уже-уже         0.0012   |   ужей            0.0012   |   \n"
     ]
    }
   ],
   "source": [
    "n_examples = 15\n",
    "normalization_factor = 0.5\n",
    "beamsize = 6\n",
    "\n",
    "curve_id_to_hypotheses = predict(\n",
    "    beam_generator__with_vocab, val_dataset, n_hypotheses=4, n_examples=n_examples,\n",
    "    generator_call_kwargs={'normalization_factor': normalization_factor, 'beamsize': beamsize}, \n",
    "    scores_to_prob =lambda pred, score: beamsearch_score_to_prob(score, len(pred) + 1, normalization_factor))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "target                pred1                        pred2                        pred3                        pred4                        \n",
      "---------------------------------------------------------------------------------------------------------------------------------------\n",
      "на                |   на              0.8729   |   нан             0.0014   |   нак             0.0013   |   наз             0.0012   |   \n",
      "все               |   все             0.8297   |   всем            0.0015   |   вссе            0.0012   |   всен            0.0012   |   \n",
      "добрый            |   добрый          0.7269   |   добрым          0.0032   |   добрыйо         0.0011   |   добрыйи         0.0010   |   \n",
      "девочка           |   девочка         0.6857   |   девочки         0.0023   |   девочку         0.0020   |   девочкам        0.0013   |   \n",
      "сказала           |   сказала         0.7000   |   сказал          0.0031   |   сказали         0.0021   |   сказалась       0.0009   |   \n",
      "скинь             |   скинь           0.7596   |   скиньт          0.0020   |   скиньно         0.0010   |   скиньль         0.0008   |   \n",
      "геев              |   геев            0.7808   |   генев           0.0082   |   геева           0.0042   |   гееев           0.0037   |   \n",
      "тобой             |   тобой           0.7647   |   тобойой         0.0010   |   тобойа          0.0011   |   тобойе          0.0011   |   \n",
      "была              |   баса            0.3738   |   быча            0.1813   |   бычка           0.0504   |   быса            0.0366   |   \n",
      "да                |   да              0.8705   |   дар             0.0013   |   дан             0.0013   |   дак             0.0013   |   \n",
      "муж               |   муж             0.7908   |   мал             0.0101   |   мудд            0.0049   |   мудл            0.0038   |   \n",
      "щас               |   щас             0.8447   |   щаса            0.0015   |   щасе            0.0013   |   щасб            0.0013   |   \n",
      "она               |   она             0.8367   |   онат            0.0012   |   онас            0.0012   |   онао            0.0012   |   \n",
      "проблема          |   проблема        0.6630   |   проблемы        0.0024   |   проблемам       0.0011   |   проблеман       0.0010   |   \n",
      "билайн            |   билайн          0.7044   |   билайк          0.0064   |   билайне         0.0023   |   билайна         0.0015   |   \n",
      "уже               |   уже             0.8313   |   ужеи            0.0013   |   ужео            0.0012   |   ужеб            0.0012   |   \n"
     ]
    }
   ],
   "source": [
    "n_examples = 15\n",
    "normalization_factor = 0.5\n",
    "\n",
    "curve_id_to_hypotheses = predict(\n",
    "    beam_generator__no_vocab, val_dataset, n_hypotheses=4, n_examples=n_examples,\n",
    "    generator_call_kwargs={'normalization_factor': normalization_factor, 'beamsize': beamsize}, \n",
    "    scores_to_prob =lambda pred, score: beamsearch_score_to_prob(score, len(pred) + 1, normalization_factor))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('на', 0.8729147558351656), ('нас', 0.001725468331042225), ('неа', 0.001583207265670061), ('нам', 0.0012425854680964096), ('нав', 0.0011460283846146304), ('наа', 0.0011295282480705778)]\n",
      "\n",
      "[('все', 0.8300481755078998), ('всем', 0.0012949032335612689), ('всен', 0.001187697357643541), ('всеб', 0.0011859830734336145), ('всех', 0.0011663816457048192), ('всек', 0.0011443660631516442)]\n"
     ]
    }
   ],
   "source": [
    "print(curve_id_to_hypotheses[0][:6])\n",
    "print()\n",
    "print(curve_id_to_hypotheses[1][:6])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---------------\n",
    "Как и ожидается, после денормализации бимсерча вероятности не такие же как в greedy\n",
    "\n",
    "Нужно отметить, что в общем случае должно быть так:\n",
    "* greedy_search_no_vocab__results == beamsearch__no_vocab__results\n",
    "* greedy_search_with_voc__results == beamsearch__with_voc__results\n",
    "* greedy_search_with_voc__results != beamsearch__no_vocab__results\n",
    "* greedy_search__no_vocab__results != beamsearch__with_voc__results\n",
    "\n",
    "\n",
    "\n",
    "Некоторые заметки:\n",
    "1. Иногда вероятность оказывается немонотонной. Это именно из-за того, что мы убрали нормализацию бимсерча. То есть убрали штраф за краткость, а при ранжировании он был\n",
    "2. Даже если beamsize = n_classes, beamsearch не оценивает явно вероятность всех слов, потому что из-за over confidence вероятности некоторых возможных токенов оказываются нулевыми и эта ветка обрывается\n",
    "---------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluate models separately and as a pair"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4\n"
     ]
    }
   ],
   "source": [
    "print(cpu_count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_default_targets = get_targets(val_default_dataset, char_tokenizer)\n",
    "val_extra_targets = get_targets(val_extra_dataset, char_tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "default_predictions = weights_to_raw_predictions(\n",
    "    grid_name = \"default\",\n",
    "    model_getter=get_m1_bigger_model,\n",
    "    weights_path = os.path.join(MODELS_ROOT, \"m1_bigger/m1_bigger_v2__2023_11_12__12_30_29__0.13121__greed_acc_0.86098__default_l2_0_ls0_switch_2.pt\"),\n",
    "    char_tokenizer=char_tokenizer,\n",
    "    dataset=val_default_dataset,\n",
    "    generator_ctor=GreedyGenerator,\n",
    "    n_workers=4\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8531223449447749"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "default_MMR =  get_mmr(default_predictions, val_default_targets)\n",
    "default_MMR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [],
   "source": [
    "default_predictions_best_bigger = default_predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [],
   "source": [
    "default_predictions_best_bigger_clean, _ = separate_out_vocab_all_crvs(default_predictions_best_bigger, vocab_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8784"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(bool(el) for el in default_predictions_best_bigger_clean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 9416/9416 [04:08<00:00, 37.90it/s]\n"
     ]
    }
   ],
   "source": [
    "default_predictions = weights_to_raw_predictions(\n",
    "    grid_name = \"default\",\n",
    "    model_getter=get_m1_smaller_model,\n",
    "    weights_path = os.path.join(MODELS_ROOT, \"m1_smaller/m1_smaller_v2_2023_11_11_17_43_35_0_33179_default_l2_1e_05_ls0_02.pt\"),\n",
    "    char_tokenizer=char_tokenizer,\n",
    "    dataset=val_default_dataset,\n",
    "    generator_ctor=GreedyGenerator,\n",
    "    n_workers=4\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [],
   "source": [
    "default_predictions_clean, _ = separate_out_vocab_all_crvs(default_predictions, vocab_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8449"
      ]
     },
     "execution_count": 184,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(bool(el) for el in default_predictions_clean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8221112999150383"
      ]
     },
     "execution_count": 185,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "default_MMR =  get_mmr(default_predictions, val_default_targets)\n",
    "default_MMR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "# grid_name = \"default\"\n",
    "# model_getter = get_m1_bigger_model\n",
    "# weights_path = os.path.join(MODELS_ROOT, \"m1_bigger/m1_bigger_v2__2023_11_11__13_17_50__0.13845_default_l2_0_ls0_switch_0.pt\")\n",
    "# model = model_getter(device, weights_path)\n",
    "# grid_name_to_greedy_generator = {grid_name: GreedyGenerator(model, char_tokenizer, device)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 9416/9416 [05:02<00:00, 31.10it/s]\n"
     ]
    }
   ],
   "source": [
    "# default_predictions = predict_greedy_raw_multiproc(val_default_dataset,\n",
    "#                                                     grid_name_to_greedy_generator,\n",
    "#                                                     num_workers=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid_name = \"extra\"\n",
    "model_getter = get_m1_model\n",
    "weights_path = os.path.join(MODELS_ROOT, \"m1_v2/m1_v2__2023_11_09__17_47_40__0.14301_extra_l2_1e-05_switch_0.pt\")\n",
    "model = model_getter(device, weights_path)\n",
    "grid_name_to_greedy_generator = {grid_name: GreedyGenerator(model, char_tokenizer, device)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 584/584 [00:18<00:00, 31.60it/s]\n"
     ]
    }
   ],
   "source": [
    "extra_predictions = predict_raw_mp(val_extra_dataset,\n",
    "                                   grid_name_to_greedy_generator,\n",
    "                                   num_workers=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.851027397260274"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "extra_MMR = get_mmr(extra_predictions, val_extra_targets)\n",
    "extra_MMR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_preds = merge_default_and_extra_preds(default_predictions, extra_predictions, val_default_dataset.grid_name_idxs, val_extra_dataset.grid_name_idxs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_targets = None\n",
    "with open(os.path.join(DATA_ROOT, \"valid.ref\"), 'r', encoding='utf-8') as f:\n",
    "    all_targets = f.read().splitlines() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8512"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "full_MMR = get_mmr(all_preds, all_targets)\n",
    "full_MMR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 365,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "19"
      ]
     },
     "execution_count": 365,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max(len(el) for el in all_targets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "{\"m1_v2/best_model__2023_11_09__10_36_02__0.14229_default_switch_2_try_2.pt\": 0.8512107051826678,\n",
    " \"m1_v2/m1_v2__2023_11_09__17_47_40__0.14301_extra_l2_1e-05_switch_0.pt\": 0.851027397260274,\n",
    " \n",
    " \"m1_bigger/m1_bigger_v2__2023_11_10__13_38_32__0.50552_default_l2_5e-05_ls0.045_switch_0.pt\": 0.810429056924384,\n",
    " \"m1_bigger/m1_bigger_v2__2023_11_10__16_36_38__0.49848_default_l2_5e-05_ls0.045_switch_0.pt\": 0.818500424808836,\n",
    " \"m1_bigger/m1_bigger_v2__2023_11_10__21_51_01__0.49382_default_l2_5e-05_ls0.045_switch_0.pt\": 0.8210492778249787,\n",
    " \n",
    " \"m1_bigger/m1_bigger_v2__2023_11_11__13_17_50__0.13845_default_l2_0_ls0_switch_0.pt\": 0.8512107051826678,\n",
    " \"m1_bigger/m1_bigger_v2__2023_11_11__14_29_37__0.13679_default_l2_0_ls0_switch_0.pt\": 0.8531223449447749,\n",
    " \n",
    " \"m1_smaller/m1_smaller_v2_2023_11_11_17_43_35_0_33179_default_l2_1e_05_ls0_02.pt\": 0.8221112999150383}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Let's create a greedy submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import Set, List, Tuple"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid_name = \"default\"\n",
    "model_getter = get_m1_model\n",
    "weights_path = os.path.join(MODELS_ROOT, \"m1_v2/m1_v2__2023_11_09__10_36_02__0.14229_default_switch_0.pt\")\n",
    "model = model_getter(device, weights_path)\n",
    "grid_name_to_greedy_generator = {grid_name: GreedyGenerator(model, char_tokenizer, device)}\n",
    "default_test_predictions = predict_raw_mp(test_default_dataset,\n",
    "                                          grid_name_to_greedy_generator,\n",
    "                                          num_workers=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 627/627 [00:20<00:00, 30.18it/s]\n"
     ]
    }
   ],
   "source": [
    "grid_name = \"extra\"\n",
    "model_getter = get_m1_model\n",
    "weights_path = os.path.join(MODELS_ROOT, \"m1_v2/m1_v2__2023_11_09__17_47_40__0.14301_extra_l2_1e-05_switch_0.pt\")\n",
    "model = model_getter(device, weights_path)\n",
    "grid_name_to_greedy_generator = {grid_name: GreedyGenerator(model, char_tokenizer, device)}\n",
    "extra_test_predictions = predict_raw_mp(test_extra_dataset,\n",
    "                                        grid_name_to_greedy_generator,\n",
    "                                        num_workers=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_test_preds = merge_default_and_extra_preds(default_test_predictions, extra_test_predictions, test_default_dataset.grid_name_idxs, test_extra_dataset.grid_name_idxs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab_set = get_vocab_set(os.path.join(DATA_ROOT, \"voc.txt\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "clean_test_preds, invalid_test_preds = separate_out_vocab_all_crvs(all_test_preds, vocab_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['на'],\n",
       " ['что'],\n",
       " ['опоздания'],\n",
       " ['сколько'],\n",
       " [],\n",
       " ['не'],\n",
       " ['как'],\n",
       " ['садовод'],\n",
       " ['заметил'],\n",
       " ['ваги']]"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clean_test_preds[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "augment_list = None\n",
    "with open(r\"..\\data\\submissions\\sample_submission.csv\", 'r', encoding = 'utf-8') as f:\n",
    "    augment_lines = f.read().splitlines()\n",
    "augment_list = [line.split(\",\") for line in augment_lines]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "augmented_test_preds = append_preds(clean_test_preds, augment_list, limit = 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['на', 'неа', 'на', 'ненка'],\n",
       " ['что', 'часто', 'частого', 'чисто'],\n",
       " ['опоздания', 'опоздания', 'опозданиям', 'оприходования'],\n",
       " ['сколько', 'сколько', 'сокольского', 'свердловского'],\n",
       " ['дремать', 'дописать', 'донимать', 'дюрренматт'],\n",
       " ['не', 'неук', 'нк', 'ненка'],\n",
       " ['как', 'как', 'капак', 'капе'],\n",
       " ['садовод', 'спародировал', 'садовод', 'сурдоперевод'],\n",
       " ['заметил', 'знаменито', 'знаменитого', 'замерил'],\n",
       " ['ваги', 'ваенги', 'венгрии', 'ванги']]"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "augmented_test_preds[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission_name = \"m1_v2__0.14229_deault__0.14301_extra__greedy.csv\"\n",
    "out_path = rf\"..\\data\\submissions\\{submission_name}\"\n",
    "create_submission(augmented_test_preds, out_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# BeamSearch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import Tuple, List\n",
    "def remove_beamsearch_probs(preds: List[List[Tuple[float, str]]]) -> List[List[str]]:\n",
    "    new_preds = []\n",
    "    for pred_line in preds:\n",
    "        new_preds_line = []\n",
    "        for _, word in pred_line:\n",
    "            new_preds_line.append(word)\n",
    "        new_preds.append(new_preds_line)\n",
    "    return new_preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def patch_wrong_prediction_shape(prediciton):\n",
    "    return [pred_el[0] for pred_el in prediciton]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Beamsearch Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 372,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_VAL_WORD_LEN = max(len(el) for el in all_targets)\n",
    "\n",
    "generator_kwargs = {\n",
    "    'max_steps_n': MAX_VAL_WORD_LEN+1,\n",
    "    'return_hypotheses_n': 7,\n",
    "    'beamsize': 6,\n",
    "    'normalization_factor': 0.5,\n",
    "    'verbose': False\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 373,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid_name_to_val_dataset = {\n",
    "    'default': val_default_dataset,\n",
    "    'extra': val_extra_dataset\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 374,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 9416/9416 [41:23<00:00,  3.79it/s] \n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "\n",
    "bs_params = [\n",
    "    (\"default\", get_m1_bigger_model, \"m1_bigger/m1_bigger_v2__2023_11_12__14_51_49__0.13115__greed_acc_0.86034__default_l2_0_ls0_switch_2.pt\"),\n",
    "]\n",
    "\n",
    "\n",
    "for grid_name, model_getter, weights_f_name in bs_params:\n",
    "\n",
    "    bs_preds_path = os.path.join(\"../data/saved_beamsearch_validation_results/\",\n",
    "                                f\"{weights_f_name.replace('/', '__')}.pkl\")\n",
    "    \n",
    "    if os.path.exists(bs_preds_path):\n",
    "        print(f\"Path {bs_preds_path} exists. Skipping.\")\n",
    "        continue\n",
    "\n",
    "    bs_predictions = weights_to_raw_predictions(\n",
    "        grid_name = grid_name,\n",
    "        model_getter=model_getter,\n",
    "        weights_path = os.path.join(MODELS_ROOT, weights_f_name),\n",
    "        char_tokenizer=char_tokenizer,\n",
    "        dataset=grid_name_to_val_dataset[grid_name],\n",
    "        generator_ctor=BeamGenerator,\n",
    "        n_workers=4,\n",
    "        generator_kwargs=generator_kwargs\n",
    "    )\n",
    "\n",
    "    with open(bs_preds_path, 'wb') as f:\n",
    "        pickle.dump(bs_predictions, f, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 375,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8929800339847141"
      ]
     },
     "execution_count": 375,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds_name = \"m1_bigger__m1_bigger_v2__2023_11_12__14_51_49__0.13115__greed_acc_0.86034__default_l2_0_ls0_switch_2.pt.pkl\"\n",
    "bs_preds_path = os.path.join(\"../data/saved_beamsearch_validation_results/\",\n",
    "                                preds_name)\n",
    "with open(bs_preds_path, 'rb') as f:\n",
    "    default_valid_preds_bs = pickle.load(f)\n",
    "\n",
    "default_valid_preds_bs = patch_wrong_prediction_shape(default_valid_preds_bs)\n",
    "default_valid_preds_bs = remove_beamsearch_probs(default_valid_preds_bs)\n",
    "default_valid_preds_bs, _ = separate_out_vocab_all_crvs(default_valid_preds_bs, vocab_set)\n",
    "get_mmr(default_valid_preds_bs, val_default_targets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 340,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8876027397260277"
      ]
     },
     "execution_count": 340,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds_name = \"m1_v2__m1_v2__2023_11_09__17_47_40__0.14301_extra_l2_1e-05_switch_0.pt.pkl\"\n",
    "bs_preds_path = os.path.join(\"../data/saved_beamsearch_validation_results/\",\n",
    "                                preds_name)\n",
    "with open(bs_preds_path, 'rb') as f:\n",
    "    extra_valid_preds_bs = pickle.load(f)\n",
    "\n",
    "extra_valid_preds_bs = patch_wrong_prediction_shape(extra_valid_preds_bs)\n",
    "extra_valid_preds_bs = remove_beamsearch_probs(extra_valid_preds_bs)\n",
    "extra_valid_preds_bs, _ = separate_out_vocab_all_crvs(extra_valid_preds_bs, vocab_set)\n",
    "get_mmr(extra_valid_preds_bs, val_extra_targets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# {\n",
    "#     \"m1_smaller__m1_smaller_v2_2023_11_12_01_21_45_0_31891_default_l2_1e_05_ls0_02.pt.pkl\": 0.8811533559898118,\n",
    "#     \"m1_smaller__m1_smaller_v2_2023_11_12_08_17_33_0_31223_default_l2_1e_05_ls0_02.pt.pkl\": 0.8835960067969484,\n",
    "#     \"m1_bigger__m1_bigger_v2__2023_11_11__22_18_35__0.13542_default_l2_0_ls0_switch_1.pt.pkl\": 0.8900881478334822,\n",
    "#     \"m1_bigger__m1_bigger_v2__2023_11_11__15_53_07__0.13636_default_l2_0_ls0_switch_0.pt.pkl\": 0.8871590909090984,\n",
    "#     \"m1_bigger__m1_bigger_v2__2023_11_12__00_39_33__0.13297_default_l2_0_ls0_switch_1.pt.pkl\": 0.887674171622777,\n",
    "#     \"m1_v2__m1_v2__2023_11_09__10_36_02__0.14229_default_switch_0.pt.pkl\": 0.8877740016992428,\n",
    "    \n",
    "    \n",
    "#     \"m1_bigger__m1_bigger_v2__2023_11_11__16_45_33__0.13721_extra_l2_0_ls0_switch_0.pt.pkl\": 0.8864383561643838,\n",
    "#     \"m1_v2__m1_v2__2023_11_09__17_47_40__0.14301_extra_l2_1e-05_switch_0.pt.pkl\": 0.8876027397260277\n",
    "    \n",
    "# }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid_name_to_ranged_bs_model_preds_paths = {\n",
    "    'default': [\n",
    "        \"m1_bigger__m1_bigger_v2__2023_11_12__14_51_49__0.13115__greed_acc_0.86034__default_l2_0_ls0_switch_2.pt.pkl\", #: 0.8929800339847141,\n",
    "        \"m1_bigger__m1_bigger_v2__2023_11_12__12_30_29__0.13121__greed_acc_0.86098__default_l2_0_ls0_switch_2.pt.pkl\", #: 0.8914698385726496,\n",
    "        \"m1_bigger__m1_bigger_v2__2023_11_11__22_18_35__0.13542_default_l2_0_ls0_switch_1.pt.pkl\",  #: 0.8900881478334822,\n",
    "        \"m1_v2__m1_v2__2023_11_09__10_36_02__0.14229_default_switch_0.pt.pkl\",  #: E 0.8877740016992428,\n",
    "        \"m1_bigger__m1_bigger_v2__2023_11_12__00_39_33__0.13297_default_l2_0_ls0_switch_1.pt.pkl\",  #: 0.887674171622777,\n",
    "        # \"m1_bigger__m1_bigger_v2__2023_11_11__15_53_07__0.13636_default_l2_0_ls0_switch_0.pt.pkl\",  #: 0.8871590909090984,\n",
    "        \"m1_smaller__m1_smaller_v2__2023_11_12__17_40_42__0.30909_default_l2_1e-05_ls0.02_switch_0.pt.pkl\",  # 0.8849384027187835\n",
    "        # \"m1_smaller__m1_smaller_v2_2023_11_12_08_17_33_0_31223_default_l2_1e_05_ls0_02.pt.pkl\",  #: 0.8835960067969484,\n",
    "        # \"m1_smaller__m1_smaller_v2_2023_11_12_01_21_45_0_31891_default_l2_1e_05_ls0_02.pt.pkl\",  #: 0.8811533559898118,\n",
    "        ],\n",
    "    'extra': [\n",
    "        \"m1_v2__m1_v2__2023_11_09__17_47_40__0.14301_extra_l2_1e-05_switch_0.pt.pkl\",\n",
    "        \"m1_bigger__m1_bigger_v2__2023_11_11__16_45_33__0.13721_extra_l2_0_ls0_switch_0.pt.pkl\"\n",
    "        ]\n",
    "}\n",
    "\n",
    "# Отранжированы по качесту beamsearch на валидации"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "default_idxs = val_default_dataset.grid_name_idxs\n",
    "extra_idxs = val_extra_dataset.grid_name_idxs \n",
    "\n",
    "grid_name_to_augmented_preds = {}\n",
    "\n",
    "for grid_name in ('default', 'extra'):\n",
    "    bs_pred_list = []\n",
    "\n",
    "    for f_name in grid_name_to_ranged_bs_model_preds_paths[grid_name]:\n",
    "        f_path = os.path.join(\"../data/saved_beamsearch_validation_results/\", f_name)\n",
    "        with open(f_path, 'rb') as f:\n",
    "            bs_pred_list.append(pickle.load(f))\n",
    "        \n",
    "    bs_pred_list = [patch_wrong_prediction_shape(bs_preds) for bs_preds in bs_pred_list] \n",
    "    bs_pred_list = [remove_beamsearch_probs(bs_preds) for bs_preds in bs_pred_list]\n",
    "    bs_pred_list = [separate_out_vocab_all_crvs(bs_preds, vocab_set)[0] for bs_preds in bs_pred_list]\n",
    "\n",
    "\n",
    "    augmented_preds = bs_pred_list.pop(0)\n",
    "\n",
    "    while bs_pred_list:\n",
    "        augmented_preds = append_preds(augmented_preds, bs_pred_list.pop(0))\n",
    "\n",
    "    grid_name_to_augmented_preds[grid_name] = augmented_preds\n",
    "\n",
    "\n",
    "full_preds = merge_default_and_extra_preds(\n",
    "    grid_name_to_augmented_preds['default'],\n",
    "    grid_name_to_augmented_preds['extra'],\n",
    "    default_idxs,\n",
    "    extra_idxs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_preds[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "\n",
    "n_preds_in_line_dict = defaultdict(int)\n",
    "\n",
    "for line in full_preds:\n",
    "    n_preds_in_line_dict[len(line)] += 1\n",
    "\n",
    "print(n_preds_in_line_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_targets = None\n",
    "with open(os.path.join(DATA_ROOT, \"valid.ref\"), 'r', encoding='utf-8') as f:\n",
    "    all_targets = f.read().splitlines() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_mmr(full_preds, all_targets)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```python\n",
    "grid_name_to_ranged_bs_model_preds_paths = {\n",
    "    'default': [\n",
    "        \"m1_bigger__m1_bigger_v2__2023_11_11__22_18_35__0.13542_default_l2_0_ls0_switch_1.pt.pkl\",#: 0.8900881478334822,\n",
    "        \"m1_v2__m1_v2__2023_11_09__10_36_02__0.14229_default_switch_0.pt.pkl\",#: 0.8877740016992428,\n",
    "        \"m1_bigger__m1_bigger_v2__2023_11_12__00_39_33__0.13297_default_l2_0_ls0_switch_1.pt.pkl\",#: 0.887674171622777,\n",
    "        \"m1_bigger__m1_bigger_v2__2023_11_11__15_53_07__0.13636_default_l2_0_ls0_switch_0.pt.pkl\",#: 0.8871590909090984,\n",
    "        \"m1_smaller__m1_smaller_v2_2023_11_12_08_17_33_0_31223_default_l2_1e_05_ls0_02.pt.pkl\",#: 0.8835960067969484,\n",
    "        \"m1_smaller__m1_smaller_v2_2023_11_12_01_21_45_0_31891_default_l2_1e_05_ls0_02.pt.pkl\",#: 0.8811533559898118,\n",
    "        ],\n",
    "    'extra': [\n",
    "        \"m1_v2__m1_v2__2023_11_09__17_47_40__0.14301_extra_l2_1e-05_switch_0.pt.pkl\",\n",
    "        \"m1_bigger__m1_bigger_v2__2023_11_11__16_45_33__0.13721_extra_l2_0_ls0_switch_0.pt.pkl\"\n",
    "        ]\n",
    "}\n",
    "```\n",
    "\n",
    "0.8936010000000082\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "``` python\n",
    "grid_name_to_ranged_bs_model_preds_paths = {\n",
    "    'default': [\n",
    "        \"m1_v2__m1_v2__2023_11_09__10_36_02__0.14229_default_switch_0.pt.pkl\",#: 0.8877740016992428,\n",
    "        \"m1_bigger__m1_bigger_v2__2023_11_11__22_18_35__0.13542_default_l2_0_ls0_switch_1.pt.pkl\",#: 0.8900881478334822,\n",
    "        \"m1_bigger__m1_bigger_v2__2023_11_12__00_39_33__0.13297_default_l2_0_ls0_switch_1.pt.pkl\",#: 0.887674171622777,\n",
    "        \"m1_bigger__m1_bigger_v2__2023_11_11__15_53_07__0.13636_default_l2_0_ls0_switch_0.pt.pkl\",#: 0.8871590909090984,\n",
    "        \"m1_smaller__m1_smaller_v2_2023_11_12_08_17_33_0_31223_default_l2_1e_05_ls0_02.pt.pkl\",#: 0.8835960067969484,\n",
    "        \"m1_smaller__m1_smaller_v2_2023_11_12_01_21_45_0_31891_default_l2_1e_05_ls0_02.pt.pkl\",#: 0.8811533559898118,\n",
    "        ],\n",
    "    'extra': [\n",
    "        \"m1_v2__m1_v2__2023_11_09__17_47_40__0.14301_extra_l2_1e-05_switch_0.pt.pkl\",\n",
    "        \"m1_bigger__m1_bigger_v2__2023_11_11__16_45_33__0.13721_extra_l2_0_ls0_switch_0.pt.pkl\"\n",
    "        ]\n",
    "}\n",
    "```\n",
    "\n",
    "0.892801000000009\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "``` python\n",
    "grid_name_to_ranged_bs_model_preds_paths = {\n",
    "    'default': [\n",
    "        \"m1_bigger__m1_bigger_v2__2023_11_11__22_18_35__0.13542_default_l2_0_ls0_switch_1.pt.pkl\",#: 0.8900881478334822,\n",
    "        \"m1_v2__m1_v2__2023_11_09__10_36_02__0.14229_default_switch_0.pt.pkl\",#: 0.8877740016992428,\n",
    "        \"m1_bigger__m1_bigger_v2__2023_11_12__00_39_33__0.13297_default_l2_0_ls0_switch_1.pt.pkl\",#: 0.887674171622777,\n",
    "        \"m1_smaller__m1_smaller_v2_2023_11_12_08_17_33_0_31223_default_l2_1e_05_ls0_02.pt.pkl\",#: 0.8835960067969484,\n",
    "        \"m1_smaller__m1_smaller_v2_2023_11_12_01_21_45_0_31891_default_l2_1e_05_ls0_02.pt.pkl\",#: 0.8811533559898118,\n",
    "        \"m1_bigger__m1_bigger_v2__2023_11_11__15_53_07__0.13636_default_l2_0_ls0_switch_0.pt.pkl\",#: 0.8871590909090984,\n",
    "        ],\n",
    "    'extra': [\n",
    "        \"m1_v2__m1_v2__2023_11_09__17_47_40__0.14301_extra_l2_1e-05_switch_0.pt.pkl\",\n",
    "        \"m1_bigger__m1_bigger_v2__2023_11_11__16_45_33__0.13721_extra_l2_0_ls0_switch_0.pt.pkl\"\n",
    "        ]\n",
    "}\n",
    "\n",
    "# должны ранжироваться по качесту beamsearch на валидации\n",
    "```\n",
    "\n",
    "0.8936000000000084"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Beamsearch test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 343,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_WORD_LEN = 36\n",
    "\n",
    "generator_kwargs = {\n",
    "    'max_steps_n': MAX_WORD_LEN - 1,\n",
    "    'return_hypotheses_n': 7,\n",
    "    'beamsize': 6,\n",
    "    'normalization_factor': 0.5,\n",
    "    'verbose': False\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 344,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid_name_to_test_dataset = {\n",
    "    'default': test_default_dataset,\n",
    "    'extra': test_extra_dataset\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 356,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 627/627 [06:02<00:00,  1.73it/s]\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "\n",
    "bs_params = [\n",
    "    (\"extra\", get_m1_bigger_model, \"m1_bigger/m1_bigger_v2__2023_11_12__02_27_14__0.13413_extra_l2_0_ls0_switch_1.pt\"),\n",
    "\n",
    "    # \"m1_smaller__m1_smaller_v2_2023_11_12_08_17_33_0_31223_default_l2_1e_05_ls0_02.pt.pkl\": 0.8835960067969484,\n",
    "    # \"m1_bigger__m1_bigger_v2__2023_11_11__15_53_07__0.13636_default_l2_0_ls0_switch_0.pt.pkl\": 0.8871590909090984,\n",
    "    # \"m1_bigger__m1_bigger_v2__2023_11_11__16_45_33__0.13721_extra_l2_0_ls0_switch_0.pt.pkl\": 0.8864383561643838,\n",
    "]\n",
    "\n",
    "\n",
    "for grid_name, model_getter, weights_f_name in bs_params:\n",
    "\n",
    "    bs_preds_path = os.path.join(\"../data/saved_beamsearch_results/\",\n",
    "                                f\"{weights_f_name.replace('/', '__')}.pkl\")\n",
    "    \n",
    "    if os.path.exists(bs_preds_path):\n",
    "        print(f\"Path {bs_preds_path} exists. Skipping.\")\n",
    "        continue\n",
    "\n",
    "    bs_predictions = weights_to_raw_predictions(\n",
    "        grid_name = grid_name,\n",
    "        model_getter=model_getter,\n",
    "        weights_path = os.path.join(MODELS_ROOT, weights_f_name),\n",
    "        char_tokenizer=char_tokenizer,\n",
    "        dataset=grid_name_to_test_dataset[grid_name],\n",
    "        generator_ctor=BeamGenerator,\n",
    "        n_workers=4,\n",
    "        generator_kwargs=generator_kwargs\n",
    "    )\n",
    "\n",
    "    with open(bs_preds_path, 'wb') as f:\n",
    "        pickle.dump(bs_predictions, f, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(9373, 9193)"
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# len(clean_default_test_predictions), sum(bool(el) for el in clean_default_test_predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab_set = get_vocab_set(os.path.join(DATA_ROOT, \"voc.txt\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "defaultdict(<class 'int'>, {4: 7404, 3: 1032, 2: 840, 1: 577, 0: 147})\n"
     ]
    }
   ],
   "source": [
    "# from collections import defaultdict\n",
    "\n",
    "# n_preds_in_line_dict = defaultdict(int)\n",
    "\n",
    "# for line in clean_test_predictions:\n",
    "#     n_preds_in_line_dict[len(line)] += 1\n",
    "\n",
    "# print(n_preds_in_line_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [],
   "source": [
    "old_preds_path = os.path.join(DATA_ROOT, \"test_raw_pred___best_model__2023_11_04__18_31_37__0.02530_default_switch_2.pt__best_model__2023_11_05__07_55_13__0.02516_extra_switch_2__with_pad_cutting.pt.pkl\")\n",
    "with open(old_preds_path, 'rb') as f:\n",
    "    old_preds_list = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [],
   "source": [
    "old_preds_list = remove_beamsearch_probs(old_preds_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [],
   "source": [
    "old_preds_list_valid, old_preds_list_invalid = separate_out_vocab_all_crvs(old_preds_list, vocab_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [],
   "source": [
    "# submission_name = \"default__m1_bigger_13679__m1_v2__14229___extra__14301___with_baseline__beam.csv\"\n",
    "# out_path = rf\"..\\data\\submissions\\{submission_name}\"\n",
    "# create_submission(clean_test_baseline_augmented, out_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Submission creation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "id 1\n",
    "\n",
    "```\n",
    "grid_name_to_ranged_bs_model_preds_paths = {\n",
    "    'default': [\n",
    "        \"m1_bigger__m1_bigger_v2__2023_11_12__12_30_29__0.13121__greed_acc_0.86098__default_l2_0_ls0_switch_2.pt.pkl\",\n",
    "        \"m1_bigger__m1_bigger_v2__2023_11_11__22_18_35__0.13542_default_l2_0_ls0_switch_1.pt.pkl\",\n",
    "        \"m1_v2__m1_v2__2023_11_09__10_36_02__0.14229_default_switch_0.pt.pkl\",\n",
    "        \"m1_bigger__m1_bigger_v2__2023_11_12__00_39_33__0.13297_default_l2_0_ls0_switch_1.pt.pkl\",\n",
    "        \"m1_bigger__m1_bigger_v2__2023_11_11__14_29_37__0.13679_default_l2_0_ls0_switch_0.pt.pkl\",\n",
    "        \n",
    "    ],\n",
    "    'extra': [\"m1_v2__m1_v2__2023_11_09__17_47_40__0.14301_extra_l2_1e-05_switch_0.pt.pkl\"]\n",
    "}\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "id 2\n",
    "\n",
    "```\n",
    "grid_name_to_ranged_bs_model_preds_paths = {\n",
    "    'default': [\n",
    "        \"m1_bigger__m1_bigger_v2__2023_11_12__12_30_29__0.13121__greed_acc_0.86098__default_l2_0_ls0_switch_2.pt.pkl\",\n",
    "        \"m1_bigger__m1_bigger_v2__2023_11_11__22_18_35__0.13542_default_l2_0_ls0_switch_1.pt.pkl\",\n",
    "        \"m1_v2__m1_v2__2023_11_09__10_36_02__0.14229_default_switch_0.pt.pkl\",\n",
    "        \"m1_bigger__m1_bigger_v2__2023_11_12__00_39_33__0.13297_default_l2_0_ls0_switch_1.pt.pkl\",\n",
    "        \"m1_bigger__m1_bigger_v2__2023_11_11__14_29_37__0.13679_default_l2_0_ls0_switch_0.pt.pkl\",\n",
    "        \n",
    "    ],\n",
    "    'extra': [\n",
    "        \"m1_v2__m1_v2__2023_11_09__17_47_40__0.14301_extra_l2_1e-05_switch_0.pt.pkl\",\n",
    "        \"m1_bigger__m1_bigger_v2__2023_11_12__02_27_14__0.13413_extra_l2_0_ls0_switch_1.pt.pkl\"\n",
    "    ]\n",
    "}\n",
    "\n",
    "# должны ранжироваться по качесту beamsearch на валидации\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "id 3\n",
    "\n",
    "```python\n",
    "grid_name_to_ranged_bs_model_preds_paths = {\n",
    "    'default': [\n",
    "        \"m1_bigger_m1_bigger_v2_2023_11_12_14_51_49_0_13115_greed_acc_0_86034.pkl\",\n",
    "        \"m1_bigger__m1_bigger_v2__2023_11_12__12_30_29__0.13121__greed_acc_0.86098__default_l2_0_ls0_switch_2.pt.pkl\",\n",
    "        \"m1_bigger__m1_bigger_v2__2023_11_11__22_18_35__0.13542_default_l2_0_ls0_switch_1.pt.pkl\",\n",
    "        \"m1_v2__m1_v2__2023_11_09__10_36_02__0.14229_default_switch_0.pt.pkl\",\n",
    "        \"m1_bigger__m1_bigger_v2__2023_11_12__00_39_33__0.13297_default_l2_0_ls0_switch_1.pt.pkl\",\n",
    "        \"m1_bigger__m1_bigger_v2__2023_11_11__14_29_37__0.13679_default_l2_0_ls0_switch_0.pt.pkl\",\n",
    "        \n",
    "    ],\n",
    "    'extra': [\n",
    "        \"m1_v2__m1_v2__2023_11_09__17_47_40__0.14301_extra_l2_1e-05_switch_0.pt.pkl\",\n",
    "        \"m1_bigger__m1_bigger_v2__2023_11_12__02_27_14__0.13413_extra_l2_0_ls0_switch_1.pt.pkl\"\n",
    "    ]\n",
    "}\n",
    "\n",
    "# должны ранжироваться по качесту beamsearch на валидации\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 385,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid_name_to_ranged_bs_model_preds_paths = {\n",
    "    'default': [\n",
    "        \"m1_bigger_m1_bigger_v2_2023_11_12_14_51_49_0_13115_greed_acc_0_86034.pkl\",\n",
    "        \"m1_bigger__m1_bigger_v2__2023_11_12__12_30_29__0.13121__greed_acc_0.86098__default_l2_0_ls0_switch_2.pt.pkl\",\n",
    "        \"m1_bigger__m1_bigger_v2__2023_11_11__22_18_35__0.13542_default_l2_0_ls0_switch_1.pt.pkl\",\n",
    "        \"m1_v2__m1_v2__2023_11_09__10_36_02__0.14229_default_switch_0.pt.pkl\",\n",
    "        \"m1_bigger__m1_bigger_v2__2023_11_12__00_39_33__0.13297_default_l2_0_ls0_switch_1.pt.pkl\",\n",
    "        \"m1_bigger__m1_bigger_v2__2023_11_11__14_29_37__0.13679_default_l2_0_ls0_switch_0.pt.pkl\",\n",
    "        \n",
    "    ],\n",
    "    'extra': [\n",
    "        \"m1_v2__m1_v2__2023_11_09__17_47_40__0.14301_extra_l2_1e-05_switch_0.pt.pkl\",\n",
    "        \"m1_bigger__m1_bigger_v2__2023_11_12__02_27_14__0.13413_extra_l2_0_ls0_switch_1.pt.pkl\"\n",
    "    ]\n",
    "}\n",
    "\n",
    "# должны ранжироваться по качесту beamsearch на валидации"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 386,
   "metadata": {},
   "outputs": [],
   "source": [
    "default_idxs = test_default_dataset.grid_name_idxs\n",
    "extra_idxs = test_extra_dataset.grid_name_idxs \n",
    "\n",
    "grid_name_to_augmented_preds = {}\n",
    "\n",
    "for grid_name in ('default', 'extra'):\n",
    "    bs_pred_list = []\n",
    "\n",
    "    for f_name in grid_name_to_ranged_bs_model_preds_paths[grid_name]:\n",
    "        f_path = os.path.join(\"../data/saved_beamsearch_results/\", f_name)\n",
    "        with open(f_path, 'rb') as f:\n",
    "            bs_pred_list.append(pickle.load(f))\n",
    "        \n",
    "    bs_pred_list = [patch_wrong_prediction_shape(bs_preds) for bs_preds in bs_pred_list] \n",
    "    bs_pred_list = [remove_beamsearch_probs(bs_preds) for bs_preds in bs_pred_list]\n",
    "    bs_pred_list = [separate_out_vocab_all_crvs(bs_preds, vocab_set)[0] for bs_preds in bs_pred_list]\n",
    "\n",
    "\n",
    "    augmented_preds = bs_pred_list.pop(0)\n",
    "\n",
    "    while bs_pred_list:\n",
    "        augmented_preds = append_preds(augmented_preds, bs_pred_list.pop(0))\n",
    "\n",
    "    grid_name_to_augmented_preds[grid_name] = augmented_preds\n",
    "\n",
    "\n",
    "full_preds = merge_default_and_extra_preds(\n",
    "    grid_name_to_augmented_preds['default'],\n",
    "    grid_name_to_augmented_preds['extra'],\n",
    "    default_idxs,\n",
    "    extra_idxs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 387,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "defaultdict(<class 'int'>, {4: 8188, 3: 706, 2: 606, 1: 407, 0: 93})\n"
     ]
    }
   ],
   "source": [
    "from collections import defaultdict\n",
    "\n",
    "n_preds_in_line_dict = defaultdict(int)\n",
    "\n",
    "for line in full_preds:\n",
    "    n_preds_in_line_dict[len(line)] += 1\n",
    "\n",
    "print(n_preds_in_line_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 388,
   "metadata": {},
   "outputs": [],
   "source": [
    "baseline_preds = None\n",
    "with open(r\"..\\data\\submissions\\sample_submission.csv\", 'r', encoding = 'utf-8') as f:\n",
    "    baseline_preds = f.read().splitlines()\n",
    "baseline_preds = [line.split(\",\") for line in augment_lines]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 389,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_preds = append_preds(full_preds, baseline_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 390,
   "metadata": {},
   "outputs": [],
   "source": [
    "create_submission(full_preds,\n",
    "                  f\"../data/submissions/id3_with_baseline_without_old_preds\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 363,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_preds_augmentations = [\n",
    "    \n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}