GitHub_Recommender_System.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cqMZ2tvyj0S_"
      },
      "source": [
        "# GitHub Recommender System\n",
        "Group members:\n",
        "- 1795119 - Matteo Orsini\n",
        "- 1815023 - Fabrizio Rossi\n",
        "\n",
        "The task we address in this project is to create a recommender system for GitHub repositories, based on the implicit feedback captured when a user stars a repository."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZimRQ6bDhwxq"
      },
      "source": [
        "# Colab requirements for PySpark\n",
        "In this block of code we setup everything needed to run PySpark on Colab."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZLLBm9xgYhvL",
        "outputId": "9b0a2b0c-363e-44a1-d4cf-a0fb62d0835b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: pyspark in /usr/local/lib/python3.7/dist-packages (3.1.2)\n",
            "Requirement already satisfied: py4j==0.10.9 in /usr/local/lib/python3.7/dist-packages (from pyspark) (0.10.9)\n"
          ]
        }
      ],
      "source": [
        "# Install Java\n",
        "!apt-get install openjdk-8-jdk-headless -qq > /dev/null\n",
        "\n",
        "# Install Spark\n",
        "!wget -q https://archive.apache.org/dist/spark/spark-3.1.2/spark-3.1.2-bin-hadoop3.2.tgz\n",
        "\n",
        "# Unzip Spark to the current folder\n",
        "!tar xf spark-3.1.2-bin-hadoop3.2.tgz\n",
        "\n",
        "# Set Spark folder in the system path\n",
        "import os\n",
        "os.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-8-openjdk-amd64\"\n",
        "os.environ[\"SPARK_HOME\"] = \"/content/spark-3.1.2-bin-hadoop3.2\"\n",
        "\n",
        "# Install findspark and PySpark using pip\n",
        "!pip install -q findspark\n",
        "!pip install pyspark\n",
        "\n",
        "# Set the dataset file paths\n",
        "users_path = \"users.csv\"\n",
        "repos_path = \"repos.csv\"\n",
        "starred_path = \"starred.csv\""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fyNsHIX5RfDq"
      },
      "source": [
        "# Setup\n",
        "In this section, we download the dataset, we import all the libraries used for this project and we setup a Spark instance."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "qlblO4jIv6kf"
      },
      "outputs": [],
      "source": [
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "import os\n",
        "\n",
        "from IPython.display import HTML"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jfABgWsLbZN7",
        "outputId": "30c627c3-dccb-4791-c604-1d3675b2d12d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "repos.csv already downloaded.\n",
            "starred.csv already downloaded.\n",
            "users.csv already downloaded.\n"
          ]
        }
      ],
      "source": [
        "#@title Download dataset from Google Drive\n",
        "from google_drive_downloader import GoogleDriveDownloader as gdd\n",
        "\n",
        "datasets = {\n",
        "  \"repos.csv\": \"13fZqmFP9Nfhu_dk59BJ5hooF3kVsJ39j\",\n",
        "  \"starred.csv\": \"1j_FxOFKVobYBc_wwoVRs9mGLO3cNj5Kc\",\n",
        "  \"users.csv\": \"18IDJA55jM_lo2M4pTTToxcGngdnRPgPO\"\n",
        "}\n",
        "for name, file_id in datasets.items():\n",
        "  if not os.path.isfile(\"/content/\" + name):\n",
        "    gdd.download_file_from_google_drive(file_id=file_id, dest_path=\"/content/\" + name, unzip=False)\n",
        "  else:\n",
        "    print(name + \" already downloaded.\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        },
        "id": "kpXJbu87QRjt",
        "outputId": "5b1135df-4973-4b7f-ab49-e7b5d5befc85"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "            <div>\n",
              "                <p><b>SparkSession - in-memory</b></p>\n",
              "                \n",
              "        <div>\n",
              "            <p><b>SparkContext</b></p>\n",
              "\n",
              "            <p><a href=\"http://93308cb2b27b:4050\">Spark UI</a></p>\n",
              "\n",
              "            <dl>\n",
              "              <dt>Version</dt>\n",
              "                <dd><code>v3.1.2</code></dd>\n",
              "              <dt>Master</dt>\n",
              "                <dd><code>local[*]</code></dd>\n",
              "              <dt>AppName</dt>\n",
              "                <dd><code>GitHub Recommender System</code></dd>\n",
              "            </dl>\n",
              "        </div>\n",
              "        \n",
              "            </div>\n",
              "        "
            ],
            "text/plain": [
              "<pyspark.sql.session.SparkSession at 0x7f1288163a10>"
            ]
          },
          "execution_count": 4,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "import findspark\n",
        "findspark.init()\n",
        "\n",
        "from pyspark.sql import SparkSession\n",
        "spark = (SparkSession.builder\n",
        "         .master(\"local[*]\")\n",
        "         .appName(\"GitHub Recommender System\")\n",
        "         .config(\"spark.ui.port\", \"4050\")\n",
        "         .config(\"spark.driver.memory\", \"12g\")\n",
        "         # Reduce the number of partitions to increase performances in single node mode\n",
        "         .config(\"spark.sql.shuffle.partitions\", 2) \n",
        "         .getOrCreate())\n",
        "\n",
        "# Set checkpoint path to avoid too long lineage errors\n",
        "!mkdir checkpoints\n",
        "spark.sparkContext.setCheckpointDir('checkpoints')\n",
        "\n",
        "from pyspark.sql import *\n",
        "from pyspark.sql.types import *\n",
        "from pyspark.sql.functions import *\n",
        "\n",
        "spark.getActiveSession()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_atfLeMguoU0"
      },
      "source": [
        "## Expose the Spark Web UI"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "F1ajwJeEWx8F"
      },
      "source": [
        "By default PySpark exposes a web interface to manage the current active session on port 4040.\n",
        "Using a tool called `ngrok` we were able to access it from Colab, in order to monitor the jobs and the resource utilization in a detailed way, which was useful for debugging."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "0Pw0p0-xlIsz"
      },
      "outputs": [],
      "source": [
        "# Change ngrok port to avoid conflicts\n",
        "!echo \"web_addr: 127.0.0.1:4080\" > ngrok.yml"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LaiRu3YEoze6",
        "outputId": "53b9ca9b-b7d4-4224-a8a0-671ab75e5043"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "File ‘ngrok-stable-linux-amd64.zip’ already there; not retrieving.\n",
            "\n",
            "Archive:  ngrok-stable-linux-amd64.zip\n",
            "Spark UI link: https://ac29-35-245-207-167.ngrok.io\n"
          ]
        }
      ],
      "source": [
        "# Install ngrok\n",
        "!wget -nc https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-amd64.zip\n",
        "!unzip -n ngrok-stable-linux-amd64.zip\n",
        "get_ipython().system_raw(\"./ngrok http 4050 -config ./ngrok.yml &\")\n",
        "\n",
        "# Wait that the process starts\n",
        "import time\n",
        "time.sleep(2)\n",
        "\n",
        "# Retrieve Spark UI link\n",
        "import requests\n",
        "print(\"Spark UI link:\", requests.get(\"http://localhost:4080/api/tunnels\").json()[\"tunnels\"][0][\"public_url\"])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VUvkdfkzR0mG"
      },
      "source": [
        "# Load Dataset\n",
        "In this section we load the dataset which is self-collected and contains information about GitHub users and their starred repositories."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "C9nmqKx2bC44"
      },
      "source": [
        "## Dataset collection\n",
        "\n",
        "The collection of the dataset was performed using some custom scripts written in Python. In order to gather the users for our recommender system, we took all the users that starred a very popular repository, [React.js](https:/facebook/react) (a JavaScript library for building user interfaces) and got all the repos starred by those users.\n",
        "\n",
        "So, we created one script (*users.py*) in order to get all the *stargazers* (the name that GitHub gave to the users that starred a repository) of React. The information of all the stargazers for React is accessible at the link *https:/facebook/react/stargazers*. We obtained and analyzed the HTML of the pages, using the *requests* and *BeautifulSoup* libraries. In this way we were able to write a simple script that goes through several pages and collect the users listed in each page. We decided to collect just 10.000 of all the 170.000 and more users that starred the React repository since the requests takes some time.\n",
        "\n",
        "Then, after the collection of all the users, we made another script (*stars.py*), which in the same way was used to gather the metadata of all the repositories starred by each user, accessible at the link *https:/user?tab=stars*. We were able to collect more than 1 million relations between users and repos that they starred and a total of 354.981 repositories on GitHub starred by those users.\n",
        "\n",
        "In order to use the dataset, we need to load it using the methods offered by Spark:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "dAnBJSa3R1oC"
      },
      "outputs": [],
      "source": [
        "users_df = spark.read.load(users_path, format=\"csv\", sep=\",\", inferSchema=\"true\", header=\"true\")\n",
        "repos_df = spark.read.load(repos_path, format=\"csv\", sep=\",\", inferSchema=\"true\", header=\"true\", quotes='\"', escape='\"')\n",
        "starred_df = spark.read.load(starred_path, format=\"csv\", sep=\",\", inferSchema=\"true\", header=\"true\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9QT91Z59gRbW"
      },
      "source": [
        "The *users_df* dataframe contains the information relative to all the users of the recommender system, in particular the `user_id` which identifies each user and their `name`.\n",
        "\n",
        "Instead, the *repos_df* dataframe contains the information relative to all the repositories of the system. Each repository is identified by a `repo_id` and contains several features:\n",
        "- `creator`: the name of the user or organization that created the repository;\n",
        "- `name`: the name of the repository;\n",
        "- `about`: a short description of the content of the repository;\n",
        "- `language`: the main programming language in which the code in the repository is written;\n",
        "- `stars`: the number of stars received by the repository;\n",
        "- `forks`: the number of forks of the repository;\n",
        "- `updated`: the last time the repository was updated;\n",
        "- `sponsor`: if the repository can be sponsored or not (a form of donations).\n",
        "\n",
        "Finally, we have the `starred_df` dataframe which contains the information relative to the repositories that each user has starred, represented as records composed by `user_id` and `repo_id`.\n",
        "\n",
        "These are some examples of the content of the dataset:\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lOQ06EaIR5re",
        "outputId": "28a49adf-9de7-4d48-87ab-c0330288bb31"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+-------+-----------------+\n",
            "|user_id|             name|\n",
            "+-------+-----------------+\n",
            "|      0|EliveltonSouzaDev|\n",
            "|      1|      Ersonusaini|\n",
            "|      2|          harixth|\n",
            "|      3|        Luckyanry|\n",
            "|      4|       biroltopuz|\n",
            "+-------+-----------------+\n",
            "only showing top 5 rows\n",
            "\n",
            "+-------+-----------------+-----------------+--------------------+----------+------+-----+-------------------+-------+\n",
            "|repo_id|          creator|             name|               about|  language| stars|forks|            updated|sponsor|\n",
            "+-------+-----------------+-----------------+--------------------+----------+------+-----+-------------------+-------+\n",
            "|      0|styled-components|styled-components|Visual primitives...|TypeScript| 33940| 2084|2021-06-16 19:12:26|      0|\n",
            "|      1|           vercel|          next.js| The React Framework|JavaScript| 69687|13111|2021-06-22 07:49:26|      0|\n",
            "|      2|         facebook|            react|A declarative, ef...|JavaScript|170264|34406|2021-06-22 06:08:58|      0|\n",
            "|      3|         rebassjs|           rebass|⚛️ React primitiv...|JavaScript|  7360|  603|2021-06-08 20:47:06|      0|\n",
            "|      4|    rafaballerini|    rafaballerini|                null|      null|   208|  473|2021-06-22 01:28:02|      0|\n",
            "+-------+-----------------+-----------------+--------------------+----------+------+-----+-------------------+-------+\n",
            "only showing top 5 rows\n",
            "\n",
            "+-------+-------+\n",
            "|user_id|repo_id|\n",
            "+-------+-------+\n",
            "|      0|      0|\n",
            "|      0|      1|\n",
            "|      0|      2|\n",
            "|      0|      3|\n",
            "|      0|      4|\n",
            "+-------+-------+\n",
            "only showing top 5 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "users_df.show(5)\n",
        "repos_df.show(5)\n",
        "starred_df.show(5)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NyevOIbNR_MV"
      },
      "source": [
        "# Check Dataset\n",
        "In this section we inspect the dataset to show some of its characteristics.\n",
        "\n",
        "In the following block is reported the structure of each dataframe:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ibv1-rcGSAj9",
        "outputId": "eca62e63-6d40-4120-9c03-f3275eba7e9e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "root\n",
            " |-- user_id: integer (nullable = true)\n",
            " |-- name: string (nullable = true)\n",
            "\n",
            "root\n",
            " |-- repo_id: integer (nullable = true)\n",
            " |-- creator: string (nullable = true)\n",
            " |-- name: string (nullable = true)\n",
            " |-- about: string (nullable = true)\n",
            " |-- language: string (nullable = true)\n",
            " |-- stars: integer (nullable = true)\n",
            " |-- forks: integer (nullable = true)\n",
            " |-- updated: timestamp (nullable = true)\n",
            " |-- sponsor: integer (nullable = true)\n",
            "\n",
            "root\n",
            " |-- user_id: integer (nullable = true)\n",
            " |-- repo_id: integer (nullable = true)\n",
            "\n"
          ]
        }
      ],
      "source": [
        "users_df.printSchema()\n",
        "repos_df.printSchema()\n",
        "starred_df.printSchema()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "50MrrbK8SWqr"
      },
      "source": [
        "## Handle missing values"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8I0Q7spihDYi"
      },
      "source": [
        "Let's check if there are missing values in each dataframe:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hsMzBEG-Scbw",
        "outputId": "02aee41c-3d5f-4651-9989-6d35ea8b4c99"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+-------+----+\n",
            "|user_id|name|\n",
            "+-------+----+\n",
            "|      0|   0|\n",
            "+-------+----+\n",
            "\n",
            "+-------+-------+----+-----+--------+-----+-----+-------+-------+\n",
            "|repo_id|creator|name|about|language|stars|forks|updated|sponsor|\n",
            "+-------+-------+----+-----+--------+-----+-----+-------+-------+\n",
            "|      0|      0|   0|36083|   28039|    0|72874|      0|      0|\n",
            "+-------+-------+----+-----+--------+-----+-----+-------+-------+\n",
            "\n",
            "+-------+-------+\n",
            "|user_id|repo_id|\n",
            "+-------+-------+\n",
            "|      0|      0|\n",
            "+-------+-------+\n",
            "\n"
          ]
        }
      ],
      "source": [
        "users_df.select([count(when(col(c).isNull(), c)).alias(c) for c in users_df.columns]).show()\n",
        "repos_df.select([count(when(col(c).isNull(), c)).alias(c) for c in repos_df.columns]).show()\n",
        "starred_df.select([count(when(col(c).isNull(), c)).alias(c) for c in starred_df.columns]).show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ipwf9izQfMGm"
      },
      "source": [
        "We can see that there are some records missing the `about`, `language` or `forks` values, so let's analyze those fields:"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gofAKqEPSgrf"
      },
      "source": [
        "When the forks value is missing it means that the repo doesn't have any fork. So, we can simply set those values to 0."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sT1bEAG8SePA",
        "outputId": "c57897c8-68f6-4e2e-d430-82912ca8ddd4"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+-------+-------+----+-----+--------+-----+-----+-------+-------+\n",
            "|repo_id|creator|name|about|language|stars|forks|updated|sponsor|\n",
            "+-------+-------+----+-----+--------+-----+-----+-------+-------+\n",
            "|      0|      0|   0|36083|   28039|    0|    0|      0|      0|\n",
            "+-------+-------+----+-----+--------+-----+-----+-------+-------+\n",
            "\n"
          ]
        }
      ],
      "source": [
        "repos_df = repos_df.na.fill(0)\n",
        "repos_df.select([count(when(col(c).isNull(), c)).alias(c) for c in repos_df.columns]).show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CJM5cZftSlNM"
      },
      "source": [
        "Regarding the language attribute, GitHub will not report the language attribute of a repo when it doesn't know which language was used (maybe the repo doesn't contain any code but just simple text).\n",
        "\n",
        "Instead, the about attribute will be `null` when the creator of the repo didn't provide any description about the repository.\n",
        "\n",
        "As a solution, we can create a default value for both: for the language we can create a special class called `UNK`, representing that we don't know the language of the repo; instead, for the about attribute, we can simply assign an empty string to the null values to handle them in the same way as the other ones."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Xvag6nFmSjV6",
        "outputId": "00bda297-5135-4f72-fdf4-366eb27dd51a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+-------+-------+----+-----+--------+-----+-----+-------+-------+\n",
            "|repo_id|creator|name|about|language|stars|forks|updated|sponsor|\n",
            "+-------+-------+----+-----+--------+-----+-----+-------+-------+\n",
            "|      0|      0|   0|    0|       0|    0|    0|      0|      0|\n",
            "+-------+-------+----+-----+--------+-----+-----+-------+-------+\n",
            "\n"
          ]
        }
      ],
      "source": [
        "repos_df = repos_df.na.fill(\"\", \"about\")\n",
        "repos_df = repos_df.na.fill(\"UNK\", \"language\")\n",
        "repos_df.select([count(when(col(c).isNull(), c)).alias(c) for c in repos_df.columns]).show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "buN6VH9wSob8"
      },
      "source": [
        "## Check duplicates\n",
        "\n",
        "Now we can inspect the dataset in order to check if there are any duplicates.\n",
        "\n",
        "First of all, let's report the total number of records for each dataframe."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bgwYjyX0SnA9",
        "outputId": "935f33f4-f203-4476-fd00-03f12575842a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Number of records for users.csv: 10000\n",
            "Number of records for repos.csv: 354981\n",
            "Number of records for starred.csv: 1083195\n"
          ]
        }
      ],
      "source": [
        "print(\"Number of records for users.csv:\", users_df.count())\n",
        "print(\"Number of records for repos.csv:\", repos_df.count())\n",
        "print(\"Number of records for starred.csv:\", starred_df.count())"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tQqhPc3gS5Y_"
      },
      "source": [
        "Now, using the `distinct()` method offered by PySpark we can count the number of distinct values, using the id of each dataframe."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1Lorn6PgS5wc",
        "outputId": "a7a6ca7e-1b83-4890-ee95-b23d466e295b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Number of distinct records for users.csv: 10000\n",
            "Number of distinct records for repos.csv: 354981\n",
            "Number of distinct records for starred.csv: 1083195\n"
          ]
        }
      ],
      "source": [
        "print(\"Number of distinct records for users.csv:\", users_df.select(\"user_id\").distinct().count())\n",
        "print(\"Number of distinct records for repos.csv:\", repos_df.select(\"repo_id\").distinct().count())\n",
        "print(\"Number of distinct records for starred.csv:\", starred_df.select(\"user_id\", \"repo_id\").distinct().count())"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p4trlRKkTAkv"
      },
      "source": [
        "As we can see from the results, the values are the same of the total number of records for each dataframe: this means that there are no duplicates.\n",
        "\n",
        "Moreover, we can also see if we have any duplicated user or repo using their name."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "b04venYiTBMn",
        "outputId": "d3566bf4-077d-4dd6-aab5-51b8e7696b57"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Number of distinct names for users.csv: 10000\n",
            "Number of distinct repo name for repos.csv: 354981\n"
          ]
        }
      ],
      "source": [
        "print(\"Number of distinct names for users.csv:\", users_df.select(\"name\").distinct().count())\n",
        "print(\"Number of distinct repo name for repos.csv:\", repos_df.select(\"creator\", \"name\").distinct().count())"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0rmKm3jQTFdT"
      },
      "source": [
        "But, again, the number of distinct record is the same as the total number of records. This means that all the users and repos have different names, indicating that they are different from each other."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RMnFuercTJ-X"
      },
      "source": [
        "## Describe repos data\n",
        "\n",
        "In the following block we compute some statistical indicators for some columns of the repos dataframe using the method `describe()`."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qbcXwyp2TKy4",
        "outputId": "f962f99a-191a-433b-d69f-a741887ee750"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+-------+------------------+-----------------+--------------------+\n",
            "|summary|             stars|            forks|             sponsor|\n",
            "+-------+------------------+-----------------+--------------------+\n",
            "|  count|            354981|           354981|              354981|\n",
            "|   mean|443.86323775075283|98.75728278414901|0.053864854738704324|\n",
            "| stddev|2337.1390773287194| 807.991949927328|  0.2257511367185888|\n",
            "|    min|                 0|                0|                   0|\n",
            "|    max|            325307|           237014|                   1|\n",
            "+-------+------------------+-----------------+--------------------+\n",
            "\n"
          ]
        }
      ],
      "source": [
        "repos_df.describe([\"stars\", \"forks\", \"sponsor\"]).show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mikzAWuPTNSt"
      },
      "source": [
        "# Exploratory Data Analysis\n",
        "In this section we dive into the dataset in order to extract some knowledge from it and answer some interesting questions."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5PhVpiHxTgEj"
      },
      "source": [
        "## What are the most used programming languages?"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JLZhf02IgMFn"
      },
      "source": [
        "In the following plot we show what are the most diffused programming languages according to the number of repos that contain mostly files of that language:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        },
        "id": "rH7g58_gTdxc",
        "outputId": "81780f6c-cf5a-41a0-b295-f925b67cc2ef"
      },
      "outputs": [
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAA4cAAAHTCAYAAACUbW1eAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdebxdZX3v8c8X4hBl1kiRQRxQiygIEWnpdQBFwCrcOuBIrnLBXnHqrVXUKioOeFsnWmulkhosFQEHomIxIqC1RQiDICASUWpQJBImscj0u3/s58jmeHJyTnL2XmcfPu/Xa7/OWs8a9m8vQnK++1nreVJVSJIkSZLu2zbougBJkiRJUvcMh5IkSZIkw6EkSZIkyXAoSZIkScJwKEmSJEnCcChJkiRJAuZ1XcCwPfShD63tt9++6zIkSZIkqRPnn3/+r6pqwfj2+1w43H777Vm+fHnXZUiSJElSJ5JcPVG7t5VKkiRJkgyHkiRJkiTDoSRJkiQJw6EkSZIkCcOhJEmSJAnDoSRJkiQJw6EkSZIkCcOhJEmSJAnDoSRJkiQJw6EkSZIkCcOhJEmSJAnDoSRJkiQJw6EkSZIkCcOhJEmSJAnDoSRJkiQJw6EkSZIkCZjXdQGzzW5/dXzXJcwa5//NwV2XIEmSJGlI7DmUJEmSJBkOJUmSJEmGQ0mSJEkShkNJkiRJEoZDSZIkSRKGQ0mSJEkSAwyHSR6X5KK+181J3pRkiyTLklzZfm7e9k+SY5KsSHJxkl37zrWo7X9lkkV97bsluaQdc0ySDOrzSJIkSdJcNrBwWFVXVNUuVbULsBvwG+BLwBHAGVW1A3BGWwfYD9ihvQ4DPgmQZAvgSOCpwO7AkWOBsu1zaN9x+w7q80iSJEnSXDas20r3Bn5cVVcDBwBLWvsS4MC2fABwfPWcA2yWZCvgOcCyqlpdVTcAy4B927ZNquqcqirg+L5zSZIkSZKmYVjh8CXA59ryllX1i7Z8LbBlW94a+FnfMStb22TtKydo/z1JDkuyPMnyVatWrc/nkCRJkqQ5aeDhMMn9gecDJ4/f1nr8atA1VNWxVbWwqhYuWLBg0G8nSZIkSSNnGD2H+wEXVNUv2/ov2y2htJ/XtfZrgG37jtumtU3Wvs0E7ZIkSZKkaRpGOHwp99xSCrAUGBtxdBFwal/7wW3U0j2Am9rtp6cD+yTZvA1Esw9wett2c5I92iilB/edS5IkSZI0DfMGefIkDwaeDbymr/lo4KQkhwBXAy9u7acB+wMr6I1s+iqAqlqd5CjgvLbfe6tqdVt+LfAZYD7w9faSJEmSJE3TQMNhVd0KPGRc2/X0Ri8dv28Bh6/hPIuBxRO0Lwd2mpFiJUmSJOk+bFijlUqSJEmSZjHDoSRJkiTJcChJkiRJMhxKkiRJkjAcSpIkSZIwHEqSJEmSMBxKkiRJkjAcSpIkSZIwHEqSJEmSMBxKkiRJkjAcSpIkSZIwHEqSJEmSMBxKkiRJkjAcSpIkSZIwHEqSJEmSMBxKkiRJkjAcSpIkSZIwHEqSJEmSMBxKkiRJkjAcSpIkSZIwHEqSJEmSMBxKkiRJkjAcSpIkSZIwHEqSJEmSMBxKkiRJkjAcSpIkSZIwHEqSJEmSMBxKkiRJkjAcSpIkSZIwHEqSJEmSMBxKkiRJkjAcSpIkSZIwHEqSJEmSMBxKkiRJkjAcSpIkSZIwHEqSJEmSMBxKkiRJkjAcSpIkSZIwHEqSJEmSMBxKkiRJkjAcSpIkSZIYcDhMslmSU5L8MMnlSf4oyRZJliW5sv3cvO2bJMckWZHk4iS79p1nUdv/yiSL+tp3S3JJO+aYJBnk55EkSZKkuWrQPYcfB/6tqh4P7AxcDhwBnFFVOwBntHWA/YAd2usw4JMASbYAjgSeCuwOHDkWKNs+h/Ydt++AP48kSZIkzUkDC4dJNgWeBhwHUFW3V9WNwAHAkrbbEuDAtnwAcHz1nANslmQr4DnAsqpaXVU3AMuAfdu2TarqnKoq4Pi+c0mSJEmSpmGQPYePBFYB/5zkwiSfTvJgYMuq+kXb51pgy7a8NfCzvuNXtrbJ2ldO0P57khyWZHmS5atWrVrPjyVJkiRJc88gw+E8YFfgk1X1ZOBW7rmFFIDW41cDrGHsfY6tqoVVtXDBggWDfjtJkiRJGjmDDIcrgZVV9b22fgq9sPjLdkso7ed1bfs1wLZ9x2/T2iZr32aCdkmSJEnSNA0sHFbVtcDPkjyuNe0NXAYsBcZGHF0EnNqWlwIHt1FL9wBuarefng7sk2TzNhDNPsDpbdvNSfZoo5Qe3HcuSZIkSdI0zBvw+V8PnJDk/sBVwKvoBdKTkhwCXA28uO17GrA/sAL4TduXqlqd5CjgvLbfe6tqdVt+LfAZYD7w9faSJEmSJE3TQMNhVV0ELJxg094T7FvA4Ws4z2Jg8QTty4Gd1rNMSZIkSbrPG/Q8h5IkSZKkEWA4lCRJkiQZDiVJkiRJhkNJkiRJEoZDSZIkSRKGQ0mSJEkShkNJkiRJEoZDSZIkSRKGQ0mSJEkShkNJkiRJEoZDSZIkSRKGQ0mSJEkShkNJkiRJEoZDSZIkSRKGQ0mSJEkShkNJkiRJEoZDSZIkSRKGQ0mSJEkShkNJkiRJEoZDSZIkSRKGQ0mSJEkShkNJkiRJEoZDSZIkSRKGQ0mSJEkShkNJkiRJEoZDSZIkSRKGQ0mSJEkShkNJkiRJEoZDSZIkSRKGQ0mSJEkShkNJkiRJEoZDSZIkSRKGQ0mSJEkShkNJkiRJEoZDSZIkSRKGQ0mSJEkShkNJkiRJEoZDSZIkSRKGQ0mSJEkShkNJkiRJEoZDSZIkSRIDDodJfprkkiQXJVne2rZIsizJle3n5q09SY5JsiLJxUl27TvPorb/lUkW9bXv1s6/oh2bQX4eSZIkSZqrhtFz+Myq2qWqFrb1I4AzqmoH4Iy2DrAfsEN7HQZ8EnphEjgSeCqwO3DkWKBs+xzad9y+g/84kiRJkjT3dHFb6QHAkra8BDiwr/346jkH2CzJVsBzgGVVtbqqbgCWAfu2bZtU1TlVVcDxfeeSJEmSJE3DoMNhAd9Icn6Sw1rbllX1i7Z8LbBlW94a+FnfsStb22TtKydolyRJkiRN07wBn/9PquqaJA8DliX5Yf/GqqokNeAaaMH0MIDttttu0G8nSZIkSSNnoD2HVXVN+3kd8CV6zwz+st0SSvt5Xdv9GmDbvsO3aW2TtW8zQftEdRxbVQurauGCBQvW92NJkiRJ0pwzsHCY5MFJNh5bBvYBfgAsBcZGHF0EnNqWlwIHt1FL9wBuarefng7sk2TzNhDNPsDpbdvNSfZoo5Qe3HcuSZIkSdI0DPK20i2BL7XZJeYB/1pV/5bkPOCkJIcAVwMvbvufBuwPrAB+A7wKoKpWJzkKOK/t996qWt2WXwt8BpgPfL29JEmSJEnTNLBwWFVXATtP0H49sPcE7QUcvoZzLQYWT9C+HNhpvYuVJEmSpPu4LqaykCRJkiTNMoZDSZIkSZLhUJIkSZJkOJQkSZIkYTiUJEmSJGE4lCRJkiRhOJQkSZIkYTiUJEmSJGE4lCRJkiRhOJQkSZIkYTiUJEmSJGE4lCRJkiRhOJQkSZIkAfO6LkBz23+994ldlzBrbPeuS7ouQZIkSVojew4lSZIkSYZDSZIkSdIUwmGSNybZJD3HJbkgyT7DKE6SJEmSNBxT6Tl8dVXdDOwDbA68Ejh6oFVJkiRJkoZqKuEw7ef+wGer6tK+NkmSJEnSHDCVcHh+km/QC4enJ9kYuHuwZUmSJEmShmkqU1kcAuwCXFVVv0nyEOBVgy1LkiRJkjRMaw2HVXV3km2AlyUBOLuqvjLwyiRJkiRJQzOV0UqPBt4IXNZeb0jygUEXJkmSJEkanqncVro/sEtV3Q2QZAlwIfD2QRYmSZIkSRqeqQxIA7BZ3/KmgyhEkiRJktSdqfQcfhC4MMmZ9KaweBpwxECrkiRJkiQN1VQGpPlckrOApwAFvLWqrh10YZIkSZKk4ZlKzyHAHwF/Qi8czgO+NLCKJEmSJElDN5XRSv8B+HPgEuAHwGuSfGLQhUmSJEmShmcqPYd7AX9YVQW/G6300oFWJUmSJEkaqqmMVroC2K5vfdvWJkmSJEmaI6bSc7gxcHmSc+k9c7g7sDzJUoCqev4A65MkSZIkDcFUwuG7Bl6FJEmSJKlTU5nK4uwkjwB2qKpvJpkPzKuqWwZfniRJkiRpGKYyWumhwCnAp1rTNsCXB1mUJEmSJGm4pjIgzeHAnsDNAFV1JfCwQRYlSZIkSRquqYTD31bV7WMrSebRG5hGkiRJkjRHTCUcnp3k7cD8JM8GTga+MtiyJEmSJEnDNJVw+FZgFXAJ8BrgNOCvB1mUJEmSJGm4Jh2tNMmGwKVV9Xjgn4ZTkiRJkiRp2CbtOayqu4Arkmw3pHokSZIkSR1Y6zyHwObApUnOBW4da6yq5w+sKkmSJEnSUE0lHL5zfd6g3Zq6HLimqv40ySOBE4GHAOcDr6yq25M8ADge2A24Hjioqn7azvE24BDgLuANVXV6a98X+DiwIfDpqjp6fWqVJEmSpPuqtYbDqjp7Pd/jjcDlwCZt/UPAR6vqxCT/SC/0fbL9vKGqHpPkJW2/g5LsCLwEeALwcOCbSR7bzvUJ4NnASuC8JEur6rL1rFeSJEmS7nOmMlrpOkuyDfBc4NNtPcBewCltlyXAgW35gLZO27532/8A4MSq+m1V/QRYAezeXiuq6qo2D+OJbV9JkiRJ0jQNNBwCHwPeAtzd1h8C3FhVd7b1lcDWbXlr4GcAbftNbf/ftY87Zk3tvyfJYUmWJ1m+atWq9f1MkiRJkjTnrDEcJjmj/fzQupw4yZ8C11XV+etY24ypqmOramFVLVywYEHX5UiSJEnSrDPZM4dbJflj4PlJTgTSv7GqLljLufdsx+4PPJDeM4cfBzZLMq/1Dm4DXNP2vwbYFliZZB6wKb2Bacbax/Qfs6Z2SZIkSdI0TBYO30VvpNJtgI+M21b0nh1co6p6G/A2gCTPAN5cVS9PcjLwQnrPCC4CTm2HLG3r/9m2f6uqKslS4F+TfITegDQ7AOfSC6s7tNFPr6E3aM3LpvCZJUmSJEnjrDEcVtUpwClJ3llVR83ge74VODHJ+4ALgeNa+3HAZ5OsAFbTC3tU1aVJTgIuA+4EDq+quwCSvA44nd5UFour6tIZrFOSJEmS7jOmMpXFUUmeDzytNZ1VVV+dzptU1VnAWW35KnojjY7f5zbgRWs4/v3A+ydoPw04bTq1SJIkSZJ+31pHK03yQXpzFV7WXm9M8oFBFyZJkiRJGp619hzSm6dwl6q6GyDJEnq3g759kIVJkiRJkoZnqvMcbta3vOkgCpEkSZIkdWcqPYcfBC5Mcia9EUKfBhwx0KokSZIkSUM1lQFpPpfkLOApremtVXXtQKuSJEmSJA3VVHoOqapf0JuHUJIkSZI0B031mUNJkiRJ0hxmOJQkSZIkTR4Ok2yY5IfDKkaSJEmS1I1Jw2FV3QVckWS7IdUjSZIkSerAVAak2Ry4NMm5wK1jjVX1/IFVJUmSJEkaqqmEw3cOvApJkiRJUqemMs/h2UkeAexQVd9M8iBgw8GXJkmSJEkalrWOVprkUOAU4FOtaWvgy4MsSpIkSZI0XFOZyuJwYE/gZoCquhJ42CCLkiRJkiQN11TC4W+r6vaxlSTzgBpcSZIkSZKkYZtKODw7yduB+UmeDZwMfGWwZUmSJEmShmkq4fAIYBVwCfAa4DTgrwdZlCRJkiRpuKYyWundSZYA36N3O+kVVeVtpZIkSZI0h6w1HCZ5LvCPwI+BAI9M8pqq+vqgi5MkSZIkDcdawyHwYeCZVbUCIMmjga8BhkNJkiRJmiOm8szhLWPBsLkKuGVA9UiSJEmSOrDGnsMkf9YWlyc5DTiJ3jOHLwLOG0JtkiRJkqQhmey20uf1Lf8SeHpbXgXMH1hFkiRJkqShW2M4rKpXDbMQSZIkSVJ3pjJa6SOB1wPb9+9fVc8fXFmSJEmSpGGaymilXwaOA74C3D3YciRJkiRJXZhKOLytqo4ZeCWSJEmSpM5MJRx+PMmRwDeA3441VtUFA6tKkiRJkjRUUwmHTwReCezFPbeVVluXJEmSJM0BUwmHLwIeVVW3D7oYSZIkSVI3NpjCPj8ANht0IZIkSZKk7kyl53Az4IdJzuPezxw6lYUkSZIkzRFTCYdHDrwKSZIkSVKn1hoOq+rsYRQiSZIkSerOWsNhklvojU4KcH/gfsCtVbXJIAuTJEmSJA3PVHoONx5bThLgAGCPQRYlSZIkSRquqYxW+jvV82XgOQOqR5IkSZLUgancVvpnfasbAAuB2wZWkSRJkiRp6KYyWunz+pbvBH5K79ZSSZIkSdIcMZVnDl81jEIkSZIkSd1ZYzhM8q5JjquqOmqyEyd5IPBt4AHtfU6pqiOTPBI4EXgIcD7wyqq6PckDgOOB3YDrgYOq6qftXG8DDgHuAt5QVae39n2BjwMbAp+uqqPX/pElSZIkSeNNNiDNrRO8oBfS3jqFc/8W2KuqdgZ2AfZNsgfwIeCjVfUY4IZ2vrHz3tDaP9r2I8mOwEuAJwD7Av+QZMMkGwKfAPYDdgRe2vaVJEmSJE3TGsNhVX147AUcC8wHXkWv1+9RaztxG9n01231fu1VwF7AKa19CXBgWz6grdO27903dcaJVfXbqvoJsALYvb1WVNVVVXV7q8tnISVJkiRpHUw6lUWSLZK8D7iY3q2hu1bVW6vquqmcvPXwXQRcBywDfgzcWFV3tl1WAlu35a2BnwG07TfRu/X0d+3jjllT+0R1HJZkeZLlq1atmkrpkiRJknSfssZwmORvgPOAW4AnVtW7q+qG6Zy8qu6qql2Abej19D1+fYpdV1V1bFUtrKqFCxYs6KIESZIkSZrVJus5/Evg4cBfAz9PcnN73ZLk5um8SVXdCJwJ/BGwWZKxgXC2Aa5py9cA2wK07ZvSG5jmd+3jjllTuyRJkiRpmiZ75nCDqppfVRtX1SZ9r42rapO1nTjJgiSbteX5wLOBy+mFxBe23RYBp7blpW2dtv1bVVWt/SVJHtBGOt0BOJder+YOSR6Z5P70Bq1ZOr2PL0mSJEmCKcxzuB62Apa0UUU3AE6qqq8muQw4sT3LeCFwXNv/OOCzSVYAq+mFParq0iQnAZcBdwKHV9VdAEleB5xObyqLxVV16QA/jyRJkiTNWQMLh1V1MfDkCdqvovf84fj224AXreFc7wfeP0H7acBp612sJEmSJN3HTTpaqSRJkiTpvsFwKEmSJEkyHEqSJEmSDIeSJEmSJAY7WqmkGbTn3+3ZdQmzxndf/92uS5AkSZpz7DmUJEmSJBkOJUmSJEmGQ0mSJEkShkNJkiRJEoZDSZIkSRKGQ0mSJEkShkNJkiRJEoZDSZIkSRKGQ0mSJEkShkNJkiRJEoZDSZIkSRKGQ0mSJEkShkNJkiRJEoZDSZIkSRKGQ0mSJEkShkNJkiRJEoZDSZIkSRKGQ0mSJEkShkNJkiRJEoZDSZIkSRKGQ0mSJEkShkNJkiRJEoZDSZIkSRKGQ0mSJEkShkNJkiRJEoZDSZIkSRKGQ0mSJEkShkNJkiRJEoZDSZIkSRKGQ0mSJEkShkNJkiRJEoZDSZIkSRKGQ0mSJEkSMK/rAiSpC2c/7eldlzBrPP3bZ3ddgiRJmgXsOZQkSZIkGQ4lSZIkSQMMh0m2TXJmksuSXJrkja19iyTLklzZfm7e2pPkmCQrklycZNe+cy1q+1+ZZFFf+25JLmnHHJMkg/o8kiRJkjSXDfKZwzuBv6yqC5JsDJyfZBnwv4AzquroJEcARwBvBfYDdmivpwKfBJ6aZAvgSGAhUO08S6vqhrbPocD3gNOAfYGvD/AzSZIm8Pd/+ZWuS5g1Xvfh53VdgiRJ62RgPYdV9YuquqAt3wJcDmwNHAAsabstAQ5sywcAx1fPOcBmSbYCngMsq6rVLRAuA/Zt2zapqnOqqoDj+84lSZIkSZqGoTxzmGR74Mn0evi2rKpftE3XAlu25a2Bn/UdtrK1Tda+coJ2SZIkSdI0DTwcJtkI+ALwpqq6uX9b6/GrIdRwWJLlSZavWrVq0G8nSZIkSSNnoOEwyf3oBcMTquqLrfmX7ZZQ2s/rWvs1wLZ9h2/T2iZr32aC9t9TVcdW1cKqWrhgwYL1+1CSJEmSNAcNcrTSAMcBl1fVR/o2LQXGRhxdBJza135wG7V0D+Cmdvvp6cA+STZvI5vuA5zett2cZI/2Xgf3nUuSJEmSNA2DHK10T+CVwCVJLmptbweOBk5KcghwNfDitu00YH9gBfAb4FUAVbU6yVHAeW2/91bV6rb8WuAzwHx6o5Q6UqkkSZIkrYOBhcOq+ndgTfMO7j3B/gUcvoZzLQYWT9C+HNhpPcqUJEmSJDGk0UolSZIkSbOb4VCSJEmSZDiUJEmSJBkOJUmSJEkYDiVJkiRJGA4lSZIkSRgOJUmSJEkYDiVJkiRJGA4lSZIkSRgOJUmSJEkYDiVJkiRJGA4lSZIkSRgOJUmSJEkYDiVJkiRJGA4lSZIkSRgOJUmSJEkYDiVJkiRJGA4lSZIkSRgOJUmSJEkYDiVJkiRJGA4lSZIkSRgOJUmSJEkYDiVJkiRJwLyuC5AkSfd4/yte2HUJs8Y7/uWUrkuQpPsUew4lSZIkSfYcSpKkuevy93+r6xJmjT98x15dlyBpljMcSpIkaUre/e53d13CrOG10FzkbaWSJEmSJHsOJUmSpC6cdPLuXZcwa7z4Red2XYKw51CSJEmShOFQkiRJkoThUJIkSZKE4VCSJEmShOFQkiRJkoThUJIkSZKEU1lIkiRJGnE7n3J61yXMGt9/4XPW+Vh7DiVJkiRJhkNJkiRJkuFQkiRJkoThUJIkSZKE4VCSJEmSxADDYZLFSa5L8oO+ti2SLEtyZfu5eWtPkmOSrEhycZJd+45Z1Pa/MsmivvbdklzSjjkmSQb1WSRJkiRprhtkz+FngH3HtR0BnFFVOwBntHWA/YAd2usw4JPQC5PAkcBTgd2BI8cCZdvn0L7jxr+XJEmSJGmKBhYOq+rbwOpxzQcAS9ryEuDAvvbjq+ccYLMkWwHPAZZV1eqqugFYBuzbtm1SVedUVQHH951LkiRJkjRNw37mcMuq+kVbvhbYsi1vDfysb7+VrW2y9pUTtEuSJEmS1kFnA9K0Hr8axnslOSzJ8iTLV61aNYy3lCRJkqSRMuxw+Mt2Syjt53Wt/Rpg2779tmltk7VvM0H7hKrq2KpaWFULFyxYsN4fQpIkSZLmmmGHw6XA2Iiji4BT+9oPbqOW7gHc1G4/PR3YJ8nmbSCafYDT27abk+zRRik9uO9ckiRJkqRpmjeoEyf5HPAM4KFJVtIbdfRo4KQkhwBXAy9uu58G7A+sAH4DvAqgqlYnOQo4r+333qoaG+TmtfRGRJ0PfL29JEmSJEnrYGDhsKpeuoZNe0+wbwGHr+E8i4HFE7QvB3ZanxolSZIkST2dDUgjSZIkSZo9DIeSJEmSJMOhJEmSJMlwKEmSJEnCcChJkiRJwnAoSZIkScJwKEmSJEnCcChJkiRJwnAoSZIkScJwKEmSJEnCcChJkiRJwnAoSZIkScJwKEmSJEnCcChJkiRJwnAoSZIkScJwKEmSJEnCcChJkiRJwnAoSZIkScJwKEmSJEnCcChJkiRJwnAoSZIkScJwKEmSJEnCcChJkiRJwnAoSZIkScJwKEmSJEnCcChJkiRJwnAoSZIkScJwKEmSJEnCcChJkiRJwnAoSZIkScJwKEmSJEnCcChJkiRJwnAoSZIkScJwKEmSJEnCcChJkiRJwnAoSZIkScJwKEmSJEnCcChJkiRJwnAoSZIkScJwKEmSJEnCcChJkiRJwnAoSZIkSWIOhMMk+ya5IsmKJEd0XY8kSZIkjaKRDodJNgQ+AewH7Ai8NMmO3VYlSZIkSaNnpMMhsDuwoqquqqrbgROBAzquSZIkSZJGTqqq6xrWWZIXAvtW1f9u668EnlpVrxu332HAYW31ccAVQy10+h4K/KrrIuYQr+fM8nrOLK/nzPFaziyv58zyes4cr+XM8nrOrFG5no+oqgXjG+d1UcmwVdWxwLFd1zFVSZZX1cKu65grvJ4zy+s5s7yeM8drObO8njPL6zlzvJYzy+s5s0b9eo76baXXANv2rW/T2iRJkiRJ0zDq4fA8YIckj0xyf+AlwNKOa5IkSZKkkTPSt5VW1Z1JXgecDmwILK6qSzsuayaMzC2wI8LrObO8njPL6zlzvJYzy+s5s7yeM8drObO8njNrpK/nSA9II0mSJEmaGaN+W6kkSZIkaQYYDiVJkiRJhkNJ0mhK8qKptEmSpKkxHM4CST40lTZp2JJsOsm2kZ3Dp2tJHpZku7FX1/WMsLdNsU2aNZL8bdc1SABJ7p9kp/a6X9f1jLIkD+26hpnigDSzQJILqmrXcW0XV9WTuqpp1CVZABwKbE/fqLxV9equahpFSc4D9qmqG8a17wMcV1XbTnykJpLk+cCHgYcD1wGPAC6vqid0WtiISbIfsD/wYuDzfZs2AXasqt07KWyOSPK2qvpg13XMVUn+q6r8Umiakry6qhZ3XcdckeQZwBLgp0DozRu+qKq+3WFZIyfJ84DFwJ3AXcCLq+o/uq1q/dhz2KEk/yfJJcDjklzc9/oJcHHX9Y24U4FNgW8CX+t7aXqOBc5sYRuAJC8DPgU8t7OqRtdRwB7Aj6rqkcDewDndljSSfg4sB24Dzu97LQWe02Fdc4W35g5Wui5glCQ5KMlWwOv62s7osKS54sP0vvx9elU9jd7fnR/tuKZR9H7gf1TVVsALgJH/Ym2k5zmcA/4V+Dq9P0hH9LXfUlWruylpznhQVb216yJGXVX9U5LbgG+13sKDgD8HnllVP+20uNF0R1Vdn2SDJBtU1ZlJPtZ1UaOmqr4PfD/Jv9L7RfvxQAFXVNXtnRYnAUm2WNMmDIfT9QfAJ4DHJvk8vS/Pt0vy4Kq6tdvSRtr9quqKsZWq+pG3lq6TO6vqhwBV9b0kG3dd0PoyHHaoqm4CbgJemmRX4E/o/YLzXTgVwmcAABODSURBVMBwuH6+mmT/qjqt60JGXVV9tgXEC4H/Av6kqn7VcVmj6sYkGwHfBk5Ich3gLzfr7tn0erF/TO8X7kcmeU1Vfb3bskZPu2Ol6F3HrZJc1Zarqh7VaXGj6XzuuZ7j3THkWkbdl6vq40kuBN4M7AxsBJyaZF5VPaPT6kbX8iSfBv6lrb+C3h0Zmp6HJfm/a1qvqo90UNN68ZnDWSDJO+k9O/PF1nQgcHJVva+7qkZbkluABwO3c88/xFVVm3RX1ehptz2P/YLzCGAVvTAz9kujz8VOQ5IHA/9N75b+l9O79fmEqrq+08JGVJIfAn9aVSva+qOBr1XV47utbLQlubCqntx1HRJAu0Nge+BxwPvo9Rx+pKp2TnK/qjJsr4MkDwAOB/ZsTd8B/sG7L6YnyZGTba+q9wyrlpliOJwFklwB7FxVt7X1+cBFVfW4bivTfV2SR0y2vaquHlYtc0H7NvHzVXVN17XMBUnOq6qn9K0HOLe/TdNnOFx/7W6gNaqqC4ZVy1yR5FLgXcBuwGuBS4DvV9XrJj1Q95LkAGCbqvpEWz8XWEDvi+C3VNUpXdan7nlb6ezwc+CB9AZXAHgA4C+P66mNDPm0tnpWVX21y3pGkeFvxm0MfCPJanqjbJ5cVb/suKZRtjzJacBJ9H6xeRFwXpI/A6iqL052sNbou10XMAcsB34AjN2C3397aQF7Db2i0XdhVX0B+EIbsfjpgF9iTN9bgJf0rd+fXuDeCPhnwHA4DUmOmWx7Vb1hWLXMFHsOZ4EkXwaeAiyj94/Gs4FzgZUwmn+wupbkaHrX9ITW9FJgeVU5B9o09D2H9LumvvWqqkcPv6rRl+RJ9Ab3eQGwsqqe1XFJIynJP0+yuZy6Rl1J8ibghfTGFTgR+FJV/brbqkZTkscAW1bVd/va9gNuBq6tqh93VtwImuCOi78f631Nck5V7dFddaMnyaK+1fcA97rNtKqWDLei9Wc4nAXG/cH6PaP4B6trSS4Gdqmqu9v6hvS+dfQZuWlI8pBxTRvQez72zcAFVfWC4Vc1+pL8Ab1erpcAG/vnUrNJkidU1aVd1zEXJHkUvf/PDwCuBj5QVRd1W9VoSfJV4G1Vdcm49ifSu57P66ay0ZRkRVU9Zg3bfuyXvuturtyS722ls4Dhb2A2455RXzftspBRNTZQSpINgFcCfwVcBDy3qi7rsrZRlOS19ML1AuBk4FCv4/QleUtV/b8kf8e9e7YB77aYAZ8FJn1mTlNTVVclORWYT+/v0MfS+ztUU7fl+GAIUFWXJNl++OWMvO8lObSq/qm/Mclr6N21pnU3J3rcDIcdSnJSVb24b0TIe7E3Yb18ELgwyZn0boV8GveeS1JT0OY8ejXwF8C/AweOjQypdbIt8CZ7Dtbb5e2nw64PhvPwradxPYY/o3dr6Qeq6r87LWw0bTbJtvlDq2Lu+Avgy0leBowNjLQbvfEuDuysKs0a3lbaoSRbVdUv1jQipIOBrJ8kW9F77hB6Ixhe22U9oyjJSuBO4GP05ji8Fwf8WDdJHkZvECoAqur3rq0m124V/1BVvbnrWuaCNhz72LQ1rwH+cWxbVb23q7pGVZK76U25cCq9Z+Pu9cvWKM591pUknwO+NUFP1/8Gnl1VB3VT2WhLshfwhLZ6aVV9q8t6RlWbOm3s7875wG/GNjGiU6jZc9ihFgw3BD5TVc/sup45aAN6I8XNAx6b5LFV9e2Oaxo1y9rPJ7UX3NOrUNwzN6emIMnzgI8ADweuozd35OXc8w+0pqiq7kqy59r31BT9tG/5DnrPx2nd9c9ttlFnVcwNbwK+lOTlwPmtbSG9UTb/Z2dVjbgWBg2E66mqNu66hplmOOxY+wXn7iSbVtVNXdczVyT5EL3RIC8F7m7NBRgOp+cH3PONGG15FfDvVfWTzqoaXe8D9gC+WVVPTvJM4BUd1zTKLkqylN7zm7eONdqjPX39z74neaPPwq+366vq77suYi5o0/38cfv7cqfW/DV7ujQbJHkg8OfAY+jdLbC4qu7stqr1YzicHX4NXJJkGff+BcdBFdbdgcDjquq3XRcy4ib6xvsRwDuSvLuqThx2QSPujqq6PskGSTaoqjOTfKzrokbYA4HrufeccfZorz+fOVx/rwYMhzOoqs4Ezuy6DmmcJfTutvgOsD+9O4He2GlF68lwODt8EX+ZmWlXAfcDDIfroareM1F7ki2Ab9IbZEFTd2OSjej9I3JCkuvo+0JI01NVr+q6hrlggnnk9m7te+I8cpI0mR2r6okASY5jDoz4ajicHU4Bbququ+B3Ay08oNuSRlPf0Pa/oXfL2Rn0BUR7Y2dGVa1OYu/C9D0fuI3et4qvADbh3s8maRqSLAHeWFU3tvXNgQ9X1au7rWzkfAx429hKVY1NAXRz2+Y8ctP3pCQ3T9A+soNUSJrQHWMLVXXnXPjVyHA4O5wBPIve7aXQG+3oG8Afd1bR6Bob2v58YOm4bQ7NO0Pasx83dF3HqOgbzexeze3nu5L8GHhHVZ0x3MpG3pPGgiFAVd2QZOQnIO6A88jNvEvmwmTYktZq574vggLMb+sj+0WQ4XB2eGBVjQVDqurXSR7UZUGjamwQhTagwsf7tyUZ6XvAu7CGOTi3AH4OHDz8ikbTZKOZtTsFdgJO4J7BFjQ1GyTZvKpugN/d7uy/a9PnPHKStA6qasOua5hp/iM6O9yaZNequgAgyW6AE+Wun0XAx8e1/a8J2jS5Px23XvRG4fM5uRnSbif/frslWtPzYeA/k5xM71vaFwLv77akkbQ8yaFrmEfu/DUco8md3HUBkrQuUuWddl1L8hR6A3v8nN4vOH8AHFRV/qM8TUleCrwM+BN6g36M2Ri4u6r27qQwSQORZEd6o5UWcGZVXdZxSSMnyZbAl4DbmWAeuaq6tqvaRlXf8+8T8vl3SbOVPYezQFWdl+TxwONa0xVVdcdkx2iN/gP4BfBQer0KY26hN/+MpBHXbru/o6ruqKrLktxFbwjxxwOGw2lyHrmBWN63/B7gyK4KkaTpsOewQ63H8Gdj38omORh4AXA18O6+EeM0TUneAHx27FkkSXNHkm8Dh1TVlW0ahnPpPbO5I3BeVR3RaYFSnyQXOjiNpFGxQdcF3Md9it5tPCR5GnA0cDxwE3Bsh3XNBQ8DzktyUpJ9nXZBmlM2r6or2/Ii4HNV9XpgP+C53ZUlTchv4SWNDMNhtzbs6x08CDi2qr5QVe8EHtNhXSOvqv4a2AE4jt5ANFcm+UCSR3damKSZ0P/L9l7AMoCquh24u5OKJEmaA3zmsFsbJplXVXcCewOH9W3zv816qqpKci1wLXAnsDlwSpJlVfWWbquTtB4uTvK3wDX0vkj7BkCSyaZkkIZm3NymDxo3D9pIzn0m6b7BANKtzwFnJ/kVvakrvgPQnqG5qcvCRl2b0/Bg4FfAp4G/qqo7kmwAXAkYDqXRdSjwRmB7YJ+q+k1r3xH4266KksZMNrepJM1mDkjTsSR7AFsB3xibOy7JY4GNxuY91PQleQ+wuKqunmDbH1bV5R2UJWmGJZkPbFdVV3RdiyRJo85nDjtWVedU1Zeq6tYkD0uyHXAbvR4vTVOSByZ5E/AQYN8kv9c7bjCU5oYkzwMuAv6tre+SZGm3VUmSNLoMh7NAkucluRL4CXB2+/n1bqsaWUvoTd58Cb2RCz88+e6SRti7gd2BGwGq6iLgUV0WJEnSKPOZw9nhfcAewDer6sltIuJXdFzTqNqxqp4IkOQ4evOfSZqb7qiqm8bNVONopZIkrSN7DmeHO6rqemCDJBtU1Zn0er80fXeMLbRRYCXNXZcmeRm9kZ93SPJ3wH90XZQkSaPKnsPZ4cYkGwHfBk5Ich1wa8c1jaqdxw0ZPr+tO3y4NPe8HngH8Ft6oz+fDhzVaUWSJI0wRyudBZI8mN5UFhsALwc2BU5ovYmSpEkk2YTelz+3dF2LJEmjzHA4CyT5v8Dnq+qarmuRpFGR5CnAYmBsTrmbgFdX1fndVSVJ0ujymcPZYWPgG0m+k+R1SbbsuiBJGgHHAa+tqu2ranvgcOCfuy1JkqTRZc/hLJLkScBBwAuAlVX1rI5LkqRZK8mFVfXkcW0XVNWuXdUkSdIoc0Ca2eU64FrgeuBhHdciSbPd2Uk+RW8wmqL35dpZSXYFqKoLuixOkqRRY8/hLJDktcCLgQXAycBJVXVZt1VJ0uyW5MxJNldV7TW0YiRJmgPsOZwdtgXeVFUXdV2IJI2QZ1XVXV0XIUnSXGHP4SyS5GHAA8fWq+q/OixHkma1JFcBXwAWV9XlXdcjSdKoc7TSWSDJ85JcCfwEOBv4KfD1TouSpNlvZ+BHwHFJzklyWJvzUJIkrQN7DmeBJN8H9gK+WVVPTvJM4BVVdUjHpUnSrJNkXlXdOa7t6cC/ApsBpwBHVdWKLuqTJGlU2XM4O9xRVdcDGyTZoKrOBBZ2XZQkzVLnAiTZMMnzk3wZ+BjwYeBRwFeA0zqsT5KkkeSANLPDjUk2Ar4DnJDkOuDWjmuSpNnuSuBM4ENV9Z997ackeVpHNUmSNLK8rXQWSPIg4DYgwCuATYATqmp1p4VJ0iyUZCXwEeBBwH/Tm+Pwd6rqI13UJUnSqLPnsENJbmHcLzX0AiLAu5L8GHhHVZ0x3MokaVbbENiI3t+XG3VciyRJc4Y9h7NUkg2Bnej1IO7UdT2SNFskuaCqdu26DkmS5hoHpJmlququqvo+8Hdd1yJJs0zWvoskSZouew4lSSMlyRY+ky1J0swzHEqSJEmSvK1UkiRJkmQ4lCRJkiRhOJQkDUCSSvIvfevzkqxK8tW1HLcwyTGDr3D2SnJaks1m4DzbJ/nBTNQkSbpvcJ5DSdIg3ArslGR+Vf038GzgmrUdVFXLgeUzXUySeVV1Z1fHT0dV7T+M95EkaTx7DiVJg3Ia8Ny2/FLgc2Mbkuye5D+TXJjkP5I8rrU/Y6x3Mcm7kyxOclaSq5K8YaI3SfLrJB9NcmmSM5IsaO1nJflYkuXAG5Ps3d7vknbeB7T99k/ywyTnJzlm3Pt/Nsl3gc+2nrjvJLmgvf64r+azk5za6jw6ycuTnNve69Ftv88k+WSSc9p+z2h1XJ7kM32f56dJHtre7/Ik/9Q+2zeSzG/7PCXJxUkuSvI3a+shXEvtZyU5pV2DE5JkCtflzX3n/kGS7dvyl9v+lyY5rG+fQ5L8qF2Tf0ry9619QZIvJDmvvfac7HNIkgbLcChJGpQTgZckeSDwJOB7fdt+CPyPqnoy8C7gA2s4x+OB5wC7A0cmud8E+zwYWF5VTwDOBo7s23b/qloIfAL4DHBQVT2R3p0z/6fV9ilgv6raDVgw7tw7As+qqpcC1wHPrqpdgYOA/ttfdwb+HPhD4JXAY6tqd+DTwOv79tsc+CPgL4ClwEeBJwBPTLLLBJ9tB+AT7bPdCLygtf8z8Jqq2gW4a4Ljxpus9icDb2qf9VHAnlO4Lmvy6rb/QuANSR6S5OHAO4E9gD3p/Tcd83Hgo1X1lPbZPj3F95EkDYC3lUqSBqKqLm49Si+l14vYb1NgSZIdgAImCn0AX6uq3wK/TXIdsCWwctw+dwOfb8v/Anyxb9tY++OAn1TVj9r6EuBw4Czgqqr6SWv/HHBY3/FL222xtBr/voW4u+D/t3f3IFJdUQDH/2fdQnFV1EYLYROChJQiagpRg11shBQSkCSIjYhil9KPQCQIVooSsLSRYEiaJU0MRJRdJIbdVSSIFhvESoOrrKB7LN4dfcrM7OjMqJD/r3rz5n6cuc1wuOe+x+pau7HMvAMQETeB38r9cWBLrd2vmZkRMQ7czczx0mcSGAauvvLbbmVm494VYLicR1yUmZfK/bPANtprF/toZk6VOK6WOKZpvy6t7IuI7eV6FVVyuwL4o/Fuyog4V5t/K/BJ2awEWBwRQ5k53cFckqQeMzmUJPXTL8AxYDOwvHb/CPB7Zm4vCeSFFv0f166f0tn/Vv0Fvg87jLOVev8DwF2qXcIBYKb2XT3O2drnWV6O+XGTNs3aNRv3KbCg08Bf0WnsnazxE16uPJoPVYkqVbL3aWY+iogLje/aGAA2ZObMHO0kSW+BZaWSpH46Axxq7JDVLOHFA2q+7nKOAeCLcv0l8GeTNjeodt0+Kp93UpWg3gA+bJyZoyq5bGUJcCczZ0v/ed2F/WYy8z7wICLWl1s7Ouj2urG3W5fbwBqAiFgDfFCb415JDD+mKiMFGAM2RcTSiBjkRWksVDusz8tuW5TWSpLeEpNDSVLfZOZUZjZ7NcUPwPcR8RfdV7E8BNaVh7J8BhxuEscM8A1wrpR0zgKnSsnoHmAkIq4AD4D/WsxzEvgqIv6mOjfX7a5kN3YBP5Yy0IW0jrnhtWKfY11+ApaVUti9QKNUdwQYjIjrwFHgchnrX6ozpaPARarksjHWPmBtebjONapzm5KkdyQyc+5WkiS9pyJiOjOHuug/lJnT5SmdJ4B/MvN47yLsvfq5vIj4FliZmfv7MUcv1qU21iBwHjiTmed7Ga8kqXvuHEqS/u92lx24SarSyNPvOJ5OfF5eYzEBbAS+68McvVyXg2WsCeAW8HMP4pMk9Zg7h5IkSZIkdw4lSZIkSSaHkiRJkiRMDiVJkiRJmBxKkiRJkjA5lCRJkiRhcihJkiRJAp4BvX6eve1nu6QAAAAASUVORK5CYII=",
            "text/plain": [
              "<Figure size 1080x504 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "n_languages = 10\n",
        "plt.figure(figsize=(15, 7))\n",
        "language_hist = repos_df.toPandas()[\"language\"].value_counts()[:n_languages]\n",
        "sns.barplot(x=language_hist.index, y=language_hist)\n",
        "plt.xticks(rotation=90)\n",
        "plt.xlabel(\"Main programming language\")\n",
        "plt.ylabel(\"Number of repos\")\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DlUV7JllgmMT"
      },
      "source": [
        "As we can see the top 3 used programming languages are:\n",
        "- Javascript\n",
        "- Python\n",
        "- Java\n",
        "\n",
        "It is not a surprise that Javascript is the most used programming language, since we collected repos starting from the users that starred the React repository, one of the most known Javascript frameworks, so this could have influenced the results.\n",
        "\n",
        "Interestingly, we can also see that there are a lot of repos that only contain text or use a language not recognized by GitHub (indicated by `UNK`)."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rvz3fC64TyTA"
      },
      "source": [
        "## How often are repos updated?"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yWCGagakjUm1"
      },
      "source": [
        "In the following plot we show how frequently repos are updated:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        },
        "id": "YniBeO6OTPo4",
        "outputId": "ca32a45d-8ca2-4773-fb5e-e26154526adf"
      },
      "outputs": [
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAA4cAAAGpCAYAAAA+1EP5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeZhcZ3mg/fupqt7U2hfLxrIt28jGZrUxjgmJIRiMgQQzmZAQMkAIEydfSMJkGSCZJHwZQgLJZOMKISEsMZmAIYSAQwxmN2Ezkm28yKu8y5YsWVJra3XX9s4f53SrLHdLLamqq6vP/buuuvrUW6dOPU9Xt1RPv1uklJAkSZIkFVup2wFIkiRJkrrP4lCSJEmSZHEoSZIkSbI4lCRJkiRhcShJkiRJAirdDmC2rVy5Mq1du7bbYUiSJElSV9xwww2Pp5RWHdpeuOJw7dq1bNiwodthSJIkSVJXRMSDU7U7rFSSJEmSZHEoSZIkSbI4lCRJkiRhcShJkiRJwuJQkiRJkoTFoSRJkiQJi0NJkiRJEhaHkiRJkiQsDiVJkiRJWBxKkiRJkrA4lCRJkiRhcShJkiRJwuJQkiRJkoTFoSRJkiQJi0NJkiRJaqvRap07t+5h33i926EcFYtDSZIkSWqjO7bs5bK/+k82PLCz26EcFYtDSZIkSWqjWqMJQH+lt8qt3opWkiRJkua4yeKw3FvlVm9FK0mSJElz3ERx2GdxKEmSJEnFVa0nwOJQkiRJkgrt4JzD6HIkR8fiUJIkSZLaqFp3WKkkSZIkFZ5zDiVJkiRJFoeSJEmSJKg2sgVp3OdQkiRJkgrMfQ4lSZIkSdQmF6RxtVJJkiRJKqxao0kElEsWh5IkSZJUWNVGoq9cIsLicFJE/EZEbIyI2yLiExExGBGnR8T1EbEpIj4ZEf35uQP5/U3542tbrvM7eftdEfGylvbL8rZNEfGOTuYiSZIkSTNRrTd7br4hdLA4jIiTgV8HLkgpPQMoA68F3gv8ZUrpqcAu4M35U94M7Mrb/zI/j4g4N3/e04HLgL+NiHJElIH3Ay8HzgV+Nj9XkiRJkrqm1mj23HxD6Pyw0gowFBEVYAGwBXgx8On88SuBV+fHl+f3yR+/JLJ+2MuBq1JK4yml+4FNwIX5bVNK6b6UUhW4Kj9XkiRJkrqm1mj23DYW0MHiMKX0CPB/gIfIisLdwA3ASEqpnp+2GTg5Pz4ZeDh/bj0/f0Vr+yHPma79SSLiiojYEBEbtm/ffvzJSZIkSdI0qo0mfQ4rPSgilpH15J0OPAUYJhsWOutSSh9MKV2QUrpg1apV3QhBkiRJUkHUGsk5h4d4CXB/Sml7SqkGfAZ4AbA0H2YKsAZ4JD9+BDgFIH98CbCjtf2Q50zXLkmSJEldU6vbc3ioh4CLImJBPnfwEuB24OvAT+XnvBH4XH58dX6f/PGvpZRS3v7afDXT04F1wPeB9cC6fPXTfrJFa67uYD6SJEmSdES1RpO+Su8tSFM58inHJqV0fUR8GrgRqAM3AR8E/gO4KiL+KG/7cP6UDwP/FBGbgJ1kxR4ppY0R8SmywrIOvCWl1ACIiF8FriVbCfUjKaWNncpHkiRJkmaiV+ccdqw4BEgpvRN45yHN95GtNHrouWPAa6a5zruBd0/Rfg1wzfFHKkmSJEntUXVYqSRJkiSp1mi6II0kSZIkFV2tkdznUJIkSZKKrtZo0lfuvQVpLA4lSZIkqY16dUGa3otYkiRJkuYw5xxKkiRJkqjVkz2HkiRJklR0tUaTvopzDiVJkiSp0NznUJIkSZJE1TmHkiRJkqRao+k+h5IkSZJUZI1moplwWKkkSZIkFVmt0QQsDiVJkiSp0KqTxaGrlUqSJElSYdXqWXHonENJkiRJKrBaIwEOK5UkSZKkQqvWnXMoSZIkSYXnnENJkiRJ0uRqpQPOOZQkSZKk4nIrC0mSJEmSxaEkSZIkCap1VyuVJEmSpMKb6Dnsr7ggjSRJkiQVlsNKJUmSJEnucyhJkiRJOrjPYb9bWUiSJElScdUa2YI0/fYcSpIkSVJxOedQkiRJktRSHLpaqSRJkiQV1uSCNM45lCRJkqTics6hJEmSJMk5h5IkSZKkbFhpKaBccs6hJEmSJBVWrdHsyT0OweJQkiRJktqm2mj25JBSsDiUJEmSpLapNZo9uRgNWBxKkiRJUtvU6smeQ0mSJEkqulqjSV+l9xajAYtDSZIkSWob5xxKkiRJkpxzKEmSJEnK9jm051CSJEmSCq7WSO5zKEmSJElFl805dEEaSZIkSSq0mgvSSJIkSZJckEaSJEmSRK2e7DmUJEmSpKKrNZr0uSCNJEmSJBWbC9JIkiRJkqjWnXMoSZIkSYVXazTd51CSJEmSiq7WcEEaSZIkSSq8qvscSpIkSVKxpZTyfQ5dkEaSJEmSCqvRTKSEPYeSJEmSVGS1RgJwn0NJkiRJKrJqownYcyhJkiRJhVatZ8Whcw4lSZIkqcBqec+h+xxKkiRJUoHVHFYqSZIkSbI4lCRJkiRRreerlVocSpIkSVJxHZxz6II0kiRJklRYDiuVJEmSJLnPoSRJkiSpZZ9Dt7KQJEmSpOKqNbIFafrtOZQkSZKk4nLOoSRJkiSppTh0tVJJkiRJKqyJOYf2HEqSJElSgU3OOXRBGkmSJEkqLuccSpIkSZKccyhJkiRJgnH3OZQkSZIkTfYclnqzzOpo1BGxNCI+HRF3RsQdEfH8iFgeEV+OiHvyr8vycyMi3hcRmyLilog4v+U6b8zPvyci3tjS/tyIuDV/zvsiojf7byVJkiT1vFqjSaUUlEq9WZZ0uqT9a+CLKaWnAc8G7gDeAXw1pbQO+Gp+H+DlwLr8dgXwAYCIWA68E/gh4ELgnRMFZX7OL7Y877IO5yNJkiRJU6o1Us8uRgMdLA4jYglwMfBhgJRSNaU0AlwOXJmfdiXw6vz4cuBjKfM9YGlEnAS8DPhySmlnSmkX8GXgsvyxxSml76WUEvCxlmtJkiRJ0qyq1ps9uxgNdLbn8HRgO/DRiLgpIj4UEcPA6pTSlvycrcDq/Phk4OGW52/O2w7XvnmK9ieJiCsiYkNEbNi+fftxpiVJkiRJT1ZrNHt2MRrobHFYAc4HPpBSOg/Yz8EhpADkPX6pgzFMvM4HU0oXpJQuWLVqVadfTpIkSVIB1RpNh5VOYzOwOaV0fX7/02TF4mP5kFDyr9vyxx8BTml5/pq87XDta6ZolyRJkqRZ55zDaaSUtgIPR8TZedMlwO3A1cDEiqNvBD6XH18NvCFftfQiYHc+/PRa4NKIWJYvRHMpcG3+2J6IuChfpfQNLdeSJEmSpFlVrff2sNJKh6//a8A/R0Q/cB/wJrKC9FMR8WbgQeCn83OvAV4BbAJG83NJKe2MiHcB6/Pz/ndKaWd+/CvAPwJDwBfymyRJkiTNumqPDyvtaHGYUvoBcMEUD10yxbkJeMs01/kI8JEp2jcAzzjOMCVJkiTpuNUaTfpdrVSSJEmSis0FaSRJkiRJ1OouSCNJkiRJhVdtNOnr4QVpejdySZIkSZpDnHMoSZIkSXLOoSRJkiSp9/c57N3IJUmSJGkOqTVckEaSJEmSCq/qsFJJkiRJkgvSSJIkSZKo1e05lCRJkqTCqzWS+xxKkiRJUpGllJxzKEmSJElFV28mAAbsOZQkSZKk4qrWmwD0uSCNJEmSJBVXrTFRHPZuidW7kUuSJEnSHFG1OJQkSZIk1RrZnMN+i0NJkiRJKq7axJzDinMOJUmSJKmwnHMoSZIkSXLOoSRJkiSpZc6h+xxKkiRJUnFN7HPogjSSJEmSVGDOOZQkSZIktcw5dLVSSZIkSSqsya0s7DmUJEmSpOJyQRpJkiRJknMOJUmSJEnOOZQkSZIkcbDncF4PK42It0bE4sh8OCJujIhLZyM4SZIkSeoFRdnn8BdSSnuAS4FlwOuB93Q0KkmSJEnqIUWZczgxaPYVwD+llDa2tEmSJElS4U2sVjrfi8MbIuJLZMXhtRGxCGh2NixJkiRJ6h3Veu8vSFOZwTlvBp4D3JdSGo2IFcCbOhuWJEmSJPWOWqNJXzmImMfFYUqpGRFrgNfliV6XUvr3jkcmSZIkST0iKw57d0gpzGy10vcAbwVuz2+/HhF/3OnAJEmSJKlX1Bqp54vDmQwrfQXwnJRSEyAirgRuAn63k4FJkiRJUq+oNpo9vcchzGxBGoClLcdLOhGIJEmSJPWqar3Z03scwsx6Dv8EuCkivk62hcXFwDs6GpUkSZIk9ZCJBWl62UwWpPlERHwDeB6QgLenlLZ2OjBJkiRJ6hXzYUGamfQcAjwf+BGy4rAC/FvHIpIkSZKkHlOt9/6CNDNZrfRvgV8GbgVuA34pIt7f6cAkSZIkqVfUGk36enxBmpn0HL4YOCellGBytdKNHY1KkiRJknpIrdGkv8fnHM6ktN0EnNpy/5S8TZIkSZJEXhwWoOdwEXBHRHyfbM7hhcCGiLgaIKX0qg7GJ0mSJElzXrWRGB6Y/8XhH3Q8CkmSJEnqYdV6AVYrTSldFxGnAetSSl+JiCGgklLa2/nwJEmSJGnuy+Yc9nZxOJPVSn8R+DTw93nTGuCznQxKkiRJknpJts/h/F+Q5i3AC4A9ACmle4ATOhmUJEmSJPWS2jwYVjqT6MdTStWJOxFRIVuYRpIkSZJEtiBNr+9zOJPor4uI3wWGIuKlwL8A/97ZsCRJkiSpdxRiziHwdmA7cCvwS8A1wO91MihJkiRJ6iXzfp/DiCgDG1NKTwP+YXZCkiRJkqTeMu8XpEkpNYC7IuLUWYpHkiRJknpKs5moNVLPL0hzxH0OgWXAxoj4PrB/ojGl9KqORSVJkiRJPaLWbAIUojj8/Y5HIUmSJEk9qtbINnPo9QVpjlgcppSum41AJEmSJKkX1eoTPYfzeM6hJEmSJOnwao28OOzx1Up7O3pJkiRJ6rJqY37MOZw2+oj4av71vbMXjiRJkiT1lok5hwM93nN4uDmHJ0XEDwOvioirgCcMoE0p3djRyCRJkiSpB9TmSc/h4YrDPyBbqXQN8BeHPJaAF3cqKEmSJEnqFdX6PC8OU0qfBj4dEb+fUnrXLMYkSZIkST3j4JzD3l6tdCZbWbwrIl4FXJw3fSOl9PnOhiVJkiRJvWFiK4te3+fwiNFHxJ8AbwVuz29vjYg/7nRgkiRJktQLJhak6fWtLI7Ycwi8EnhOSqkJEBFXAjcBv9vJwCRJkiSpF8yXBWlmGv3SluMlnQhEkiRJknpRYeYcAn8C3BQRXyfbzuJi4B0djUqSJEmSesREz+F83ucQgJTSJyLiG8Dz8qa3p5S2djQqSZIkSeoR82VY6Ux6DkkpbQGu7nAskiRJktRz5ss+h70dvSRJkiR1WXVitVKLQ0mSJEkqrkLscxgR5Yi4c7aCkSRJkqReMznnsNLbq5UetjhMKTWAuyLi1FmKR5IkSZJ6ynxZkGYm0S8DNkbEVyPi6onbTF8g7328KSI+n98/PSKuj4hNEfHJiOjP2wfy+5vyx9e2XON38va7IuJlLe2X5W2bIsLtNSRJkiTNuok5h5VSb/cczmS10t8/ztd4K3AHsDi//17gL1NKV0XE3wFvBj6Qf92VUnpqRLw2P+9nIuJc4LXA04GnAF+JiLPya70feCmwGVgfEVenlG4/znglSZIkacZqjSb9lRIRvV0cHrHnMKV0HfAA0JcfrwdunMnFI2IN8ErgQ/n9AF4MfDo/5Urg1fnx5fl98scvyc+/HLgqpTSeUrof2ARcmN82pZTuSylVgavycyVJkiRp1tTqzZ5fjAZmUBxGxC+SFWt/nzedDHx2htf/K+BtQDO/vwIYSSnV8/ub8+tNXPdhgPzx3fn5k+2HPGe69qlyuCIiNkTEhu3bt88wdEmSJEk6smqjSV+5t3sNYWZzDt8CvADYA5BSugc44UhPiogfB7allG44rgjbIKX0wZTSBSmlC1atWtXtcCRJkiTNI7VGs+cXo4GZzTkcTylVJ8bPRkQFSDN43guAV0XEK4BBsjmHfw0sjYhK3ju4BngkP/8R4BRgc/4aS4AdLe0TWp8zXbskSZIkzYpqPc2L4nAmGVwXEb8LDEXES4F/Af79SE9KKf1OSmlNSmkt2YIyX0sp/RzwdeCn8tPeCHwuP746v0/++NdSSilvf22+munpwDrg+2RzH9flq5/2568x41VUJUmSJKkdJhak6XUz6Tl8B9lKorcCvwRcQ77AzDF6O3BVRPwRcBPw4bz9w8A/RcQmYCdZsUdKaWNEfAq4HagDb8n3XyQifhW4FigDH0kpbTyOuCRJkiTpqNXmyZzDIxaHKaVmRFwJXE82nPSuvEdvxlJK3wC+kR/fR7bS6KHnjAGvmeb57wbePUX7NWTFqiRJkiR1RWF6DiPilcDfAfcCAZweEb+UUvpCp4OTJEmSpLmu2pgfcw5nMqz0z4EfSyltAoiIM4H/ACwOJUmSJBVerT4/ViudSQZ7JwrD3H3A3g7FI0mSJEk9pdpo0j8PisNpew4j4ifzww0RcQ3wKbI5h68hWylUkiRJkgqv1miyeHAmgzLntsNl8BMtx48BL8yPtwNDHYtIkiRJknpIdZ4MK522OEwpvWk2A5EkSZKkXlRrNOkryGqlpwO/BqxtPT+l9KrOhSVJkiRJvaHWSPN7zmGLz5JtUP/vQLOz4UiSJElSb6nN9wVpWoyllN7X8UgkSZIkqQdlw0qj22Ect5kUh38dEe8EvgSMTzSmlG7sWFSSJEmS1CPm/YI0LZ4JvB54MQeHlab8viRJkiQV2rzf57DFa4AzUkrVTgcjSZIkSb2m1kjzoudwJhncBiztdCCSJEmS1GvqjSaNZqK/CFtZkBWGd0bEep4459CtLCRJkiQV2u4DNQCWDPV1OZLjN5Pi8J0dj0KSJEmSetBIXhwuXVCA4jCldN1sBCJJkiRJvWZktEA9hxGxl2x1UoB+oA/Yn1Ja3MnAJEmSJGmu230gW7dz2YL+Lkdy/GbSc7ho4jgiArgcuKiTQUmSJElSL9i1f/4MKz2qJXVS5rPAyzoUjyRJkiT1jMk5h0MF6DmMiJ9suVsCLgDGOhaRJEmSJPWI3aNVSgGLBmey1ufcNpMMfqLluA48QDa0VJIkSZIKbeRAjSVDfZRK0e1QjttM5hy+aTYCkSRJkqRes2u0xtJ5sBgNHKY4jIg/OMzzUkrpXR2IR5IkSZJ6xshodV5sYwGH7zncP0XbMPBmYAVgcShJkiSp0HYfqLF8eJ73HKaU/nziOCIWAW8F3gRcBfz5dM+TJEmSpKIYGa1x5qqF3Q6jLQ475zAilgO/CfwccCVwfkpp12wEJkmSJElz3a4iDCuNiD8DfhL4IPDMlNK+WYtKkiRJkua4eqPJ3rE6SxfMj+KwdJjHfgt4CvB7wKMRsSe/7Y2IPbMTniRJkiTNTXvG6gAsne89hymlwxWOkiRJklRoI6NVAJbNkwVpLAAlSZIk6RjsGq0BzJs5hxaHkiRJknQMdh/Ieg6XLrDnUJIkSZIKayTvOZwvcw4tDiVJkiTpGEwMK11mz6EkSZIkFdfu0SoRsGjwsNvH9wyLQ0mSJEk6BiMHaiwZ6qNUim6H0hYWh5IkSZJ0DEZGa/NmviFYHEqSJEnSMdk1Wp03K5WCxaEkSZIkHZPdB2osXWDPoSRJkiQVmsNKJUmSJEmMOKxUkiRJkoqt3miyZ6zusFJJkiRJKrI9Y3UAh5VKkiRJUpGNjFYBHFYqSZIkSUU2cqAGwBKHlUqSJElScU32HDqsVJIkSZKKa2Q06zlc5rBSSZIkSSquieLQ1UolSZIkqcBGDtSIgEWDFoeSJEmSVFgjo1UWD/ZRLkW3Q2kbi0NJkiRJOkojozWWzaMhpWBxKEmSJElHbeRAjSXzaDEasDiUJEmSpKM2MlqdV9tYgMWhJEmSJB21kdHavFqpFCwOJUmSJOmojYxW59Ueh2BxKEmSJElHpdFM7Bmrs8RhpZIkSZJUXLsP1AAcVipJkiRJRTYyWgUsDiVJkiSp0EYmew6dcyhJkiRJhbV7NC8OnXMoSZIkScW1a3JYqT2HkiRJklRYI/YcSpIkSZJGDtSIgMUWh5IkSZJUXLtHqywe7KNcim6H0lYWh5IkSZJ0FHaN1ubdNhZgcShJkiRJR2XkQG3ezTcEi0NJkiRJOiq7R6vzbqVSsDiUJEmSpKMycsBhpZIkSZJUeLv2Vx1WKkmSJElF1mgm9ozVWeKwUkmSJEkqrj0HagAsc1ipJEmSJBXXSF4cOudQkiRJkgps12gVgKVDDiuVJEmSpMLaPZr1HC6x53DmIuKUiPh6RNweERsj4q15+/KI+HJE3JN/XZa3R0S8LyI2RcQtEXF+y7XemJ9/T0S8saX9uRFxa/6c90VEdCofSZIkSRo5kPUcLnNBmqNSB34rpXQucBHwlog4F3gH8NWU0jrgq/l9gJcD6/LbFcAHICsmgXcCPwRcCLxzoqDMz/nFludd1sF8JEmSJBXcrv35nEO3spi5lNKWlNKN+fFe4A7gZOBy4Mr8tCuBV+fHlwMfS5nvAUsj4iTgZcCXU0o7U0q7gC8Dl+WPLU4pfS+llICPtVxLkiRJktpuYkGaxRaHxyYi1gLnAdcDq1NKW/KHtgKr8+OTgYdbnrY5bztc++Yp2qd6/SsiYkNEbNi+fftx5SJJkiSpuHaPVlk8WKFcmn8z2jpeHEbEQuBfgf+RUtrT+lje45c6HUNK6YMppQtSShesWrWq0y8nSZIkaZ4aOVBj2fD8m28IHS4OI6KPrDD855TSZ/Lmx/IhoeRft+XtjwCntDx9Td52uPY1U7RLkiRJUkfsGq3Ny/mG0NnVSgP4MHBHSukvWh66GphYcfSNwOda2t+Qr1p6EbA7H356LXBpRCzLF6K5FLg2f2xPRFyUv9YbWq4lSZIkSW23bc8YS+fhSqUAlQ5e+wXA64FbI+IHedvvAu8BPhURbwYeBH46f+wa4BXAJmAUeBNASmlnRLwLWJ+f979TSjvz418B/hEYAr6Q3yRJkiSp7R4ZOcCdW/dy+XOmXOqk53WsOEwpfQuYbpbmJVOcn4C3THOtjwAfmaJ9A/CM4whTkiRJkmbkSxu3AvCyp68+wpm9aVZWK5UkSZKkXvfF27Zy1uqFnLFqYbdD6QiLQ0mSJEk6gh37xln/wE4ue/qJ3Q6lYywOJUmSJOkIvnLHYzQTvOwZFoeSJEmSVFhfvG0rpywf4tyTFnc7lI6xOJQkSZKkw9gzVuPbm3bwsnNPJNtFb36yOJQkSZKkw/j6nduoNppcNo+HlILFoSRJkiQd1rUbt7Jq0QDnn7qs26F0lMWhJEmSJE1jrNbgG3dt59JzV1Mqzd8hpWBxKEmSJEnT+s97Hme02uBl83gLiwkWh5IkSZI0jS/etpXFgxUuOmNFt0PpOItDSZIkSZpCrdHkK3c8xkvOWU1/Zf6XTvM/Q0mSJEk6Bt+/fye7D9Tm9cb3rSwOJUmSJGkK127cymBfiYvXrep2KLPC4lCSJEmSDpFS4ht3becFZ65kqL/c7XBmhcWhJEmSJB3igR2jPLRzlBeeXYxeQ7A4lCRJkqQnue6ubQC88CyLQ0mSJEkqrOvu3s7pK4c5bcVwt0OZNRaHkiRJktRirNbgu/ftKFSvIVgcSpIkSdITfP/+nYzVmhaHkiRJklRk1929nf5KiYvOWNHtUGaVxaEkSZIktbju7u380OnLC7OFxQSLQ0mSJEnKbd41yqZt+wo3pBQsDiVJkiRp0nV3bwfgRQXa33CCxaEkSZIk5a67azsnLx3izFULux3KrLM4lCRJkiSgWm/ynXt38MKzVxER3Q5n1lkcSpIkSRJww4O72DdeL+R8Q7A4lCRJkiQgm29YKQUveOrKbofSFRaHkiRJkgpn6+4xxmqNJ7Rdd/d2Lli7jIUDlS5F1V3FzFqSJElSYd32yG5e9TffohTBWasX8exTlrDuhEXcsWUPb7/sad0Or2ssDiVJkiQVyt9ddy/D/RVe//zTuGXzbv7jli3sGXsYgB97WjHnG4LFoSRJkqQCeXDHfq65dQtXXHwmb8t7CVNKPLhjlF2jVZ524uIuR9g9FoeSJEmSCuND/3k/lVKJN71g7WRbRLB25TBrGe5eYHOAC9JIkiRJKoTH943zqQ0P81/OO5nViwe7Hc6cY3EoSZIkqRA+9p0HqDaaXPHCM7odypxkcShJkiRp3ts/XufK7z7IS89ZzZmrFnY7nDnJ4lCSJEnSvPfJ9Q+z+0CNX37Rmd0OZc6yOJQkSZI0r9UaTT78rfu5cO1yzj91WbfDmbMsDiVJkiTNa5+/5VEeGTnAL7/IuYaHY3EoSZIkad4arzf426/fy1mrF/Kis07odjhzmsWhJEmSpJ70tTsf4//7vzew+0Bt2nP+5Jo7uWfbPt72sqdRKsUsRtd7Kt0OQJIkSZKO1litwe/92208unuMXaNVrvyFCxmolJ9wzhdu3cI/fucBfuEFp/OSc1d3KdLeYc+hJEmSpJ7z8esf4tHdY7z+otP43n07+c1P3UyzmSYff3DHft726Vt49ilLecfLn9bFSHuHPYeSJEmSesr+8Tp/+41N/PCZK3jXq5/BmmVD/MkX7mT1okF+/8fPodpo8paP30gE/M3Pnkd/xT6xmbA4lCRJktRT/vE7D/D4vioffMPZAFxx8Rls3TPGR759PycuGWDzrgPc9sge/uENF3DK8gVdjrZ3WBxKkiRJ6hm7R2v83XX38pJzTpjcszAi+P1Xnsu2veP88TV3AvCLP3o6L3We4VGxOJQkSZLUM/7+m/eyd6zOb1169hPaS6XgL3762YyO16k2mrztMucZHi2LQ0mSJEk9YfvecT767Qf4iWc/hXNOWvykxwcqZT76pgtJKRHhthVHy5mZkiRJknrC+7++iWqjyW+8ZN1hz7MwPDYWh5IkSZLmvDu27OHj1z/Ea567hjNWLex2OPOSxaEkSZKkOe0/btnCf/3Ad1iyoI+3HqHXUMfOOYeSJEmS5qR6o8mffeku/v66+zj/1KV84L89l9WLB7sd1rxlcShJkiSpq9Y/sJPH946zeskgJy4e5IRFA+wdq/Nrn7iJb6ceRqUAABqmSURBVG16nJ/7oVN550883c3sO8ziUJIkSVJX3LV1L+++5g6+eff2J7RHQH+5RAL+9L8+i59+3indCbBgLA4lSZIkzarte8f5iy/fzSfXP8TCgQq/98pzuOiMFWzbO8bW3eNs3TPGyGiVn3ruGp61Zmm3wy0Mi0NJkiRJbZdS4ubNu9nwwE72jtXZO1Zn33iNvWN1vnn3dsbrTd7w/LW89ZJ1LBvuz5+1pKsxF53FoSRJkqS2qDearH9gF9du3Mq1G7eyZffY5GMLByrZbbDCi552Ar/10rPckmKOsTiUJEmSdNy+efd2fvNTN/P4vnEGKiVeeNYqfvvSs7n4rFUsH+6nXHJj+rnO4lCSJEnScfnUhof53c/cylNPWMi7Ln86Lzx7FQv6LTV6je+YJEmSpGOSUuIvv3IP7/vqPfzoupX87c+dz6LBvm6HpWNkcShJkiTpqFXrTd7xmVv4zI2P8JrnruGPf/KZ9JXdh7CXWRxKkiRJmrH943W+euc2rvzOA9zw4C5+86Vn8WsvfioRzinsdRaHkiRJ0jyzedcov/rxm1i2oI83/PBaXrhuFaUZLgizb7zO5l2jVEpBKYJKqUSpBDc/vJvP3/IoX7tzG+P1JicsGuAvf+bZ/Jfz1nQ4G80Wi0NJkiRpHrntkd286R/XM1ZrMNhX5k0fXc9pKxbw+otO4zUXnMKSoSfOCazWm/zg4RG+telxvr3pcW5+eIR6M0157VWLBnjt807hlc96ChectmzGBad6Q6Q09Rs/X11wwQVpw4YN3Q5DkiRJPWjPWI3h/sq02zI0molv3LWN79+/k8VDfSxb0M/y4exrf6X0pM3gzzxhIRevW9W2bR6+ftc23vLPN7JsQT8ffdPzWLtimC9u3MrHvvMAGx7cRX+lxKKBrH9oogrYP15nvN6kFPDMNUt5wZkrOPcpi0kpy6fRTNSbTU5bMczz1i53S4p5ICJuSCldcGi7PYeSJEkqjId2jPK+r93DV+94jHWrF3HeKUs579SlnHfqMlYvHpzyOdv2jPGF27byH7duYf0DO1k61Mcl56zmpeeu5uJ1qxjqL7Nl9wE+uf5hPrn+YbbsHqNSiml73w514uJBfuq5a/jpC07h1BULjjm3T3z/IX7vs7fxtBMX8dGffx4n5Pm86tlP4VXPfgq3PbKbq29+lNFqHYAgK/IG+0pcsHY5F52x4km9iioWew4lSZI07z28c5S/+dom/vXGzZRLwaVPP5GHd46y8dHd1BrZ5+Hlw/2sXNjP8uF+ViwcYMVwP3du3cv6B3aSEpy1eiGXnnsim3eN8rU7t7FnrM5ApcQ5Jy3mls0jJOBH163idReewiXnrKbRTIyM1ti5v8rO/VVqjSaLBissHKywaLCPBX1lvnvfDj614WG+efd2mgkuXLucxUN9jIxW2TVaZddojb1jNQAiglJAOYJyKVjQX2FBf5mh/jKVcombHx7hRWev4v2vO5/hAfuANL3peg4tDiVJktQWzWY64hy03QdqjNUaLB/un3Lbg/F6g+17xxkZrbF4sI/lC/sZ7i8fdiXMlBK7Rmts3T3GY3vH2HOgxr7xOvvH6+wbb7B55yhX3/wopVLwugtP5VdedOZkr9pYrcHtW/Zw00MjbNq2j537x9mxLyvmduyvcuLiQV7+zBN55TNPYt3qRZOvWWs0WX//Tr50+2Pc+NAufuSpK/nZC0/llOXH1vO3ZfcB/vWGzXz+li0A+XDUfpYu6GPxUB8BNBM0U6LZTNSbidFqndFqgwPVBqPVBs86ZQn/89KzqbidhI7A4jBncShJktQ++8frfP6WR/n49x/m9kd3c9EZK3jJOat5ybmrOXnpEADb9o7xpY2P8YXbtvC9+3bSyIdbLl3Qx4rhfpYt6GfvWJ1te8fYNVp70mv0V0qsGO5n8WAfh9aI+6t1HtszTrXenDbGRQMVXn3eyfzKj53JSUuG2pe81KMsDnMWh5IkaS5rNhNb94xx/+P7ue/x/ew5UOOMlcOsW72Q01YMd2yT8Z37q9y3fR/3bd/P1j1j7BqtTg6J3H2gxrIFfZy2YphTly/gtBULWDhQ4eqbH+VzP3iUfeN11p2wkIvOWMG3732c+7bvB+DpT1nMcH+F9Q9mwzLPWDnMy595IictGWLHvio7WnrpFg5WWL14gBMWDbJ68QBLhvrZO3ZwSOaO/dXJ4ZWtH18X9JdZvWSQExcPsjq/LV3Qx8KBCsMDFRb0lV1RUzqEC9JIkiTNQfvH61x//w7+857Huf6+ndz/+H4O1BpTnlspBWtXZgXa5CqYec9bQDbEsNZgrNY4eJx/Ha02GK83CIJSCUp5F9y+8Tr3P76fkUN67BYNVFiar7K5ZKiPx/aMs/6BXewbr0+eM1Ap8cpnncTrLjyV5562bHLo573b9/GV2x/jK3c8xr7xOr/+4nW84pkncdbqhW6ULs1h9hxKkiR12Gi1zuN7q2zfN86OfeM8vq/K1j1jXH/fDm58aBe1RspWjDxtOWefuIjTVw5zxqphzli5kEWDFe7bvp97tu1l07Z93P3YPrbsPsCu/dliJVMVkhEw1FdmQX+ZwfzrUF+ZgUoZyOetpUQzZeedvmqYM1YOc+YJCzlz5UJOXDJIf+XJPZQpJXbsr/LgjlG27x3n+WesYMkCV7eUeo09h5IkqdBSSozXm4zVJnrXmhzIe9XGaw0aKVEuBZVSiXIpWw1y31idHfvH2ZUPbdw1enChk/3VbLGTar3JQKXEYF+Jwb6sCKs3E4/vG+fxfdmwydHq1D2BT3/KYn7hR07n4nWreO5pyxjsK0953jPXLOGZa5ZM+diBaoNdo1UgK/SG+ssMVEod6aGLCFYuHGDlwoG2X1tS91kcSpKkaVXrTUZGqySy3qggX0q/FAxUyvRXSlNuiN1oJqr1JtVGk2q9Sa2R3cbrTR7fN862PeM8tmeMx/aMs3P/OH3lEkN579ZEgTRRXD2+r8r2veOM1xsMVLLCZ6CvRH+5RDPBeL3JeL3BeC37euigqGZKjNWajE3x2NGIgCVD2Vy2iflsiwcrDFTKk6+/c3+VsVqDUl5EnXbqAlYuHGDFwgFWLuxn5aIBVg4PsHJRPyuGB6bsnTtaQ/1lhvpdZEXS8ev54jAiLgP+GigDH0opvafLIUmSdNyazcT+ap1mEwb7s0JooidorNbgnsf2cefWPdy1dS/3bt9HI0FfKaiUg0o5O79Syo77yllvGDDZS3Ygv02sGjmxfxrA3rE6O/aN5wuA1KeMr1VfOSsUI5gsBGe49zcL+sssH+6n3kiTc+XG81Unly3oY9WirJfqvFOXMlgpU220FoJNSqXIisVKqaVYfeJrBMFQf5nBSonBlgJ04utEj1+5FDSaiUa+TUCj2WThQB/Lh/tYPjzAkqG+KQthSZovero4jIgy8H7gpcBmYH1EXJ1Sur27kUnS3NVspqw3J+/RmbzlPTsDlWx+0sTGyn2lEgdqDfZX6xyoNtg/ni1qcSQz7aCZWU9OYrzWZH+1Mbmv12i1QTmgv1KmrxyTPVhjtWZW+FTrHKhmhUTr3mCNlEgp69manHfVzJbKH2qZn9VfLjFarbNnrM7u0Rp7xrLhhFkv1cT3rUEC+ssl+isl+vKiLJGoNRK1ll6zUrQWalmxlsjmfDWaaXLI496xOnvHauwdrz/he1MuRV7MlNi5vzpZfA1USpyxaiH95aDWSNSbTeqNRG3ia0sbkBVF/aXJwqhcClLK3698DtrioQrPXLOUFcP9rBjuZ+lwP+UImimLMwH1RvZzNNFbN15v0mgmBiot34vJ70k8oW35cP/kqpILp9ioe+L74V5tkjS7ero4BC4ENqWU7gOIiKuAy4GeKQ7Hag1ufGjXkx+Y5sPSdJ+hpvtwlaZ4xvTnTnftqR+Z9vPcEWKZuFzrZScOW1/rYNuTLzx5jSedM3XOkP3lGJjcH6n1b78Hp2W0tqYnvE724elgDq2vcuj3aKp4ntD2pPifnDfT5BQRlCObCzOxNHftkCFbE+eVIluNrhQT9w+2RWRxNFP2YbmZsjiyD8xZATGxWMHEh+pmgkZKBEzOxylFTP6VfuL70kwt37vJa2d5pPx1Wr+nE6+RJl+XyQ+hk6+ff4BvTMaSaDwhtuz5E9+Xie9RM6X8A3KTejP72mqiJ6ZWP/gBd7zepN5o0lfOPsz2VbIPt0E8aYgcQKmUfdgvl7Iei2aTvOehmfc+JCqloK9cmiwOSnEwtolzU/4TOPHeBdn7FBF5+8H3buK8ifdxYh7VxK2WFwKtP9oTw/zqM+3SmScO/qxCeeL3oHRwaOTE97vWyDaUHqs98WdkqK/M4qEKiwf7WDhYYaBSYslQH/3lrLeKyH5+WovtUgSDfSUWD1Ymf46a6YmFWr3ZpBSlyfd0Yq7b4sEKi4f6WDSYvWapFNkcuerB3r5VCwc4+8RFnH3iItauGJ53vVlZPvMrJ0nqBb1eHJ4MPNxyfzPwQ12K5Zhs3zvO6/7h+m6HIc261gKnnH84PrR4LeXFZ8BkwVdqaZ/4wD9xDFkBNFFANlLKemvywqyvXMqLgSyG1iJ/oFxieKDC8uFsaFq5FNSbTar1NFkINlNicX/fZC9IpVQiAurN7PUmCsGJ1yznPUTliMnCdKI4mCgYy6VS/jX/hrQU0RMFb1ZIpycV3xPtACsr5clhc0P95ckhhHDwepVS5IVu1nvzhB6eyV6eYLzenOyZG6tli20s6C+zYKDCcH+ZBf2VyaLoiO/zjH8ejnzmQKXEcH+FBQNlhvsrDPWVaaZ0sChrZN/XySGD+ffjaHufms2Di5YsGDi4uqMkSfNdrxeHMxIRVwBXAJx66qldjuaJVi0a4KorLprysek+Kk33IWq6z1ZTNU//Oez4r52dP811DrletFxhqqccel7rOYd97JDrHNrD2NoTN3VPZmrptTn4OhMFzcHXe3L8Bx9veWyKHKeK+0nXOuQaiYM9ehND4gD6y2X6KgcLoFIw2RPY2vOWJo+z65RKrb2LLcctvSqTvRpx8Bhai7CsN28ixtZzJnu6eGLvl3tcaS4rlSJf4MOiUJJULL1eHD4CnNJyf03e9gQppQ8CH4Rsn8PZCW1mBvvKXHTGim6HIR21Sjl6/h8QSZIkHdTrM73XA+si4vSI6AdeC1zd5ZgkSZIkqef09B/+U0r1iPhV4FqyrSw+klLa2OWwJEmSJKnn9HRxCJBSuga4pttxSJIkSVIv6/VhpZIkSZKkNrA4lCRJkiRZHEqSJEmSLA4lSZIkSVgcSpIkSZKwOJQkSZIkYXEoSZIkScLiUJIkSZKExaEkSZIkCYtDSZIkSRIWh5IkSZIkLA4lSZIkSUCklLodw6yKiO3Ag92OY55ZCTze7SBmQRHyLEKOUIw8i5AjFCNPc5w/ipBnEXKEYuRZhByhOHke6rSU0qpDGwtXHKr9ImJDSumCbsfRaUXIswg5QjHyLEKOUIw8zXH+KEKeRcgRipFnEXKE4uQ5Uw4rlSRJkiRZHEqSJEmSLA7VHh/sdgCzpAh5FiFHKEaeRcgRipGnOc4fRcizCDlCMfIsQo5QnDxnxDmHkiRJkiR7DiVJkiRJFoeSJEmSJCwONY2IOCUivh4Rt0fExoh4a96+PCK+HBH35F+X5e0REe+LiE0RcUtEnN9yrfdGxG357We6ldOhjiHHp0XEdyNiPCJ++5BrXRYRd+X5v6Mb+UylzTl+JCK2RcRt3cjlcNqV53TXmQvamONgRHw/Im7Or/OH3cppKu38mc0fL0fETRHx+dnOZTpt/r18ICJujYgfRMSGbuQznTbnuTQiPh0Rd0bEHRHx/G7kdKg2/l6enb+HE7c9EfE/upVXqza/j7+RX+O2iPhERAx2I6eptDnPt+Y5bpwr7yMcU44/F9lnulsj4jsR8eyWa83Jzz3Q9jzn7GefjkkpefP2pBtwEnB+frwIuBs4F/hT4B15+zuA9+bHrwC+AARwEXB93v5K4MtABRgG1gOLu53fMeZ4AvA84N3Ab7dcpwzcC5wB9AM3A+d2O7925pg/djFwPnBbt/Pq4Hs55XW6nV+bcwxgYX7cB1wPXNTt/DrxM5s//pvAx4HPdzu3TuQIPACs7HZOs5DnlcB/z4/7gaXdzq8TP6/5OWVgK9kG1fMmR+Bk4H5gKL//KeDnu51fB/J8BnAbsIDss89XgKd2O79jzPGHgWX58cs5+Nluzn7uaWee+f05+9mnUzd7DjWllNKWlNKN+fFe4A6yf9gvJ/tPmvzrq/Pjy4GPpcz3gKURcRLZL+M3U0r1lNJ+4BbgsllMZVpHm2NKaVtKaT1QO+RSFwKbUkr3pZSqwFX5NbqujTmSUvomsHM24j5a7crzMNfpujbmmFJK+/K7ffltzqxM1s6f2YhYQ/YHqg/NQugz1s4c57J25RkRS8g+oH04P6+aUhqZlSSOoEPv5SXAvSmlBzsW+FFoc44VYCgiKmTF06MdDn/G2pjnOWTFxWhKqQ5cB/zkLKRwRMeQ43dSSrvy9u8Ba/LjOfu5B9qa55z+7NMpFoc6oohYC5xH1sOwOqW0JX9oK7A6Pz4ZeLjlaZvztpuByyJiQUSsBH4MOGUWwj4qM8xxOtPlPqccZ449o115HnKdOeV4c4xsqOUPgG3Al1NKcy5HaMt7+VfA24BmJ+JrhzbkmIAvRcQNEXFFR4Jsg+PM83RgO/DRyIYIfygihjsV67Fq47+xrwU+0dbg2uR4ckwpPQL8H+AhYAuwO6X0pY4FexyO8728DfjRiFgREQvIRlbNh889byYbIQY98rkHjjvPQrI41GFFxELgX4H/kVLa0/pYSilxhB6H/B/+a4DvkP1n912g0Zloj83x5tgLipAjtC/Pw12n29qRY0qpkVJ6DtlfRy+MiGd0JNjjcLx5RsSPA9tSSjd0Lsrj06af1x9JKZ1PNhTqLRFxcfsjPT5tyLNCNqzrAyml84D9ZEPC5ow2/tvTD7wK+Je2B3mc2vA7uYys5+Z04CnAcET8tw6Fe8za8LnnDuC9wJeALwI/oMc/90TEj5EVTW+ftSDboCh5tpvFoaYVEX1kv1T/nFL6TN78WD5clPzrtrz9EZ74l7E1eRsppXenlJ6TUnop2Xynu2cj/pk4yhynM23uc0Gbcpzz2pXnNNeZE9r9XuZD877OHBnqPaFNeb4AeFVEPEA25OnFEfF/OxTyUWvXe5n3xpBS2gb8G9lwrzmjTXluBja39HB/mqxYnBPa/Hv5cuDGlNJj7Y/02LUpx5cA96eUtqeUasBnyOZ6zRlt/L38cErpuSmli4Fd9PDnnoh4FtnQ/MtTSjvy5jn9uQfalmchWRxqShERZPM77kgp/UXLQ1cDb8yP3wh8rqX9DZG5iGy4yJZ8+NqK/JrPAp5F9te0rjuGHKezHlgXEafnf/V9bX6NrmtjjnNau/I8zHW6ro05roqIpfnxEPBS4M72R3xs2pVnSul3UkprUkpryX4nv5ZSmhO9FG18L4cjYtHEMXAp2ZC2OaGN7+VW4OGIODtvugS4vc3hHpMO/Bv7s8yxIaVtzPEh4KLIppkE2ft4R7vjPVbtfC8j4oT866lk8w0/3t5oj83R5pjH/xng9Sml1gJ3zn7ugbbmWUxpDqyK423u3YAfIetuv4VsSMQPyMbNrwC+CtxDtgLX8vz8AN5PtnrVrcAFefsg2X/it5NN8n1Ot3M7jhxPJPsL9h5gJD9enD/2CrK/DN4L/K9u59ahHD9BNk+klre/udv5tTvP6a7T7fzanOOzgJvy69wG/EG3c+vUz2zLNV/E3FqttF3v5Rlk87pvBjbOpX972v1eAs8BNuTX+iz5yoLdvrU5x2FgB7Ck23l1MMc/JPtj1G3APwED3c6vQ3n+J9nnnpuBS7qd23Hk+CGyns+Jcze0XGtOfu7pQJ5z9rNPp26RJy5JkiRJKjCHlUqSJEmSLA4lSZIkSRaHkiRJkiQsDiVJkiRJWBxKkiRJkrA4lCQVTETsO87nr42I17Urnimuf9j4ImJpRPzKMVz3/4+I3z72yCRJ853FoSRJR2ct0LHicAaWAkddHEqSdCQWh5KkwouIn4iI6yPipoj4SkSszttfGBE/yG83RcQi4D3Aj+Ztv3HIdV4UEZ9vuf83EfHz+fEDEfGnEXFrRHw/Ip6at58eEd/N2/+o5bkLI+KrEXFj/tjl+UPvAc7MX//P8nP/Z0Ssj4hbIuIPW67xvyLi7oj4FnB2J753kqT5o9LtACRJmgO+BVyUUkoR8d+BtwG/Bfw28JaU0rcjYiH8v/btIMSqMgzj+P/Z2ChFCmrEBBEZmCQMGG0EIYJZTVC00FAKJHDhohaFLV2lbqxQcCVEMsYkGE2LoY0LYRhBnWGaCdplaIukiEplovR1cb+B63SniUlwMf8fHM57vnM438fdHB6+9zIHvA+8W1VDy5jnt6ramuQN4CNgCPgYOFFVnybZ3/XsHPBqVf2eZD1wIclom/+5qhoASDIIPAO8AAQYTbIDuAnsAgbofO8ngcvLWLMkaYUwHEqSBE8AI0keB1YB37fxceBokmHgbFVdS/J/5vms6/xhq7cDr7X6FHCk1QE+aEHvDtAPPNbjnYPtmGrXD9MJi48AX1TVLYAWLCVJWpRtpZIkwTHgeFVtBfYBfQBVdRh4C1gNjCfZvMR7/ubeb2vfgvv1H+p5u4ENwLa2S/hTj/dBJ0QeqqqBdmyqqpNLrFOSpH8wHEqSBI8CP7b6zfnBJE9X1UxVHQEuApuBP+jsyvXyA7AlyUNJ1gIvLbi/s+s80epxOu2f0AmE3Wu6XlV/JXkReLKNL5z/a2Bva3slSX+SjcB54JUkq9t/JV/+119AkrTi2VYqSVpp1iS51nV9FDgInEnyK3AOeKrde6cFszvAt8BYq28nmQY+qar59lCq6mqSz4FZOq2pU9xrXZJvgD+B19vY28DpJAeAL7ueHQa+SjIDXAK+a3P8kmQ8ySwwVlXvJXkWmGgtrzeAPVU1mWQEmAau0wm3kiQtKlW9OlkkSdL9lOQK8HxV/fyg1yJJUi+2lUqSJEmS3DmUJEmSJLlzKEmSJEnCcChJkiRJwnAoSZIkScJwKEmSJEnCcChJkiRJAu4CurlU6ssKTo8AAAAASUVORK5CYII=",
            "text/plain": [
              "<Figure size 1080x504 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "repos_with_month_df = repos_df.withColumn(\"year\", year(\"updated\")).withColumn(\"month\", month(\"updated\"))\n",
        "number_of_repos_per_month = repos_with_month_df.groupBy(\"year\", \"month\").count().orderBy(\"year\", \"month\")\n",
        "\n",
        "number_of_repos_per_month = number_of_repos_per_month.withColumn(\"year month\", concat(col(\"year\"), lit(\" \"), col(\"month\")))\n",
        "data = number_of_repos_per_month.toPandas()\n",
        "\n",
        "plt.figure(figsize=(15, 7))\n",
        "sns.lineplot(data=data, x=\"year month\", y=\"count\")\n",
        "starting_year, starting_month = data.iloc[0][\"year\"], data.iloc[0][\"month\"]\n",
        "ending_year, ending_month = data.iloc[-1][\"year\"], data.iloc[-1][\"month\"]\n",
        "\n",
        "prev_ticks, prev_labels = plt.xticks()\n",
        "\n",
        "ticks = []\n",
        "for i in range(0, (ending_year - starting_year) * 12 + 1, 12):\n",
        "  value = i + (13 - starting_month)\n",
        "  if value > 0 and value < prev_ticks[-1]:\n",
        "    ticks.append(value)\n",
        "\n",
        "plt.xticks(ticks, labels=range(2009, 2022))\n",
        "plt.xlabel(\"Last updated\")\n",
        "plt.ylabel(\"Number of repos\")\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wDqGPwWWjeRv"
      },
      "source": [
        "As we can see the repos we collected are very frequently updated: the majority of them have the last commit made in the last year or two."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y3T0yKVfUFuC"
      },
      "source": [
        "## What is the distribution of stars?"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "a-D-44EJkH5u"
      },
      "source": [
        "In the following plot we show the distribution of stars, meaning how many repos have collected a certain number of stars:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        },
        "id": "pgawxye_UGUI",
        "outputId": "06de8cec-7d39-49ac-9408-be869c8bc44e"
      },
      "outputs": [
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAABDAAAAHwCAYAAABQRJ8FAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de9RldXkn+O9DlRhiRFAJCymMqJWkiRlRCSEmsY0mWhhbiMsLdEZopAU7mJhOMi06SUzH2NFOGzuuMUQUWswyXIJRSBpFmkbMjINSisPFS1PiBaoRSkEx2qLIM3+c/eqhqHrrLajznn2Kz2ets969n317jmt7Vq0ve/9+1d0BAAAAGLM95t0AAAAAwI4IMAAAAIDRE2AAAAAAoyfAAAAAAEZPgAEAAACM3tp5NzAWGzZs6A984APzbgMAAAAe6GpbRU9gDL7yla/MuwUAAABgOwQYAAAAwOgJMAAAAIDRE2AAAAAAoyfAAAAAAEZPgAEAAACMngADAAAAGD0BBgAAADB6AgwAAABg9AQYAAAAwOgJMAAAAIDRE2AAAAAAoyfAAAAAAEZPgAEAAACMngADAAAAGD0BBgAAADB6AgwAAABg9AQYAAAAwOgJMAAAAIDRE2AAAAAAoyfAAAAAAEZPgLHgDjzo0amqhfoceNCj5/0/GwAAAAtm7bwb4P75nzfdmBe/7SPzbmOnnHvyU+fdAgAAAAvGExgAAADA6AkwAAAAgNETYAAAAACjJ8AAAAAARk+AAQAAAIzezAKMqjqzqm6tqmunaudW1SeHzxeq6pND/TFV9b+mtv3V1DFPqaprqmpTVb2lqmqoP7yqLqmq64e/+w71GvbbVFVXV9WTZ/UdAQAAgNUxyycw3plkw3Shu1/c3Yd296FJ3pPk76Y2f25pW3e/fKp+WpKXJVk/fJbOeWqSS7t7fZJLh/UkOXJq35OG4wEAAIAFNrMAo7s/nOS2bW0bnqJ4UZKzlztHVR2QZO/uvqK7O8m7khw9bD4qyVnD8llb1d/VE1ck2Wc4DwAAALCg5jUGxi8muaW7r5+qHVxVV1XV5VX1i0PtwCQ3Te1z01BLkv27++Zh+ctJ9p865sbtHHMPVXVSVW2sqo1btmy5H18HAAAAmKV5BRjH5p5PX9yc5NHd/aQkv5Pkb6pq75WebHg6o3e2ie4+vbsP6+7D9ttvv509HAAAAFgla1f7glW1Nsnzkzxlqdbddya5c1j+eFV9LsmPJ9mcZN3U4euGWpLcUlUHdPfNwysitw71zUkO2s4xAAAAwAKaxxMYv5zkM939/VdDqmq/qlozLD82kwE4bxheEbmjqo4Yxs04LskFw2EXJjl+WD5+q/pxw2wkRyT5+tSrJgAAAMACmuU0qmcn+X+T/ERV3VRVJw6bjsm9B+98WpKrh2lVz0/y8u5eGgD0N5K8I8mmJJ9L8v6h/oYkv1JV12cSirxhqF+U5IZh/7cPxwMAAAALbGavkHT3sdup/6tt1N6TybSq29p/Y5InbKP+1STP3Ea9k5yyk+0CAAAAIzavQTwBAAAAVkyAAQAAAIyeAAMAAAAYPQEGAAAAMHoCDAAAAGD0BBgAAADA6AkwAAAAgNETYAAAAACjJ8AAAAAARk+AAQAAAIyeAAMAAAAYPQEGAAAAMHoCDAAAAGD0BBgAAADA6AkwAAAAgNETYAAAAACjJ8AAAAAARk+AAQAAAIyeAAMAAAAYPQEGAAAAMHoCDAAAAGD0BBgAAADA6AkwAAAAgNETYAAAAACjJ8AAAAAARk+AAQAAAIyeAAMAAAAYPQEGAAAAMHoCDAAAAGD0BBgAAADA6AkwAAAAgNETYAAAAACjJ8AAAAAARk+AAQAAAIyeAAMAAAAYPQEGAAAAMHoCDAAAAGD0BBgAAADA6AkwAAAAgNETYAAAAACjJ8AAAAAARk+AAQAAAIyeAAMAAAAYPQEGAAAAMHoCDAAAAGD0BBgAAADA6AkwAAAAgNETYAAAAACjJ8AAAAAARk+AAQAAAIyeAAMAAAAYvZkFGFV1ZlXdWlXXTtX+qKo2V9Unh89zpra9uqo2VdVnq+rZU/UNQ21TVZ06VT+4qj461M+tqj2H+oOH9U3D9sfM6jsCAAAAq2OWT2C8M8mGbdTf3N2HDp+LkqSqDklyTJKfGo75y6paU1Vrkrw1yZFJDkly7LBvkrxxONfjk9ye5MShfmKS24f6m4f9AAAAgAU2swCjuz+c5LYV7n5UknO6+87u/nySTUkOHz6buvuG7v5OknOSHFVVleQZSc4fjj8rydFT5zprWD4/yTOH/QEAAIAFNY8xMF5RVVcPr5jsO9QOTHLj1D43DbXt1R+R5GvdfddW9Xuca9j+9WH/e6mqk6pqY1Vt3LJly/3/ZgAAAMBMrHaAcVqSxyU5NMnNSd60yte/h+4+vbsP6+7D9ttvv3m2AgAAACxjVQOM7r6lu7/X3XcneXsmr4gkyeYkB03tum6oba/+1ST7VNXarer3ONew/WHD/gAAAMCCWtUAo6oOmFr9tSRLM5RcmOSYYQaRg5OsT/KxJFcmWT/MOLJnJgN9XtjdneSyJC8Yjj8+yQVT5zp+WH5Bkv8+7A8AAAAsqLU73uW+qaqzkzw9ySOr6qYkr03y9Ko6NEkn+UKSk5Oku6+rqvOSfCrJXUlO6e7vDed5RZKLk6xJcmZ3Xzdc4lVJzqmqP0lyVZIzhvoZSf66qjZlMojoMbP6jgAAAMDqmFmA0d3HbqN8xjZqS/u/Psnrt1G/KMlF26jfkB+8gjJd/3aSF+5UswAAAMCozWMWEgAAAICdIsAAAAAARk+AAQAAAIyeAAMAAAAYPQEGAAAAMHoCDAAAAGD0BBgAAADA6AkwAAAAgNETYAAAAACjJ8AAAAAARk+AAQAAAIyeAAMAAAAYPQEGAAAAMHoCDAAAAGD0BBgAAADA6AkwAAAAgNETYAAAAACjJ8AAAAAARk+AAQAAAIyeAAMAAAAYPQEGAAAAMHoCDAAAAGD0BBgAAADA6AkwAAAAgNETYAAAAACjJ8AAAAAARk+AAQAAAIyeAAMAAAAYPQEGAAAAMHoCDAAAAGD0BBgAAADA6AkwAAAAgNETYAAAAACjJ8AAAAAARk+AAQAAAIyeAAMAAAAYPQEGAAAAMHoCDAAAAGD0BBgAAADA6AkwAAAAgNETYAAAAACjJ8AAAAAARk+AAQAAAIyeAAMAAAAYPQEGAAAAMHoCDAAAAGD0BBgAAADA6AkwAAAAgNETYAAAAACjJ8AAAAAARk+AAQAAAIzezAKMqjqzqm6tqmunan9WVZ+pqqur6r1Vtc9Qf0xV/a+q+uTw+aupY55SVddU1aaqektV1VB/eFVdUlXXD3/3Heo17LdpuM6TZ/UdAQAAgNUxyycw3plkw1a1S5I8obv/tyT/I8mrp7Z9rrsPHT4vn6qfluRlSdYPn6Vznprk0u5en+TSYT1Jjpza96TheAAAAGCBzSzA6O4PJ7ltq9oHu/uuYfWKJOuWO0dVHZBk7+6+ors7ybuSHD1sPirJWcPyWVvV39UTVyTZZzgPAAAAsKDmOQbGS5O8f2r94Kq6qqour6pfHGoHJrlpap+bhlqS7N/dNw/LX06y/9QxN27nmHuoqpOqamNVbdyyZcv9+CoAAADALM0lwKiq/zPJXUnePZRuTvLo7n5Skt9J8jdVtfdKzzc8ndE720d3n97dh3X3Yfvtt9/OHg4AAACskrWrfcGq+ldJnpvkmUPwkO6+M8mdw/LHq+pzSX48yebc8zWTdUMtSW6pqgO6++bhFZFbh/rmJAdt5xgAAABgAa3qExhVtSHJv0vyvO7+1lR9v6paMyw/NpMBOG8YXhG5o6qOGGYfOS7JBcNhFyY5flg+fqv6ccNsJEck+frUqyYAAADAAprZExhVdXaSpyd5ZFXdlOS1mcw68uAklwyzoV4xzDjytCR/XFXfTXJ3kpd399IAoL+RyYwme2UyZsbSuBlvSHJeVZ2Y5ItJXjTUL0rynCSbknwryQmz+o4AAADA6phZgNHdx26jfMZ29n1PkvdsZ9vGJE/YRv2rSZ65jXonOWWnmgUAAABGbZ6zkAAAAACsiAADAAAAGD0BBgAAADB6AgwAAABg9AQYAAAAwOgJMAAAAIDRE2AAAAAAoyfAAAAAAEZPgAEAAACMngADAAAAGD0BBgAAADB6AgwAAABg9AQYAAAAwOgJMAAAAIDRE2AAAAAAoyfAAAAAAEZPgAEAAACMngADAAAAGD0BBgAAADB6AgwAAABg9AQYAAAAwOgJMAAAAIDRE2AAAAAAoyfAAAAAAEZPgAEAAACMngADAAAAGD0BBgAAADB6AgwAAABg9AQYAAAAwOjtMMCoqldW1d41cUZVfaKqnrUazQEAAAAkK3sC46XdfUeSZyXZN8lLkrxhpl0BAAAATFlJgFHD3+ck+evuvm6qBgAAADBzKwkwPl5VH8wkwLi4qh6a5O7ZtgUAAADwA2tXsM+JSQ5NckN3f6uqHpHkhNm2BQAAAPADOwwwuvvuqlqX5F9WVZJc3t1/P/POAAAAAAYrmYXkDUlemeRTw+e3quo/zLoxAAAAgCUreYXkOUkO7e67k6SqzkpyVZLXzLIxAAAAgCUrGcQzSfaZWn7YLBoBAAAA2J6VPIHxp0muqqrLMpk+9WlJTp1pVwAAAABTVjKI59lV9aEkP5Okk7yqu78868YAAAAAlqzkCYwk+bkkv5BJgLE2yXtn1hEAAADAVlYyC8lfJnl5kmuSXJvk5Kp666wbAwAAAFiykicwnpHkn3V3J9+fheS6mXYFAAAAMGUls5BsSvLoqfWDhhoAAADAqljJExgPTfLpqvpYJmNgHJ5kY1VdmCTd/bwZ9gcAAACwogDjD2feBQAAAMAyVjKN6uVV9WNJ1nf3f6uqvZKs7e5vzL49AAAAgJXNQvKyJOcnedtQWpfkfbNsCgAAAGDaSgbxPCXJzye5I0m6+/okPzrLpgAAAACmrSTAuLO7v7O0UlVrMxnMEwAAAGBVrCTAuLyqXpNkr6r6lSR/m+TvZ9sWAAAAwA+sJMB4VZItSa5JcnKSi5L8/kpOXlVnVtWtVXXtVO3hVXVJVV0//N13qFdVvaWqNlXV1VX15Kljjh/2v76qjp+qP6WqrhmOeUtV1XLXAAAAABbTsgFGVa1J8unufnt3v7C7XzAsr/QVkncm2bBV7dQkl3b3+iSXDutJcmSS9cPnpCSnDT08PMlrk/xsksOTvHYqkDgtycumjtuwg2sAAAAAC2jZAKO7v5fks1X16Pty8u7+cJLbtiofleSsYfmsJEdP1d/VE1ck2aeqDkjy7CSXdPdt3X17kkuSbBi27d3dVwyByru2Ote2rgEAAAAsoLUr2GffJNdV1ceSfHOp2N3Pu4/X3L+7bx6Wv5xk/2H5wCQ3Tu1301Bbrn7TNurLXeMequqkTJ72yKMffZ8yGgAAAGAVrCTA+INZXby7u6pmOqPJctfo7tOTnJ4khx12mJlVAAAAYKR2GGB09+W7+Jq3VNUB3X3z8BrIrUN9c5KDpvZbN9Q2J3n6VvUPDfV129h/uWsAAAAAC2gls5DsahcmWZpJ5PgkF0zVjxtmIzkiydeH10AuTvKsqtp3GLzzWUkuHrbdUVVHDLOPHLfVubZ1DQAAAGABreQVkvusqs7O5OmJR1bVTZnMJvKGJOdV1YlJvpjkRcPuFyV5TpJNSb6V5IQk6e7bqup1Sa4c9vvj7l4aGPQ3MpnpZK8k7x8+WeYaAAAAwALaboBRVZd29zOr6o3d/ar7cvLuPnY7m565jX07ySnbOc+ZSc7cRn1jkidso/7VbV0DAAAAWEzLPYFxQFU9NcnzquqcJDW9sbs/MdPOAAAAAAbLBRh/mMkMJOuS/PlW2zrJM2bVFAAAAMC07QYY3X1+kvOr6g+6+3Wr2BMAAADAPaxkGtXXVdXzkjxtKH2ou/9htm0BAAAA/MAOp1Gtqj9N8soknxo+r6yq/zDrxgAAAACWrGQa1V9Ncmh3350kVXVWkquSvGaWjQEAAAAs2eETGIN9ppYfNotGAAAAALZnJU9g/GmSq6rqskymUn1aklNn2hUAAADAlJUM4nl2VX0oyc8MpVd195dn2hUAAADAlJU8gZHuvjnJhTPuBQAAAGCbVjoGBgAAAMDcCDAAAACA0Vs2wKiqNVX1mdVqBgAAAGBblg0wuvt7ST5bVY9epX4AAAAA7mUlg3jum+S6qvpYkm8uFbv7eTPrCgAAAGDKSgKMP5h5FwAAAADL2GGA0d2XV9WPJVnf3f+tqn44yZrZtwYAAAAwscNZSKrqZUnOT/K2oXRgkvfNsikAAACAaSuZRvWUJD+f5I4k6e7rk/zoLJsCAAAAmLaSAOPO7v7O0kpVrU3Ss2sJAAAA4J5WEmBcXlWvSbJXVf1Kkr9N8vezbQsAAADgB1YSYJyaZEuSa5KcnOSiJL8/y6YAAAAApq1kFpK7q+qsJB/N5NWRz3a3V0gAAACAVbPDAKOqfjXJXyX5XJJKcnBVndzd7591cwAAAADJCgKMJG9K8kvdvSlJqupxSf5rEgEGAAAAsCpWMgbGN5bCi8ENSb4xo34AAAAA7mW7T2BU1fOHxY1VdVGS8zIZA+OFSa5chd4AAAAAkiz/Csm/mFq+Jck/H5a3JNlrZh0BAAAAbGW7AUZ3n7CajQAAAABsz0pmITk4yW8mecz0/t39vNm1BQAAAPADK5mF5H1Jzkjy90nunm07AAAAAPe2kgDj2939lpl3AgAAALAdKwkw/qKqXpvkg0nuXCp29ydm1hUAAADAlJUEGD+d5CVJnpEfvELSwzoAAADAzK0kwHhhksd293dm3QwAAADAtuyxgn2uTbLPrBsBAAAA2J6VPIGxT5LPVNWVuecYGKZRBQAAAFbFSgKM1868CwAAAIBl7DDA6O7LV6MRAAAAgO3ZYYBRVd/IZNaRJNkzyYOSfLO7955lYwAAAABLVvIExkOXlquqkhyV5IhZNgUAAAAwbSWzkHxfT7wvybNn1A8AAADAvazkFZLnT63ukeSwJN+eWUcAAAAAW1nJLCT/Ymr5riRfyOQ1EgAAAIBVsZIxME5YjUYAAAAAtme7AUZV/eEyx3V3v24G/QAAAADcy3JPYHxzG7WHJDkxySOSCDAAAACAVbHdAKO737S0XFUPTfLKJCckOSfJm7Z3HAAAAMCutuwYGFX18CS/k+TXk5yV5MndfftqNAYAAACwZLkxMP4syfOTnJ7kp7v7n1atKwAAAIApeyyz7XeTPCrJ7yf5n1V1x/D5RlXdsTrtAQAAACwTYHT3Ht29V3c/tLv3nvo8tLv3vq8XrKqfqKpPTn3uqKrfrqo/qqrNU/XnTB3z6qraVFWfrapnT9U3DLVNVXXqVP3gqvroUD+3qva8r/0CAAAA87fcExgz0d2f7e5Du/vQJE9J8q0k7x02v3lpW3dflCRVdUiSY5L8VJINSf6yqtZU1Zokb01yZJJDkhw77JskbxzO9fgkt2cycwoAAACwoFY9wNjKM5N8rru/uMw+RyU5p7vv7O7PJ9mU5PDhs6m7b+ju72QyO8pRVVVJnpHk/OH4s5IcPbNvAAAAAMzcvAOMY5KcPbX+iqq6uqrOrKp9h9qBSW6c2uemoba9+iOSfK2779qqfi9VdVJVbayqjVu2bLn/3wYAAACYibkFGMO4FM9L8rdD6bQkj0tyaJKbk7xp1j109+ndfVh3H7bffvvN+nIAAADAfbTdaVRXwZFJPtHdtyTJ0t8kqaq3J/mHYXVzkoOmjls31LKd+leT7FNVa4enMKb3BwAAABbQPF8hOTZTr49U1QFT234tybXD8oVJjqmqB1fVwUnWJ/lYkiuTrB9mHNkzk9dRLuzuTnJZkhcMxx+f5IKZfhMAAABgpubyBEZVPSTJryQ5ear8H6vq0CSd5AtL27r7uqo6L8mnktyV5JTu/t5wnlckuTjJmiRndvd1w7leleScqvqTJFclOWPmXwoAAACYmbkEGN39zUwG25yuvWSZ/V+f5PXbqF+U5KJt1G/IZJYSAAAAYDcw71lIAAAAAHZIgAEAAACMngADAAAAGD0BBgAAADB6AgwAAABg9AQYAAAAwOgJMAAAAIDRE2AAAAAAoyfAAAAAAEZPgAEAAACMngADAAAAGD0BBgAAADB6AgwAAABg9AQYAAAAwOgJMAAAAIDRE2AAAAAAoyfAAAAAAEZPgAEAAACMngADAAAAGD0BBgAAADB6AgwAAABg9AQYAAAAwOgJMAAAAIDRE2AAAAAAoyfAAAAAAEZPgAEAAACMngADAAAAGD0BBgAAADB6AgwAAABg9AQYAAAAwOgJMAAAAIDRE2AAAAAAoyfAAAAAAEZPgAEAAACMngADAAAAGD0BBgAAADB6AgwAAABg9AQYAAAAwOgJMAAAAIDRE2AAAAAAoyfAAAAAAEZPgAEAAACMngADAAAAGD0BBgAAADB6AgwAAABg9AQYAAAAwOgJMAAAAIDRE2AAAAAAoyfAAAAAAEZPgAEAAACMngADAAAAGL25BRhV9YWquqaqPllVG4faw6vqkqq6fvi771CvqnpLVW2qqqur6slT5zl+2P/6qjp+qv6U4fybhmNr9b8lAAAAsCvM+wmMX+ruQ7v7sGH91CSXdvf6JJcO60lyZJL1w+ekJKclk8AjyWuT/GySw5O8din0GPZ52dRxG2b/dQAAAIBZmHeAsbWjkpw1LJ+V5Oip+rt64ook+1TVAUmeneSS7r6tu29PckmSDcO2vbv7iu7uJO+aOhcAAACwYOYZYHSSD1bVx6vqpKG2f3ffPCx/Ocn+w/KBSW6cOvamobZc/aZt1O+hqk6qqo1VtXHLli339/sAAAAAM7J2jtf+he7eXFU/muSSqvrM9Mbu7qrqWTbQ3acnOT1JDjvssJleCwAAALjv5vYERndvHv7emuS9mYxhccvw+keGv7cOu29OctDU4euG2nL1dduoAwAAAAtoLgFGVT2kqh66tJzkWUmuTXJhkqWZRI5PcsGwfGGS44bZSI5I8vXhVZOLkzyrqvYdBu98VpKLh213VNURw+wjx02dCwAAAFgw83qFZP8k7x1mNl2b5G+6+wNVdWWS86rqxCRfTPKiYf+LkjwnyaYk30pyQpJ0921V9bokVw77/XF33zYs/0aSdybZK8n7hw8AAACwgOYSYHT3DUmeuI36V5M8cxv1TnLKds51ZpIzt1HfmOQJ97tZAAAAYO7GNo0qAAAAwL0IMAAAAIDRE2AAAAAAoyfAAAAAAEZPgAEAAACMngADAAAAGD0BBgAAADB6AgwAAABg9AQYAAAAwOgJMAAAAIDRE2AAAAAAoyfAAAAAAEZPgAEAAACMngADAAAAGD0BBgAAADB6AgwAAABg9AQYAAAAwOgJMAAAAIDRE2AAAAAAoyfAAAAAAEZPgAEAAACMngADAAAAGD0BBgAAADB6AgwAAABg9NbOuwEegPZYm6qadxc75VHrDsrmG7807zYAAAAesAQYrL6778qL3/aReXexU849+anzbgEAAOABzSskAAAAwOgJMAAAAIDRE2AAAAAAoyfAAAAAAEZPgAEAAACMngADAAAAGD0BBgAAADB6AgwAAABg9AQYAAAAwOgJMAAAAIDRE2AAAAAAoyfAAAAAAEZPgAEAAACMngADAAAAGD0BBgAAADB6AgwAAABg9AQYAAAAwOgJMAAAAIDRE2AAAAAAoyfAAAAAAEZPgAEAAACMngADAAAAGD0BBgAAADB6AgwAAABg9AQYAAAAwOiteoBRVQdV1WVV9amquq6qXjnU/6iqNlfVJ4fPc6aOeXVVbaqqz1bVs6fqG4bapqo6dap+cFV9dKifW1V7ru63BAAAAHaleTyBcVeS3+3uQ5IckeSUqjpk2Pbm7j50+FyUJMO2Y5L8VJINSf6yqtZU1Zokb01yZJJDkhw7dZ43Dud6fJLbk5y4Wl8OAAAA2PVWPcDo7pu7+xPD8jeSfDrJgcscclSSc7r7zu7+fJJNSQ4fPpu6+4bu/k6Sc5IcVVWV5BlJzh+OPyvJ0bP5NgAAAMBqmOsYGFX1mCRPSvLRofSKqrq6qs6sqn2H2oFJbpw67Kahtr36I5J8rbvv2qq+reufVFUbq2rjli1bdsE3AgAAAGZhbgFGVf1Ikvck+e3uviPJaUkel+TQJDcnedOse+ju07v7sO4+bL/99pv15QAAAID7aO08LlpVD8okvHh3d/9dknT3LVPb357kH4bVzUkOmjp83VDLdupfTbJPVa0dnsKY3h8AAABYQPOYhaSSnJHk093951P1A6Z2+7Uk1w7LFyY5pqoeXFUHJ1mf5GNJrkyyfphxZM9MBvq8sLs7yWVJXjAcf3ySC2b5nQAAAIDZmscTGD+f5CVJrqmqTw6112Qyi8ihSTrJF5KcnCTdfV1VnZfkU5nMYHJKd38vSarqFUkuTrImyZndfd1wvlclOaeq/iTJVZkEJgAAAMCCWvUAo7v/7yS1jU0XLXPM65O8fhv1i7Z1XHffkMksJbBr7LE2k4eHFsuj1h2UzTd+ad5tAAAA3G9zGQMDFs7dd+XFb/vIvLvYaeee/NR5twAAALBLzHUaVQAAAICVEGAAAAAAoyfAAAAAAEZPgAEAAACMngADAAAAGD0BBgAAADB6AgwAAABg9AQYAAAAwOgJMAAAAIDRE2AAAAAAoyfAAAAAAEZPgAEAAACMngADAAAAGD0BBgAAADB6a+fdADBDe6xNVc27i53yqHUHZfONX5p3GwAAwMgIMGB3dvddefHbPjLvLnbKuSc/dd4tAAAAI+QVEgAAAGD0BBgAAADA6AkwAAAAgNETYAAAAACjJ8AAAAAARk+AAQAAAIyeAAMAAAAYPQEGAAAAMHpr590AwD3ssTZVNe8udtqj1h2UzTd+ad5tAADAbkuAAYzL3XflxW/7yLy72GnnnvzUebcAAAC7Na+QAAAAAKMnwAAAAABGT4ABAAAAjJ4AAwAAABg9g3gC7AoLOHuKmVMAAFgkAgyAXWEBZ08xcwoAAIvEKyQAAADA6AkwAAAAgNETYAAAAACjZwwMgAeqBRx4NDH4KADAA5UAA9sOGQQAAAqcSURBVOCBagEHHk0MPgoA8EDlFRIAAABg9DyBAcBiWcBXX7z2AgBw/wkwAFgsC/jqi9deAADuPwEGAMyap0YAAO43AQYAzJqnRgAA7jcBBgBwbwv41EjiyREA2J0JMACAe1vAp0aS5Nx/87SFC16ELgCwMgIMAGD3sYDByyKGLongBYDVJ8AAAJinBQxdEuOkALD6BBgAAOy8BRwnxVMjAItNgAEAwM5bwCdHFvV1nTUPenC+9907593GThEWAbMgwAAA4IFhAUOXZPK6zqL1vYhh0SIGRYmwiAcWAQYAALBrLWBYtIhBUSIsWk2L2PfuFnDttgFGVW1I8hdJ1iR5R3e/Yc4tAQAA7FrColWziH3vbgMu7zHvBmahqtYkeWuSI5MckuTYqjpkvl0BAAAA99VuGWAkOTzJpu6+obu/k+ScJEfNuScAAADgPqrunncPu1xVvSDJhu7+18P6S5L8bHe/Yqv9Tkpy0rD6E0k+u6qN7hqPTPKVeTcBu5B7mt2R+5rdkfua3ZH7mt3RIt7XX+nuDVsXd9sxMFaiu09Pcvq8+7g/qmpjdx827z5gV3FPsztyX7M7cl+zO3Jfszvane7r3fUVks1JDppaXzfUAAAAgAW0uwYYVyZZX1UHV9WeSY5JcuGcewIAAADuo93yFZLuvquqXpHk4kymUT2zu6+bc1uzstCvwMA2uKfZHbmv2R25r9kdua/ZHe029/VuOYgnAAAAsHvZXV8hAQAAAHYjAgwAAABg9AQYC6qqNlTVZ6tqU1WdOu9+YKWq6qCquqyqPlVV11XVK4f6w6vqkqq6fvi771CvqnrLcK9fXVVPnu83gG2rqjVVdVVV/cOwfnBVfXS4d88dBpVOVT14WN80bH/MPPuG5VTVPlV1flV9pqo+XVU/5/eaRVZV/3b498e1VXV2Vf2Q32sWUVWdWVW3VtW1U7Wd/n2uquOH/a+vquPn8V12hgBjAVXVmiRvTXJkkkOSHFtVh8y3K1ixu5L8bncfkuSIJKcM9++pSS7t7vVJLh3Wk8l9vn74nJTktNVvGVbklUk+PbX+xiRv7u7HJ7k9yYlD/cQktw/1Nw/7wVj9RZIPdPdPJnliJve432sWUlUdmOS3khzW3U/IZLD/Y+L3msX0ziQbtqrt1O9zVT08yWuT/GySw5O8din0GCsBxmI6PMmm7r6hu7+T5JwkR825J1iR7r65uz8xLH8jk38MH5jJPXzWsNtZSY4elo9K8q6euCLJPlV1wCq3DcuqqnVJfjXJO4b1SvKMJOcPu2x9Ty/d6+cneeawP4xKVT0sydOSnJEk3f2d7v5a/F6z2NYm2auq1ib54SQ3x+81C6i7P5zktq3KO/v7/Owkl3T3bd19e5JLcu9QZFQEGIvpwCQ3Tq3fNNRgoQyPYj4pyUeT7N/dNw+bvpxk/2HZ/c4i+M9J/l2Su4f1RyT5WnffNaxP37ffv6eH7V8f9oexOTjJliT/ZXg96h1V9ZD4vWZBdffmJP8pyZcyCS6+nuTj8XvN7mNnf58X7ndbgAHMRVX9SJL3JPnt7r5jeltP5nc2xzMLoaqem+TW7v74vHuBXWxtkicnOa27n5Tkm/nB48hJ/F6zWIZH44/KJJx7VJKHZOT/tRnuq93191mAsZg2Jzloan3dUIOFUFUPyiS8eHd3/91QvmXpUePh761D3f3O2P18kudV1RcyeaXvGZmMG7DP8Ihycs/79vv39LD9YUm+upoNwwrdlOSm7v7osH5+JoGG32sW1S8n+Xx3b+nu7yb5u0x+w/1es7vY2d/nhfvdFmAspiuTrB9GTN4zk8GHLpxzT7Aiw7ujZyT5dHf/+dSmC5MsjXx8fJILpurHDaMnH5Hk61OPxsHcdferu3tddz8mk9/j/97dv57ksiQvGHbb+p5eutdfMOy/2/0XEhZfd385yY1V9RND6ZlJPhW/1yyuLyU5oqp+ePj3yNI97fea3cXO/j5fnORZVbXv8ITSs4baaJX/Dy6mqnpOJu9cr0lyZne/fs4twYpU1S8k+cck1+QH4wW8JpNxMM5L8ugkX0zyou6+bfgHxv+VySOe30pyQndvXPXGYQWq6ulJfq+7n1tVj83kiYyHJ7kqyf/e3XdW1Q8l+etMxn+5Lckx3X3DvHqG5VTVoZkMTrtnkhuSnJDJfwDze81Cqqp/n+TFmcyKdlWSf53JO/9+r1koVXV2kqcneWSSWzKZTeR92cnf56p6aSb/Fk+S13f3f1nN77GzBBgAAADA6HmFBAAAABg9AQYAAAAwegIMAAAAYPQEGAAAAMDoCTAAAACA0RNgAAA7paq6qt40tf57VfVHu+jc76yqF+yKc+3gOi+sqk9X1WUr3P81O94LAJglAQYAsLPuTPL8qnrkvBuZVlVrd2L3E5O8rLt/aYX773SAUVVrdvYYAGD7BBgAwM66K8npSf7t1hu2foKiqv5p+Pv0qrq8qi6oqhuq6g1V9etV9bGquqaqHjd1ml+uqo1V9T+q6rnD8Wuq6s+q6sqqurqqTp467z9W1YVJPrWNfo4dzn9tVb1xqP1hkl9IckZV/dlW+x9QVR+uqk8Ox/xiVb0hyV5D7d3Dfu+rqo9X1XVVddL0962qN1XV/5fk54bv+amh5/903/7nBgCSZGf+SwUAwJK3Jrm6qv7jThzzxCT/LMltSW5I8o7uPryqXpnkN5P89rDfY5IcnuRxSS6rqscnOS7J17v7Z6rqwUn+n6r64LD/k5M8obs/P32xqnpUkjcmeUqS25N8sKqO7u4/rqpnJPm97t64VY//MsnF3f364QmKH+7uf6yqV3T3oVP7vbS7b6uqvZJcWVXv6e6vJnlIko929+9W1SOSnJHkJ7u7q2qfnfjfCgDYiicwAICd1t13JHlXkt/aicOu7O6bu/vOJJ9LshRAXJNJaLHkvO6+u7uvzyTo+Mkkz0pyXFV9MslHkzwiyfph/49tHV4MfibJh7p7S3ffleTdSZ62ox6TnDCM6fHT3f2N7ez3W8NTFlckOWiql+8lec+w/PUk387kSY/nJ/nWDq4NACxDgAEA3Ff/OZOxJB4yVbsrw78vqmqPJHtObbtzavnuqfW7c8+nQnur63SSSvKb3X3o8Dm4u5cCkG/er28xfaHuD2cScmxO8s6qOm7rfarq6Ul+OcnPdfcTk1yV5IeGzd/u7u8N57orkydJzk/y3CQf2FV9AsADkQADALhPuvu2JOdlEmIs+UImr2wkyfOSPOg+nPqFVbXHMC7GY5N8NsnFSf5NVT0oSarqx6vqIcudJMnHkvzzqnrk8DrIsUkuX+6AqvqxJLd099uTvCOT11OS5LtL107ysCS3d/e3quonkxyxnXP9SJKHdfdFmYwX8sQd9AsALMMYGADA/fGmJK+YWn97kguG1ys+kPv2dMSXMgkf9k7y8u7+dlW9I5PXTD5RVZVkS5KjlztJd99cVacmuSyTJzj+a3dfsINrPz3J/1FV303yT5mMvZFMBi29uqo+keSlSV5eVZ/OJFy5Yjvnemgm/1v80HD939nBtQGAZVT31k9pAgAAAIyLV0gAAACA0RNgAAAAAKMnwAAAAABGT4ABAAAAjJ4AAwAAABg9AQYAAAAwegIMAAAAYPT+f786tR9lmhlbAAAAAElFTkSuQmCC",
            "text/plain": [
              "<Figure size 1080x504 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "stars_dist = repos_df.toPandas()[\"stars\"]\n",
        "stars_dist = stars_dist[stars_dist < 1_000]\n",
        "\n",
        "sns.displot(stars_dist, bins=20, height=7, aspect=15/7)\n",
        "plt.xlabel(\"Number of stars\")\n",
        "plt.ylabel(\"Number of repos\")\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PLg3xs-ukVQD"
      },
      "source": [
        "As we can see, the distribution is really skewed towards the 0, meaning that the majority of the repos have less than 50 stars.\n",
        "\n",
        "For this reason, we decided to cut the graph to only 1000 stars because, even if there are repos with a higher number of stars, they are very few compared to the ones depicted in the plot."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nK76EJpUUK1w"
      },
      "source": [
        "# Data preparation\n",
        "In this section we prepare the dataset in order to be processed by our models."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TpnWv_DUUOzK"
      },
      "source": [
        "## Dataset pruning\n",
        "We decided to cut the dataset into a smaller subset of samples, since we don't have enough computational power to handle the whole dataset for all the models.\n",
        "\n",
        "In fact, to perform some comparisons, we would obtain DataFrames with more than 100 billions of records and unfortunately, neither Colab or Databricks can provide us enough resources to process this amount of data for free."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kKQ6L812lqFK"
      },
      "source": [
        "Let's first see the total size of the dataset before the operation:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LrAylBM6Ui89",
        "outputId": "0e500f0d-eda9-43a4-a8ff-2e703187ee06"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Old repos count: 354981\n",
            "Old users count: 10000\n",
            "Old starred count: 1083195\n"
          ]
        }
      ],
      "source": [
        "print(\"Old repos count:\", repos_df.count())\n",
        "print(\"Old users count:\", users_df.count())\n",
        "print(\"Old starred count:\", starred_df.count())"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "30Xt0FFjib8-"
      },
      "source": [
        "To prune the dataset, we decided to fix the number of users and repositories that we are going to use.\n",
        "\n",
        "To do so, first of all, we took the most starring users. Then, we took the most starred repositories by those users because they are the elements that can give us more information.\n",
        "\n",
        "Finally, we removed the user-repo relationships that involved users or repos which were not selected in the previous step.\n",
        "\n",
        "The final size of our datasets can be seen in the output of the next block."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "krcRLFbBUIPa",
        "outputId": "a03af352-f474-4ff1-fadd-7c6478e4d6ed"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "New repos count: 1000\n",
            "New users count: 1000\n",
            "New starred count: 104499\n"
          ]
        }
      ],
      "source": [
        "NEW_USER_COUNT = 1000\n",
        "NEW_REPOS_COUNT = 1000\n",
        "\n",
        "most_starring_users = starred_df.groupBy(\"user_id\").count().sort(\"count\", ascending=False).limit(NEW_USER_COUNT).select(\"user_id\")\n",
        "most_starred_repos = most_starring_users.join(starred_df, \"user_id\").groupBy(\"repo_id\").count().sort(\"count\", ascending=False).limit(NEW_REPOS_COUNT).select(\"repo_id\")\n",
        "\n",
        "new_users_df = most_starring_users.join(users_df, \"user_id\").cache()\n",
        "new_starred_df = most_starring_users.join(starred_df, \"user_id\").join(most_starred_repos, \"repo_id\").cache()\n",
        "new_repos_df = most_starred_repos.join(repos_df, \"repo_id\").cache()\n",
        "\n",
        "print(\"New repos count:\", new_repos_df.count())\n",
        "print(\"New users count:\", new_users_df.count())\n",
        "print(\"New starred count:\", new_starred_df.count())"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FBHXoxozU4Rd"
      },
      "source": [
        "## Train/Test split\n",
        "\n",
        "At this point, we can split the dataset into a training set which is going to be used for setting up our models, and a test set, which is going to be used in order to evaluate their performances.\n",
        "\n",
        "Moreover, we cache them in order to speed up future computations."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "id": "hQMetXrjUUou"
      },
      "outputs": [],
      "source": [
        "train, test = new_starred_df.randomSplit([0.7, 0.3], seed=42)\n",
        "train = train.cache()\n",
        "test = test.cache()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ynMjjT7mWf6R"
      },
      "source": [
        "# Models\n",
        "In this section we build the models and we evaluate them computing recommendations for all the users in the pruned dataset."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AMQAVIHskbDP"
      },
      "source": [
        "## Similarities\n",
        "\n",
        "Many models use a similarity function in order to compare elements and find the most similar ones.\n",
        "\n",
        "Here we are going to define the functions that we will use in some of them."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gaqkzEnXk0AP"
      },
      "source": [
        "### Jaccard Similarity\n",
        "\n",
        "The Jaccard Index is a similarity measure between two sets and is typically used when you want to find the similarity between two binary vectors.\n",
        "\n",
        "Given two sets $A$ and $B$, it is computed as follows:\n",
        "\n",
        "$$\n",
        "J(A,B) = \\frac{A \\cap B}{A \\cup B}\n",
        "$$"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "id": "ZL4EtioXk2Di"
      },
      "outputs": [],
      "source": [
        "def jaccard_similarity(array1, array2):\n",
        "  set1 = set(array1)\n",
        "  set2 = set(array2)\n",
        "  return len(set1.intersection(set2)) / len(set1.union(set2))\n",
        "\n",
        "jaccard_similarity_udf = udf(jaccard_similarity, DoubleType())"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "O_p2-9Nvl1_H"
      },
      "source": [
        "### Cosine Similarity"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aXLZZk2yl9VZ"
      },
      "source": [
        "The Cosine similarity is typically used in Information Retrieval to compare how similar are two documents.\n",
        "\n",
        "Given two vectors $u$ and $v$, the cosine similarity is defined as follows:\n",
        "\n",
        "$$\n",
        "\\cos(u,v) = \\frac{u \\cdot v}{||u||_2 ||v||_2}\n",
        "$$"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "id": "KC_vyFB3muOG"
      },
      "outputs": [],
      "source": [
        "def cosine_similarity(u, v):\n",
        "  dot = u.dot(v)\n",
        "  len = u.norm(2) * v.norm(2)\n",
        "  if len == 0: return 0.0\n",
        "  return float(dot / len)\n",
        "\n",
        "cosine_similarity_udf = udf(cosine_similarity, DoubleType())"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jX_FzPyMX2tW"
      },
      "source": [
        "In order to compute the personalization metric, we also derived an alternative formulation of the cosine similarity to speed up the computation.\n",
        "\n",
        "Let's assume we have two binary vectors $u$ and $v$ where the $i$-th component is 1 if the $i$-th element is present, and 0 otherwise.\n",
        "\n",
        "In our code, those vectors are instead represented as sets by taking only the index of elements that have value 1 in the vector:\n",
        "\n",
        "$$\n",
        "A = \\{i : u_i = 1 \\} \\qquad B = \\{j : v_j = 1 \\}\n",
        "$$\n",
        "\n",
        "When we perform the dot product between such vectors, we are going to sum 1 for each element which is in common, and 0 for those that are not, so the result is going to be the number of common elements.\n",
        "\n",
        "Moreover, the L2-norm of a binary vector is going to simply be the square root of the number of components set to 1.\n",
        "\n",
        "Given the previous observations, we can re-define in this case the cosine similarity as following:\n",
        "\n",
        "$$\n",
        "cos(A, B) = \\frac{|A \\cap B|}{\\sqrt{|A| |B|}}\n",
        "$$"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {
        "id": "jm1vR3NrX0uc"
      },
      "outputs": [],
      "source": [
        "from math import sqrt\n",
        "\n",
        "def cosine_similarity_lists(list1, list2):\n",
        "  set1 = set(list1)\n",
        "  set2 = set(list2)\n",
        "  return len(set1.intersection(set2)) / (sqrt(len(set1)) * sqrt(len(set2)))\n",
        "\n",
        "cosine_similarity_lists_udf = udf(cosine_similarity_lists, DoubleType())"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tMbHdvF6XiAA"
      },
      "source": [
        "## Evaluation metrics\n",
        "In order to evaluate our models, we choose to use the following metrics:"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ggheU75QXptO"
      },
      "source": [
        "#### MAP@K"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NNLpXOKE5TIa"
      },
      "source": [
        "Mean Average Precision at K (MAP@K) is the mean of the Average Precision at K (AP@K) metric computed for each user. AP@K is a metric commonly used for information retrieval which takes into consideration the precision of the system and the order of the items recommended.\n",
        "\n",
        "Given that our system returns $K$ repos, where $|Rel|$ are actually relevant, and $1_{Rel}(K)$ is the indicator function that tells us if the $K$-th repo is relevant for a user $u$, AP@N is defined as:\n",
        "\n",
        "$$\n",
        "AP@N(u) = \\frac{1}{|Rel|} \\sum_{K=1}^N P@K(u) \\times 1_{Rel}(K)\n",
        "$$\n",
        "\n",
        "And so MAP@N, assuming we have $|U|$ users, can be computed as:\n",
        "$$\n",
        "MAP@N = \\frac{1}{|U|} \\sum_{u=1}^{|U|} AP@N(u)\n",
        "$$"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {
        "cellView": "form",
        "id": "0OkVeuNkYbp8"
      },
      "outputs": [],
      "source": [
        "k = 5 #@param {type:\"integer\"}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {
        "id": "0-rANKY5Xv4s"
      },
      "outputs": [],
      "source": [
        "from pyspark.ml.evaluation import RankingEvaluator\n",
        "\n",
        "def map_at_k(recommendations, k):\n",
        "  evaluator = RankingEvaluator(metricName=\"meanAveragePrecisionAtK\", k=k, predictionCol=\"recommended_repos\", labelCol=\"labels\")\n",
        "  return evaluator.evaluate(recommendations)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ebw4KRNMXsH9"
      },
      "source": [
        "#### Personalization\n",
        "Personalization is a metric that tells us if the recommender system produces items which are personalized for each user, or if it recommends always the same items to different users.\n",
        "\n",
        "It is defined as the average of the dissimilarity between users lists of recommendations:\n",
        "$$\n",
        "Personalization = 1 - \\left( \\frac{1}{|U|} \\sum_{u_i, u_j \\in U \\land i < j} cos(u_i, u_j) \\right)\n",
        "$$\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {
        "id": "o78GBAO9YP-R"
      },
      "outputs": [],
      "source": [
        "def personalization(recommendations):\n",
        "  target_recommendations = recommendations.withColumnRenamed(\"user_id\", \"user2_id\").withColumnRenamed(\"recommended_repos\", \"target_recommendations\")\n",
        "  cosine_matrix = recommendations.join(target_recommendations, col(\"user_id\") < col(\"user2_id\"))\n",
        "  cosine_df = cosine_matrix.withColumn(\"cosine_similarity\", cosine_similarity_lists_udf(\"recommended_repos\", \"target_recommendations\"))\n",
        "  avg_cos = cosine_df.agg(avg(\"cosine_similarity\").alias(\"avg_cos\")).collect()[0][\"avg_cos\"]\n",
        "  return 1 - avg_cos"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NHC7AqModP5z"
      },
      "source": [
        "### Evaluation code\n",
        "\n",
        "With the following code we compute the MAP@K for $K = 1 ... k$ and the Personalization metric for the dataframe containing the list of repos that the model recommended to each user."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 29,
      "metadata": {
        "id": "IZ4gR5a3Z4gB"
      },
      "outputs": [],
      "source": [
        "fields = [StructField(\"Model\", StringType(), False)]\n",
        "fields += [StructField(\"MAP@\" + str(i), DoubleType(), False) for i in range(1, k+1)]\n",
        "fields += [StructField(\"Personalization\", DoubleType(), False)]\n",
        "metrics_df = spark.createDataFrame([], StructType(fields))\n",
        "\n",
        "# Ground truth dataframe\n",
        "gt = test.withColumn(\"repo_id\", test.repo_id.cast(\"double\")).groupBy(\"user_id\").agg(collect_set(\"repo_id\").alias(\"labels\"))\n",
        "\n",
        "def evaluate_df(model_name, recommendations, k):\n",
        "  mapks = [map_at_k(recommendations, _k) for _k in range(1, k+1)]\n",
        "  p = personalization(recommendations)\n",
        "  schema = [\"Model\"] + [\"MAP@\" + str(i) for i in range(1, k+1)] + [\"Personalization\"]\n",
        "  data = [tuple([model_name] + mapks + [p])]\n",
        "  metrics_df = spark.createDataFrame(data, schema)\n",
        "  return metrics_df\n",
        "\n",
        "def round_df(dataframe, decimals, dtypes = [\"double\", \"float\"]):\n",
        "  return dataframe.select([round(col, decimals).alias(col) if dtype in dtypes else col for col, dtype in dataframe.dtypes])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KF2vRPKZXYod"
      },
      "source": [
        "## Content-based filtering\n",
        "\n",
        "Content-based filtering is a method which uses the features of the items rated by a user in order to find out the items that are similar to those already rated by him."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 30,
      "metadata": {
        "cellView": "form",
        "id": "25xypi-7DBAo"
      },
      "outputs": [],
      "source": [
        "#@title Settings\n",
        "VOCAB_SIZE = 1000 #@param\n",
        "MIN_DOC_FREQ = 10 #@param\n",
        "N_ITEMS = 5 #@param"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-vsCxoDkDqJ0"
      },
      "source": [
        "### Building Item profiles\n",
        "\n",
        "The first step for this method, is to construct the item profiles, that are a vectorized representation of each item, in our case, for each repo.\n",
        "\n",
        "From the features of each repo we removed the `creator` and `name` fields, because they do not add any information to the representation. For the `about` field we used the TF-IDF weighting scheme after a pre-processing phase that cleaned the text. Moreover, we used a one-hot encoding representation for the `language` field and normalized the fields `stars`, `forks` and `updated`. The latter was converted simply as the number of seconds in order to be machine processable. Finally, the `sponsor` field was left as a binary feature.\n",
        "\n",
        "We start by pre-processing the text in the `about` column."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 31,
      "metadata": {
        "cellView": "form",
        "id": "nqpn3y_de8mQ"
      },
      "outputs": [],
      "source": [
        "#@title Text preprocessing functions\n",
        "from pyspark import StorageLevel\n",
        "\n",
        "def clean_text(df, column_name):\n",
        "    from pyspark.sql.functions import udf, col, lower, trim, regexp_replace\n",
        "    from pyspark.ml.feature import Tokenizer, StopWordsRemover\n",
        "    from nltk.stem.snowball import SnowballStemmer\n",
        "\n",
        "    lower_case_df = df.withColumn(column_name, lower(col(column_name)))\n",
        "    \n",
        "    trimmed_df = lower_case_df.withColumn(column_name, trim(col(column_name)))\n",
        "    \n",
        "    no_punct_df = trimmed_df.withColumn(column_name, regexp_replace(col(column_name), \"[^a-zA-Z\\\\s]\", \"\"))\n",
        "    \n",
        "    cleaned_df = no_punct_df.withColumn(column_name, trim(regexp_replace(col(column_name), \" +\", \" \")))\n",
        "\n",
        "    tokenizer = Tokenizer(inputCol=column_name, outputCol=\"tokens\")\n",
        "    tokens_df = tokenizer.transform(cleaned_df)\n",
        "\n",
        "    stopwords_remover = StopWordsRemover(inputCol=\"tokens\", outputCol=\"terms\")\n",
        "    terms_df = stopwords_remover.transform(tokens_df)\n",
        "\n",
        "    stemmer = SnowballStemmer(language=\"english\")\n",
        "    stemmer_udf = udf(lambda tokens: [stemmer.stem(token) for token in tokens], ArrayType(StringType()))\n",
        "    terms_stemmed_df = terms_df.withColumn(column_name, stemmer_udf(\"terms\"))\n",
        "    \n",
        "    return terms_stemmed_df.select(df.columns)\n",
        "\n",
        "def extract_tfidf_features(df, column_name):\n",
        "    from pyspark.ml.feature import CountVectorizer, IDF\n",
        "    from pyspark.ml import Pipeline\n",
        "\n",
        "    cv = CountVectorizer(inputCol=column_name, outputCol=column_name + \"_tf\", vocabSize=VOCAB_SIZE, minDF=MIN_DOC_FREQ)\n",
        "    idf = IDF(inputCol=column_name + \"_tf\", outputCol=column_name + \"_tfidf\")\n",
        "\n",
        "    pipeline = Pipeline(stages=[cv, idf])\n",
        "    features = pipeline.fit(df)\n",
        "    tf_idf_features_df = features.transform(df)\n",
        "    \n",
        "    result = tf_idf_features_df.withColumn(column_name, col(column_name + \"_tfidf\"))\n",
        "\n",
        "    return result.select(df.columns)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 32,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AchSO1DSDbZb",
        "outputId": "68470b9a-e4ef-410e-c504-ddfd85e78476"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+-------+-----------------+--------------------+--------------------+----------+------+-----+-------------------+-------+\n",
            "|repo_id|          creator|                name|               about|  language| stars|forks|            updated|sponsor|\n",
            "+-------+-----------------+--------------------+--------------------+----------+------+-----+-------------------+-------+\n",
            "|      0|styled-components|   styled-components|[visual, primit, ...|TypeScript| 33940| 2084|2021-06-16 19:12:26|      0|\n",
            "|      1|           vercel|             next.js|  [react, framework]|JavaScript| 69687|13111|2021-06-22 07:49:26|      0|\n",
            "|      2|         facebook|               react|[declar, effici, ...|JavaScript|170264|34406|2021-06-22 06:08:58|      0|\n",
            "|      7|        chakra-ui|           chakra-ui|[simpl, modular, ...|TypeScript| 18902| 1527|2021-06-22 07:12:59|      0|\n",
            "|     11|         yangshun|front-end-intervi...|[nobullshit, answ...|JavaScript| 29404| 4363|2021-06-21 20:00:42|      1|\n",
            "|     16|     sindresorhus|             awesome|[awesom, list, ki...|     Shell|164302|20994|2021-06-21 07:27:22|      1|\n",
            "|     24|         VSCodium|            vscodium|[binari, releas, ...|     Shell| 13234|  484|2021-06-21 03:21:57|      0|\n",
            "|     28|    kamranahmedse|   developer-roadmap|[roadmap, becom, ...|       UNK|163490|23657|2021-06-12 10:15:27|      1|\n",
            "|     35|        microsoft|              vscode|[visual, studio, ...|TypeScript|117529|19179|2021-06-22 08:42:46|      0|\n",
            "|     43|           airbnb|      lottie-android|[render, effect, ...|      Java| 31317| 5023|2021-06-21 19:11:06|      0|\n",
            "|     44|           airbnb|          lottie-web|[render, effect, ...|JavaScript| 24486| 2434|2021-06-04 11:36:15|      0|\n",
            "|     46|     bradtraversy|design-resources-...|[curat, list, des...|       UNK| 30202| 6892|2021-06-22 08:25:05|      0|\n",
            "|     48|         facebook|        react-native|[framework, build...|JavaScript| 96207|20943|2021-06-22 06:00:50|      0|\n",
            "|     50|           frappe|             erpnext|[world, best, fre...|    Python|  8925| 3666|2021-06-22 08:15:25|      0|\n",
            "|     53|         sveltejs|              svelte|[cybernet, enhanc...|TypeScript| 47896| 2264|2021-06-22 07:26:38|      0|\n",
            "|     58|      highlightjs|        highlight.js|[javascript, synt...|JavaScript| 18568| 3129|2021-06-22 06:55:32|      0|\n",
            "|     64|         appwrite|            appwrite|[appwrit, secur, ...|JavaScript|  9884|  759|2021-06-22 08:39:52|      0|\n",
            "|     72|          lensapp|                lens|[len, kubernet, ide]|TypeScript| 14345|  707|2021-06-22 08:17:15|      0|\n",
            "|     77|         gatsbyjs|              gatsby|[build, blaze, fa...|JavaScript| 50577| 9662|2021-06-22 08:35:17|      0|\n",
            "|     80|         gohugoio|                hugo|[world, fastest, ...|        Go| 52665| 5968|2021-06-22 08:26:07|      0|\n",
            "+-------+-----------------+--------------------+--------------------+----------+------+-----+-------------------+-------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "features_df = clean_text(new_repos_df, \"about\")\n",
        "features_df.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "O8gFtTs6ggxz"
      },
      "source": [
        "Given the normalized text, we can compute TF-IDF on the same column."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 33,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YnKBDFvDDeAa",
        "outputId": "ee38a2fc-1b0e-4091-e783-dfc80d176dc9"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+-------+-----------------+--------------------+--------------------+----------+------+-----+-------------------+-------+\n",
            "|repo_id|          creator|                name|               about|  language| stars|forks|            updated|sponsor|\n",
            "+-------+-----------------+--------------------+--------------------+----------+------+-----+-------------------+-------+\n",
            "|      0|styled-components|   styled-components|(132,[10,16,32,33...|TypeScript| 33940| 2084|2021-06-16 19:12:26|      0|\n",
            "|      1|           vercel|             next.js|(132,[2,4],[2.397...|JavaScript| 69687|13111|2021-06-22 07:49:26|      0|\n",
            "|      2|         facebook|               react|(132,[1,3,6,123],...|JavaScript|170264|34406|2021-06-22 06:08:58|      0|\n",
            "|      7|        chakra-ui|           chakra-ui|(132,[4,14,22,29,...|TypeScript| 18902| 1527|2021-06-22 07:12:59|      0|\n",
            "|     11|         yangshun|front-end-intervi...|(132,[93,127],[4....|JavaScript| 29404| 4363|2021-06-21 20:00:42|      1|\n",
            "|     16|     sindresorhus|             awesome|(132,[5,21],[2.83...|     Shell|164302|20994|2021-06-21 07:27:22|      1|\n",
            "|     24|         VSCodium|            vscodium|(132,[17],[3.1475...|     Shell| 13234|  484|2021-06-21 03:21:57|      0|\n",
            "|     28|    kamranahmedse|   developer-roadmap|(132,[0,13],[2.40...|       UNK|163490|23657|2021-06-12 10:15:27|      1|\n",
            "|     35|        microsoft|              vscode|(132,[17,105],[3....|TypeScript|117529|19179|2021-06-22 08:42:46|      0|\n",
            "|     43|           airbnb|      lottie-android|(132,[0,4,37,50,8...|      Java| 31317| 5023|2021-06-21 19:11:06|      0|\n",
            "|     44|           airbnb|          lottie-web|(132,[0,4,37,50,8...|JavaScript| 24486| 2434|2021-06-04 11:36:15|      0|\n",
            "|     46|     bradtraversy|design-resources-...|(132,[0,2,3,5,9,2...|       UNK| 30202| 6892|2021-06-22 08:25:05|      0|\n",
            "|     48|         facebook|        react-native|(132,[2,4,6,10,37...|JavaScript| 96207|20943|2021-06-22 06:00:50|      0|\n",
            "|     50|           frappe|             erpnext|(132,[7,8,40,71],...|    Python|  8925| 3666|2021-06-22 08:15:25|      0|\n",
            "|     53|         sveltejs|              svelte|(132,[0,10],[2.40...|TypeScript| 47896| 2264|2021-06-22 07:26:38|      0|\n",
            "|     58|      highlightjs|        highlight.js|(132,[1,23],[2.42...|JavaScript| 18568| 3129|2021-06-22 06:55:32|      0|\n",
            "|     64|         appwrite|            appwrite|(132,[0,13,44,55,...|JavaScript|  9884|  759|2021-06-22 08:39:52|      0|\n",
            "|     72|          lensapp|                lens|         (132,[],[])|TypeScript| 14345|  707|2021-06-22 08:17:15|      0|\n",
            "|     77|         gatsbyjs|              gatsby|(132,[4,6,10,12,2...|JavaScript| 50577| 9662|2021-06-22 08:35:17|      0|\n",
            "|     80|         gohugoio|                hugo|(132,[2,6],[2.397...|        Go| 52665| 5968|2021-06-22 08:26:07|      0|\n",
            "+-------+-----------------+--------------------+--------------------+----------+------+-----+-------------------+-------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "features_df = extract_tfidf_features(features_df, \"about\")\n",
        "features_df.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VnumTbpTgmtR"
      },
      "source": [
        "In order to convert the `language` field we use the `StringIndexer` to convert \n",
        "strings to indexes and then we use the `OneHotEncoder` class to transform the integer representation into one-hot vectors."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 34,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "I-hwFpaRDg5z",
        "outputId": "da872d4a-446d-4205-fb6c-d438d21c6faa"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+-------+-----------------+--------------------+--------------------+--------------+------+-----+-------------------+-------+\n",
            "|repo_id|          creator|                name|               about|      language| stars|forks|            updated|sponsor|\n",
            "+-------+-----------------+--------------------+--------------------+--------------+------+-----+-------------------+-------+\n",
            "|      0|styled-components|   styled-components|(132,[10,16,32,33...|(47,[1],[1.0])| 33940| 2084|2021-06-16 19:12:26|      0|\n",
            "|      1|           vercel|             next.js|(132,[2,4],[2.397...|(47,[0],[1.0])| 69687|13111|2021-06-22 07:49:26|      0|\n",
            "|      2|         facebook|               react|(132,[1,3,6,123],...|(47,[0],[1.0])|170264|34406|2021-06-22 06:08:58|      0|\n",
            "|      7|        chakra-ui|           chakra-ui|(132,[4,14,22,29,...|(47,[1],[1.0])| 18902| 1527|2021-06-22 07:12:59|      0|\n",
            "|     11|         yangshun|front-end-intervi...|(132,[93,127],[4....|(47,[0],[1.0])| 29404| 4363|2021-06-21 20:00:42|      1|\n",
            "|     16|     sindresorhus|             awesome|(132,[5,21],[2.83...|(47,[9],[1.0])|164302|20994|2021-06-21 07:27:22|      1|\n",
            "|     24|         VSCodium|            vscodium|(132,[17],[3.1475...|(47,[9],[1.0])| 13234|  484|2021-06-21 03:21:57|      0|\n",
            "|     28|    kamranahmedse|   developer-roadmap|(132,[0,13],[2.40...|(47,[3],[1.0])|163490|23657|2021-06-12 10:15:27|      1|\n",
            "|     35|        microsoft|              vscode|(132,[17,105],[3....|(47,[1],[1.0])|117529|19179|2021-06-22 08:42:46|      0|\n",
            "|     43|           airbnb|      lottie-android|(132,[0,4,37,50,8...|(47,[6],[1.0])| 31317| 5023|2021-06-21 19:11:06|      0|\n",
            "|     44|           airbnb|          lottie-web|(132,[0,4,37,50,8...|(47,[0],[1.0])| 24486| 2434|2021-06-04 11:36:15|      0|\n",
            "|     46|     bradtraversy|design-resources-...|(132,[0,2,3,5,9,2...|(47,[3],[1.0])| 30202| 6892|2021-06-22 08:25:05|      0|\n",
            "|     48|         facebook|        react-native|(132,[2,4,6,10,37...|(47,[0],[1.0])| 96207|20943|2021-06-22 06:00:50|      0|\n",
            "|     50|           frappe|             erpnext|(132,[7,8,40,71],...|(47,[2],[1.0])|  8925| 3666|2021-06-22 08:15:25|      0|\n",
            "|     53|         sveltejs|              svelte|(132,[0,10],[2.40...|(47,[1],[1.0])| 47896| 2264|2021-06-22 07:26:38|      0|\n",
            "|     58|      highlightjs|        highlight.js|(132,[1,23],[2.42...|(47,[0],[1.0])| 18568| 3129|2021-06-22 06:55:32|      0|\n",
            "|     64|         appwrite|            appwrite|(132,[0,13,44,55,...|(47,[0],[1.0])|  9884|  759|2021-06-22 08:39:52|      0|\n",
            "|     72|          lensapp|                lens|         (132,[],[])|(47,[1],[1.0])| 14345|  707|2021-06-22 08:17:15|      0|\n",
            "|     77|         gatsbyjs|              gatsby|(132,[4,6,10,12,2...|(47,[0],[1.0])| 50577| 9662|2021-06-22 08:35:17|      0|\n",
            "|     80|         gohugoio|                hugo|(132,[2,6],[2.397...|(47,[4],[1.0])| 52665| 5968|2021-06-22 08:26:07|      0|\n",
            "+-------+-----------------+--------------------+--------------------+--------------+------+-----+-------------------+-------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "from pyspark.ml.feature import OneHotEncoder, StringIndexer\n",
        "\n",
        "cols = features_df.columns\n",
        "\n",
        "# Convert the language field to indexes\n",
        "str_indexer = StringIndexer(inputCol=\"language\", outputCol=\"language_id\")\n",
        "model = str_indexer.fit(features_df)\n",
        "features_df = model.transform(features_df)\n",
        "\n",
        "# Use the one-hot encoding on the language field\n",
        "enc = OneHotEncoder(inputCol=\"language_id\", outputCol=\"language_cat\")\n",
        "model = enc.fit(features_df)\n",
        "features_df = model.transform(features_df)\n",
        "\n",
        "features_df = features_df.withColumn(\"language\", col(\"language_cat\")).select(cols)\n",
        "features_df.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jezm3i_HgHiV"
      },
      "source": [
        "Then, as we said, we convert the `updated` field in seconds."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 35,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zk-9NpcJDjMc",
        "outputId": "5247544d-6242-44b1-f4a1-4217f2860972"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+-------+-----------------+--------------------+--------------------+--------------+------+-----+----------+-------+\n",
            "|repo_id|          creator|                name|               about|      language| stars|forks|   updated|sponsor|\n",
            "+-------+-----------------+--------------------+--------------------+--------------+------+-----+----------+-------+\n",
            "|      0|styled-components|   styled-components|(132,[10,16,32,33...|(47,[1],[1.0])| 33940| 2084|1623870746|      0|\n",
            "|      1|           vercel|             next.js|(132,[2,4],[2.397...|(47,[0],[1.0])| 69687|13111|1624348166|      0|\n",
            "|      2|         facebook|               react|(132,[1,3,6,123],...|(47,[0],[1.0])|170264|34406|1624342138|      0|\n",
            "|      7|        chakra-ui|           chakra-ui|(132,[4,14,22,29,...|(47,[1],[1.0])| 18902| 1527|1624345979|      0|\n",
            "|     11|         yangshun|front-end-intervi...|(132,[93,127],[4....|(47,[0],[1.0])| 29404| 4363|1624305642|      1|\n",
            "|     16|     sindresorhus|             awesome|(132,[5,21],[2.83...|(47,[9],[1.0])|164302|20994|1624260442|      1|\n",
            "|     24|         VSCodium|            vscodium|(132,[17],[3.1475...|(47,[9],[1.0])| 13234|  484|1624245717|      0|\n",
            "|     28|    kamranahmedse|   developer-roadmap|(132,[0,13],[2.40...|(47,[3],[1.0])|163490|23657|1623492927|      1|\n",
            "|     35|        microsoft|              vscode|(132,[17,105],[3....|(47,[1],[1.0])|117529|19179|1624351366|      0|\n",
            "|     43|           airbnb|      lottie-android|(132,[0,4,37,50,8...|(47,[6],[1.0])| 31317| 5023|1624302666|      0|\n",
            "|     44|           airbnb|          lottie-web|(132,[0,4,37,50,8...|(47,[0],[1.0])| 24486| 2434|1622806575|      0|\n",
            "|     46|     bradtraversy|design-resources-...|(132,[0,2,3,5,9,2...|(47,[3],[1.0])| 30202| 6892|1624350305|      0|\n",
            "|     48|         facebook|        react-native|(132,[2,4,6,10,37...|(47,[0],[1.0])| 96207|20943|1624341650|      0|\n",
            "|     50|           frappe|             erpnext|(132,[7,8,40,71],...|(47,[2],[1.0])|  8925| 3666|1624349725|      0|\n",
            "|     53|         sveltejs|              svelte|(132,[0,10],[2.40...|(47,[1],[1.0])| 47896| 2264|1624346798|      0|\n",
            "|     58|      highlightjs|        highlight.js|(132,[1,23],[2.42...|(47,[0],[1.0])| 18568| 3129|1624344932|      0|\n",
            "|     64|         appwrite|            appwrite|(132,[0,13,44,55,...|(47,[0],[1.0])|  9884|  759|1624351192|      0|\n",
            "|     72|          lensapp|                lens|         (132,[],[])|(47,[1],[1.0])| 14345|  707|1624349835|      0|\n",
            "|     77|         gatsbyjs|              gatsby|(132,[4,6,10,12,2...|(47,[0],[1.0])| 50577| 9662|1624350917|      0|\n",
            "|     80|         gohugoio|                hugo|(132,[2,6],[2.397...|(47,[4],[1.0])| 52665| 5968|1624350367|      0|\n",
            "+-------+-----------------+--------------------+--------------------+--------------+------+-----+----------+-------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "features_df = features_df.withColumn(\"updated\", unix_timestamp(\"updated\"))\n",
        "features_df.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "u3ihGxscf7CH"
      },
      "source": [
        "Now, we proceed with the normalization step on the `stars`, `forks`, `updated` and `sponsor` fields."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 36,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZcPwcI8l5oPR",
        "outputId": "670dafc5-551c-4ac2-cd6a-4af1ab307d0c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+-------+--------------------+--------------+--------------------+--------------------+\n",
            "|repo_id|               about|      language|           repo_info|    scaled_repo_info|\n",
            "+-------+--------------------+--------------+--------------------+--------------------+\n",
            "|      0|(132,[10,16,32,33...|(47,[1],[1.0])|[33940.0,2084.0,1...|[0.09672969194187...|\n",
            "|      1|(132,[2,4],[2.397...|(47,[0],[1.0])|[69687.0,13111.0,...|[0.20754939253307...|\n",
            "|      2|(132,[1,3,6,123],...|(47,[0],[1.0])|[170264.0,34406.0...|[0.51934934851148...|\n",
            "|      7|(132,[4,14,22,29,...|(47,[1],[1.0])|[18902.0,1527.0,1...|[0.05011020897854...|\n",
            "|     11|(132,[93,127],[4....|(47,[0],[1.0])|[29404.0,4363.0,1...|[0.08266758429979...|\n",
            "|     16|(132,[5,21],[2.83...|(47,[9],[1.0])|[164302.0,20994.0...|[0.50086648127997...|\n",
            "|     24|(132,[17],[3.1475...|(47,[9],[1.0])|[13234.0,484.0,1....|[0.03253877464976...|\n",
            "|     28|(132,[0,13],[2.40...|(47,[3],[1.0])|[163490.0,23657.0...|[0.49834919040577...|\n",
            "|     35|(132,[17,105],[3....|(47,[1],[1.0])|[117529.0,19179.0...|[0.35586494672457...|\n",
            "|     43|(132,[0,4,37,50,8...|(47,[6],[1.0])|[31317.0,5023.0,1...|[0.08859809839135...|\n",
            "|     44|(132,[0,4,37,50,8...|(47,[0],[1.0])|[24486.0,2434.0,1...|[0.06742123390654...|\n",
            "|     46|(132,[0,2,3,5,9,2...|(47,[3],[1.0])|[30202.0,6892.0,1...|[0.08514147360719...|\n",
            "|     48|(132,[2,4,6,10,37...|(47,[0],[1.0])|[96207.0,20943.0,...|[0.28976436049341...|\n",
            "|     50|(132,[7,8,40,71],...|(47,[2],[1.0])|[8925.0,3666.0,1....|[0.01918039241216...|\n",
            "|     53|(132,[0,10],[2.40...|(47,[1],[1.0])|[47896.0,2264.0,1...|[0.13999485381422...|\n",
            "|     58|(132,[1,23],[2.42...|(47,[0],[1.0])|[18568.0,3129.0,1...|[0.04907477159925...|\n",
            "|     64|(132,[0,13,44,55,...|(47,[0],[1.0])|[9884.0,759.0,1.6...|[0.02215339973773...|\n",
            "|     72|         (132,[],[])|(47,[1],[1.0])|[14345.0,707.0,1....|[0.03598299898626...|\n",
            "|     77|(132,[4,6,10,12,2...|(47,[0],[1.0])|[50577.0,9662.0,1...|[0.14830625385576...|\n",
            "|     80|(132,[2,6],[2.397...|(47,[4],[1.0])|[52665.0,5968.0,1...|[0.15477928753227...|\n",
            "+-------+--------------------+--------------+--------------------+--------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "from pyspark.ml.feature import MinMaxScaler, VectorAssembler\n",
        "\n",
        "# Aggregate columns for the normalization\n",
        "va = VectorAssembler(inputCols=[\"stars\", \"forks\", \"updated\", \"sponsor\"], outputCol=\"repo_info\")\n",
        "item_profiles = va.transform(features_df).select(\"repo_id\", \"about\", \"language\", \"repo_info\")\n",
        "\n",
        "# Apply MinMax normalization\n",
        "scaler = MinMaxScaler(inputCol=\"repo_info\", outputCol=\"scaled_repo_info\")\n",
        "model = scaler.fit(item_profiles)\n",
        "item_profiles = model.transform(item_profiles)\n",
        "item_profiles.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "E0tnM5AUf4t8"
      },
      "source": [
        "As final step, we aggregate all the features in a single vector using the `VectorAssembler` class."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 37,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IxzpRNUjDoC8",
        "outputId": "88357742-faa9-4296-ce70-aa54222108f1"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+-------+--------------------+\n",
            "|repo_id|        item_profile|\n",
            "+-------+--------------------+\n",
            "|      0|(183,[10,16,32,33...|\n",
            "|      1|(183,[2,4,132,179...|\n",
            "|      2|(183,[1,3,6,123,1...|\n",
            "|      7|(183,[4,14,22,29,...|\n",
            "|     11|(183,[93,127,132,...|\n",
            "|     16|(183,[5,21,141,17...|\n",
            "|     24|(183,[17,141,179,...|\n",
            "|     28|(183,[0,13,135,17...|\n",
            "|     35|(183,[17,105,133,...|\n",
            "|     43|(183,[0,4,37,50,8...|\n",
            "|     44|(183,[0,4,37,50,8...|\n",
            "|     46|(183,[0,2,3,5,9,2...|\n",
            "|     48|(183,[2,4,6,10,37...|\n",
            "|     50|(183,[7,8,40,71,1...|\n",
            "|     53|(183,[0,10,133,17...|\n",
            "|     58|(183,[1,23,132,17...|\n",
            "|     64|(183,[0,13,44,55,...|\n",
            "|     72|(183,[133,179,180...|\n",
            "|     77|(183,[4,6,10,12,2...|\n",
            "|     80|(183,[2,6,136,179...|\n",
            "+-------+--------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "va = VectorAssembler(inputCols=[\"about\", \"language\", \"scaled_repo_info\"], outputCol=\"item_profile\")\n",
        "item_profiles = va.transform(item_profiles).select(\"repo_id\", \"item_profile\").persist(StorageLevel.MEMORY_ONLY)\n",
        "item_profiles.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jc2gSQYOfbQA"
      },
      "source": [
        "Finally, we can save the item profiles on disk in order to use them later."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 38,
      "metadata": {
        "id": "vdUnOLXBqUBE"
      },
      "outputs": [],
      "source": [
        "item_profiles.write.mode(\"overwrite\").save(\"content_based-item_profiles.parquet\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xfYTQXtZDt-j"
      },
      "source": [
        "### Building User profiles\n",
        "\n",
        "In this section, we create the user profiles for all the users in the train set, computed as the average of the item profiles that that user has starred.\n",
        "\n",
        "So, given $\\mathcal{I}_u$ which is the set of items vectors starred by user $u$, the user profile of the $i$-th user is defined as:\n",
        "\n",
        "$$\n",
        "u = \\frac{1}{|\\mathcal{I}_u|} \\sum_{i \\in \\mathcal{I}_u} i\n",
        "$$\n",
        "\n",
        "To compute them, first of all we add the item profiles to each record of the train set."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 39,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NeYvHzmSDtJk",
        "outputId": "168e2e78-533d-4c49-8a57-598761185721"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+-------+-------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
            "|repo_id|user_id|item_profile                                                                                                                                                                                                             |\n",
            "+-------+-------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
            "|0      |145    |(183,[10,16,32,33,71,105,133,179,180,181],[2.9967317738870745,3.0801133828261253,3.5765502691400166,3.5075573976530654,4.13616605707544,4.4238481295272205,1.0,0.0967296919418791,0.0235612888024975,0.9793678412424156])|\n",
            "|0      |322    |(183,[10,16,32,33,71,105,133,179,180,181],[2.9967317738870745,3.0801133828261253,3.5765502691400166,3.5075573976530654,4.13616605707544,4.4238481295272205,1.0,0.0967296919418791,0.0235612888024975,0.9793678412424156])|\n",
            "|0      |726    |(183,[10,16,32,33,71,105,133,179,180,181],[2.9967317738870745,3.0801133828261253,3.5765502691400166,3.5075573976530654,4.13616605707544,4.4238481295272205,1.0,0.0967296919418791,0.0235612888024975,0.9793678412424156])|\n",
            "|0      |746    |(183,[10,16,32,33,71,105,133,179,180,181],[2.9967317738870745,3.0801133828261253,3.5765502691400166,3.5075573976530654,4.13616605707544,4.4238481295272205,1.0,0.0967296919418791,0.0235612888024975,0.9793678412424156])|\n",
            "|0      |766    |(183,[10,16,32,33,71,105,133,179,180,181],[2.9967317738870745,3.0801133828261253,3.5765502691400166,3.5075573976530654,4.13616605707544,4.4238481295272205,1.0,0.0967296919418791,0.0235612888024975,0.9793678412424156])|\n",
            "|0      |971    |(183,[10,16,32,33,71,105,133,179,180,181],[2.9967317738870745,3.0801133828261253,3.5765502691400166,3.5075573976530654,4.13616605707544,4.4238481295272205,1.0,0.0967296919418791,0.0235612888024975,0.9793678412424156])|\n",
            "|0      |1567   |(183,[10,16,32,33,71,105,133,179,180,181],[2.9967317738870745,3.0801133828261253,3.5765502691400166,3.5075573976530654,4.13616605707544,4.4238481295272205,1.0,0.0967296919418791,0.0235612888024975,0.9793678412424156])|\n",
            "|0      |1638   |(183,[10,16,32,33,71,105,133,179,180,181],[2.9967317738870745,3.0801133828261253,3.5765502691400166,3.5075573976530654,4.13616605707544,4.4238481295272205,1.0,0.0967296919418791,0.0235612888024975,0.9793678412424156])|\n",
            "|0      |1657   |(183,[10,16,32,33,71,105,133,179,180,181],[2.9967317738870745,3.0801133828261253,3.5765502691400166,3.5075573976530654,4.13616605707544,4.4238481295272205,1.0,0.0967296919418791,0.0235612888024975,0.9793678412424156])|\n",
            "|0      |1977   |(183,[10,16,32,33,71,105,133,179,180,181],[2.9967317738870745,3.0801133828261253,3.5765502691400166,3.5075573976530654,4.13616605707544,4.4238481295272205,1.0,0.0967296919418791,0.0235612888024975,0.9793678412424156])|\n",
            "|0      |2011   |(183,[10,16,32,33,71,105,133,179,180,181],[2.9967317738870745,3.0801133828261253,3.5765502691400166,3.5075573976530654,4.13616605707544,4.4238481295272205,1.0,0.0967296919418791,0.0235612888024975,0.9793678412424156])|\n",
            "|0      |2061   |(183,[10,16,32,33,71,105,133,179,180,181],[2.9967317738870745,3.0801133828261253,3.5765502691400166,3.5075573976530654,4.13616605707544,4.4238481295272205,1.0,0.0967296919418791,0.0235612888024975,0.9793678412424156])|\n",
            "|0      |2126   |(183,[10,16,32,33,71,105,133,179,180,181],[2.9967317738870745,3.0801133828261253,3.5765502691400166,3.5075573976530654,4.13616605707544,4.4238481295272205,1.0,0.0967296919418791,0.0235612888024975,0.9793678412424156])|\n",
            "|0      |2226   |(183,[10,16,32,33,71,105,133,179,180,181],[2.9967317738870745,3.0801133828261253,3.5765502691400166,3.5075573976530654,4.13616605707544,4.4238481295272205,1.0,0.0967296919418791,0.0235612888024975,0.9793678412424156])|\n",
            "|0      |2464   |(183,[10,16,32,33,71,105,133,179,180,181],[2.9967317738870745,3.0801133828261253,3.5765502691400166,3.5075573976530654,4.13616605707544,4.4238481295272205,1.0,0.0967296919418791,0.0235612888024975,0.9793678412424156])|\n",
            "|0      |2517   |(183,[10,16,32,33,71,105,133,179,180,181],[2.9967317738870745,3.0801133828261253,3.5765502691400166,3.5075573976530654,4.13616605707544,4.4238481295272205,1.0,0.0967296919418791,0.0235612888024975,0.9793678412424156])|\n",
            "|0      |2665   |(183,[10,16,32,33,71,105,133,179,180,181],[2.9967317738870745,3.0801133828261253,3.5765502691400166,3.5075573976530654,4.13616605707544,4.4238481295272205,1.0,0.0967296919418791,0.0235612888024975,0.9793678412424156])|\n",
            "|0      |2911   |(183,[10,16,32,33,71,105,133,179,180,181],[2.9967317738870745,3.0801133828261253,3.5765502691400166,3.5075573976530654,4.13616605707544,4.4238481295272205,1.0,0.0967296919418791,0.0235612888024975,0.9793678412424156])|\n",
            "|0      |2931   |(183,[10,16,32,33,71,105,133,179,180,181],[2.9967317738870745,3.0801133828261253,3.5765502691400166,3.5075573976530654,4.13616605707544,4.4238481295272205,1.0,0.0967296919418791,0.0235612888024975,0.9793678412424156])|\n",
            "|0      |3108   |(183,[10,16,32,33,71,105,133,179,180,181],[2.9967317738870745,3.0801133828261253,3.5765502691400166,3.5075573976530654,4.13616605707544,4.4238481295272205,1.0,0.0967296919418791,0.0235612888024975,0.9793678412424156])|\n",
            "+-------+-------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "repo_item_profiles_df = train.join(item_profiles, on=\"repo_id\")\n",
        "repo_item_profiles_df.show(truncate=False)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ppFfXslOfLBg"
      },
      "source": [
        "Then, we can compute the average of all the item profiles of the repos that each user has starred using the `Summarizer` class."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 40,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mrr7ptalDycJ",
        "outputId": "a7a795b6-a6ab-4527-e457-84bac7ed3364"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+-------+--------------------+--------------------+\n",
            "|user_id|        user_profile|        starred_repo|\n",
            "+-------+--------------------+--------------------+\n",
            "|     90|[0.18249584158976...|[1047, 662, 743, ...|\n",
            "|    123|[0.28580704682872...|[2440, 7660, 2, 1...|\n",
            "|    140|[0.20648100934156...|[8604, 204, 662, ...|\n",
            "|    150|[0.42825690826399...|[1148, 438, 6949,...|\n",
            "|    191|[0.33226829089447...|[691, 1, 743, 2, ...|\n",
            "|    210|[0.16728785479062...|[12095, 5199, 259...|\n",
            "|    263|[0.39276278950841...|[2440, 589, 12907...|\n",
            "|    320|[0.15055906931155...|[13293, 691, 438,...|\n",
            "|    336|[0.26766056766499...|[2581, 7692, 2, 1...|\n",
            "|    369|[0.30111813862311...|[5859, 9409, 1463...|\n",
            "|    384|[0.25357316936683...|[1148, 8604, 357,...|\n",
            "|    393|[0.19890372459508...|[7210, 204, 17975...|\n",
            "|    406|[0.33457570958124...|[3355, 7210, 3682...|\n",
            "|    419|[0.17130276330559...|[5859, 13061, 146...|\n",
            "|    426|[0.10107461995741...|[10802, 17402, 60...|\n",
            "|    431|[0.26279401188926...|[2207, 335, 691, ...|\n",
            "|    463|[0.0,1.2100592047...|[25974, 169, 2, 1...|\n",
            "|    483|[0.28104359604824...|[5859, 1395, 2, 6...|\n",
            "|    499|[0.35877905878499...|[2258, 6977, 1057...|\n",
            "|    546|[0.76071950810051...|[2599, 691, 4081,...|\n",
            "+-------+--------------------+--------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "from pyspark.ml.stat import Summarizer\n",
        "\n",
        "user_profiles = repo_item_profiles_df.groupBy(\"user_id\").agg(Summarizer.mean(col(\"item_profile\")).alias(\"user_profile\"), collect_set(\"repo_id\").alias(\"starred_repo\")).persist(StorageLevel.MEMORY_ONLY)\n",
        "user_profiles.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yM74pKm3D54q"
      },
      "source": [
        "### Predicting new repos\n",
        "\n",
        "For the prediction, we need to find the $k$ items that are more similar to the user profile for each user, applying the cosine similarity between the user profile vector and the item profile for each item not rated yet.\n",
        "\n",
        "So, given a user $u$, we compute the similarity values between his user profile and all the items $i$ not rated yet by $u$.\n",
        "\n",
        "$$\n",
        "S_u = \\{(i, sim(u, i)) : i \\in \\mathcal{I} - \\mathcal{I}_u\\}\n",
        "$$\n",
        "\n",
        "Then, we **order** the values of $S_u$ based on the similarity value and we get the top-$k$ items.\n",
        "\n",
        "$$\n",
        "R_{u,k} = \\{S_u[j] : j = 1 ... k \\}\n",
        "$$\n",
        "\n",
        "In order to put in relation each user profile with all the item profiles we apply the cartesian product between the train set and the item profiles."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 41,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QWP48xksD7Mp",
        "outputId": "9b252ddb-daad-431a-af4e-272601215016"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+-------+--------------------+--------------------+-------+--------------------+\n",
            "|user_id|        user_profile|        starred_repo|repo_id|        item_profile|\n",
            "+-------+--------------------+--------------------+-------+--------------------+\n",
            "|     90|[0.18249584158976...|[1047, 662, 743, ...|      0|(183,[10,16,32,33...|\n",
            "|     90|[0.18249584158976...|[1047, 662, 743, ...|      1|(183,[2,4,132,179...|\n",
            "|     90|[0.18249584158976...|[1047, 662, 743, ...|      2|(183,[1,3,6,123,1...|\n",
            "|     90|[0.18249584158976...|[1047, 662, 743, ...|      7|(183,[4,14,22,29,...|\n",
            "|     90|[0.18249584158976...|[1047, 662, 743, ...|     11|(183,[93,127,132,...|\n",
            "|     90|[0.18249584158976...|[1047, 662, 743, ...|     16|(183,[5,21,141,17...|\n",
            "|     90|[0.18249584158976...|[1047, 662, 743, ...|     24|(183,[17,141,179,...|\n",
            "|     90|[0.18249584158976...|[1047, 662, 743, ...|     28|(183,[0,13,135,17...|\n",
            "|     90|[0.18249584158976...|[1047, 662, 743, ...|     35|(183,[17,105,133,...|\n",
            "|     90|[0.18249584158976...|[1047, 662, 743, ...|     43|(183,[0,4,37,50,8...|\n",
            "|     90|[0.18249584158976...|[1047, 662, 743, ...|     44|(183,[0,4,37,50,8...|\n",
            "|     90|[0.18249584158976...|[1047, 662, 743, ...|     46|(183,[0,2,3,5,9,2...|\n",
            "|     90|[0.18249584158976...|[1047, 662, 743, ...|     48|(183,[2,4,6,10,37...|\n",
            "|     90|[0.18249584158976...|[1047, 662, 743, ...|     50|(183,[7,8,40,71,1...|\n",
            "|     90|[0.18249584158976...|[1047, 662, 743, ...|     53|(183,[0,10,133,17...|\n",
            "|     90|[0.18249584158976...|[1047, 662, 743, ...|     58|(183,[1,23,132,17...|\n",
            "|     90|[0.18249584158976...|[1047, 662, 743, ...|     64|(183,[0,13,44,55,...|\n",
            "|     90|[0.18249584158976...|[1047, 662, 743, ...|     72|(183,[133,179,180...|\n",
            "|     90|[0.18249584158976...|[1047, 662, 743, ...|     77|(183,[4,6,10,12,2...|\n",
            "|     90|[0.18249584158976...|[1047, 662, 743, ...|     80|(183,[2,6,136,179...|\n",
            "+-------+--------------------+--------------------+-------+--------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "user_item_profiles = user_profiles.join(item_profiles)\n",
        "user_item_profiles.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6JNJ3M12esEd"
      },
      "source": [
        "Before computing the similarity between each item we have to remove the item profiles of the repos already starred by the user, otherwise we will recommend repos that the user has already starred."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 42,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jWiNZhBJBlZ2",
        "outputId": "97570152-253a-44b6-c5a1-c6368096ff7d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+-------+--------------------+--------------------+-------+--------------------+\n",
            "|user_id|        user_profile|        starred_repo|repo_id|        item_profile|\n",
            "+-------+--------------------+--------------------+-------+--------------------+\n",
            "|     90|[0.18249584158976...|[1047, 662, 743, ...|      0|(183,[10,16,32,33...|\n",
            "|     90|[0.18249584158976...|[1047, 662, 743, ...|      7|(183,[4,14,22,29,...|\n",
            "|     90|[0.18249584158976...|[1047, 662, 743, ...|     11|(183,[93,127,132,...|\n",
            "|     90|[0.18249584158976...|[1047, 662, 743, ...|     24|(183,[17,141,179,...|\n",
            "|     90|[0.18249584158976...|[1047, 662, 743, ...|     28|(183,[0,13,135,17...|\n",
            "|     90|[0.18249584158976...|[1047, 662, 743, ...|     35|(183,[17,105,133,...|\n",
            "|     90|[0.18249584158976...|[1047, 662, 743, ...|     43|(183,[0,4,37,50,8...|\n",
            "|     90|[0.18249584158976...|[1047, 662, 743, ...|     44|(183,[0,4,37,50,8...|\n",
            "|     90|[0.18249584158976...|[1047, 662, 743, ...|     46|(183,[0,2,3,5,9,2...|\n",
            "|     90|[0.18249584158976...|[1047, 662, 743, ...|     48|(183,[2,4,6,10,37...|\n",
            "|     90|[0.18249584158976...|[1047, 662, 743, ...|     50|(183,[7,8,40,71,1...|\n",
            "|     90|[0.18249584158976...|[1047, 662, 743, ...|     53|(183,[0,10,133,17...|\n",
            "|     90|[0.18249584158976...|[1047, 662, 743, ...|     58|(183,[1,23,132,17...|\n",
            "|     90|[0.18249584158976...|[1047, 662, 743, ...|     64|(183,[0,13,44,55,...|\n",
            "|     90|[0.18249584158976...|[1047, 662, 743, ...|     72|(183,[133,179,180...|\n",
            "|     90|[0.18249584158976...|[1047, 662, 743, ...|     77|(183,[4,6,10,12,2...|\n",
            "|     90|[0.18249584158976...|[1047, 662, 743, ...|     80|(183,[2,6,136,179...|\n",
            "|     90|[0.18249584158976...|[1047, 662, 743, ...|     92|(183,[113,133,179...|\n",
            "|     90|[0.18249584158976...|[1047, 662, 743, ...|    100|(183,[9,28,71,102...|\n",
            "|     90|[0.18249584158976...|[1047, 662, 743, ...|    103|(183,[4,21,26,83,...|\n",
            "+-------+--------------------+--------------------+-------+--------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "user_item_profiles = user_item_profiles.filter(~array_contains(col(\"starred_repo\"), col(\"repo_id\")))\n",
        "user_item_profiles.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rdQvRb4XeexV"
      },
      "source": [
        "Now we can compute the cosine similarity using the previously defined udf functions."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 43,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IKu_JHF_D_AG",
        "outputId": "acf153a7-db53-4f56-b47b-1f1a21a604db"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+-------+-------+-------------------+\n",
            "|user_id|repo_id|         similarity|\n",
            "+-------+-------+-------------------+\n",
            "|     90|      0|0.19263542812959603|\n",
            "|     90|      7|0.22949633312015053|\n",
            "|     90|     11| 0.1631732096607974|\n",
            "|     90|     24|  0.275482419219751|\n",
            "|     90|     28| 0.3472998353046473|\n",
            "|     90|     35|0.21522446037345336|\n",
            "|     90|     43|0.10339401291029349|\n",
            "|     90|     44| 0.1156640802140694|\n",
            "|     90|     46|0.29223742036241324|\n",
            "|     90|     48| 0.2554083335991793|\n",
            "|     90|     50|0.25352432546786025|\n",
            "|     90|     53|0.22718420889810703|\n",
            "|     90|     58| 0.2433376423737014|\n",
            "|     90|     64| 0.2472584416780459|\n",
            "|     90|     72| 0.4332705921803832|\n",
            "|     90|     77| 0.2510632058887067|\n",
            "|     90|     80|0.31914765763311753|\n",
            "|     90|     92| 0.1647830438442851|\n",
            "|     90|    100|0.20013086943478758|\n",
            "|     90|    103| 0.1911303884440288|\n",
            "+-------+-------+-------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "similarity = cosine_similarity_udf(\"user_profile\", \"item_profile\").alias(\"similarity\")\n",
        "prediction_df = user_item_profiles.select(\"user_id\", \"repo_id\", similarity).persist(StorageLevel.MEMORY_ONLY)\n",
        "prediction_df.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "v-idqRhRePCB"
      },
      "source": [
        "Given the similarities computed in the previous step, we can take the top-k items for each user.\n",
        "\n",
        "To do that, we use a `Window function` in which we can perform some aggregation operations for subsets of users.\n",
        "\n",
        "In this case, for each user we order the records by similarity and, after adding the row number for each grouping, we filter them by taking the first `N_ITEMS` records."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 44,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zne8wxvGEH-U",
        "outputId": "b512596c-bd20-47d6-f6ec-070466f188c0"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+-------+-------+-------------------+----+\n",
            "|user_id|repo_id|         similarity|rank|\n",
            "+-------+-------+-------------------+----+\n",
            "|     90|  11903|0.48425226445217956|   1|\n",
            "|     90|  14274|0.48200720798957775|   2|\n",
            "|     90|   7854| 0.4806740134282395|   3|\n",
            "|     90|   8418|  0.480479224263995|   4|\n",
            "|     90|    619|  0.480397710468917|   5|\n",
            "|    123|  11903| 0.5193505622041305|   1|\n",
            "|    123|  14274| 0.5177934184352835|   2|\n",
            "|    123|    619| 0.5164651457869246|   3|\n",
            "|    123|   8418| 0.5164383330962349|   4|\n",
            "|    123|   6355| 0.5157807868617589|   5|\n",
            "|    140|   5859|0.41908032109972254|   1|\n",
            "|    140|    716| 0.4092775801854016|   2|\n",
            "|    140|   2265| 0.4060659936550205|   3|\n",
            "|    140|   6189|0.40326978334418195|   4|\n",
            "|    140|   1375| 0.4006986845082195|   5|\n",
            "|    150|   4672| 0.6085488732632729|   1|\n",
            "|    150|    418|  0.605380443078601|   2|\n",
            "|    150|  11903| 0.5802743824972105|   3|\n",
            "|    150|    643| 0.5768581948324893|   4|\n",
            "|    150|  14274| 0.5765535234549904|   5|\n",
            "+-------+-------+-------------------+----+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "from pyspark.sql.window import Window\n",
        "\n",
        "window_user = Window.partitionBy(\"user_id\").orderBy(col(\"similarity\").desc())\n",
        "top_k_repos = prediction_df.withColumn(\"rank\", row_number().over(window_user)).filter(col(\"rank\") <= N_ITEMS)\n",
        "top_k_repos.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Qe-gyrfAeJjI"
      },
      "source": [
        "Now, for the evaluation phase, we can finally aggregate the recommendations for each user in a single vector."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 45,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_2czpussAgiw",
        "outputId": "94432e7a-aad3-456e-c980-2aeeceafa1b6"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+-------+------------------------------------------+\n",
            "|user_id|recommended_repos                         |\n",
            "+-------+------------------------------------------+\n",
            "|90     |[11903.0, 14274.0, 7854.0, 8418.0, 619.0] |\n",
            "|123    |[11903.0, 14274.0, 619.0, 8418.0, 6355.0] |\n",
            "|140    |[5859.0, 716.0, 2265.0, 6189.0, 1375.0]   |\n",
            "|150    |[4672.0, 418.0, 11903.0, 643.0, 14274.0]  |\n",
            "|191    |[4672.0, 170.0, 418.0, 6958.0, 438.0]     |\n",
            "|210    |[11903.0, 1124.0, 8027.0, 14274.0, 7854.0]|\n",
            "|263    |[4672.0, 418.0, 170.0, 11903.0, 14274.0]  |\n",
            "|320    |[4672.0, 418.0, 170.0, 526.0, 6958.0]     |\n",
            "|336    |[141.0, 442.0, 570.0, 17402.0, 80.0]      |\n",
            "|369    |[72.0, 1254.0, 11903.0, 14274.0, 7854.0]  |\n",
            "|384    |[5853.0, 16063.0, 431.0, 4672.0, 170.0]   |\n",
            "|393    |[11903.0, 14274.0, 619.0, 8418.0, 6355.0] |\n",
            "|406    |[11903.0, 14274.0, 643.0, 7854.0, 619.0]  |\n",
            "|419    |[11903.0, 14274.0, 8418.0, 619.0, 6355.0] |\n",
            "|426    |[5880.0, 1254.0, 335.0, 11903.0, 14274.0] |\n",
            "|431    |[11903.0, 14274.0, 7854.0, 619.0, 8418.0] |\n",
            "|463    |[8815.0, 1583.0, 473.0, 5922.0, 160.0]    |\n",
            "|483    |[11903.0, 14274.0, 7854.0, 8418.0, 619.0] |\n",
            "|499    |[14274.0, 8418.0, 619.0, 6355.0, 3637.0]  |\n",
            "|546    |[570.0, 11975.0, 16931.0, 6101.0, 697.0]  |\n",
            "+-------+------------------------------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "recommendations_list = top_k_repos.withColumn(\"repo_id\", col(\"repo_id\").cast(\"double\")).groupBy(\"user_id\").agg(collect_list(\"repo_id\").alias(\"recommended_repos\"))\n",
        "recommendations_list.show(truncate=False)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WjKwA9yVBHvd"
      },
      "source": [
        "### Evaluation\n",
        "\n",
        "In this section, we carry out the evaluation phase for the Content-based filtering model."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 46,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NnPTH52MBGlP",
        "outputId": "71ed0805-3e46-4e7f-8036-aed0c8497317"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+-------+------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
            "|user_id|recommended_repos                         |labels                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |\n",
            "+-------+------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
            "|90     |[11903.0, 14274.0, 7854.0, 8418.0, 619.0] |[1258.0, 5968.0, 6319.0, 6251.0, 1409.0, 6300.0, 1067.0, 2709.0, 6259.0, 506.0, 335.0, 147.0, 6316.0, 1965.0, 4104.0, 1937.0, 797.0, 28.0, 821.0, 2256.0, 2192.0, 6156.0, 6163.0, 6201.0, 6254.0, 2258.0, 6330.0]                                                                                                                                                                                                                                                                                                                                                                           |\n",
            "|123    |[11903.0, 14274.0, 619.0, 8418.0, 6355.0] |[238.0, 123.0, 1049.0, 6300.0, 122.0, 1280.0, 1411.0, 483.0, 1041.0, 6935.0, 2851.0, 3375.0, 48.0, 2857.0, 7539.0, 2275.0, 4140.0, 2701.0, 2634.0, 3953.0, 706.0, 526.0, 910.0, 2069.0, 442.0, 797.0, 1042.0, 7814.0, 77.0, 871.0, 7750.0, 1257.0, 6435.0]                                                                                                                                                                                                                                                                                                                                  |\n",
            "|140    |[5859.0, 716.0, 2265.0, 6189.0, 1375.0]   |[4815.0, 2.0, 2967.0, 2266.0, 8603.0, 5859.0, 433.0, 2839.0, 1510.0, 6256.0, 1965.0, 109.0, 413.0, 2866.0, 616.0, 3520.0, 8654.0, 8583.0, 8558.0, 1042.0, 8675.0, 8539.0, 8528.0, 4016.0, 126.0]                                                                                                                                                                                                                                                                                                                                                                                            |\n",
            "|150    |[4672.0, 418.0, 11903.0, 643.0, 14274.0]  |[2162.0, 8332.0, 2609.0, 3375.0, 9183.0, 533.0, 2384.0, 668.0, 1053.0, 8006.0, 746.0, 1054.0, 9077.0]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |\n",
            "|191    |[4672.0, 170.0, 418.0, 6958.0, 438.0]     |[383.0, 414.0, 35.0, 122.0, 4028.0, 7814.0, 871.0, 476.0, 2275.0]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           |\n",
            "|210    |[11903.0, 1124.0, 8027.0, 14274.0, 7854.0]|[238.0, 3374.0, 703.0, 4989.0, 1378.0, 6289.0, 2074.0, 1510.0, 2385.0, 1444.0, 6135.0, 648.0, 454.0, 381.0, 365.0, 48.0, 386.0, 716.0, 4200.0]                                                                                                                                                                                                                                                                                                                                                                                                                                              |\n",
            "|263    |[4672.0, 418.0, 170.0, 11903.0, 14274.0]  |[11903.0, 988.0, 347.0, 671.0, 506.0, 721.0, 1411.0, 1377.0, 6938.0, 11186.0, 320.0, 731.0, 5940.0, 418.0, 691.0, 1677.0, 5948.0, 1061.0, 11691.0, 126.0, 35.0, 117.0, 14274.0, 1148.0, 441.0, 3953.0, 16.0, 1119.0, 3682.0, 5680.0, 7854.0, 5872.0, 442.0, 683.0, 1612.0, 716.0, 435.0, 77.0, 871.0, 1339.0]                                                                                                                                                                                                                                                                               |\n",
            "|320    |[4672.0, 418.0, 170.0, 526.0, 6958.0]     |[1236.0, 6935.0, 510.0, 834.0, 1385.0, 3426.0, 7660.0, 2599.0]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              |\n",
            "|336    |[141.0, 442.0, 570.0, 17402.0, 80.0]      |[2851.0, 357.0]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |\n",
            "|369    |[72.0, 1254.0, 11903.0, 14274.0, 7854.0]  |[7197.0, 2967.0, 408.0, 671.0, 331.0, 2709.0, 110.0, 1737.0, 1429.0, 11469.0, 483.0, 1056.0, 1382.0, 6935.0, 3375.0, 48.0, 691.0, 1433.0, 4033.0, 4727.0, 8797.0, 1405.0, 669.0, 4140.0, 125.0, 4378.0, 19465.0, 614.0, 1148.0, 6749.0, 697.0, 706.0, 11975.0, 442.0, 6747.0, 4848.0, 6254.0, 160.0, 2207.0, 8476.0, 4803.0]                                                                                                                                                                                                                                                                |\n",
            "|384    |[5853.0, 16063.0, 431.0, 4672.0, 170.0]   |[527.0, 11.0, 1067.0, 441.0, 12146.0, 1070.0, 2256.0, 11186.0, 20507.0, 103.0, 1047.0, 2081.0]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              |\n",
            "|393    |[11903.0, 14274.0, 619.0, 8418.0, 6355.0] |[383.0, 645.0, 4989.0, 449.0, 2045.0, 2581.0, 3930.0, 21321.0, 106.0, 1249.0, 352.0, 8455.0, 2704.0, 14512.0, 5940.0, 4874.0, 16593.0, 6042.0, 357.0, 691.0, 2646.0, 668.0, 2173.0, 1061.0, 7539.0, 16591.0, 2328.0, 16063.0, 2275.0, 15630.0, 35.0, 2044.0, 8339.0, 647.0, 453.0, 145.0, 3953.0, 4394.0, 16.0, 648.0, 6633.0, 8254.0, 8891.0, 471.0, 666.0, 410.0, 21455.0, 12282.0, 6212.0, 643.0, 20774.0, 21661.0, 419.0, 1044.0, 6910.0, 1583.0]                                                                                                                                       |\n",
            "|406    |[11903.0, 14274.0, 643.0, 7854.0, 619.0]  |[2162.0, 35.0, 117.0, 119.0, 2149.0, 122.0, 1280.0, 721.0, 2187.0, 6915.0, 6802.0, 7567.0, 15732.0, 8202.0, 3669.0, 3226.0, 357.0, 691.0, 3426.0, 4471.0, 160.0, 1110.0, 4016.0, 6273.0, 791.0, 5896.0]                                                                                                                                                                                                                                                                                                                                                                                     |\n",
            "|419    |[11903.0, 14274.0, 8418.0, 619.0, 6355.0] |[6251.0, 10136.0, 6949.0, 1444.0, 22921.0, 2278.0, 8095.0, 3520.0, 7519.0, 1254.0, 1321.0, 5858.0, 11095.0, 334.0, 864.0, 2275.0, 1993.0, 1258.0, 1063.0, 6432.0, 9039.0, 10544.0, 3067.0, 19835.0, 964.0, 7887.0, 18500.0, 5095.0, 5996.0, 2033.0, 1853.0, 674.0, 10107.0, 1612.0, 707.0, 4989.0, 581.0, 1344.0, 12912.0, 23716.0, 2457.0, 454.0, 2314.0, 527.0, 3401.0, 6976.0, 16931.0, 6547.0, 19555.0, 24052.0, 3833.0, 8983.0, 1302.0, 459.0, 863.0, 11819.0, 13869.0, 337.0, 23277.0, 117.0, 1995.0, 4654.0, 6150.0, 4715.0, 7997.0, 526.0, 16344.0, 333.0, 7814.0, 10474.0, 17387.0]|\n",
            "|426    |[5880.0, 1254.0, 335.0, 11903.0, 14274.0] |[383.0, 1342.0, 1994.0, 2884.0, 3521.0, 2400.0, 7134.0, 320.0, 348.0, 15348.0, 357.0, 15701.0, 2256.0, 18508.0, 8093.0, 20064.0, 9039.0, 792.0, 2594.0, 8136.0, 6202.0, 120.0, 8339.0, 1053.0, 615.0, 8812.0, 3682.0, 1186.0, 430.0, 340.0, 2069.0, 512.0, 160.0, 77.0, 3383.0, 1422.0, 2294.0, 743.0, 1421.0, 318.0, 1899.0, 449.0, 2858.0, 110.0, 4413.0, 1380.0, 5930.0, 1046.0, 464.0, 2701.0, 5886.0, 9553.0, 6482.0, 5822.0, 7036.0, 6150.0, 9675.0, 145.0, 2926.0, 4715.0, 706.0, 12095.0, 419.0, 798.0, 80.0, 865.0]                                                                |\n",
            "|431    |[11903.0, 14274.0, 7854.0, 619.0, 8418.0] |[669.0, 4140.0, 7977.0, 2.0, 4378.0, 4107.0, 371.0, 3374.0, 3730.0, 6734.0, 2349.0, 1475.0, 13293.0, 656.0, 413.0, 802.0, 352.0, 6746.0, 6404.0, 8798.0, 77.0, 1340.0, 505.0, 1941.0, 22107.0]                                                                                                                                                                                                                                                                                                                                                                                              |\n",
            "|463    |[8815.0, 1583.0, 473.0, 5922.0, 160.0]    |[9684.0, 1279.0, 120.0, 2045.0, 204.0, 1369.0, 961.0, 8254.0]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               |\n",
            "|483    |[11903.0, 14274.0, 7854.0, 8418.0, 619.0] |[8293.0, 1211.0, 3521.0, 581.0, 118.0, 204.0, 11183.0, 804.0, 2851.0, 3458.0, 18819.0, 528.0, 2256.0, 6163.0, 5926.0, 1423.0, 1258.0, 2162.0, 5878.0, 5968.0, 8573.0, 489.0, 8339.0, 533.0, 1022.0, 1376.0, 10601.0, 5125.0, 802.0, 526.0, 2069.0, 6449.0, 1042.0, 716.0, 5820.0, 80.0, 1422.0]                                                                                                                                                                                                                                                                                             |\n",
            "|499    |[14274.0, 8418.0, 619.0, 6355.0, 3637.0]  |[238.0, 8429.0, 988.0, 629.0, 4412.0, 671.0, 581.0, 6406.0, 106.0, 147.0, 631.0, 2440.0, 3968.0, 4364.0, 336.0, 8583.0, 6547.0, 682.0, 666.0, 28.0, 6302.0, 14672.0, 1355.0, 2599.0, 187.0, 791.0]                                                                                                                                                                                                                                                                                                                                                                                          |\n",
            "|546    |[570.0, 11975.0, 16931.0, 6101.0, 697.0]  |[2.0, 5968.0, 2967.0, 5859.0, 24937.0, 2255.0, 1460.0, 15332.0, 438.0, 666.0, 77.0, 6910.0, 199.0, 2275.0]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  |\n",
            "+-------+------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "recommendations = recommendations_list.join(gt, \"user_id\").cache()\n",
        "recommendations.show(truncate=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 47,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UOV1XwsiBOZr",
        "outputId": "28b8581b-8ee0-430f-fda2-ade5fda0ae23"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+-----------------------+-----+-----+-----+-----+-----+---------------+\n",
            "|Model                  |MAP@1|MAP@2|MAP@3|MAP@4|MAP@5|Personalization|\n",
            "+-----------------------+-----+-----+-----+-----+-----+---------------+\n",
            "|Content-Based Filtering|0.067|0.051|0.043|0.037|0.033|0.676          |\n",
            "+-----------------------+-----+-----+-----+-----+-----+---------------+\n",
            "\n"
          ]
        }
      ],
      "source": [
        "evaluation = evaluate_df(\"Content-Based Filtering\", recommendations, k)\n",
        "metrics_df = metrics_df.union(evaluation)\n",
        "round_df(evaluation, 3).show(truncate=False)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZgwaSQB7s_0W"
      },
      "source": [
        "### Free resources"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 48,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0Vk97uZ5tBud",
        "outputId": "1d37bf71-bd2b-44d0-e186-d472fcb671de"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "717"
            ]
          },
          "execution_count": 48,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "item_profiles.unpersist()\n",
        "user_profiles.unpersist()\n",
        "prediction_df.unpersist()\n",
        "\n",
        "# Free RAM\n",
        "import gc\n",
        "gc.collect()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sYFLSFjo9uDM"
      },
      "source": [
        "## User-based CF\n",
        "In the User-Based Collaborative Filtering technique, we try to recommend new items according to user rating vectors, which in this case are composed by all the IDs corresponding to the repos starred by one user."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 49,
      "metadata": {
        "cellView": "form",
        "id": "eziNgd8dbK01"
      },
      "outputs": [],
      "source": [
        "#@title Settings\n",
        "NEIGHBOURS = 5 #@param\n",
        "N_ITEMS = 5 #@param"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bUmyPtqPZIoG"
      },
      "source": [
        "### Building Similarity matrix\n",
        "\n",
        "First of all, we create a dataframe where every row contains a user id and the set of the repos id that the corresponding user has starred:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 50,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "t56J8Koz4fOX",
        "outputId": "fab32aa8-47f9-47ad-b9d8-9cf89885f087"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+-------+--------------------+\n",
            "|user_id|       starred_repos|\n",
            "+-------+--------------------+\n",
            "|     90|[1047, 662, 743, ...|\n",
            "|    123|[2440, 7660, 2, 1...|\n",
            "|    140|[8604, 204, 662, ...|\n",
            "|    150|[1148, 438, 6949,...|\n",
            "|    191|[691, 1, 743, 2, ...|\n",
            "|    210|[12095, 5199, 259...|\n",
            "|    263|[2440, 589, 12907...|\n",
            "|    320|[13293, 691, 438,...|\n",
            "|    336|[2581, 7692, 2, 1...|\n",
            "|    369|[5859, 9409, 1463...|\n",
            "|    384|[1148, 8604, 357,...|\n",
            "|    393|[7210, 204, 17975...|\n",
            "|    406|[3355, 7210, 3682...|\n",
            "|    419|[5859, 13061, 146...|\n",
            "|    426|[10802, 17402, 60...|\n",
            "|    431|[2207, 335, 691, ...|\n",
            "|    463|[25974, 169, 2, 1...|\n",
            "|    483|[5859, 1395, 2, 6...|\n",
            "|    499|[2258, 6977, 1057...|\n",
            "|    546|[2599, 691, 4081,...|\n",
            "+-------+--------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "starred_repos = train.groupBy(\"user_id\").agg(collect_set(\"repo_id\").alias(\"starred_repos\"))\n",
        "starred_repos.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-DY0rPQ8Ar7W"
      },
      "source": [
        "Then, we compute the Jaccard similarity between each user pair using their vectors:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 51,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9KWe54B4-1BB",
        "outputId": "ea87730e-c733-4432-9c7f-15b44f89c5fc"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+--------------+--------------------+--------------+--------------------+--------------------+\n",
            "|source_user_id|source_starred_repos|target_user_id|target_starred_repos|  jaccard_similarity|\n",
            "+--------------+--------------------+--------------+--------------------+--------------------+\n",
            "|            90|[1047, 662, 743, ...|           123|[2440, 7660, 2, 1...| 0.07758620689655173|\n",
            "|            90|[1047, 662, 743, ...|           140|[8604, 204, 662, ...|                0.01|\n",
            "|            90|[1047, 662, 743, ...|           150|[1148, 438, 6949,...|0.027777777777777776|\n",
            "|            90|[1047, 662, 743, ...|           191|[691, 1, 743, 2, ...| 0.09195402298850575|\n",
            "|            90|[1047, 662, 743, ...|           210|[12095, 5199, 259...| 0.05343511450381679|\n",
            "|            90|[1047, 662, 743, ...|           263|[2440, 589, 12907...|  0.0896551724137931|\n",
            "|            90|[1047, 662, 743, ...|           320|[13293, 691, 438,...|0.031578947368421054|\n",
            "|            90|[1047, 662, 743, ...|           336|[2581, 7692, 2, 1...|0.013513513513513514|\n",
            "|            90|[1047, 662, 743, ...|           369|[5859, 9409, 1463...| 0.07777777777777778|\n",
            "|            90|[1047, 662, 743, ...|           384|[1148, 8604, 357,...| 0.09473684210526316|\n",
            "|            90|[1047, 662, 743, ...|           393|[7210, 204, 17975...|0.023391812865497075|\n",
            "|            90|[1047, 662, 743, ...|           406|[3355, 7210, 3682...| 0.05154639175257732|\n",
            "|            90|[1047, 662, 743, ...|           419|[5859, 13061, 146...| 0.06985294117647059|\n",
            "|            90|[1047, 662, 743, ...|           426|[10802, 17402, 60...| 0.05025125628140704|\n",
            "|            90|[1047, 662, 743, ...|           431|[2207, 335, 691, ...| 0.07079646017699115|\n",
            "|            90|[1047, 662, 743, ...|           463|[25974, 169, 2, 1...|0.014492753623188406|\n",
            "|            90|[1047, 662, 743, ...|           483|[5859, 1395, 2, 6...| 0.10059171597633136|\n",
            "|            90|[1047, 662, 743, ...|           499|[2258, 6977, 1057...|0.056074766355140186|\n",
            "|            90|[1047, 662, 743, ...|           546|[2599, 691, 4081,...|0.011904761904761904|\n",
            "|            90|[1047, 662, 743, ...|           548|[7210, 691, 1, 61...| 0.12962962962962962|\n",
            "+--------------+--------------------+--------------+--------------------+--------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "source_users = starred_repos.withColumnRenamed(\"user_id\", \"source_user_id\").withColumnRenamed(\"starred_repos\", \"source_starred_repos\")\n",
        "target_users = starred_repos.withColumnRenamed(\"user_id\", \"target_user_id\").withColumnRenamed(\"starred_repos\", \"target_starred_repos\")\n",
        "users_join = source_users.join(target_users, source_users.source_user_id != target_users.target_user_id)\n",
        "jaccard = users_join.withColumn(\"jaccard_similarity\", jaccard_similarity_udf(\"source_starred_repos\", \"target_starred_repos\"))\n",
        "jaccard.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ivh2_D61ZbGy"
      },
      "source": [
        "### Taking top k similar users\n",
        "Now we need to take the top-$k$ most similar users for each user.\n",
        "\n",
        "They are formally computed as follows, using the similarity matrix previously built:\n",
        "\n",
        "$$\n",
        "\\mathcal{U}^k = argmax_{\\mathcal{U}' \\subseteq \\mathcal{U} / u, |\\mathcal{U}'| = k} \\sum_{u' \\in \\mathcal{U}'} sim(u, u')\n",
        "$$"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "px-4Qf-2A4aj"
      },
      "source": [
        "In our case, we take the `NEIGHBOURS` most similar users to each user in the dataframe to later use their starred repos:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 52,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YOE5_uvd9ycb",
        "outputId": "00efd2e4-77b9-4232-bebe-9f62f820c22d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+--------------+--------------------+--------------+--------------------+-------------------+----+\n",
            "|source_user_id|source_starred_repos|target_user_id|target_starred_repos| jaccard_similarity|rank|\n",
            "+--------------+--------------------+--------------+--------------------+-------------------+----+\n",
            "|            90|[1047, 662, 743, ...|          4885|[204, 1984, 743, ...|           0.203125|   1|\n",
            "|            90|[1047, 662, 743, ...|          7579|[6977, 691, 865, ...|0.15463917525773196|   2|\n",
            "|            90|[1047, 662, 743, ...|          9864|[743, 1853, 2, 11...|0.13690476190476192|   3|\n",
            "|            90|[1047, 662, 743, ...|           550|[1984, 1853, 2, 7...|0.13513513513513514|   4|\n",
            "|            90|[1047, 662, 743, ...|          5027|[743, 1853, 2, 71...|0.13131313131313133|   5|\n",
            "|           123|[2440, 7660, 2, 1...|          6932|[5859, 743, 2, 12...|0.16363636363636364|   1|\n",
            "|           123|[2440, 7660, 2, 1...|          1257|[489, 2, 125, 418...|0.12389380530973451|   2|\n",
            "|           123|[2440, 7660, 2, 1...|          7109|[5859, 204, 5940,...|0.12280701754385964|   3|\n",
            "|           123|[2440, 7660, 2, 1...|          2420|[5830, 133, 2, 12...|0.11864406779661017|   4|\n",
            "|           123|[2440, 7660, 2, 1...|          9727|[10802, 5940, 766...|0.11711711711711711|   5|\n",
            "|           140|[8604, 204, 662, ...|          1043|[2, 11183, 2457, ...|0.11904761904761904|   1|\n",
            "|           140|[8604, 204, 662, ...|          7780|[0, 17402, 5859, ...|0.10457516339869281|   2|\n",
            "|           140|[8604, 204, 662, ...|          3662|[5859, 5680, 1169...|0.09734513274336283|   3|\n",
            "|           140|[8604, 204, 662, ...|          5087|[28907, 8604, 106...|0.09722222222222222|   4|\n",
            "|           140|[8604, 204, 662, ...|          4026|[5859, 438, 2, 62...|0.09523809523809523|   5|\n",
            "|           150|[1148, 438, 6949,...|          8706|[2, 2180, 5853, 1...| 0.1839080459770115|   1|\n",
            "|           150|[1148, 438, 6949,...|          4978|[641, 6042, 379, ...|0.14705882352941177|   2|\n",
            "|           150|[1148, 438, 6949,...|          6434|[8937, 1236, 1797...|0.14516129032258066|   3|\n",
            "|           150|[1148, 438, 6949,...|          2609|[8604, 103, 2, 37...|0.13513513513513514|   4|\n",
            "|           150|[1148, 438, 6949,...|          6090|[8451, 7210, 0, 4...|0.13157894736842105|   5|\n",
            "+--------------+--------------------+--------------+--------------------+-------------------+----+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "window = Window.partitionBy(jaccard[\"source_user_id\"]).orderBy(jaccard[\"jaccard_similarity\"].desc())\n",
        "most_similar_users = (jaccard.select(\"*\", row_number().over(window).alias(\"rank\"))\n",
        "  .filter(col(\"rank\") <= NEIGHBOURS))\n",
        "most_similar_users.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "w7PDH4KYZkvC"
      },
      "source": [
        "### Predicting new repos\n",
        "\n",
        "Identified the users neighbours, we can compute the predictions taking the items rated by the user's neighbours.\n",
        "\n",
        "$$\n",
        "\\mathcal{I}^k = \\{ i \\in \\mathcal{I} : r_{u,i} = (defined) \\land u \\in \\mathcal{U}^k\\}\n",
        "$$\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KaOkqN5fBMlR"
      },
      "source": [
        "So, we take all the repos starred by the most similar users and we remove the repos already starred by the source user:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 53,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0jjJ_NLDPGPJ",
        "outputId": "c4e8a5d0-17bc-4d1d-f864-6f7fcb35cab4"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+--------------+--------------------+--------------+--------------------+------------------+----+------------+\n",
            "|source_user_id|source_starred_repos|target_user_id|target_starred_repos|jaccard_similarity|rank|starred_repo|\n",
            "+--------------+--------------------+--------------+--------------------+------------------+----+------------+\n",
            "|            90|[1047, 662, 743, ...|          4885|[204, 1984, 743, ...|          0.203125|   1|         204|\n",
            "|            90|[1047, 662, 743, ...|          4885|[204, 1984, 743, ...|          0.203125|   1|        1984|\n",
            "|            90|[1047, 662, 743, ...|          4885|[204, 1984, 743, ...|          0.203125|   1|        1853|\n",
            "|            90|[1047, 662, 743, ...|          4885|[204, 1984, 743, ...|          0.203125|   1|        5861|\n",
            "|            90|[1047, 662, 743, ...|          4885|[204, 1984, 743, ...|          0.203125|   1|        2180|\n",
            "|            90|[1047, 662, 743, ...|          4885|[204, 1984, 743, ...|          0.203125|   1|        5680|\n",
            "|            90|[1047, 662, 743, ...|          4885|[204, 1984, 743, ...|          0.203125|   1|         483|\n",
            "|            90|[1047, 662, 743, ...|          4885|[204, 1984, 743, ...|          0.203125|   1|        1826|\n",
            "|            90|[1047, 662, 743, ...|          4885|[204, 1984, 743, ...|          0.203125|   1|        1339|\n",
            "|            90|[1047, 662, 743, ...|          4885|[204, 1984, 743, ...|          0.203125|   1|        4258|\n",
            "|            90|[1047, 662, 743, ...|          4885|[204, 1984, 743, ...|          0.203125|   1|         433|\n",
            "|            90|[1047, 662, 743, ...|          4885|[204, 1984, 743, ...|          0.203125|   1|        1899|\n",
            "|            90|[1047, 662, 743, ...|          4885|[204, 1984, 743, ...|          0.203125|   1|         527|\n",
            "|            90|[1047, 662, 743, ...|          4885|[204, 1984, 743, ...|          0.203125|   1|         100|\n",
            "|            90|[1047, 662, 743, ...|          4885|[204, 1984, 743, ...|          0.203125|   1|         398|\n",
            "|            90|[1047, 662, 743, ...|          4885|[204, 1984, 743, ...|          0.203125|   1|        6532|\n",
            "|            90|[1047, 662, 743, ...|          4885|[204, 1984, 743, ...|          0.203125|   1|        9351|\n",
            "|            90|[1047, 662, 743, ...|          4885|[204, 1984, 743, ...|          0.203125|   1|         706|\n",
            "|            90|[1047, 662, 743, ...|          4885|[204, 1984, 743, ...|          0.203125|   1|          46|\n",
            "|            90|[1047, 662, 743, ...|          4885|[204, 1984, 743, ...|          0.203125|   1|       10107|\n",
            "+--------------+--------------------+--------------+--------------------+------------------+----+------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "possible_repos = most_similar_users.select(\"*\", explode(\"target_starred_repos\").alias(\"starred_repo\"))\n",
        "repos_without_source_repos = possible_repos.filter(~array_contains(col(\"source_starred_repos\"), col(\"starred_repo\")))\n",
        "repos_without_source_repos.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5X1P0HK4Bc1y"
      },
      "source": [
        "Now, we aggregate the neighbours ratings using the plain average between the ratings.\n",
        "\n",
        "$$\n",
        "r_{u,i} = \\frac{1}{k} \\sum_{v \\in \\mathcal{U}^k} r_{v,i}\n",
        "$$\n",
        "\n",
        "where $r_{v,i}$ is the rating already given by the user $v$ to the item $i$.\n",
        "\n",
        "In our case, we don't have proper integer ratings but just a binary implicit feedback, given by the fact that a user starred a repo or not. So, we can change the aggregation formula to simplify the process in this scenario.\n",
        "\n",
        "First of all, since we only care about the relative order of the items and not the exact value, we can omit the normalization factor, i.e. $\\frac{1}{k}$. Moreover, the rating for each user can be 1 or 0, so we can simply count the number of users that starred that item to obtain the sum of the ratings.\n",
        "\n",
        "So, we compute the score of each repo starred by one of the top-$k$ most similar users, by computing how many times the repo has been starred by them:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 54,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zCSLbId7W9jq",
        "outputId": "1fc92883-9e26-4ce0-a840-2a28bd3f05d7"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+--------------+------------+-----+\n",
            "|source_user_id|starred_repo|count|\n",
            "+--------------+------------+-----+\n",
            "|            90|         204|    1|\n",
            "|            90|        1984|    3|\n",
            "|            90|        1853|    4|\n",
            "|            90|        5861|    1|\n",
            "|            90|        2180|    1|\n",
            "|            90|        5680|    2|\n",
            "|            90|         483|    3|\n",
            "|            90|        1826|    1|\n",
            "|            90|        1339|    3|\n",
            "|            90|        4258|    2|\n",
            "|            90|         433|    1|\n",
            "|            90|        1899|    1|\n",
            "|            90|         527|    2|\n",
            "|            90|         100|    1|\n",
            "|            90|         398|    1|\n",
            "|            90|        6532|    2|\n",
            "|            90|        9351|    2|\n",
            "|            90|         706|    1|\n",
            "|            90|          46|    1|\n",
            "|            90|       10107|    1|\n",
            "+--------------+------------+-----+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "repos_occurrencies = repos_without_source_repos.groupBy(\"source_user_id\", \"starred_repo\").count()\n",
        "repos_occurrencies.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FJFXIP1msLxt"
      },
      "source": [
        "Then, we take the `N_ITEMS` top scoring repos using the usual Window function:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 55,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_nRR8S0baRS4",
        "outputId": "6ee02e50-a9b3-4261-ac32-98ed29adab8c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+--------------+------------+-----+----+\n",
            "|source_user_id|starred_repo|count|rank|\n",
            "+--------------+------------+-----+----+\n",
            "|            90|        1853|    4|   1|\n",
            "|            90|        1984|    3|   2|\n",
            "|            90|         483|    3|   3|\n",
            "|            90|        1339|    3|   4|\n",
            "|            90|       15729|    3|   5|\n",
            "|           123|         238|    4|   1|\n",
            "|           123|         482|    4|   2|\n",
            "|           123|         122|    3|   3|\n",
            "|           123|          16|    3|   4|\n",
            "|           123|         120|    3|   5|\n",
            "|           140|         424|    5|   1|\n",
            "|           140|          28|    4|   2|\n",
            "|           140|           2|    3|   3|\n",
            "|           140|         746|    3|   4|\n",
            "|           140|        2162|    3|   5|\n",
            "|           150|         192|    3|   1|\n",
            "|           150|         120|    3|   2|\n",
            "|           150|        3375|    3|   3|\n",
            "|           150|        2609|    3|   4|\n",
            "|           150|        8937|    3|   5|\n",
            "+--------------+------------+-----+----+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "window = Window.partitionBy(repos_occurrencies[\"source_user_id\"]).orderBy(repos_occurrencies[\"count\"].desc())\n",
        "recommendations = (repos_occurrencies.select(\"*\", row_number().over(window).alias(\"rank\")) \n",
        "  .filter(col(\"rank\") <= N_ITEMS))\n",
        "recommendations.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_hRMMASExGEw"
      },
      "source": [
        "Finally, we compute the set of repos recommended by the model for each user:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 56,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VF8EafA8bPYi",
        "outputId": "2fa3b2b3-5888-4ead-e467-4c029de6859a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+--------------+--------------------+\n",
            "|source_user_id|   recommended_repos|\n",
            "+--------------+--------------------+\n",
            "|            90|[1853.0, 1984.0, ...|\n",
            "|           123|[238.0, 482.0, 12...|\n",
            "|           140|[424.0, 28.0, 2.0...|\n",
            "|           150|[192.0, 120.0, 33...|\n",
            "|           191|[35.0, 1148.0, 13...|\n",
            "|           210|[2400.0, 204.0, 5...|\n",
            "|           263|[1839.0, 5144.0, ...|\n",
            "|           320|[35.0, 1070.0, 11...|\n",
            "|           336|[526.0, 7959.0, 6...|\n",
            "|           369|[7567.0, 10544.0,...|\n",
            "|           384|[1110.0, 441.0, 5...|\n",
            "|           393|[16.0, 2361.0, 54...|\n",
            "|           406|[833.0, 11844.0, ...|\n",
            "|           419|[352.0, 1995.0, 6...|\n",
            "|           426|[238.0, 28.0, 357...|\n",
            "|           431|[489.0, 2.0, 527....|\n",
            "|           463|[647.0, 4394.0, 2...|\n",
            "|           483|[192.0, 11469.0, ...|\n",
            "|           499|[6005.0, 458.0, 5...|\n",
            "|           546|[2.0, 158.0, 7.0,...|\n",
            "+--------------+--------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "recommendations_list = (recommendations\n",
        "                        .withColumn(\"starred_repo\", col(\"starred_repo\")\n",
        "                        .cast(\"double\")).groupBy(\"source_user_id\")\n",
        "                        .agg(collect_list(\"starred_repo\").alias(\"recommended_repos\")))\n",
        "recommendations_list.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4Sz_1UZV9rJ0"
      },
      "source": [
        "### Evaluation\n",
        "\n",
        "In this section, we carry out the evaluation phase for the User-based Collaborative Filtering model."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 57,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UwyCdKn-u081",
        "outputId": "5b0ee9f8-72ff-4ce1-b5c0-e06fa204bb6e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+-------+--------------------+--------------------+\n",
            "|user_id|   recommended_repos|              labels|\n",
            "+-------+--------------------+--------------------+\n",
            "|     90|[1853.0, 1984.0, ...|[1258.0, 5968.0, ...|\n",
            "|    123|[238.0, 482.0, 12...|[238.0, 123.0, 10...|\n",
            "|    140|[424.0, 28.0, 2.0...|[4815.0, 2.0, 296...|\n",
            "|    150|[192.0, 120.0, 33...|[2162.0, 8332.0, ...|\n",
            "|    191|[35.0, 1148.0, 13...|[383.0, 414.0, 35...|\n",
            "|    210|[2400.0, 204.0, 5...|[238.0, 3374.0, 7...|\n",
            "|    263|[1839.0, 5144.0, ...|[11903.0, 988.0, ...|\n",
            "|    320|[35.0, 1070.0, 11...|[1236.0, 6935.0, ...|\n",
            "|    336|[526.0, 7959.0, 6...|     [2851.0, 357.0]|\n",
            "|    369|[7567.0, 10544.0,...|[7197.0, 2967.0, ...|\n",
            "|    384|[1110.0, 441.0, 5...|[527.0, 11.0, 106...|\n",
            "|    393|[16.0, 2361.0, 54...|[383.0, 645.0, 49...|\n",
            "|    406|[833.0, 11844.0, ...|[2162.0, 35.0, 11...|\n",
            "|    419|[352.0, 1995.0, 6...|[6251.0, 10136.0,...|\n",
            "|    426|[238.0, 28.0, 357...|[383.0, 1342.0, 1...|\n",
            "|    431|[489.0, 2.0, 527....|[669.0, 4140.0, 7...|\n",
            "|    463|[647.0, 4394.0, 2...|[9684.0, 1279.0, ...|\n",
            "|    483|[192.0, 11469.0, ...|[8293.0, 1211.0, ...|\n",
            "|    499|[6005.0, 458.0, 5...|[238.0, 8429.0, 9...|\n",
            "|    546|[2.0, 158.0, 7.0,...|[2.0, 5968.0, 296...|\n",
            "+-------+--------------------+--------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "recommendations = recommendations_list.withColumnRenamed(\"source_user_id\", \"user_id\").join(gt, \"user_id\").cache()\n",
        "recommendations.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 58,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IYwKlZ-AdyYh",
        "outputId": "58ec8189-10f2-4c96-b4f9-62d3bb97a357"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+----------------------------------+-----+-----+-----+-----+-----+---------------+\n",
            "|Model                             |MAP@1|MAP@2|MAP@3|MAP@4|MAP@5|Personalization|\n",
            "+----------------------------------+-----+-----+-----+-----+-----+---------------+\n",
            "|User-Based Collaborative Filtering|0.357|0.261|0.213|0.183|0.164|0.965          |\n",
            "+----------------------------------+-----+-----+-----+-----+-----+---------------+\n",
            "\n"
          ]
        }
      ],
      "source": [
        "evaluation = evaluate_df(\"User-Based Collaborative Filtering\", recommendations, k)\n",
        "metrics_df = metrics_df.union(evaluation)\n",
        "round_df(evaluation, 3).show(truncate=False)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "e5gl2bWi1FG7"
      },
      "source": [
        "### Free resources"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 59,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PybNzOV81G2j",
        "outputId": "6efb31b0-9bf8-4239-df4f-6f0a8164973f"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "4150"
            ]
          },
          "execution_count": 59,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Free RAM\n",
        "import gc\n",
        "gc.collect()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vQ_qg-oPgcsf"
      },
      "source": [
        "## Item-based CF\n",
        "\n",
        "In the Item-based Collaborative Filtering technique, we try to recommend new items exploiting the similarity between the vector representation of the items that a user has already starred and the ones that has not starred yet.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 60,
      "metadata": {
        "cellView": "form",
        "id": "RW-Q_6URw1FQ"
      },
      "outputs": [],
      "source": [
        "#@title Settings\n",
        "NEIGHBOURS =  5#@param\n",
        "N_ITEMS =  5#@param"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PXy7lPy0h8IM"
      },
      "source": [
        "### Compute Similarity\n",
        "\n",
        "In order to build the similarity matrix, we compute the item vectors which are the lists of users that starred that repo."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 61,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GS298U-ZIGGd",
        "outputId": "3be39f52-1c67-4037-9da6-2ae67dd7e5e6"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+-------+--------------------+\n",
            "|repo_id|         item_vector|\n",
            "+-------+--------------------+\n",
            "|      2|[8951, 9380, 8982...|\n",
            "|     28|[9380, 785, 3450,...|\n",
            "|     46|[1199, 5147, 7202...|\n",
            "|     50|[3601, 6164, 2543...|\n",
            "|     58|[7681, 8343, 6471...|\n",
            "|     77|[5147, 7202, 2992...|\n",
            "|     92|[6164, 4995, 6287...|\n",
            "|    109|[5534, 4416, 6623...|\n",
            "|    110|[5359, 4995, 7681...|\n",
            "|    118|[6956, 2665, 9380...|\n",
            "|    123|[8748, 1657, 9765...|\n",
            "|    126|[2992, 7275, 5534...|\n",
            "|    133|[5359, 7914, 9634...|\n",
            "|    168|[8951, 9634, 8751...|\n",
            "|    169|[8951, 7202, 1657...|\n",
            "|    176|[4995, 4110, 2992...|\n",
            "|    187|[5990, 7994, 9634...|\n",
            "|    198|[7202, 9380, 785,...|\n",
            "|    202|[785, 5432, 9411,...|\n",
            "|    203|[9380, 5534, 8343...|\n",
            "+-------+--------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "item_vectors = train.groupBy(\"repo_id\").agg(collect_set(\"user_id\").alias(\"item_vector\"))\n",
        "item_vectors.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ySTjc6zAkNId"
      },
      "source": [
        "Then, we add the item vectors to each repo in the train set."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 62,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fCknTCp6iX40",
        "outputId": "31e740d8-eee3-45d6-c29d-32ca415e911a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+-------+-------+--------------------+\n",
            "|repo_id|user_id|         item_vector|\n",
            "+-------+-------+--------------------+\n",
            "|      0|    145|[5990, 2665, 1657...|\n",
            "|      0|    322|[5990, 2665, 1657...|\n",
            "|      0|    726|[5990, 2665, 1657...|\n",
            "|      0|    746|[5990, 2665, 1657...|\n",
            "|      0|    766|[5990, 2665, 1657...|\n",
            "|      0|    971|[5990, 2665, 1657...|\n",
            "|      0|   1567|[5990, 2665, 1657...|\n",
            "|      0|   1638|[5990, 2665, 1657...|\n",
            "|      0|   1657|[5990, 2665, 1657...|\n",
            "|      0|   1977|[5990, 2665, 1657...|\n",
            "|      0|   2011|[5990, 2665, 1657...|\n",
            "|      0|   2061|[5990, 2665, 1657...|\n",
            "|      0|   2126|[5990, 2665, 1657...|\n",
            "|      0|   2226|[5990, 2665, 1657...|\n",
            "|      0|   2464|[5990, 2665, 1657...|\n",
            "|      0|   2517|[5990, 2665, 1657...|\n",
            "|      0|   2665|[5990, 2665, 1657...|\n",
            "|      0|   2911|[5990, 2665, 1657...|\n",
            "|      0|   2931|[5990, 2665, 1657...|\n",
            "|      0|   3108|[5990, 2665, 1657...|\n",
            "+-------+-------+--------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "user_item = train.join(item_vectors, \"repo_id\")\n",
        "user_item.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YScRYv5YPqA8"
      },
      "source": [
        "Now, in order to add the list of starred repos to each user record of the train set, we take all the repos starred by each user.\n",
        "\n",
        "$$\n",
        "\\mathcal{I}_u = \\{ i \\in \\mathcal{I} : r_{u,i} = (defined) \\}\n",
        "$$"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 63,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "czQjZ15hPpvE",
        "outputId": "a0402b7d-6e0a-4670-c284-cd931e11c134"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+-------+--------------------+\n",
            "|user_id|       starred_repos|\n",
            "+-------+--------------------+\n",
            "|     90|[1047, 662, 743, ...|\n",
            "|    123|[2440, 7660, 2, 1...|\n",
            "|    140|[8604, 204, 662, ...|\n",
            "|    150|[1148, 438, 6949,...|\n",
            "|    191|[691, 1, 743, 2, ...|\n",
            "|    210|[12095, 5199, 259...|\n",
            "|    263|[2440, 589, 12907...|\n",
            "|    320|[13293, 691, 438,...|\n",
            "|    336|[2581, 7692, 2, 1...|\n",
            "|    369|[5859, 9409, 1463...|\n",
            "|    384|[1148, 8604, 357,...|\n",
            "|    393|[7210, 204, 17975...|\n",
            "|    406|[3355, 7210, 3682...|\n",
            "|    419|[5859, 13061, 146...|\n",
            "|    426|[10802, 17402, 60...|\n",
            "|    431|[2207, 335, 691, ...|\n",
            "|    463|[25974, 169, 2, 1...|\n",
            "|    483|[5859, 1395, 2, 6...|\n",
            "|    499|[2258, 6977, 1057...|\n",
            "|    546|[2599, 691, 4081,...|\n",
            "+-------+--------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "user_starred_repos = train.groupBy(\"user_id\").agg(collect_set(\"repo_id\").alias(\"starred_repos\"))\n",
        "user_starred_repos.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PkZPArrbQBSD"
      },
      "source": [
        "Then, we can finally add the starred repos to each record user-item."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 64,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XfvX2fYUmFIM",
        "outputId": "d72dd948-5635-409e-bb9a-6c7570b11180"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+-------+-------+--------------------+--------------------+\n",
            "|user_id|repo_id|         item_vector|       starred_repos|\n",
            "+-------+-------+--------------------+--------------------+\n",
            "|    145|      0|[5990, 2665, 1657...|[0, 133, 489, 2, ...|\n",
            "|    322|      0|[5990, 2665, 1657...|[0, 1322, 4081, 7...|\n",
            "|    726|      0|[5990, 2665, 1657...|[0, 17402, 4059, ...|\n",
            "|    746|      0|[5990, 2665, 1657...|[10802, 0, 743, 1...|\n",
            "|    766|      0|[5990, 2665, 1657...|[0, 5859, 17975, ...|\n",
            "|    971|      0|[5990, 2665, 1657...|[0, 4379, 16, 179...|\n",
            "|   1567|      0|[5990, 2665, 1657...|[0, 8603, 8798, 3...|\n",
            "|   1638|      0|[5990, 2665, 1657...|[2207, 0, 17975, ...|\n",
            "|   1657|      0|[5990, 2665, 1657...|[0, 6977, 13061, ...|\n",
            "|   1977|      0|[5990, 2665, 1657...|[1148, 0, 458, 21...|\n",
            "|   2011|      0|[5990, 2665, 1657...|[6316, 4393, 407,...|\n",
            "|   2061|      0|[5990, 2665, 1657...|[0, 14672, 757, 3...|\n",
            "|   2126|      0|[5990, 2665, 1657...|[2440, 0, 510, 74...|\n",
            "|   2226|      0|[5990, 2665, 1657...|[0, 458, 691, 205...|\n",
            "|   2464|      0|[5990, 2665, 1657...|[2207, 0, 691, 58...|\n",
            "|   2517|      0|[5990, 2665, 1657...|[1561, 1612, 0, 1...|\n",
            "|   2665|      0|[5990, 2665, 1657...|[0, 407, 2, 8445,...|\n",
            "|   2911|      0|[5990, 2665, 1657...|[0, 1322, 5076, 2...|\n",
            "|   2931|      0|[5990, 2665, 1657...|[2440, 0, 2266, 1...|\n",
            "|   3108|      0|[5990, 2665, 1657...|[0, 8445, 1172, 1...|\n",
            "+-------+-------+--------------------+--------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "user_item = user_item.join(user_starred_repos, \"user_id\")\n",
        "user_item.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5t5d55QZmFnU"
      },
      "source": [
        "Now, we associate the items on which will be computed the similarity value. We need to put in relation all the items starred by the user $u$ with all the items not starred yet by $u$.\n",
        "\n",
        "$$\n",
        "S_u = \\{ (i, i') : i \\in I_u \\land i' \\in I - I_u \\}\n",
        "$$\n",
        "\n",
        "In practice, we join the items starred by each user with all the item vectors. Notice that it is important to remove the items already starred by the user, i.e. $i \\in I_u$ and to do so we can use the list of starred repos added before."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 65,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "i8NNkH2_N-Rl",
        "outputId": "c69029bf-ffcc-4c8f-b564-c2ea31bf85a2"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+-------+-------+--------------------+--------------------+------------------+----------------------+\n",
            "|user_id|repo_id|         item_vector|       starred_repos|no_starred_repo_id|no_starred_item_vector|\n",
            "+-------+-------+--------------------+--------------------+------------------+----------------------+\n",
            "|    145|      0|[5990, 2665, 1657...|[0, 133, 489, 2, ...|                50|  [3601, 6164, 2543...|\n",
            "|    145|      0|[5990, 2665, 1657...|[0, 133, 489, 2, ...|                58|  [7681, 8343, 6471...|\n",
            "|    145|      0|[5990, 2665, 1657...|[0, 133, 489, 2, ...|                77|  [5147, 7202, 2992...|\n",
            "|    145|      0|[5990, 2665, 1657...|[0, 133, 489, 2, ...|                92|  [6164, 4995, 6287...|\n",
            "|    145|      0|[5990, 2665, 1657...|[0, 133, 489, 2, ...|               109|  [5534, 4416, 6623...|\n",
            "|    145|      0|[5990, 2665, 1657...|[0, 133, 489, 2, ...|               110|  [5359, 4995, 7681...|\n",
            "|    145|      0|[5990, 2665, 1657...|[0, 133, 489, 2, ...|               118|  [6956, 2665, 9380...|\n",
            "|    145|      0|[5990, 2665, 1657...|[0, 133, 489, 2, ...|               168|  [8951, 9634, 8751...|\n",
            "|    145|      0|[5990, 2665, 1657...|[0, 133, 489, 2, ...|               169|  [8951, 7202, 1657...|\n",
            "|    145|      0|[5990, 2665, 1657...|[0, 133, 489, 2, ...|               176|  [4995, 4110, 2992...|\n",
            "|    145|      0|[5990, 2665, 1657...|[0, 133, 489, 2, ...|               187|  [5990, 7994, 9634...|\n",
            "|    145|      0|[5990, 2665, 1657...|[0, 133, 489, 2, ...|               198|  [7202, 9380, 785,...|\n",
            "|    145|      0|[5990, 2665, 1657...|[0, 133, 489, 2, ...|               202|  [785, 5432, 9411,...|\n",
            "|    145|      0|[5990, 2665, 1657...|[0, 133, 489, 2, ...|               203|  [9380, 5534, 8343...|\n",
            "|    145|      0|[5990, 2665, 1657...|[0, 133, 489, 2, ...|               238|  [7202, 9380, 785,...|\n",
            "|    145|      0|[5990, 2665, 1657...|[0, 133, 489, 2, ...|               297|  [2665, 4995, 7326...|\n",
            "|    145|      0|[5990, 2665, 1657...|[0, 133, 489, 2, ...|               300|  [3682, 7711, 2971...|\n",
            "|    145|      0|[5990, 2665, 1657...|[0, 133, 489, 2, ...|               328|  [8951, 1657, 2992...|\n",
            "|    145|      0|[5990, 2665, 1657...|[0, 133, 489, 2, ...|               333|  [5990, 6956, 7681...|\n",
            "|    145|      0|[5990, 2665, 1657...|[0, 133, 489, 2, ...|               336|  [5147, 5359, 8982...|\n",
            "+-------+-------+--------------------+--------------------+------------------+----------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "other_items = item_vectors.select(col(\"repo_id\").alias(\"no_starred_repo_id\"), col(\"item_vector\").alias(\"no_starred_item_vector\"))\n",
        "user_item_item = user_item.join(other_items, ~array_contains(col(\"starred_repos\"), col(\"no_starred_repo_id\")))\n",
        "user_item_item.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A6mlf397iF5n"
      },
      "source": [
        "Since we will have many correlations item-to-item repeated for different users, we precompute the similarity matrix among all the pairs of items simply computing the Jaccard similarity."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 66,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "M5eDQaFQihmJ",
        "outputId": "96b0d47f-7494-42f7-9f1f-6d9adf7cec3f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+-------+--------------------+------------------+----------------------+--------------------+\n",
            "|repo_id|         item_vector|no_starred_repo_id|no_starred_item_vector|  jaccard_similarity|\n",
            "+-------+--------------------+------------------+----------------------+--------------------+\n",
            "|      2|[8951, 9380, 8982...|                28|  [9380, 785, 3450,...|  0.2503128911138924|\n",
            "|      2|[8951, 9380, 8982...|                46|  [1199, 5147, 7202...| 0.11637347767253045|\n",
            "|      2|[8951, 9380, 8982...|                50|  [3601, 6164, 2543...|0.036885245901639344|\n",
            "|      2|[8951, 9380, 8982...|                58|  [7681, 8343, 6471...| 0.05327868852459016|\n",
            "|      2|[8951, 9380, 8982...|                77|  [5147, 7202, 2992...| 0.15013054830287206|\n",
            "|      2|[8951, 9380, 8982...|                92|  [6164, 4995, 6287...|0.059474412171507604|\n",
            "|      2|[8951, 9380, 8982...|               109|  [5534, 4416, 6623...| 0.12234042553191489|\n",
            "|      2|[8951, 9380, 8982...|               110|  [5359, 4995, 7681...| 0.07417582417582418|\n",
            "|      2|[8951, 9380, 8982...|               118|  [6956, 2665, 9380...| 0.18075422626788037|\n",
            "|      2|[8951, 9380, 8982...|               123|  [8748, 1657, 9765...| 0.04945054945054945|\n",
            "|      2|[8951, 9380, 8982...|               126|  [2992, 7275, 5534...|  0.1455026455026455|\n",
            "|      2|[8951, 9380, 8982...|               133|  [5359, 7914, 9634...| 0.06925207756232687|\n",
            "|      2|[8951, 9380, 8982...|               168|  [8951, 9634, 8751...| 0.11126005361930295|\n",
            "|      2|[8951, 9380, 8982...|               169|  [8951, 7202, 1657...| 0.10591471801925723|\n",
            "|      2|[8951, 9380, 8982...|               176|  [4995, 4110, 2992...| 0.05006954102920723|\n",
            "|      2|[8951, 9380, 8982...|               187|  [5990, 7994, 9634...|0.044382801664355064|\n",
            "|      2|[8951, 9380, 8982...|               198|  [7202, 9380, 785,...|  0.1934640522875817|\n",
            "|      2|[8951, 9380, 8982...|               202|  [785, 5432, 9411,...| 0.06796116504854369|\n",
            "|      2|[8951, 9380, 8982...|               203|  [9380, 5534, 8343...|0.054945054945054944|\n",
            "|      2|[8951, 9380, 8982...|               238|  [7202, 9380, 785,...| 0.26674786845310594|\n",
            "+-------+--------------------+------------------+----------------------+--------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "temp = item_vectors.select(col(\"repo_id\").alias(\"no_starred_repo_id\"), col(\"item_vector\").alias(\"no_starred_item_vector\"))\n",
        "similarity = item_vectors.join(temp, col(\"repo_id\") != col(\"no_starred_repo_id\"))\n",
        "\n",
        "similarity = similarity.withColumn(\"jaccard_similarity\", jaccard_similarity_udf(\"item_vector\", \"no_starred_item_vector\"))\n",
        "similarity.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AGTIZW1Sit8t"
      },
      "source": [
        "As said, we use the previous matrix in order to compute the similarity between all the repos for each user."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 67,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "07hfu17RjV9n",
        "outputId": "c7c370e0-c600-4cd0-f9ff-b764c373028b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+-------+-------+------------------+--------------------+\n",
            "|user_id|repo_id|no_starred_repo_id|  jaccard_similarity|\n",
            "+-------+-------+------------------+--------------------+\n",
            "|    145|      0|                50| 0.04316546762589928|\n",
            "|    145|      0|                58|0.060810810810810814|\n",
            "|    145|      0|                77| 0.12658227848101267|\n",
            "|    145|      0|                92|  0.0410958904109589|\n",
            "|    145|      0|               109| 0.07476635514018691|\n",
            "|    145|      0|               110| 0.07006369426751592|\n",
            "|    145|      0|               118| 0.06521739130434782|\n",
            "|    145|      0|               168| 0.11398963730569948|\n",
            "|    145|      0|               169|  0.1377245508982036|\n",
            "|    145|      0|               176|0.044444444444444446|\n",
            "|    145|      0|               187| 0.06923076923076923|\n",
            "|    145|      0|               198| 0.07553956834532374|\n",
            "|    145|      0|               202|                0.04|\n",
            "|    145|      0|               203| 0.03355704697986577|\n",
            "|    145|      0|               238| 0.10077519379844961|\n",
            "|    145|      0|               297| 0.02097902097902098|\n",
            "|    145|      0|               300|0.044444444444444446|\n",
            "|    145|      0|               328|  0.1366906474820144|\n",
            "|    145|      0|               333|0.056962025316455694|\n",
            "|    145|      0|               336| 0.07317073170731707|\n",
            "+-------+-------+------------------+--------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "from pyspark.sql.functions import broadcast\n",
        "\n",
        "similarity_matrix = similarity.select(col(\"repo_id\").alias(\"sim_repo_id\"), col(\"no_starred_repo_id\").alias(\"sim_no_starred_repo_id\"), \"jaccard_similarity\")\n",
        "jaccard = user_item_item.join(broadcast(similarity_matrix), [col(\"repo_id\") == col(\"sim_repo_id\"), col(\"no_starred_repo_id\") == col(\"sim_no_starred_repo_id\")])\n",
        "jaccard = jaccard.select(\"user_id\", \"repo_id\", \"no_starred_repo_id\", \"jaccard_similarity\")\n",
        "jaccard.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dYjw1Hp0h0xc"
      },
      "source": [
        "### Taking top k similar items\n",
        "\n",
        "Now, in order to predict the score for a given item $i$ we compute the top-$k$ similar items starred by $u$ to the item $i$.\n",
        "\n",
        "$$\n",
        "\\mathcal{I}_u^k = argmax_{\\mathcal{I}_u' \\subseteq \\mathcal{I}_u, |\\mathcal{I}_u'| = k} \\sum_{i' \\in \\mathcal{I}_u'} sim(i, i')\n",
        "$$"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 68,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "heV-Ol0UwWV3",
        "outputId": "434452cc-593e-4c14-b711-fcc4e65caff5"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+-------+-------+------------------+-------------------+----+\n",
            "|user_id|repo_id|no_starred_repo_id| jaccard_similarity|rank|\n",
            "+-------+-------+------------------+-------------------+----+\n",
            "|     44|   1510|                 0|               0.12|   1|\n",
            "|     44|    442|                 0|0.11764705882352941|   2|\n",
            "|     44|      2|                 0|0.09878213802435724|   3|\n",
            "|     44|    526|                 0|0.09293680297397769|   4|\n",
            "|     44|    615|                 0| 0.0712166172106825|   5|\n",
            "|     44|    528|                11|0.14906832298136646|   1|\n",
            "|     44|    526|                11|0.13877551020408163|   2|\n",
            "|     44|    879|                11| 0.1349206349206349|   3|\n",
            "|     44|   1609|                11| 0.1044776119402985|   4|\n",
            "|     44|    442|                11| 0.0946969696969697|   5|\n",
            "|     44|      2|                16|0.25660377358490566|   1|\n",
            "|     44|    442|                16|0.20638820638820637|   2|\n",
            "|     44|    615|                16|0.20175438596491227|   3|\n",
            "|     44|    526|                16|0.17317073170731706|   4|\n",
            "|     44|    871|                16| 0.1712707182320442|   5|\n",
            "|     44|    871|                24|0.12637362637362637|   1|\n",
            "|     44|   1677|                24|0.11564625850340136|   2|\n",
            "|     44|   1022|                24|0.10778443113772455|   3|\n",
            "|     44|    442|                24|0.10569105691056911|   4|\n",
            "|     44|    879|                24|0.09565217391304348|   5|\n",
            "+-------+-------+------------------+-------------------+----+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "window = Window.partitionBy(jaccard[\"user_id\"], jaccard[\"no_starred_repo_id\"]).orderBy(jaccard[\"jaccard_similarity\"].desc())\n",
        "top_k_repos = jaccard.select(\"*\", row_number().over(window).alias(\"rank\")).filter(col(\"rank\") <= NEIGHBOURS)\n",
        "top_k_repos.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Iopwhnx7j3r8"
      },
      "source": [
        "### Predicting new repos\n",
        "\n",
        "At this point, we need to aggregate the scores of the $k$ items found, but a plain average will not work in our case: the only rating that can appear for the items rated by the user $u$ is 1, since we are considering a simple binary implicit feedback. So, we used the average using the similarity as defined below.\n",
        "\n",
        "$$\n",
        "r_{u,i} = \\frac{\\sum_{i' \\in \\mathcal{I}_u^k} sim(i, i')}{k}\n",
        "$$\n",
        "\n",
        "where $k$ is the number of neighbours we are considering.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 69,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YjkyymrgZ4gK",
        "outputId": "feafdd07-cce7-4809-c983-db2df1a041ec"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+-------+------------------+-------------------+\n",
            "|user_id|no_starred_repo_id|              score|\n",
            "+-------+------------------+-------------------+\n",
            "|     44|                 0|0.10011652340650937|\n",
            "|     44|                11|0.12438780994867024|\n",
            "|     44|                16| 0.2018375631754771|\n",
            "|     44|                24|0.11022950936767298|\n",
            "|     44|                28| 0.1818073826917406|\n",
            "|     44|                44|0.07396391035965834|\n",
            "|     44|                64|0.15888102394160838|\n",
            "|     44|                77|0.14432752748636773|\n",
            "|     44|                92|0.08991605503664538|\n",
            "|     44|               100|0.13464067031674037|\n",
            "|     44|               103|0.12287995864638199|\n",
            "|     44|               105| 0.0559815800324021|\n",
            "|     44|               109|0.14219041528150095|\n",
            "|     44|               110|0.12495900305493532|\n",
            "|     44|               117|0.16565828738226474|\n",
            "|     44|               118|0.18089617314022183|\n",
            "|     44|               119|0.20520966941343274|\n",
            "|     44|               125|0.17732232510662982|\n",
            "|     44|               147|0.13629592770152935|\n",
            "|     44|               158|0.14505781778903398|\n",
            "+-------+------------------+-------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "repo_scores = top_k_repos.groupBy(\"user_id\", \"no_starred_repo_id\").agg(mean(\"jaccard_similarity\").alias(\"score\"))\n",
        "repo_scores.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n3el-f3BjXvy"
      },
      "source": [
        "And take the top-k items for each user based on the previously computed score."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 70,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "655olqMGZ4gL",
        "outputId": "94864114-73a2-41f6-8a8a-015ead1872d3"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+-------+------------------+-------------------+----+\n",
            "|user_id|no_starred_repo_id|              score|rank|\n",
            "+-------+------------------+-------------------+----+\n",
            "|     90|                35| 0.2661138474702245|   1|\n",
            "|     90|               721|0.25232144871625534|   2|\n",
            "|     90|               238| 0.2500617220006752|   3|\n",
            "|     90|               666|0.24981770825868094|   4|\n",
            "|     90|                28|0.24801501995847888|   5|\n",
            "|    123|               238|0.26438959561139347|   1|\n",
            "|    123|                 1|0.25163484181731915|   2|\n",
            "|    123|               398|0.24390532800978665|   3|\n",
            "|    123|               120|0.23142484587209666|   4|\n",
            "|    123|               476|0.22800927165819615|   5|\n",
            "|    140|                 2|0.21983671799856802|   1|\n",
            "|    140|                 1| 0.2163965637884247|   2|\n",
            "|    140|              1110|0.21459631995431208|   3|\n",
            "|    140|               721|0.21189484222334082|   4|\n",
            "|    140|               674| 0.2068299357223093|   5|\n",
            "|    150|                35| 0.2501153333683712|   1|\n",
            "|    150|               238|0.24559168415546867|   2|\n",
            "|    150|                 1| 0.2358293550686062|   3|\n",
            "|    150|              1110|0.23048746744161958|   4|\n",
            "|    150|               527|0.22770183667288305|   5|\n",
            "+-------+------------------+-------------------+----+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "window = Window.partitionBy(repo_scores[\"user_id\"]).orderBy(repo_scores[\"score\"].desc())\n",
        "recommendations = repo_scores.select(\"*\", row_number().over(window).alias(\"rank\")).filter(col(\"rank\") <= N_ITEMS)\n",
        "recommendations.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UNImbQMLjipL"
      },
      "source": [
        "Finally, we construct the list of repos recommended for each user."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 71,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "g7nIrvtiZ4gL",
        "outputId": "2e71ae4f-b27b-4e17-9743-3e5b6e780826"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+-------+--------------------+\n",
            "|user_id|   recommended_repos|\n",
            "+-------+--------------------+\n",
            "|     90|[35.0, 721.0, 238...|\n",
            "|    123|[238.0, 1.0, 398....|\n",
            "|    140|[2.0, 1.0, 1110.0...|\n",
            "|    150|[35.0, 238.0, 1.0...|\n",
            "|    191|[35.0, 352.0, 198...|\n",
            "|    210|[238.0, 1.0, 527....|\n",
            "|    263|[35.0, 28.0, 1110...|\n",
            "|    320|[35.0, 119.0, 1.0...|\n",
            "|    336|[1.0, 35.0, 16.0,...|\n",
            "|    369|[1110.0, 198.0, 6...|\n",
            "|    384|[527.0, 1110.0, 3...|\n",
            "|    393|[35.0, 16.0, 238....|\n",
            "|    406|[1110.0, 721.0, 5...|\n",
            "|    419|[28.0, 1.0, 527.0...|\n",
            "|    426|[238.0, 1.0, 120....|\n",
            "|    431|[2.0, 120.0, 35.0...|\n",
            "|    463|[8687.0, 35.0, 35...|\n",
            "|    483|[35.0, 238.0, 674...|\n",
            "|    499|[16.0, 666.0, 28....|\n",
            "|    546|[2.0, 35.0, 238.0...|\n",
            "+-------+--------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "recommendations_list = recommendations.withColumn(\"no_starred_repo_id\", col(\"no_starred_repo_id\").cast(\"double\")).groupBy(\"user_id\").agg(collect_list(\"no_starred_repo_id\").alias(\"recommended_repos\"))\n",
        "recommendations_list.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WI9RtuGdwp6O"
      },
      "source": [
        "### Evaluation\n",
        "\n",
        "In this section, we carry out the evaluation phase for the Item-based Collaborative Filtering model."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 72,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2c2LGmeBomr9",
        "outputId": "79df5e0e-4c47-427d-dd7f-e40f00294e55"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+-------+-------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
            "|user_id|recommended_repos                    |labels                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |\n",
            "+-------+-------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
            "|90     |[35.0, 721.0, 238.0, 666.0, 28.0]    |[1258.0, 5968.0, 6319.0, 6251.0, 1409.0, 6300.0, 1067.0, 2709.0, 6259.0, 506.0, 335.0, 147.0, 6316.0, 1965.0, 4104.0, 1937.0, 797.0, 28.0, 821.0, 2256.0, 2192.0, 6156.0, 6163.0, 6201.0, 6254.0, 2258.0, 6330.0]                                                                                                                                                                                                                                                                                                                                                                           |\n",
            "|123    |[238.0, 1.0, 398.0, 120.0, 476.0]    |[238.0, 123.0, 1049.0, 6300.0, 122.0, 1280.0, 1411.0, 483.0, 1041.0, 6935.0, 2851.0, 3375.0, 48.0, 2857.0, 7539.0, 2275.0, 4140.0, 2701.0, 2634.0, 3953.0, 706.0, 526.0, 910.0, 2069.0, 442.0, 797.0, 1042.0, 7814.0, 77.0, 871.0, 7750.0, 1257.0, 6435.0]                                                                                                                                                                                                                                                                                                                                  |\n",
            "|140    |[2.0, 1.0, 1110.0, 721.0, 674.0]     |[4815.0, 2.0, 2967.0, 2266.0, 8603.0, 5859.0, 433.0, 2839.0, 1510.0, 6256.0, 1965.0, 109.0, 413.0, 2866.0, 616.0, 3520.0, 8654.0, 8583.0, 8558.0, 1042.0, 8675.0, 8539.0, 8528.0, 4016.0, 126.0]                                                                                                                                                                                                                                                                                                                                                                                            |\n",
            "|150    |[35.0, 238.0, 1.0, 1110.0, 527.0]    |[2162.0, 8332.0, 2609.0, 3375.0, 9183.0, 533.0, 2384.0, 668.0, 1053.0, 8006.0, 746.0, 1054.0, 9077.0]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |\n",
            "|191    |[35.0, 352.0, 198.0, 1110.0, 482.0]  |[383.0, 414.0, 35.0, 122.0, 4028.0, 7814.0, 871.0, 476.0, 2275.0]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           |\n",
            "|210    |[238.0, 1.0, 527.0, 16.0, 1110.0]    |[238.0, 3374.0, 703.0, 4989.0, 1378.0, 6289.0, 2074.0, 1510.0, 2385.0, 1444.0, 6135.0, 648.0, 454.0, 381.0, 365.0, 48.0, 386.0, 716.0, 4200.0]                                                                                                                                                                                                                                                                                                                                                                                                                                              |\n",
            "|263    |[35.0, 28.0, 1110.0, 666.0, 16.0]    |[11903.0, 988.0, 347.0, 671.0, 506.0, 721.0, 1411.0, 1377.0, 6938.0, 11186.0, 320.0, 731.0, 5940.0, 418.0, 691.0, 1677.0, 5948.0, 1061.0, 11691.0, 126.0, 35.0, 117.0, 14274.0, 1148.0, 441.0, 3953.0, 16.0, 1119.0, 3682.0, 5680.0, 7854.0, 5872.0, 442.0, 683.0, 1612.0, 716.0, 435.0, 77.0, 871.0, 1339.0]                                                                                                                                                                                                                                                                               |\n",
            "|320    |[35.0, 119.0, 1.0, 28.0, 238.0]      |[1236.0, 6935.0, 510.0, 834.0, 1385.0, 3426.0, 7660.0, 2599.0]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              |\n",
            "|336    |[1.0, 35.0, 16.0, 119.0, 126.0]      |[2851.0, 357.0]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |\n",
            "|369    |[1110.0, 198.0, 666.0, 527.0, 352.0] |[7197.0, 2967.0, 408.0, 671.0, 331.0, 2709.0, 110.0, 1737.0, 1429.0, 11469.0, 483.0, 1056.0, 1382.0, 6935.0, 3375.0, 48.0, 691.0, 1433.0, 4033.0, 4727.0, 8797.0, 1405.0, 669.0, 4140.0, 125.0, 4378.0, 19465.0, 614.0, 1148.0, 6749.0, 697.0, 706.0, 11975.0, 442.0, 6747.0, 4848.0, 6254.0, 160.0, 2207.0, 8476.0, 4803.0]                                                                                                                                                                                                                                                                |\n",
            "|384    |[527.0, 1110.0, 3426.0, 35.0, 441.0] |[527.0, 11.0, 1067.0, 441.0, 12146.0, 1070.0, 2256.0, 11186.0, 20507.0, 103.0, 1047.0, 2081.0]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              |\n",
            "|393    |[35.0, 16.0, 238.0, 1.0, 721.0]      |[383.0, 645.0, 4989.0, 449.0, 2045.0, 2581.0, 3930.0, 21321.0, 106.0, 1249.0, 352.0, 8455.0, 2704.0, 14512.0, 5940.0, 4874.0, 16593.0, 6042.0, 357.0, 691.0, 2646.0, 668.0, 2173.0, 1061.0, 7539.0, 16591.0, 2328.0, 16063.0, 2275.0, 15630.0, 35.0, 2044.0, 8339.0, 647.0, 453.0, 145.0, 3953.0, 4394.0, 16.0, 648.0, 6633.0, 8254.0, 8891.0, 471.0, 666.0, 410.0, 21455.0, 12282.0, 6212.0, 643.0, 20774.0, 21661.0, 419.0, 1044.0, 6910.0, 1583.0]                                                                                                                                       |\n",
            "|406    |[1110.0, 721.0, 527.0, 35.0, 238.0]  |[2162.0, 35.0, 117.0, 119.0, 2149.0, 122.0, 1280.0, 721.0, 2187.0, 6915.0, 6802.0, 7567.0, 15732.0, 8202.0, 3669.0, 3226.0, 357.0, 691.0, 3426.0, 4471.0, 160.0, 1110.0, 4016.0, 6273.0, 791.0, 5896.0]                                                                                                                                                                                                                                                                                                                                                                                     |\n",
            "|419    |[28.0, 1.0, 527.0, 541.0, 117.0]     |[6251.0, 10136.0, 6949.0, 1444.0, 22921.0, 2278.0, 8095.0, 3520.0, 7519.0, 1254.0, 1321.0, 5858.0, 11095.0, 334.0, 864.0, 2275.0, 1993.0, 1258.0, 1063.0, 6432.0, 9039.0, 10544.0, 3067.0, 19835.0, 964.0, 7887.0, 18500.0, 5095.0, 5996.0, 2033.0, 1853.0, 674.0, 10107.0, 1612.0, 707.0, 4989.0, 581.0, 1344.0, 12912.0, 23716.0, 2457.0, 454.0, 2314.0, 527.0, 3401.0, 6976.0, 16931.0, 6547.0, 19555.0, 24052.0, 3833.0, 8983.0, 1302.0, 459.0, 863.0, 11819.0, 13869.0, 337.0, 23277.0, 117.0, 1995.0, 4654.0, 6150.0, 4715.0, 7997.0, 526.0, 16344.0, 333.0, 7814.0, 10474.0, 17387.0]|\n",
            "|426    |[238.0, 1.0, 120.0, 527.0, 16.0]     |[383.0, 1342.0, 1994.0, 2884.0, 3521.0, 2400.0, 7134.0, 320.0, 348.0, 15348.0, 357.0, 15701.0, 2256.0, 18508.0, 8093.0, 20064.0, 9039.0, 792.0, 2594.0, 8136.0, 6202.0, 120.0, 8339.0, 1053.0, 615.0, 8812.0, 3682.0, 1186.0, 430.0, 340.0, 2069.0, 512.0, 160.0, 77.0, 3383.0, 1422.0, 2294.0, 743.0, 1421.0, 318.0, 1899.0, 449.0, 2858.0, 110.0, 4413.0, 1380.0, 5930.0, 1046.0, 464.0, 2701.0, 5886.0, 9553.0, 6482.0, 5822.0, 7036.0, 6150.0, 9675.0, 145.0, 2926.0, 4715.0, 706.0, 12095.0, 419.0, 798.0, 80.0, 865.0]                                                                |\n",
            "|431    |[2.0, 120.0, 35.0, 238.0, 352.0]     |[669.0, 4140.0, 7977.0, 2.0, 4378.0, 4107.0, 371.0, 3374.0, 3730.0, 6734.0, 2349.0, 1475.0, 13293.0, 656.0, 413.0, 802.0, 352.0, 6746.0, 6404.0, 8798.0, 77.0, 1340.0, 505.0, 1941.0, 22107.0]                                                                                                                                                                                                                                                                                                                                                                                              |\n",
            "|463    |[8687.0, 35.0, 352.0, 238.0, 11342.0]|[9684.0, 1279.0, 120.0, 2045.0, 204.0, 1369.0, 961.0, 8254.0]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               |\n",
            "|483    |[35.0, 238.0, 674.0, 120.0, 320.0]   |[8293.0, 1211.0, 3521.0, 581.0, 118.0, 204.0, 11183.0, 804.0, 2851.0, 3458.0, 18819.0, 528.0, 2256.0, 6163.0, 5926.0, 1423.0, 1258.0, 2162.0, 5878.0, 5968.0, 8573.0, 489.0, 8339.0, 533.0, 1022.0, 1376.0, 10601.0, 5125.0, 802.0, 526.0, 2069.0, 6449.0, 1042.0, 716.0, 5820.0, 80.0, 1422.0]                                                                                                                                                                                                                                                                                             |\n",
            "|499    |[16.0, 666.0, 28.0, 527.0, 3426.0]   |[238.0, 8429.0, 988.0, 629.0, 4412.0, 671.0, 581.0, 6406.0, 106.0, 147.0, 631.0, 2440.0, 3968.0, 4364.0, 336.0, 8583.0, 6547.0, 682.0, 666.0, 28.0, 6302.0, 14672.0, 1355.0, 2599.0, 187.0, 791.0]                                                                                                                                                                                                                                                                                                                                                                                          |\n",
            "|546    |[2.0, 35.0, 238.0, 1.0, 398.0]       |[2.0, 5968.0, 2967.0, 5859.0, 24937.0, 2255.0, 1460.0, 15332.0, 438.0, 666.0, 77.0, 6910.0, 199.0, 2275.0]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  |\n",
            "+-------+-------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "recommendations = recommendations_list.join(gt, \"user_id\").cache()\n",
        "recommendations.show(truncate=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 73,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Oe9fMJtgomr-",
        "outputId": "bcc4125d-e0ef-4e61-8f54-a36748d22587"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+----------------------------------+-----+-----+-----+-----+-----+---------------+\n",
            "|Model                             |MAP@1|MAP@2|MAP@3|MAP@4|MAP@5|Personalization|\n",
            "+----------------------------------+-----+-----+-----+-----+-----+---------------+\n",
            "|Item-Based Collaborative Filtering|0.388|0.308|0.263|0.229|0.205|0.68           |\n",
            "+----------------------------------+-----+-----+-----+-----+-----+---------------+\n",
            "\n"
          ]
        }
      ],
      "source": [
        "evaluation = evaluate_df(\"Item-Based Collaborative Filtering\", recommendations, k)\n",
        "metrics_df = metrics_df.union(evaluation)\n",
        "round_df(evaluation, 3).show(truncate=False)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2NlQlH7g0xzC"
      },
      "source": [
        "### Free resources"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 74,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SeRpAhSy00XK",
        "outputId": "ae02ee96-dc08-499a-f7f9-671a703c773d"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "491"
            ]
          },
          "execution_count": 74,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "import gc\n",
        "gc.collect()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RvgTFkKD3_7h"
      },
      "source": [
        "## Matrix Factorization\n",
        "\n",
        "In the Matrix Factorization technique, we try to estimate the ranking matrix as a product of two other matrices, the users and items matrices, which both have a lower rank than the starting matrix and can be learned from the observed ratings.\n",
        "\n",
        "Those matrices contain latent factors and a single prediction is made by computing the dot product between a user vector $x_u$ and an item vector $w_i$.\n",
        "\n",
        "$$\n",
        "r_{u,i} = x^T_u \\cdot w_i = \\sum_{j=0}^d x_{u,j} w_{j, i}\n",
        "$$\n",
        "\n",
        "where $d$ is the dimension of the latent space.\n",
        "\n",
        "So, in order to obtain the recommended repos for a user $u$ we have to build the matrix $R \\in \\mathbb{R}_{m \\times n}$ which is given by the matrix multiplication between the user vectors matrix $X \\in \\mathbb{R}_{m \\times d}$ and the item vectors matrix $W \\in \\mathbb{R}_{d \\times n}$ transposed.\n",
        "\n",
        "$$\n",
        "R = X \\times W^T\n",
        "$$\n",
        "\n",
        "This time, the model is already implemented in the PySpark library, so less code is required."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 75,
      "metadata": {
        "cellView": "form",
        "id": "1RcTFsgxbmkZ"
      },
      "outputs": [],
      "source": [
        "#@title Settings\n",
        "N_ITEMS = 5 #@param"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WfDKiOJpH34F"
      },
      "source": [
        "### Training"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KHEePk3AG1Vd"
      },
      "source": [
        "The $R$ matrix is partially known from the training set and so, the model takes in input all the previous ratings from all the users. In order to do this, we need to build the entire user-item matrix $M$ as follows:\n",
        "\n",
        "$$\n",
        "m_{ij} = \\begin{cases}\n",
        "1 & \\text{if } r_{i,j} = (defined) \\\\\n",
        "0 & \\text{otherwise}\n",
        "\\end{cases}\n",
        "$$\n",
        "\n",
        "In order to produce the predictions, we build a test set, which will contain all the user-repo pairs such that the user has not starred the repo.\n",
        "\n",
        "$$\n",
        "TEST = \\{ (u, i) : u \\in \\mathcal{U} \\land i \\in \\mathcal{I} \\land r_{u,i} \\neq (defined) \\}\n",
        "$$"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 76,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JhYW55neTvRl",
        "outputId": "120a855d-d828-45bd-b3b7-1fca03c1d52d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+-------+-------+-------+\n",
            "|repo_id|user_id|starred|\n",
            "+-------+-------+-------+\n",
            "|      0|    145|      1|\n",
            "|      0|    322|      1|\n",
            "|      0|    726|      1|\n",
            "|      0|    746|      1|\n",
            "|      0|    766|      1|\n",
            "|      0|    971|      1|\n",
            "|      0|   1567|      1|\n",
            "|      0|   1638|      1|\n",
            "|      0|   1657|      1|\n",
            "|      0|   1977|      1|\n",
            "|      0|   2011|      1|\n",
            "|      0|   2061|      1|\n",
            "|      0|   2126|      1|\n",
            "|      0|   2226|      1|\n",
            "|      0|   2464|      1|\n",
            "|      0|   2517|      1|\n",
            "|      0|   2665|      1|\n",
            "|      0|   2911|      1|\n",
            "|      0|   2931|      1|\n",
            "|      0|   3108|      1|\n",
            "+-------+-------+-------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "from pyspark import StorageLevel\n",
        "\n",
        "non_starred_train_df = new_repos_df.select(\"repo_id\").join(new_users_df.select(\"user_id\")).exceptAll(train)\n",
        "non_starred_train_df = non_starred_train_df.persist(StorageLevel.MEMORY_ONLY)\n",
        "train_matrix = train.withColumn(\"starred\", lit(1)).union(non_starred_train_df.withColumn(\"starred\", lit(0)))\n",
        "train_matrix = train_matrix.persist(StorageLevel.MEMORY_ONLY)\n",
        "train_matrix.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nH0DH9zWbtiK"
      },
      "source": [
        "In order to compute the $R$ matrix we set up an optimization problem based on the following loss function.\n",
        "\n",
        "$$\n",
        "L(X, W) = \\sum_{(u,i) \\in TRAIN} (m_{u,i} - x_u^T \\cdot w_i)^2 + \\text{regularization}\n",
        "$$\n",
        "\n",
        "So, we want to minimize the distance between the known ratings in the matrix $M$, and the predicted ratings, given by the dot product between the two matrices.\n",
        "\n",
        "$$\n",
        "X^*, W^* = argmin_{X, W} L(X,W)\n",
        "$$\n",
        "\n",
        "Now, we can address this optimization problem in several ways. The method used by PySpark is the **Alternating Least Squares (ALS)**.\n",
        "\n",
        "The ALS method is based on the idea to fix one latent matrix and compute the optimal for the other one, then we swap the two roles and update the other matrix.\n",
        "\n",
        "Let's now set the parameters of the model, which have been tuned using manual experiments, and train it:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 77,
      "metadata": {
        "id": "ZVCe_2ro4CNV"
      },
      "outputs": [],
      "source": [
        "from pyspark.ml.recommendation import ALS\n",
        "from pyspark.ml.evaluation import RankingEvaluator\n",
        "from pyspark.ml.tuning import CrossValidator, ParamGridBuilder\n",
        "\n",
        "train_with_ratings = train.withColumn(\"starred\", lit(1))\n",
        "\n",
        "als_params = {\n",
        "    \"rank\": 5,\n",
        "    \"maxIter\": 20, \n",
        "    \"userCol\": \"user_id\", \n",
        "    \"itemCol\": \"repo_id\", \n",
        "    \"ratingCol\": \"starred\", \n",
        "    \"coldStartStrategy\": \"drop\",\n",
        "    \"seed\": 42,\n",
        "    \"regParam\": 0.1,\n",
        "    \"implicitPrefs\": True,\n",
        "    \"nonnegative\": False,\n",
        "    \"alpha\": 1\n",
        "}\n",
        "als = ALS(**als_params)\n",
        "als_model = als.fit(train_matrix)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7kV8GtGVG81Q"
      },
      "source": [
        "Here is a discussion about the parameter tuning we carried out:\n",
        "  - `rank` is the rank of the two matrices. We tried both to increase and decrease this number, but the lower it is and the less accurate results we get, while if we increase it, we don't achieve higher scores.\n",
        "  - `maxIter` is the maximum number of iterations which is performed to compute the matrices.\n",
        "  - `coldStartStrategy` this is to indicate that we should drop records for new users not seen in the training. It is used to perform a correct evaluation.\n",
        "  - `regParam` is the regularization parameter. We tried 0.001, 0.01, 0.1 and 1, and 0.1 worked best.\n",
        "  - `implicitPrefs` indicates that we use implicit preferences meaning that they are not ratings but an evidence on how \"strong\" is the observation. This also uses some tweaks in the implementations to obtain better results in our case.\n",
        "  - `nonnegative` indicates if some non negative constraints are added to the optimization problem. Since we only need to rank items, we saw that disabling this achieved better performances.\n",
        "  - `alpha` indicates how confident we are in the rating. In our case we know for sure that a user starred a repo, so we left it at 1, even if we experimented with lower values, obtaining worse results."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Q_kyqc0S_k27"
      },
      "source": [
        "We also save the model to use it later in the demo:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 78,
      "metadata": {
        "id": "O2A9BU8z_n2u"
      },
      "outputs": [],
      "source": [
        "als_model.write().overwrite().save(\"als_model\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "I9hH8yH-H6xP"
      },
      "source": [
        "### Prediction"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jtH8b-Bu5KVm"
      },
      "source": [
        "We can now obtain the recommendations from our model by predicting a score for each non-starred repo for each user:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 79,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RStq8DOb4GCV",
        "outputId": "8cd5afb7-7528-447d-f2ac-a46293d4aea3"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+-------+-------+----------+\n",
            "|repo_id|user_id|prediction|\n",
            "+-------+-------+----------+\n",
            "|      2|    140|0.23797423|\n",
            "|      2|    431|0.47488958|\n",
            "|      2|    546|0.26229107|\n",
            "|      2|    586|0.61152077|\n",
            "|      2|    656|0.38166583|\n",
            "|      2|    723|0.38425478|\n",
            "|      2|    744| 1.1301888|\n",
            "|      2|    823|0.58897686|\n",
            "|      2|    835|0.71540534|\n",
            "|      2|    850| 0.2564758|\n",
            "|      2|    947|0.61876297|\n",
            "|      2|   1041| 0.5520584|\n",
            "|      2|   1065|0.44424093|\n",
            "|      2|   1069| 1.2033892|\n",
            "|      2|   1104|0.49694663|\n",
            "|      2|   1171| 1.1404679|\n",
            "|      2|   1262| 0.6350749|\n",
            "|      2|   1277|0.23192126|\n",
            "|      2|   1329|0.44869748|\n",
            "|      2|   1359|0.39930958|\n",
            "+-------+-------+----------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "predict = als_model.transform(non_starred_train_df).cache()\n",
        "predict.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4HoYKQlJ5REi"
      },
      "source": [
        "Let's now take the `N_ITEMS` top scoring repos for each user:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 80,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DWrNqTZY-I9T",
        "outputId": "8ca68126-e849-41f7-c4d5-24c115177754"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+-------+-------+----------+----+\n",
            "|repo_id|user_id|prediction|rank|\n",
            "+-------+-------+----------+----+\n",
            "|     35|     90| 0.4050569|   1|\n",
            "|    320|     90|0.38768515|   2|\n",
            "|    666|     90| 0.3849909|   3|\n",
            "|    527|     90|0.36306375|   4|\n",
            "|      7|     90|0.35994244|   5|\n",
            "|      1|    123| 0.4360758|   1|\n",
            "|    398|    123|0.43481076|   2|\n",
            "|    120|    123|0.41536427|   3|\n",
            "|    238|    123|0.40301564|   4|\n",
            "|     48|    123| 0.3501742|   5|\n",
            "|      2|    140|0.23797423|   1|\n",
            "|    482|    140| 0.1968115|   2|\n",
            "|   1110|    140|0.18946297|   3|\n",
            "|     16|    140|0.18742013|   4|\n",
            "|    721|    140|0.18241842|   5|\n",
            "|     35|    150|0.44719887|   1|\n",
            "|    160|    150|0.41495872|   2|\n",
            "|    398|    150| 0.3903003|   3|\n",
            "|    120|    150| 0.3847297|   4|\n",
            "|    615|    150| 0.3797544|   5|\n",
            "+-------+-------+----------+----+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "window = Window.partitionBy(predict[\"user_id\"]).orderBy(predict[\"prediction\"].desc())\n",
        "top_k_items = (predict.select(\"*\", row_number().over(window).alias(\"rank\")) \n",
        "  .filter(col(\"rank\") <= N_ITEMS))\n",
        "top_k_items.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mSegmhjq4LJJ"
      },
      "source": [
        "### Evaluation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7S3pb0Z--JTJ"
      },
      "source": [
        "Finally, we convert the dataframe to a format suitable for the evaluation:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 81,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pePHKLVB4Ic9",
        "outputId": "5bfa775f-0bdf-4830-d824-0fc508fdf6ed"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+-------+---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
            "|user_id|recommended_repos                      |labels                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |\n",
            "+-------+---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
            "|90     |[35.0, 320.0, 666.0, 527.0, 7.0]       |[1258.0, 5968.0, 6319.0, 6251.0, 1409.0, 6300.0, 1067.0, 2709.0, 6259.0, 506.0, 335.0, 147.0, 6316.0, 1965.0, 4104.0, 1937.0, 797.0, 28.0, 821.0, 2256.0, 2192.0, 6156.0, 6163.0, 6201.0, 6254.0, 2258.0, 6330.0]                                                                                                                                                                                                                                                                                                                                                                           |\n",
            "|123    |[1.0, 398.0, 120.0, 238.0, 48.0]       |[238.0, 123.0, 1049.0, 6300.0, 122.0, 1280.0, 1411.0, 483.0, 1041.0, 6935.0, 2851.0, 3375.0, 48.0, 2857.0, 7539.0, 2275.0, 4140.0, 2701.0, 2634.0, 3953.0, 706.0, 526.0, 910.0, 2069.0, 442.0, 797.0, 1042.0, 7814.0, 77.0, 871.0, 7750.0, 1257.0, 6435.0]                                                                                                                                                                                                                                                                                                                                  |\n",
            "|140    |[2.0, 482.0, 1110.0, 16.0, 721.0]      |[4815.0, 2.0, 2967.0, 2266.0, 8603.0, 5859.0, 433.0, 2839.0, 1510.0, 6256.0, 1965.0, 109.0, 413.0, 2866.0, 616.0, 3520.0, 8654.0, 8583.0, 8558.0, 1042.0, 8675.0, 8539.0, 8528.0, 4016.0, 126.0]                                                                                                                                                                                                                                                                                                                                                                                            |\n",
            "|150    |[35.0, 160.0, 398.0, 120.0, 615.0]     |[2162.0, 8332.0, 2609.0, 3375.0, 9183.0, 533.0, 2384.0, 668.0, 1053.0, 8006.0, 746.0, 1054.0, 9077.0]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |\n",
            "|191    |[35.0, 198.0, 476.0, 352.0, 482.0]     |[383.0, 414.0, 35.0, 122.0, 4028.0, 7814.0, 871.0, 476.0, 2275.0]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           |\n",
            "|210    |[238.0, 117.0, 615.0, 1302.0, 476.0]   |[238.0, 3374.0, 703.0, 4989.0, 1378.0, 6289.0, 2074.0, 1510.0, 2385.0, 1444.0, 6135.0, 648.0, 454.0, 381.0, 365.0, 48.0, 386.0, 716.0, 4200.0]                                                                                                                                                                                                                                                                                                                                                                                                                                              |\n",
            "|263    |[35.0, 28.0, 125.0, 320.0, 16.0]       |[11903.0, 988.0, 347.0, 671.0, 506.0, 721.0, 1411.0, 1377.0, 6938.0, 11186.0, 320.0, 731.0, 5940.0, 418.0, 691.0, 1677.0, 5948.0, 1061.0, 11691.0, 126.0, 35.0, 117.0, 14274.0, 1148.0, 441.0, 3953.0, 16.0, 1119.0, 3682.0, 5680.0, 7854.0, 5872.0, 442.0, 683.0, 1612.0, 716.0, 435.0, 77.0, 871.0, 1339.0]                                                                                                                                                                                                                                                                               |\n",
            "|320    |[35.0, 1.0, 119.0, 122.0, 120.0]       |[1236.0, 6935.0, 510.0, 834.0, 1385.0, 3426.0, 7660.0, 2599.0]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              |\n",
            "|336    |[35.0, 1.0, 119.0, 352.0, 398.0]       |[2851.0, 357.0]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |\n",
            "|369    |[320.0, 797.0, 6273.0, 442.0, 1411.0]  |[7197.0, 2967.0, 408.0, 671.0, 331.0, 2709.0, 110.0, 1737.0, 1429.0, 11469.0, 483.0, 1056.0, 1382.0, 6935.0, 3375.0, 48.0, 691.0, 1433.0, 4033.0, 4727.0, 8797.0, 1405.0, 669.0, 4140.0, 125.0, 4378.0, 19465.0, 614.0, 1148.0, 6749.0, 697.0, 706.0, 11975.0, 442.0, 6747.0, 4848.0, 6254.0, 160.0, 2207.0, 8476.0, 4803.0]                                                                                                                                                                                                                                                                |\n",
            "|384    |[527.0, 1110.0, 703.0, 683.0, 541.0]   |[527.0, 11.0, 1067.0, 441.0, 12146.0, 1070.0, 2256.0, 11186.0, 20507.0, 103.0, 1047.0, 2081.0]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              |\n",
            "|393    |[352.0, 238.0, 482.0, 5859.0, 16.0]    |[383.0, 645.0, 4989.0, 449.0, 2045.0, 2581.0, 3930.0, 21321.0, 106.0, 1249.0, 352.0, 8455.0, 2704.0, 14512.0, 5940.0, 4874.0, 16593.0, 6042.0, 357.0, 691.0, 2646.0, 668.0, 2173.0, 1061.0, 7539.0, 16591.0, 2328.0, 16063.0, 2275.0, 15630.0, 35.0, 2044.0, 8339.0, 647.0, 453.0, 145.0, 3953.0, 4394.0, 16.0, 648.0, 6633.0, 8254.0, 8891.0, 471.0, 666.0, 410.0, 21455.0, 12282.0, 6212.0, 643.0, 20774.0, 21661.0, 419.0, 1044.0, 6910.0, 1583.0]                                                                                                                                       |\n",
            "|406    |[35.0, 238.0, 398.0, 120.0, 119.0]     |[2162.0, 35.0, 117.0, 119.0, 2149.0, 122.0, 1280.0, 721.0, 2187.0, 6915.0, 6802.0, 7567.0, 15732.0, 8202.0, 3669.0, 3226.0, 357.0, 691.0, 3426.0, 4471.0, 160.0, 1110.0, 4016.0, 6273.0, 791.0, 5896.0]                                                                                                                                                                                                                                                                                                                                                                                     |\n",
            "|419    |[117.0, 1.0, 119.0, 398.0, 120.0]      |[6251.0, 10136.0, 6949.0, 1444.0, 22921.0, 2278.0, 8095.0, 3520.0, 7519.0, 1254.0, 1321.0, 5858.0, 11095.0, 334.0, 864.0, 2275.0, 1993.0, 1258.0, 1063.0, 6432.0, 9039.0, 10544.0, 3067.0, 19835.0, 964.0, 7887.0, 18500.0, 5095.0, 5996.0, 2033.0, 1853.0, 674.0, 10107.0, 1612.0, 707.0, 4989.0, 581.0, 1344.0, 12912.0, 23716.0, 2457.0, 454.0, 2314.0, 527.0, 3401.0, 6976.0, 16931.0, 6547.0, 19555.0, 24052.0, 3833.0, 8983.0, 1302.0, 459.0, 863.0, 11819.0, 13869.0, 337.0, 23277.0, 117.0, 1995.0, 4654.0, 6150.0, 4715.0, 7997.0, 526.0, 16344.0, 333.0, 7814.0, 10474.0, 17387.0]|\n",
            "|426    |[656.0, 4715.0, 1418.0, 7572.0, 5341.0]|[383.0, 1342.0, 1994.0, 2884.0, 3521.0, 2400.0, 7134.0, 320.0, 348.0, 15348.0, 357.0, 15701.0, 2256.0, 18508.0, 8093.0, 20064.0, 9039.0, 792.0, 2594.0, 8136.0, 6202.0, 120.0, 8339.0, 1053.0, 615.0, 8812.0, 3682.0, 1186.0, 430.0, 340.0, 2069.0, 512.0, 160.0, 77.0, 3383.0, 1422.0, 2294.0, 743.0, 1421.0, 318.0, 1899.0, 449.0, 2858.0, 110.0, 4413.0, 1380.0, 5930.0, 1046.0, 464.0, 2701.0, 5886.0, 9553.0, 6482.0, 5822.0, 7036.0, 6150.0, 9675.0, 145.0, 2926.0, 4715.0, 706.0, 12095.0, 419.0, 798.0, 80.0, 865.0]                                                                |\n",
            "|431    |[2.0, 35.0, 398.0, 320.0, 120.0]       |[669.0, 4140.0, 7977.0, 2.0, 4378.0, 4107.0, 371.0, 3374.0, 3730.0, 6734.0, 2349.0, 1475.0, 13293.0, 656.0, 413.0, 802.0, 352.0, 6746.0, 6404.0, 8798.0, 77.0, 1340.0, 505.0, 1941.0, 22107.0]                                                                                                                                                                                                                                                                                                                                                                                              |\n",
            "|463    |[35.0, 352.0, 120.0, 398.0, 615.0]     |[9684.0, 1279.0, 120.0, 2045.0, 204.0, 1369.0, 961.0, 8254.0]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               |\n",
            "|483    |[238.0, 35.0, 674.0, 482.0, 424.0]     |[8293.0, 1211.0, 3521.0, 581.0, 118.0, 204.0, 11183.0, 804.0, 2851.0, 3458.0, 18819.0, 528.0, 2256.0, 6163.0, 5926.0, 1423.0, 1258.0, 2162.0, 5878.0, 5968.0, 8573.0, 489.0, 8339.0, 533.0, 1022.0, 1376.0, 10601.0, 5125.0, 802.0, 526.0, 2069.0, 6449.0, 1042.0, 716.0, 5820.0, 80.0, 1422.0]                                                                                                                                                                                                                                                                                             |\n",
            "|499    |[352.0, 160.0, 28.0, 35.0, 125.0]      |[238.0, 8429.0, 988.0, 629.0, 4412.0, 671.0, 581.0, 6406.0, 106.0, 147.0, 631.0, 2440.0, 3968.0, 4364.0, 336.0, 8583.0, 6547.0, 682.0, 666.0, 28.0, 6302.0, 14672.0, 1355.0, 2599.0, 187.0, 791.0]                                                                                                                                                                                                                                                                                                                                                                                          |\n",
            "|546    |[2.0, 35.0, 238.0, 398.0, 120.0]       |[2.0, 5968.0, 2967.0, 5859.0, 24937.0, 2255.0, 1460.0, 15332.0, 438.0, 666.0, 77.0, 6910.0, 199.0, 2275.0]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  |\n",
            "+-------+---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "recommendations = top_k_items.withColumn(\"repo_id\", col(\"repo_id\").cast(\"double\")).groupBy(\"user_id\").agg(collect_list(\"repo_id\").alias(\"recommended_repos\"))\n",
        "recommendations = recommendations.join(gt, \"user_id\").cache()\n",
        "recommendations.show(truncate=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 82,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "X1uBLxUe7uDl",
        "outputId": "be523d2c-4904-4395-ee47-e39cf1b4bd0a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+--------------------+-----+-----+-----+-----+-----+---------------+\n",
            "|Model               |MAP@1|MAP@2|MAP@3|MAP@4|MAP@5|Personalization|\n",
            "+--------------------+-----+-----+-----+-----+-----+---------------+\n",
            "|Matrix Factorization|0.506|0.393|0.332|0.298|0.268|0.864          |\n",
            "+--------------------+-----+-----+-----+-----+-----+---------------+\n",
            "\n"
          ]
        }
      ],
      "source": [
        "evaluation = evaluate_df(\"Matrix Factorization\", recommendations, k)\n",
        "metrics_df = metrics_df.union(evaluation)\n",
        "round_df(evaluation, 3).show(truncate=False)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4fJ0mRmetc4d"
      },
      "source": [
        "### Free resources"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 83,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pVbYcMaitebb",
        "outputId": "59797eb3-316e-42db-dcb4-3cb5c9f0f42a"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "DataFrame[repo_id: int, user_id: int, starred: int]"
            ]
          },
          "execution_count": 83,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "non_starred_train_df.unpersist()\n",
        "train_matrix.unpersist()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "U92b8l_ofl65"
      },
      "source": [
        "## Models comparison"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kGKg7GCZBO0X"
      },
      "source": [
        "Let's now show the metrics collected by all the models in order to compare them:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 84,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "l9-ktP40fpsM",
        "outputId": "a90bde5e-dd67-4b35-b4c3-5b0eb55a42ee"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "+----------------------------------+-----+-----+-----+-----+-----+---------------+\n",
            "|Model                             |MAP@1|MAP@2|MAP@3|MAP@4|MAP@5|Personalization|\n",
            "+----------------------------------+-----+-----+-----+-----+-----+---------------+\n",
            "|Content-Based Filtering           |0.067|0.051|0.043|0.037|0.033|0.676          |\n",
            "|User-Based Collaborative Filtering|0.357|0.261|0.213|0.183|0.164|0.965          |\n",
            "|Item-Based Collaborative Filtering|0.388|0.308|0.263|0.229|0.205|0.68           |\n",
            "|Matrix Factorization              |0.506|0.393|0.332|0.298|0.268|0.864          |\n",
            "+----------------------------------+-----+-----+-----+-----+-----+---------------+\n",
            "\n"
          ]
        }
      ],
      "source": [
        "round_df(metrics_df, 3).show(truncate=False)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qzvHqHAS-XRY"
      },
      "source": [
        "As we can see from the comparative table above, the best performing model according to MAP@K is the **Matrix Factorization**, while the worst one is the **Content-Based Filtering**, probably due to the low discriminance of the repos features we extracted. In particular, the about field, which should give us more information than other fields, in general is very short and contains recurring terms which means we will obtain similar TF-IDF vectors for different repos but with similar descriptions.\n",
        "\n",
        "The User and Item based approaches instead obtain similar performances from the point of view of MAP@K, but while the Item-based obtains a slightly higher MAP@K in general, the User-based approach obtains a higher Personalization score, which moreover is the best of all.\n",
        "\n",
        "From the table we can see that the scores obtained for the MAP@K decrease with the increase of $K$. This was an expected behaviour, since using a higher $K$ means we have more chances to recommend a non-interesting repository."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "T1RddR3qagVG"
      },
      "source": [
        "# Demo\n",
        "\n",
        "In this section we setup a working demo, including a web server, which allows to experiment with all the recommender systems from a web interface.\n",
        "\n",
        "For each model we also include an example of what results it produces using the same user/repos."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KmwEwKw50il8"
      },
      "source": [
        "## Models definitions\n",
        "\n",
        "First we define a base class for each model, which can be used to predict repos for both an already existing user, indicated by his id, or a new user represented by the repos he has starred."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 85,
      "metadata": {
        "id": "H4zbDZj_1RgR"
      },
      "outputs": [],
      "source": [
        "class RecommenderModel():\n",
        "\n",
        "  def __init__(self, users_df, repos_df, starred_repos_df):\n",
        "    self.users = users_df\n",
        "    self.repos = repos_df\n",
        "    self.starred_repos = starred_repos_df\n",
        "\n",
        "  def predict_from_user(user_id):\n",
        "    pass\n",
        "\n",
        "  def predict_from_repos(starred_repos_list):\n",
        "    pass"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yvCK4Jmd3niA"
      },
      "source": [
        "### Content-based"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 86,
      "metadata": {
        "id": "CWmqbJ-Hobn9"
      },
      "outputs": [],
      "source": [
        "from pyspark.ml.stat import Summarizer\n",
        "\n",
        "class ContentBased(RecommenderModel):\n",
        "\n",
        "  def __init__(self, users_df, repos_df, starred_repos_df):\n",
        "    super().__init__(users_df, repos_df, starred_repos_df)\n",
        "    # Load the item profiles previously saved in the notebook\n",
        "    self.item_profiles = spark.read.load(\"content_based-item_profiles.parquet\")\n",
        "\n",
        "  def predict_from_user_profile(self, user_profile, K):\n",
        "    # Perform a cartesian product between the user profile and all the item profiles\n",
        "    user_item_profiles = user_profile.join(self.item_profiles)\n",
        "\n",
        "    # Remove the repos already starred by the user\n",
        "    user_item_profiles = user_item_profiles.filter(~array_contains(col(\"starred_repo\"), col(\"repo_id\")))\n",
        "\n",
        "    # Compute the similarity between the user profile and each item profile\n",
        "    similarity = cosine_similarity_udf(\"user_profile\", \"item_profile\").alias(\"score\")\n",
        "    prediction_df = user_item_profiles.select(\"repo_id\", similarity)\n",
        "\n",
        "    # Take the top-k items\n",
        "    top_k_repos = prediction_df.orderBy(col(\"score\").desc()).limit(K)\n",
        "\n",
        "    return top_k_repos\n",
        "\n",
        "  def predict_from_user(self, user_id, K):\n",
        "    # Get the item profiles of the repos starred by the user\n",
        "    join_df = train.filter(col(\"user_id\") == user_id).join(self.item_profiles, on=\"repo_id\")\n",
        "\n",
        "    # Compute the user profile as the mean of the item profiles of the starred repos\n",
        "    user_profile = join_df.groupBy(\"user_id\").agg(Summarizer.mean(col(\"item_profile\")).alias(\"user_profile\"), collect_set(\"repo_id\").alias(\"starred_repo\"))\n",
        "\n",
        "    return self.predict_from_user_profile(user_profile, K)\n",
        "\n",
        "  def predict_from_repos(self, starred_repos_list, K):\n",
        "    # Take all the item profiles of the repos starred\n",
        "    starred_items_profiles = self.item_profiles.filter(col(\"repo_id\").isin(starred_repos_list))\n",
        "\n",
        "    # Compute the user profile as the mean of the item profiles of the starred repos\n",
        "    user_profile = starred_items_profiles.agg(Summarizer.mean(col(\"item_profile\")).alias(\"user_profile\"), collect_set(\"repo_id\").alias(\"starred_repo\"))\n",
        "\n",
        "    return self.predict_from_user_profile(user_profile, K)\n",
        "\n",
        "cbm = ContentBased(new_users_df, new_repos_df, train)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 87,
      "metadata": {
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "JpkDRsfwXK9k",
        "outputId": "b8717d22-e8ef-4c21-a85a-3b847a11e80d"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<h3>Repos previously starred by user 463</h3>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>repo_id</th>\n",
              "      <th>user_id</th>\n",
              "      <th>creator</th>\n",
              "      <th>name</th>\n",
              "      <th>about</th>\n",
              "      <th>language</th>\n",
              "      <th>stars</th>\n",
              "      <th>forks</th>\n",
              "      <th>updated</th>\n",
              "      <th>sponsor</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2</td>\n",
              "      <td>463</td>\n",
              "      <td>facebook</td>\n",
              "      <td>react</td>\n",
              "      <td>A declarative, efficient, and flexible JavaScr...</td>\n",
              "      <td>JavaScript</td>\n",
              "      <td>170264</td>\n",
              "      <td>34406</td>\n",
              "      <td>2021-06-22 06:08:58</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>169</td>\n",
              "      <td>463</td>\n",
              "      <td>date-fns</td>\n",
              "      <td>date-fns</td>\n",
              "      <td>⏳ Modern JavaScript date utility library ⌛️</td>\n",
              "      <td>JavaScript</td>\n",
              "      <td>26155</td>\n",
              "      <td>1260</td>\n",
              "      <td>2021-06-21 12:37:46</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>13629</td>\n",
              "      <td>463</td>\n",
              "      <td>square</td>\n",
              "      <td>okhttp</td>\n",
              "      <td>Square’s meticulous HTTP client for the JVM, A...</td>\n",
              "      <td>Java</td>\n",
              "      <td>40244</td>\n",
              "      <td>8538</td>\n",
              "      <td>2021-06-14 08:10:00</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>25974</td>\n",
              "      <td>463</td>\n",
              "      <td>square</td>\n",
              "      <td>retrofit</td>\n",
              "      <td>A type-safe HTTP client for Android and the JVM</td>\n",
              "      <td>Java</td>\n",
              "      <td>38265</td>\n",
              "      <td>6872</td>\n",
              "      <td>2021-06-24 19:20:04</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   repo_id  user_id   creator  ...  forks             updated sponsor\n",
              "0        2      463  facebook  ...  34406 2021-06-22 06:08:58       0\n",
              "1      169      463  date-fns  ...   1260 2021-06-21 12:37:46       0\n",
              "2    13629      463    square  ...   8538 2021-06-14 08:10:00       0\n",
              "3    25974      463    square  ...   6872 2021-06-24 19:20:04       0\n",
              "\n",
              "[4 rows x 10 columns]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<br /><h3>Predicted repos for user 463</h3>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>repo_id</th>\n",
              "      <th>score</th>\n",
              "      <th>creator</th>\n",
              "      <th>name</th>\n",
              "      <th>about</th>\n",
              "      <th>language</th>\n",
              "      <th>stars</th>\n",
              "      <th>forks</th>\n",
              "      <th>updated</th>\n",
              "      <th>sponsor</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>8815</td>\n",
              "      <td>0.669058</td>\n",
              "      <td>request</td>\n",
              "      <td>request</td>\n",
              "      <td>🏊🏾 Simplified HTTP request client.</td>\n",
              "      <td>JavaScript</td>\n",
              "      <td>25189</td>\n",
              "      <td>3062</td>\n",
              "      <td>2021-05-20 12:36:23</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1583</td>\n",
              "      <td>0.480193</td>\n",
              "      <td>signalapp</td>\n",
              "      <td>Signal-Android</td>\n",
              "      <td>A private messenger for Android.</td>\n",
              "      <td>Java</td>\n",
              "      <td>20775</td>\n",
              "      <td>4982</td>\n",
              "      <td>2021-06-22 00:27:46</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>473</td>\n",
              "      <td>0.479396</td>\n",
              "      <td>qbittorrent</td>\n",
              "      <td>qBittorrent</td>\n",
              "      <td>qBittorrent BitTorrent client</td>\n",
              "      <td>C++</td>\n",
              "      <td>12288</td>\n",
              "      <td>2201</td>\n",
              "      <td>2021-06-22 05:15:45</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>5922</td>\n",
              "      <td>0.479263</td>\n",
              "      <td>go-redis</td>\n",
              "      <td>redis</td>\n",
              "      <td>Type-safe Redis client for Golang</td>\n",
              "      <td>Go</td>\n",
              "      <td>11762</td>\n",
              "      <td>1498</td>\n",
              "      <td>2021-06-21 06:45:54</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>160</td>\n",
              "      <td>0.472465</td>\n",
              "      <td>axios</td>\n",
              "      <td>axios</td>\n",
              "      <td>Promise based HTTP client for the browser and ...</td>\n",
              "      <td>JavaScript</td>\n",
              "      <td>85310</td>\n",
              "      <td>8327</td>\n",
              "      <td>2021-06-22 05:55:53</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   repo_id     score      creator  ... forks             updated sponsor\n",
              "4     8815  0.669058      request  ...  3062 2021-05-20 12:36:23       0\n",
              "2     1583  0.480193    signalapp  ...  4982 2021-06-22 00:27:46       0\n",
              "1      473  0.479396  qbittorrent  ...  2201 2021-06-22 05:15:45       0\n",
              "3     5922  0.479263     go-redis  ...  1498 2021-06-21 06:45:54       0\n",
              "0      160  0.472465        axios  ...  8327 2021-06-22 05:55:53       0\n",
              "\n",
              "[5 rows x 10 columns]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<br /><h3>Ground Truth for user 463</h3>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>repo_id</th>\n",
              "      <th>user_id</th>\n",
              "      <th>creator</th>\n",
              "      <th>name</th>\n",
              "      <th>about</th>\n",
              "      <th>language</th>\n",
              "      <th>stars</th>\n",
              "      <th>forks</th>\n",
              "      <th>updated</th>\n",
              "      <th>sponsor</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>120</td>\n",
              "      <td>463</td>\n",
              "      <td>nodejs</td>\n",
              "      <td>node</td>\n",
              "      <td>Node.js JavaScript runtime ✨🐢🚀✨</td>\n",
              "      <td>JavaScript</td>\n",
              "      <td>79974</td>\n",
              "      <td>20533</td>\n",
              "      <td>2021-06-22 08:41:01</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>204</td>\n",
              "      <td>463</td>\n",
              "      <td>django</td>\n",
              "      <td>django</td>\n",
              "      <td>The Web framework for perfectionists with dead...</td>\n",
              "      <td>Python</td>\n",
              "      <td>58081</td>\n",
              "      <td>24842</td>\n",
              "      <td>2021-06-22 08:28:44</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>961</td>\n",
              "      <td>463</td>\n",
              "      <td>bumptech</td>\n",
              "      <td>glide</td>\n",
              "      <td>An image loading and caching library for Andro...</td>\n",
              "      <td>Java</td>\n",
              "      <td>31342</td>\n",
              "      <td>5710</td>\n",
              "      <td>2021-06-11 23:50:48</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1279</td>\n",
              "      <td>463</td>\n",
              "      <td>ReactiveX</td>\n",
              "      <td>rxjs</td>\n",
              "      <td>A reactive programming library for JavaScript</td>\n",
              "      <td>TypeScript</td>\n",
              "      <td>25031</td>\n",
              "      <td>2575</td>\n",
              "      <td>2021-06-21 19:24:28</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1369</td>\n",
              "      <td>463</td>\n",
              "      <td>r-spacex</td>\n",
              "      <td>SpaceX-API</td>\n",
              "      <td>🚀 Open Source REST API for SpaceX launch, rock...</td>\n",
              "      <td>JavaScript</td>\n",
              "      <td>8548</td>\n",
              "      <td>686</td>\n",
              "      <td>2021-06-19 16:04:27</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>2045</td>\n",
              "      <td>463</td>\n",
              "      <td>DrKLO</td>\n",
              "      <td>Telegram</td>\n",
              "      <td>Telegram for Android source</td>\n",
              "      <td>C++</td>\n",
              "      <td>17143</td>\n",
              "      <td>5993</td>\n",
              "      <td>2021-06-03 05:06:31</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>8254</td>\n",
              "      <td>463</td>\n",
              "      <td>google</td>\n",
              "      <td>gson</td>\n",
              "      <td>A Java serialization/deserialization library t...</td>\n",
              "      <td>Java</td>\n",
              "      <td>19688</td>\n",
              "      <td>3818</td>\n",
              "      <td>2021-06-19 12:39:51</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>9684</td>\n",
              "      <td>463</td>\n",
              "      <td>jquense</td>\n",
              "      <td>yup</td>\n",
              "      <td>Dead simple Object schema validation</td>\n",
              "      <td>TypeScript</td>\n",
              "      <td>13746</td>\n",
              "      <td>616</td>\n",
              "      <td>2021-06-21 18:30:52</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   repo_id  user_id    creator  ...  forks             updated sponsor\n",
              "0      120      463     nodejs  ...  20533 2021-06-22 08:41:01       0\n",
              "1      204      463     django  ...  24842 2021-06-22 08:28:44       1\n",
              "2      961      463   bumptech  ...   5710 2021-06-11 23:50:48       0\n",
              "3     1279      463  ReactiveX  ...   2575 2021-06-21 19:24:28       0\n",
              "4     1369      463   r-spacex  ...    686 2021-06-19 16:04:27       0\n",
              "5     2045      463      DrKLO  ...   5993 2021-06-03 05:06:31       0\n",
              "6     8254      463     google  ...   3818 2021-06-19 12:39:51       0\n",
              "7     9684      463    jquense  ...    616 2021-06-21 18:30:52       0\n",
              "\n",
              "[8 rows x 10 columns]"
            ]
          },
          "execution_count": 87,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#@title Predict from user id\n",
        "user_id = 463 #@param {type: \"integer\"}\n",
        "n_items = 5 #@param {type: \"integer\"}\n",
        "recommendations = cbm.predict_from_user(user_id, n_items)\n",
        "\n",
        "display(HTML(\"<h3>Repos previously starred by user \" + str(user_id) + \"</h3>\"))\n",
        "display(train.filter(col(\"user_id\") == user_id) \\\n",
        "  .join(new_repos_df, \"repo_id\") \\\n",
        "  .toPandas())\n",
        "\n",
        "display(HTML(\"<br /><h3>Predicted repos for user \" + str(user_id) + \"</h3>\"))\n",
        "display(recommendations.join(new_repos_df, \"repo_id\").toPandas().sort_values(\"score\", ascending=False))\n",
        "\n",
        "display(HTML(\"<br /><h3>Ground Truth for user \" + str(user_id) + \"</h3>\"))\n",
        "test.filter(col(\"user_id\") == user_id) \\\n",
        "  .join(new_repos_df, \"repo_id\") \\\n",
        "  .toPandas()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 88,
      "metadata": {
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 450
        },
        "id": "7_6SwtBxXp_j",
        "outputId": "6287b918-b757-4af7-dc73-f10890c0e6e3"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<h3>Repos previously starred</h3>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>repo_id</th>\n",
              "      <th>creator</th>\n",
              "      <th>name</th>\n",
              "      <th>about</th>\n",
              "      <th>language</th>\n",
              "      <th>stars</th>\n",
              "      <th>forks</th>\n",
              "      <th>updated</th>\n",
              "      <th>sponsor</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>vercel</td>\n",
              "      <td>next.js</td>\n",
              "      <td>The React Framework</td>\n",
              "      <td>JavaScript</td>\n",
              "      <td>69687</td>\n",
              "      <td>13111</td>\n",
              "      <td>2021-06-22 07:49:26</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   repo_id creator     name  ...  forks             updated  sponsor\n",
              "0        1  vercel  next.js  ...  13111 2021-06-22 07:49:26        0\n",
              "\n",
              "[1 rows x 9 columns]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<br /><h3>Predicted repos</h3>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>repo_id</th>\n",
              "      <th>score</th>\n",
              "      <th>creator</th>\n",
              "      <th>name</th>\n",
              "      <th>about</th>\n",
              "      <th>language</th>\n",
              "      <th>stars</th>\n",
              "      <th>forks</th>\n",
              "      <th>updated</th>\n",
              "      <th>sponsor</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1143</td>\n",
              "      <td>0.798809</td>\n",
              "      <td>ReactTraining</td>\n",
              "      <td>react-router</td>\n",
              "      <td>Declarative routing for React</td>\n",
              "      <td>JavaScript</td>\n",
              "      <td>43426</td>\n",
              "      <td>8466</td>\n",
              "      <td>2021-06-15 22:38:05</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2393</td>\n",
              "      <td>0.784845</td>\n",
              "      <td>alibaba</td>\n",
              "      <td>ice</td>\n",
              "      <td>🚀 The Progressive Framework Based On React（基于 ...</td>\n",
              "      <td>TypeScript</td>\n",
              "      <td>16379</td>\n",
              "      <td>1951</td>\n",
              "      <td>2021-06-17 09:38:33</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>15532</td>\n",
              "      <td>0.743813</td>\n",
              "      <td>umijs</td>\n",
              "      <td>umi</td>\n",
              "      <td>🌋 Pluggable enterprise-level react application...</td>\n",
              "      <td>TypeScript</td>\n",
              "      <td>10697</td>\n",
              "      <td>1687</td>\n",
              "      <td>2021-06-25 06:53:08</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>7977</td>\n",
              "      <td>0.718425</td>\n",
              "      <td>react-dnd</td>\n",
              "      <td>react-dnd</td>\n",
              "      <td>Drag and Drop for React</td>\n",
              "      <td>TypeScript</td>\n",
              "      <td>15992</td>\n",
              "      <td>1668</td>\n",
              "      <td>2021-06-17 17:20:10</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>553</td>\n",
              "      <td>0.718241</td>\n",
              "      <td>pmndrs</td>\n",
              "      <td>react-three-fiber</td>\n",
              "      <td>🇨🇭 A React renderer for Three.js</td>\n",
              "      <td>TypeScript</td>\n",
              "      <td>13725</td>\n",
              "      <td>728</td>\n",
              "      <td>2021-06-19 12:17:17</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   repo_id     score        creator  ... forks             updated sponsor\n",
              "1     1143  0.798809  ReactTraining  ...  8466 2021-06-15 22:38:05       0\n",
              "2     2393  0.784845        alibaba  ...  1951 2021-06-17 09:38:33       0\n",
              "4    15532  0.743813          umijs  ...  1687 2021-06-25 06:53:08       0\n",
              "3     7977  0.718425      react-dnd  ...  1668 2021-06-17 17:20:10       0\n",
              "0      553  0.718241         pmndrs  ...   728 2021-06-19 12:17:17       0\n",
              "\n",
              "[5 rows x 10 columns]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "#@title Predict from repos id\n",
        "repos_id = [1, 5, 6, 8] #@param {type:\"raw\"}\n",
        "n_items = 5 #@param {type: \"integer\"}\n",
        "recommendations = cbm.predict_from_repos(repos_id, n_items)\n",
        "\n",
        "display(HTML(\"<h3>Repos previously starred</h3>\"))\n",
        "display(new_repos_df.filter(col(\"repo_id\").isin(repos_id)).toPandas())\n",
        "\n",
        "display(HTML(\"<br /><h3>Predicted repos</h3>\"))\n",
        "display(recommendations.join(new_repos_df, \"repo_id\").toPandas().sort_values(\"score\", ascending=False))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rS87VnDp3rth"
      },
      "source": [
        "### User-based CF"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 89,
      "metadata": {
        "id": "LElLOA9X2Hjo"
      },
      "outputs": [],
      "source": [
        "class UserBasedCF(RecommenderModel):\n",
        "\n",
        "  def predict_from_user(self, user_id, K, neighbours = 5):\n",
        "    # Compute the starred repos by each user\n",
        "    starred_repos_set = self.starred_repos.groupBy(\"user_id\").agg(collect_set(\"repo_id\").alias(\"starred_repos\"))\n",
        "\n",
        "    user_starred_repos = starred_repos_set.filter(col(\"user_id\") == user_id)\n",
        "    target_starred_repos = starred_repos_set.withColumnRenamed(\"starred_repos\", \"target_starred_repos\").withColumnRenamed(\"user_id\", \"target_user_id\")\n",
        "\n",
        "    # Prepare to compare the user rating vector with any other user\n",
        "    users_comparisons = user_starred_repos.join(target_starred_repos, col(\"user_id\") != col(\"target_user_id\"))\n",
        "\n",
        "    return self.predict(users_comparisons, K, neighbours)\n",
        "\n",
        "  def predict_from_repos(self, starred_repos_list, K, neighbours = 5):\n",
        "    # Create a Dataframe with the starred repos\n",
        "    user_starred_repos = spark.createDataFrame([(starred_repos_list, )], [\"starred_repos\"])\n",
        "\n",
        "    target_starred_repos = self.starred_repos.groupBy(\"user_id\").agg(collect_set(\"repo_id\").alias(\"target_starred_repos\"))\n",
        "    # Prepare to compare the user rating vector with any other user\n",
        "    users_comparisons = user_starred_repos.join(target_starred_repos)\n",
        "    return self.predict(users_comparisons, K, neighbours)\n",
        "\n",
        "  def predict(self, users_comparisons, K, neighbours = 5):\n",
        "    # Compute the Jaccard similarity among the repos starred by the user and those starred by any other user\n",
        "    jaccard = users_comparisons.withColumn(\"jaccard_similarity\", jaccard_similarity_udf(\"starred_repos\", \"target_starred_repos\"))\n",
        "\n",
        "    # Compute the `neighbours` most similar users\n",
        "    top_k_users = jaccard.sort(\"jaccard_similarity\", ascending=False).limit(neighbours)\n",
        "    \n",
        "    # Remove the repos already starred\n",
        "    possible_repos = top_k_users.select(\"*\", explode(\"target_starred_repos\").alias(\"repo_id\"))\n",
        "    repos_without_source_repos = possible_repos.filter(~array_contains(col(\"starred_repos\"), col(\"repo_id\")))\n",
        "\n",
        "    # Compute how many times each repo has been starred by other users\n",
        "    repos_occurrencies = repos_without_source_repos.groupBy(\"repo_id\").count()\n",
        "\n",
        "    # Compute the k most starred repos\n",
        "    recommendations = repos_occurrencies.sort(\"count\", ascending=False).limit(K)\n",
        "    # Normalize the count dividing by k to obtain a score\n",
        "    recommendations = recommendations.withColumn(\"score\", col(\"count\") / neighbours).drop(\"count\")\n",
        "\n",
        "    return recommendations\n",
        "\n",
        "ubm = UserBasedCF(new_users_df, new_repos_df, train)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 90,
      "metadata": {
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "FYzwrhoAf4Hj",
        "outputId": "372fdbc7-5e5f-49ae-b8a9-6bd8c8107bc9"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<h3>Repos previously starred by user 463</h3>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>repo_id</th>\n",
              "      <th>user_id</th>\n",
              "      <th>creator</th>\n",
              "      <th>name</th>\n",
              "      <th>about</th>\n",
              "      <th>language</th>\n",
              "      <th>stars</th>\n",
              "      <th>forks</th>\n",
              "      <th>updated</th>\n",
              "      <th>sponsor</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2</td>\n",
              "      <td>463</td>\n",
              "      <td>facebook</td>\n",
              "      <td>react</td>\n",
              "      <td>A declarative, efficient, and flexible JavaScr...</td>\n",
              "      <td>JavaScript</td>\n",
              "      <td>170264</td>\n",
              "      <td>34406</td>\n",
              "      <td>2021-06-22 06:08:58</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>169</td>\n",
              "      <td>463</td>\n",
              "      <td>date-fns</td>\n",
              "      <td>date-fns</td>\n",
              "      <td>⏳ Modern JavaScript date utility library ⌛️</td>\n",
              "      <td>JavaScript</td>\n",
              "      <td>26155</td>\n",
              "      <td>1260</td>\n",
              "      <td>2021-06-21 12:37:46</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>13629</td>\n",
              "      <td>463</td>\n",
              "      <td>square</td>\n",
              "      <td>okhttp</td>\n",
              "      <td>Square’s meticulous HTTP client for the JVM, A...</td>\n",
              "      <td>Java</td>\n",
              "      <td>40244</td>\n",
              "      <td>8538</td>\n",
              "      <td>2021-06-14 08:10:00</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>25974</td>\n",
              "      <td>463</td>\n",
              "      <td>square</td>\n",
              "      <td>retrofit</td>\n",
              "      <td>A type-safe HTTP client for Android and the JVM</td>\n",
              "      <td>Java</td>\n",
              "      <td>38265</td>\n",
              "      <td>6872</td>\n",
              "      <td>2021-06-24 19:20:04</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   repo_id  user_id   creator  ...  forks             updated sponsor\n",
              "0        2      463  facebook  ...  34406 2021-06-22 06:08:58       0\n",
              "1      169      463  date-fns  ...   1260 2021-06-21 12:37:46       0\n",
              "2    13629      463    square  ...   8538 2021-06-14 08:10:00       0\n",
              "3    25974      463    square  ...   6872 2021-06-24 19:20:04       0\n",
              "\n",
              "[4 rows x 10 columns]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<br /><h3>Predicted repos for user 463</h3>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>repo_id</th>\n",
              "      <th>score</th>\n",
              "      <th>creator</th>\n",
              "      <th>name</th>\n",
              "      <th>about</th>\n",
              "      <th>language</th>\n",
              "      <th>stars</th>\n",
              "      <th>forks</th>\n",
              "      <th>updated</th>\n",
              "      <th>sponsor</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>647</td>\n",
              "      <td>0.2</td>\n",
              "      <td>justjavac</td>\n",
              "      <td>free-programming-books-zh_CN</td>\n",
              "      <td>📚 免费的计算机编程类中文书籍，欢迎投稿</td>\n",
              "      <td>UNK</td>\n",
              "      <td>80728</td>\n",
              "      <td>23574</td>\n",
              "      <td>2021-05-25 00:52:10</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2042</td>\n",
              "      <td>0.2</td>\n",
              "      <td>iluwatar</td>\n",
              "      <td>java-design-patterns</td>\n",
              "      <td>Design patterns implemented in Java</td>\n",
              "      <td>Java</td>\n",
              "      <td>68497</td>\n",
              "      <td>21345</td>\n",
              "      <td>2021-06-22 05:03:56</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>4394</td>\n",
              "      <td>0.2</td>\n",
              "      <td>zxing</td>\n",
              "      <td>zxing</td>\n",
              "      <td>ZXing (\"Zebra Crossing\") barcode scanning libr...</td>\n",
              "      <td>Java</td>\n",
              "      <td>27934</td>\n",
              "      <td>9025</td>\n",
              "      <td>2021-06-07 12:11:10</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>5956</td>\n",
              "      <td>0.2</td>\n",
              "      <td>MrRio</td>\n",
              "      <td>jsPDF</td>\n",
              "      <td>Client-side JavaScript PDF generation for ever...</td>\n",
              "      <td>JavaScript</td>\n",
              "      <td>22230</td>\n",
              "      <td>4027</td>\n",
              "      <td>2021-06-18 08:42:15</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>21661</td>\n",
              "      <td>0.2</td>\n",
              "      <td>bilibili</td>\n",
              "      <td>ijkplayer</td>\n",
              "      <td>Android/iOS video player based on FFmpeg n3.4,...</td>\n",
              "      <td>C</td>\n",
              "      <td>29014</td>\n",
              "      <td>7524</td>\n",
              "      <td>2021-06-13 14:34:08</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   repo_id  score    creator  ...  forks             updated sponsor\n",
              "0      647    0.2  justjavac  ...  23574 2021-05-25 00:52:10       0\n",
              "1     2042    0.2   iluwatar  ...  21345 2021-06-22 05:03:56       1\n",
              "2     4394    0.2      zxing  ...   9025 2021-06-07 12:11:10       0\n",
              "3     5956    0.2      MrRio  ...   4027 2021-06-18 08:42:15       0\n",
              "4    21661    0.2   bilibili  ...   7524 2021-06-13 14:34:08       0\n",
              "\n",
              "[5 rows x 10 columns]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<br /><h3>Ground Truth for user 463</h3>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>repo_id</th>\n",
              "      <th>user_id</th>\n",
              "      <th>creator</th>\n",
              "      <th>name</th>\n",
              "      <th>about</th>\n",
              "      <th>language</th>\n",
              "      <th>stars</th>\n",
              "      <th>forks</th>\n",
              "      <th>updated</th>\n",
              "      <th>sponsor</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>120</td>\n",
              "      <td>463</td>\n",
              "      <td>nodejs</td>\n",
              "      <td>node</td>\n",
              "      <td>Node.js JavaScript runtime ✨🐢🚀✨</td>\n",
              "      <td>JavaScript</td>\n",
              "      <td>79974</td>\n",
              "      <td>20533</td>\n",
              "      <td>2021-06-22 08:41:01</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>204</td>\n",
              "      <td>463</td>\n",
              "      <td>django</td>\n",
              "      <td>django</td>\n",
              "      <td>The Web framework for perfectionists with dead...</td>\n",
              "      <td>Python</td>\n",
              "      <td>58081</td>\n",
              "      <td>24842</td>\n",
              "      <td>2021-06-22 08:28:44</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>961</td>\n",
              "      <td>463</td>\n",
              "      <td>bumptech</td>\n",
              "      <td>glide</td>\n",
              "      <td>An image loading and caching library for Andro...</td>\n",
              "      <td>Java</td>\n",
              "      <td>31342</td>\n",
              "      <td>5710</td>\n",
              "      <td>2021-06-11 23:50:48</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1279</td>\n",
              "      <td>463</td>\n",
              "      <td>ReactiveX</td>\n",
              "      <td>rxjs</td>\n",
              "      <td>A reactive programming library for JavaScript</td>\n",
              "      <td>TypeScript</td>\n",
              "      <td>25031</td>\n",
              "      <td>2575</td>\n",
              "      <td>2021-06-21 19:24:28</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1369</td>\n",
              "      <td>463</td>\n",
              "      <td>r-spacex</td>\n",
              "      <td>SpaceX-API</td>\n",
              "      <td>🚀 Open Source REST API for SpaceX launch, rock...</td>\n",
              "      <td>JavaScript</td>\n",
              "      <td>8548</td>\n",
              "      <td>686</td>\n",
              "      <td>2021-06-19 16:04:27</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>2045</td>\n",
              "      <td>463</td>\n",
              "      <td>DrKLO</td>\n",
              "      <td>Telegram</td>\n",
              "      <td>Telegram for Android source</td>\n",
              "      <td>C++</td>\n",
              "      <td>17143</td>\n",
              "      <td>5993</td>\n",
              "      <td>2021-06-03 05:06:31</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>8254</td>\n",
              "      <td>463</td>\n",
              "      <td>google</td>\n",
              "      <td>gson</td>\n",
              "      <td>A Java serialization/deserialization library t...</td>\n",
              "      <td>Java</td>\n",
              "      <td>19688</td>\n",
              "      <td>3818</td>\n",
              "      <td>2021-06-19 12:39:51</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>9684</td>\n",
              "      <td>463</td>\n",
              "      <td>jquense</td>\n",
              "      <td>yup</td>\n",
              "      <td>Dead simple Object schema validation</td>\n",
              "      <td>TypeScript</td>\n",
              "      <td>13746</td>\n",
              "      <td>616</td>\n",
              "      <td>2021-06-21 18:30:52</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   repo_id  user_id    creator  ...  forks             updated sponsor\n",
              "0      120      463     nodejs  ...  20533 2021-06-22 08:41:01       0\n",
              "1      204      463     django  ...  24842 2021-06-22 08:28:44       1\n",
              "2      961      463   bumptech  ...   5710 2021-06-11 23:50:48       0\n",
              "3     1279      463  ReactiveX  ...   2575 2021-06-21 19:24:28       0\n",
              "4     1369      463   r-spacex  ...    686 2021-06-19 16:04:27       0\n",
              "5     2045      463      DrKLO  ...   5993 2021-06-03 05:06:31       0\n",
              "6     8254      463     google  ...   3818 2021-06-19 12:39:51       0\n",
              "7     9684      463    jquense  ...    616 2021-06-21 18:30:52       0\n",
              "\n",
              "[8 rows x 10 columns]"
            ]
          },
          "execution_count": 90,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#@title Predict from user id\n",
        "user_id = 463 #@param {type: \"integer\"}\n",
        "n_items = 5 #@param {type: \"integer\"}\n",
        "neighbours = 5 #@param {type: \"integer\"}\n",
        "recommendations = ubm.predict_from_user(user_id, n_items, neighbours)\n",
        "\n",
        "display(HTML(\"<h3>Repos previously starred by user \" + str(user_id) + \"</h3>\"))\n",
        "display(train.filter(col(\"user_id\") == user_id) \\\n",
        "  .join(new_repos_df, \"repo_id\") \\\n",
        "  .toPandas())\n",
        "\n",
        "display(HTML(\"<br /><h3>Predicted repos for user \" + str(user_id) + \"</h3>\"))\n",
        "display(recommendations.join(new_repos_df, \"repo_id\").toPandas().sort_values(\"score\", ascending=False))\n",
        "\n",
        "display(HTML(\"<br /><h3>Ground Truth for user \" + str(user_id) + \"</h3>\"))\n",
        "test.filter(col(\"user_id\") == user_id) \\\n",
        "  .join(new_repos_df, \"repo_id\") \\\n",
        "  .toPandas()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 91,
      "metadata": {
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 450
        },
        "id": "FXClFRAUf4Hm",
        "outputId": "b65beb77-22ea-4e26-f5ef-236a86d519bf"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<h3>Repos previously starred</h3>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>repo_id</th>\n",
              "      <th>creator</th>\n",
              "      <th>name</th>\n",
              "      <th>about</th>\n",
              "      <th>language</th>\n",
              "      <th>stars</th>\n",
              "      <th>forks</th>\n",
              "      <th>updated</th>\n",
              "      <th>sponsor</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>vercel</td>\n",
              "      <td>next.js</td>\n",
              "      <td>The React Framework</td>\n",
              "      <td>JavaScript</td>\n",
              "      <td>69687</td>\n",
              "      <td>13111</td>\n",
              "      <td>2021-06-22 07:49:26</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   repo_id creator     name  ...  forks             updated  sponsor\n",
              "0        1  vercel  next.js  ...  13111 2021-06-22 07:49:26        0\n",
              "\n",
              "[1 rows x 9 columns]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<br /><h3>Predicted repos</h3>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>repo_id</th>\n",
              "      <th>score</th>\n",
              "      <th>creator</th>\n",
              "      <th>name</th>\n",
              "      <th>about</th>\n",
              "      <th>language</th>\n",
              "      <th>stars</th>\n",
              "      <th>forks</th>\n",
              "      <th>updated</th>\n",
              "      <th>sponsor</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2</td>\n",
              "      <td>1.0</td>\n",
              "      <td>facebook</td>\n",
              "      <td>react</td>\n",
              "      <td>A declarative, efficient, and flexible JavaScr...</td>\n",
              "      <td>JavaScript</td>\n",
              "      <td>170264</td>\n",
              "      <td>34406</td>\n",
              "      <td>2021-06-22 06:08:58</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>119</td>\n",
              "      <td>0.6</td>\n",
              "      <td>denoland</td>\n",
              "      <td>deno</td>\n",
              "      <td>A secure JavaScript and TypeScript runtime</td>\n",
              "      <td>Rust</td>\n",
              "      <td>76219</td>\n",
              "      <td>4010</td>\n",
              "      <td>2021-06-22 07:18:01</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>103</td>\n",
              "      <td>0.4</td>\n",
              "      <td>enaqx</td>\n",
              "      <td>awesome-react</td>\n",
              "      <td>A collection of awesome things regarding React...</td>\n",
              "      <td>UNK</td>\n",
              "      <td>44028</td>\n",
              "      <td>5314</td>\n",
              "      <td>2021-06-21 08:30:17</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>48</td>\n",
              "      <td>0.2</td>\n",
              "      <td>facebook</td>\n",
              "      <td>react-native</td>\n",
              "      <td>A framework for building native apps with React.</td>\n",
              "      <td>JavaScript</td>\n",
              "      <td>96207</td>\n",
              "      <td>20943</td>\n",
              "      <td>2021-06-22 06:00:50</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1320</td>\n",
              "      <td>0.2</td>\n",
              "      <td>Semantic-Org</td>\n",
              "      <td>Semantic-UI</td>\n",
              "      <td>Semantic is a UI component framework based aro...</td>\n",
              "      <td>JavaScript</td>\n",
              "      <td>49338</td>\n",
              "      <td>5128</td>\n",
              "      <td>2021-04-18 15:16:46</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   repo_id  score       creator  ...  forks             updated sponsor\n",
              "0        2    1.0      facebook  ...  34406 2021-06-22 06:08:58       0\n",
              "3      119    0.6      denoland  ...   4010 2021-06-22 07:18:01       1\n",
              "2      103    0.4         enaqx  ...   5314 2021-06-21 08:30:17       0\n",
              "1       48    0.2      facebook  ...  20943 2021-06-22 06:00:50       0\n",
              "4     1320    0.2  Semantic-Org  ...   5128 2021-04-18 15:16:46       0\n",
              "\n",
              "[5 rows x 10 columns]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "#@title Predict from repos id\n",
        "repos_id = [1, 5, 6, 8] #@param {type:\"raw\"}\n",
        "n_items = 5 #@param {type: \"integer\"}\n",
        "neighbours = 5 #@param {type: \"integer\"}\n",
        "recommendations = ubm.predict_from_repos(repos_id, n_items, neighbours)\n",
        "\n",
        "display(HTML(\"<h3>Repos previously starred</h3>\"))\n",
        "display(new_repos_df.filter(col(\"repo_id\").isin(repos_id)).toPandas())\n",
        "\n",
        "display(HTML(\"<br /><h3>Predicted repos</h3>\"))\n",
        "display(recommendations.join(new_repos_df, \"repo_id\").toPandas().sort_values(\"score\", ascending=False))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LoPbbP3M3yX6"
      },
      "source": [
        "### Item-based CF"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 92,
      "metadata": {
        "id": "NPtzX21r0nNJ"
      },
      "outputs": [],
      "source": [
        "from pyspark.sql.functions import broadcast\n",
        "\n",
        "class ItemBasedCF(RecommenderModel):\n",
        "\n",
        "  def __init__(self, users_df, repos_df, starred_df):\n",
        "    super().__init__(users_df, repos_df, starred_df)\n",
        "\n",
        "    # Compute the item vector (list of users that starred that repo) for each repo\n",
        "    self.item_vectors = self.starred_repos.groupBy(\"repo_id\").agg(collect_set(\"user_id\").alias(\"item_vector\"))\n",
        "\n",
        "  def predict_from_user(self, user_id, K, neighbours = 5):\n",
        "    # Add the item vectors info to the starred repos\n",
        "    user_items = self.starred_repos.filter(col(\"user_id\") == user_id).join(self.item_vectors, \"repo_id\")\n",
        "    \n",
        "    # Add the starred repos array to each record\n",
        "    user_starred_repos = self.starred_repos.filter(col(\"user_id\") == user_id).groupBy(\"user_id\").agg(collect_set(\"repo_id\").alias(\"starred_repos\"))\n",
        "    user_items = user_items.join(user_starred_repos, \"user_id\")\n",
        "    \n",
        "    # Remove all the items that the user has already starred\n",
        "    other_items = self.item_vectors.select(col(\"repo_id\").alias(\"no_starred_repo_id\"), col(\"item_vector\").alias(\"no_starred_item_vector\"))\n",
        "    user_item_item = user_items.join(other_items, ~array_contains(col(\"starred_repos\"), col(\"no_starred_repo_id\")))\n",
        "\n",
        "    # Compute the similarity of each pair of repos\n",
        "    jaccard = user_item_item.withColumn(\"jaccard_similarity\", jaccard_similarity_udf(\"item_vector\", \"no_starred_item_vector\")).select(\"repo_id\", \"no_starred_repo_id\", \"jaccard_similarity\")\n",
        "\n",
        "    # Select the top `neighbours` based on the similarity\n",
        "    window = Window.partitionBy(jaccard[\"no_starred_repo_id\"]).orderBy(jaccard[\"jaccard_similarity\"].desc())\n",
        "    top_k_repos = jaccard.select(\"*\", row_number().over(window).alias(\"rank\")).filter(col(\"rank\") <= neighbours)\n",
        "\n",
        "    # Aggregate the repos we found\n",
        "    repo_scores = top_k_repos.groupBy(\"no_starred_repo_id\").agg(mean(\"jaccard_similarity\").alias(\"score\"))\n",
        "\n",
        "    # Compute the top-k repos\n",
        "    recommendations = repo_scores.sort(\"score\", ascending=False).limit(K).withColumnRenamed(\"no_starred_repo_id\", \"repo_id\")\n",
        "\n",
        "    return recommendations\n",
        "\n",
        "  def predict_from_repos(self, starred_repos_list, K, neighbours = 5):\n",
        "    # Add the item vectors info to the starred repos\n",
        "    user_items = self.item_vectors.filter(col(\"repo_id\").isin(starred_repos_list))\n",
        "\n",
        "    # Add the starred repos array to each record\n",
        "    user_starred_repos = array([lit(repo_id) for repo_id in starred_repos_list])\n",
        "    user_items = user_items.withColumn(\"starred_repos\", user_starred_repos)\n",
        "\n",
        "    # Remove all the items that the user has already starred\n",
        "    other_items = self.item_vectors.select(col(\"repo_id\").alias(\"no_starred_repo_id\"), col(\"item_vector\").alias(\"no_starred_item_vector\"))\n",
        "    user_item_item = user_items.join(other_items, ~array_contains(col(\"starred_repos\"), col(\"no_starred_repo_id\")))\n",
        "\n",
        "    # Compute the similarity of each pair of repos\n",
        "    jaccard = user_item_item.withColumn(\"jaccard_similarity\", jaccard_similarity_udf(\"item_vector\", \"no_starred_item_vector\")).select(\"repo_id\", \"no_starred_repo_id\", \"jaccard_similarity\")\n",
        "\n",
        "    # Select the top `neighbours` based on the similarity\n",
        "    window = Window.partitionBy(jaccard[\"no_starred_repo_id\"]).orderBy(jaccard[\"jaccard_similarity\"].desc())\n",
        "    top_k_repos = jaccard.select(\"*\", row_number().over(window).alias(\"rank\")).filter(col(\"rank\") <= neighbours)\n",
        "\n",
        "    # Aggregate the repos we found\n",
        "    repo_scores = top_k_repos.groupBy(\"no_starred_repo_id\").agg(mean(\"jaccard_similarity\").alias(\"score\"))\n",
        "\n",
        "    # Compute the top-k repos\n",
        "    recommendations = repo_scores.sort(\"score\", ascending=False).limit(K).withColumnRenamed(\"no_starred_repo_id\", \"repo_id\")\n",
        "\n",
        "    return recommendations\n",
        "\n",
        "ibm = ItemBasedCF(new_users_df, new_repos_df, train)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 93,
      "metadata": {
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "CakS1b6mgQ-B",
        "outputId": "40f38279-e5e2-4143-bc7b-9663e081b815"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<h3>Repos previously starred by user 463</h3>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>repo_id</th>\n",
              "      <th>user_id</th>\n",
              "      <th>creator</th>\n",
              "      <th>name</th>\n",
              "      <th>about</th>\n",
              "      <th>language</th>\n",
              "      <th>stars</th>\n",
              "      <th>forks</th>\n",
              "      <th>updated</th>\n",
              "      <th>sponsor</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2</td>\n",
              "      <td>463</td>\n",
              "      <td>facebook</td>\n",
              "      <td>react</td>\n",
              "      <td>A declarative, efficient, and flexible JavaScr...</td>\n",
              "      <td>JavaScript</td>\n",
              "      <td>170264</td>\n",
              "      <td>34406</td>\n",
              "      <td>2021-06-22 06:08:58</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>169</td>\n",
              "      <td>463</td>\n",
              "      <td>date-fns</td>\n",
              "      <td>date-fns</td>\n",
              "      <td>⏳ Modern JavaScript date utility library ⌛️</td>\n",
              "      <td>JavaScript</td>\n",
              "      <td>26155</td>\n",
              "      <td>1260</td>\n",
              "      <td>2021-06-21 12:37:46</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>13629</td>\n",
              "      <td>463</td>\n",
              "      <td>square</td>\n",
              "      <td>okhttp</td>\n",
              "      <td>Square’s meticulous HTTP client for the JVM, A...</td>\n",
              "      <td>Java</td>\n",
              "      <td>40244</td>\n",
              "      <td>8538</td>\n",
              "      <td>2021-06-14 08:10:00</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>25974</td>\n",
              "      <td>463</td>\n",
              "      <td>square</td>\n",
              "      <td>retrofit</td>\n",
              "      <td>A type-safe HTTP client for Android and the JVM</td>\n",
              "      <td>Java</td>\n",
              "      <td>38265</td>\n",
              "      <td>6872</td>\n",
              "      <td>2021-06-24 19:20:04</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   repo_id  user_id   creator  ...  forks             updated sponsor\n",
              "0        2      463  facebook  ...  34406 2021-06-22 06:08:58       0\n",
              "1      169      463  date-fns  ...   1260 2021-06-21 12:37:46       0\n",
              "2    13629      463    square  ...   8538 2021-06-14 08:10:00       0\n",
              "3    25974      463    square  ...   6872 2021-06-24 19:20:04       0\n",
              "\n",
              "[4 rows x 10 columns]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<br /><h3>Predicted repos for user 463</h3>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>repo_id</th>\n",
              "      <th>score</th>\n",
              "      <th>creator</th>\n",
              "      <th>name</th>\n",
              "      <th>about</th>\n",
              "      <th>language</th>\n",
              "      <th>stars</th>\n",
              "      <th>forks</th>\n",
              "      <th>updated</th>\n",
              "      <th>sponsor</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>8687</td>\n",
              "      <td>0.142996</td>\n",
              "      <td>ReactiveX</td>\n",
              "      <td>RxJava</td>\n",
              "      <td>RxJava – Reactive Extensions for the JVM – a l...</td>\n",
              "      <td>Java</td>\n",
              "      <td>44791</td>\n",
              "      <td>7437</td>\n",
              "      <td>2021-06-22 06:09:01</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>35</td>\n",
              "      <td>0.142924</td>\n",
              "      <td>microsoft</td>\n",
              "      <td>vscode</td>\n",
              "      <td>Visual Studio Code</td>\n",
              "      <td>TypeScript</td>\n",
              "      <td>117529</td>\n",
              "      <td>19179</td>\n",
              "      <td>2021-06-22 08:42:46</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>352</td>\n",
              "      <td>0.134404</td>\n",
              "      <td>vuejs</td>\n",
              "      <td>vue</td>\n",
              "      <td>🖖 Vue.js is a progressive, incrementally-adopt...</td>\n",
              "      <td>JavaScript</td>\n",
              "      <td>184750</td>\n",
              "      <td>29397</td>\n",
              "      <td>2021-06-20 19:50:52</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>238</td>\n",
              "      <td>0.124381</td>\n",
              "      <td>flutter</td>\n",
              "      <td>flutter</td>\n",
              "      <td>Flutter makes it easy and fast to build beauti...</td>\n",
              "      <td>Dart</td>\n",
              "      <td>122655</td>\n",
              "      <td>17619</td>\n",
              "      <td>2021-06-22 08:20:39</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>11342</td>\n",
              "      <td>0.123942</td>\n",
              "      <td>google</td>\n",
              "      <td>guava</td>\n",
              "      <td>Google core libraries for Java</td>\n",
              "      <td>Java</td>\n",
              "      <td>41479</td>\n",
              "      <td>9236</td>\n",
              "      <td>2021-06-21 15:53:07</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   repo_id     score    creator  ...  forks             updated sponsor\n",
              "3     8687  0.142996  ReactiveX  ...   7437 2021-06-22 06:09:01       0\n",
              "0       35  0.142924  microsoft  ...  19179 2021-06-22 08:42:46       0\n",
              "2      352  0.134404      vuejs  ...  29397 2021-06-20 19:50:52       0\n",
              "1      238  0.124381    flutter  ...  17619 2021-06-22 08:20:39       0\n",
              "4    11342  0.123942     google  ...   9236 2021-06-21 15:53:07       0\n",
              "\n",
              "[5 rows x 10 columns]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<br /><h3>Ground Truth for user 463</h3>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>repo_id</th>\n",
              "      <th>user_id</th>\n",
              "      <th>creator</th>\n",
              "      <th>name</th>\n",
              "      <th>about</th>\n",
              "      <th>language</th>\n",
              "      <th>stars</th>\n",
              "      <th>forks</th>\n",
              "      <th>updated</th>\n",
              "      <th>sponsor</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>120</td>\n",
              "      <td>463</td>\n",
              "      <td>nodejs</td>\n",
              "      <td>node</td>\n",
              "      <td>Node.js JavaScript runtime ✨🐢🚀✨</td>\n",
              "      <td>JavaScript</td>\n",
              "      <td>79974</td>\n",
              "      <td>20533</td>\n",
              "      <td>2021-06-22 08:41:01</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>204</td>\n",
              "      <td>463</td>\n",
              "      <td>django</td>\n",
              "      <td>django</td>\n",
              "      <td>The Web framework for perfectionists with dead...</td>\n",
              "      <td>Python</td>\n",
              "      <td>58081</td>\n",
              "      <td>24842</td>\n",
              "      <td>2021-06-22 08:28:44</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>961</td>\n",
              "      <td>463</td>\n",
              "      <td>bumptech</td>\n",
              "      <td>glide</td>\n",
              "      <td>An image loading and caching library for Andro...</td>\n",
              "      <td>Java</td>\n",
              "      <td>31342</td>\n",
              "      <td>5710</td>\n",
              "      <td>2021-06-11 23:50:48</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1279</td>\n",
              "      <td>463</td>\n",
              "      <td>ReactiveX</td>\n",
              "      <td>rxjs</td>\n",
              "      <td>A reactive programming library for JavaScript</td>\n",
              "      <td>TypeScript</td>\n",
              "      <td>25031</td>\n",
              "      <td>2575</td>\n",
              "      <td>2021-06-21 19:24:28</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1369</td>\n",
              "      <td>463</td>\n",
              "      <td>r-spacex</td>\n",
              "      <td>SpaceX-API</td>\n",
              "      <td>🚀 Open Source REST API for SpaceX launch, rock...</td>\n",
              "      <td>JavaScript</td>\n",
              "      <td>8548</td>\n",
              "      <td>686</td>\n",
              "      <td>2021-06-19 16:04:27</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>2045</td>\n",
              "      <td>463</td>\n",
              "      <td>DrKLO</td>\n",
              "      <td>Telegram</td>\n",
              "      <td>Telegram for Android source</td>\n",
              "      <td>C++</td>\n",
              "      <td>17143</td>\n",
              "      <td>5993</td>\n",
              "      <td>2021-06-03 05:06:31</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>8254</td>\n",
              "      <td>463</td>\n",
              "      <td>google</td>\n",
              "      <td>gson</td>\n",
              "      <td>A Java serialization/deserialization library t...</td>\n",
              "      <td>Java</td>\n",
              "      <td>19688</td>\n",
              "      <td>3818</td>\n",
              "      <td>2021-06-19 12:39:51</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>9684</td>\n",
              "      <td>463</td>\n",
              "      <td>jquense</td>\n",
              "      <td>yup</td>\n",
              "      <td>Dead simple Object schema validation</td>\n",
              "      <td>TypeScript</td>\n",
              "      <td>13746</td>\n",
              "      <td>616</td>\n",
              "      <td>2021-06-21 18:30:52</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   repo_id  user_id    creator  ...  forks             updated sponsor\n",
              "0      120      463     nodejs  ...  20533 2021-06-22 08:41:01       0\n",
              "1      204      463     django  ...  24842 2021-06-22 08:28:44       1\n",
              "2      961      463   bumptech  ...   5710 2021-06-11 23:50:48       0\n",
              "3     1279      463  ReactiveX  ...   2575 2021-06-21 19:24:28       0\n",
              "4     1369      463   r-spacex  ...    686 2021-06-19 16:04:27       0\n",
              "5     2045      463      DrKLO  ...   5993 2021-06-03 05:06:31       0\n",
              "6     8254      463     google  ...   3818 2021-06-19 12:39:51       0\n",
              "7     9684      463    jquense  ...    616 2021-06-21 18:30:52       0\n",
              "\n",
              "[8 rows x 10 columns]"
            ]
          },
          "execution_count": 93,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#@title Predict from user id\n",
        "user_id = 463 #@param {type: \"integer\"}\n",
        "n_items = 5 #@param {type: \"integer\"}\n",
        "neighbours = 5 #@param {type: \"integer\"}\n",
        "recommendations = ibm.predict_from_user(user_id, n_items, neighbours)\n",
        "\n",
        "display(HTML(\"<h3>Repos previously starred by user \" + str(user_id) + \"</h3>\"))\n",
        "display(train.filter(col(\"user_id\") == user_id) \\\n",
        "  .join(new_repos_df, \"repo_id\") \\\n",
        "  .toPandas())\n",
        "\n",
        "display(HTML(\"<br /><h3>Predicted repos for user \" + str(user_id) + \"</h3>\"))\n",
        "display(recommendations.join(new_repos_df, \"repo_id\").toPandas().sort_values(\"score\", ascending=False))\n",
        "\n",
        "display(HTML(\"<br /><h3>Ground Truth for user \" + str(user_id) + \"</h3>\"))\n",
        "test.filter(col(\"user_id\") == user_id) \\\n",
        "  .join(new_repos_df, \"repo_id\") \\\n",
        "  .toPandas()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 94,
      "metadata": {
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 450
        },
        "id": "XHYDiMvJgQ-D",
        "outputId": "27d40d9d-17b1-4adc-870d-5cb2b97603c9"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<h3>Repos previously starred</h3>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>repo_id</th>\n",
              "      <th>creator</th>\n",
              "      <th>name</th>\n",
              "      <th>about</th>\n",
              "      <th>language</th>\n",
              "      <th>stars</th>\n",
              "      <th>forks</th>\n",
              "      <th>updated</th>\n",
              "      <th>sponsor</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>vercel</td>\n",
              "      <td>next.js</td>\n",
              "      <td>The React Framework</td>\n",
              "      <td>JavaScript</td>\n",
              "      <td>69687</td>\n",
              "      <td>13111</td>\n",
              "      <td>2021-06-22 07:49:26</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   repo_id creator     name  ...  forks             updated  sponsor\n",
              "0        1  vercel  next.js  ...  13111 2021-06-22 07:49:26        0\n",
              "\n",
              "[1 rows x 9 columns]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<br /><h3>Predicted repos</h3>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>repo_id</th>\n",
              "      <th>score</th>\n",
              "      <th>creator</th>\n",
              "      <th>name</th>\n",
              "      <th>about</th>\n",
              "      <th>language</th>\n",
              "      <th>stars</th>\n",
              "      <th>forks</th>\n",
              "      <th>updated</th>\n",
              "      <th>sponsor</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2</td>\n",
              "      <td>0.289873</td>\n",
              "      <td>facebook</td>\n",
              "      <td>react</td>\n",
              "      <td>A declarative, efficient, and flexible JavaScr...</td>\n",
              "      <td>JavaScript</td>\n",
              "      <td>170264</td>\n",
              "      <td>34406</td>\n",
              "      <td>2021-06-22 06:08:58</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>119</td>\n",
              "      <td>0.260128</td>\n",
              "      <td>denoland</td>\n",
              "      <td>deno</td>\n",
              "      <td>A secure JavaScript and TypeScript runtime</td>\n",
              "      <td>Rust</td>\n",
              "      <td>76219</td>\n",
              "      <td>4010</td>\n",
              "      <td>2021-06-22 07:18:01</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>833</td>\n",
              "      <td>0.257757</td>\n",
              "      <td>mui-org</td>\n",
              "      <td>material-ui</td>\n",
              "      <td>Material-UI is a simple and customizable compo...</td>\n",
              "      <td>JavaScript</td>\n",
              "      <td>68945</td>\n",
              "      <td>22541</td>\n",
              "      <td>2021-06-22 07:35:38</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>35</td>\n",
              "      <td>0.251386</td>\n",
              "      <td>microsoft</td>\n",
              "      <td>vscode</td>\n",
              "      <td>Visual Studio Code</td>\n",
              "      <td>TypeScript</td>\n",
              "      <td>117529</td>\n",
              "      <td>19179</td>\n",
              "      <td>2021-06-22 08:42:46</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>53</td>\n",
              "      <td>0.249370</td>\n",
              "      <td>sveltejs</td>\n",
              "      <td>svelte</td>\n",
              "      <td>Cybernetically enhanced web apps</td>\n",
              "      <td>TypeScript</td>\n",
              "      <td>47896</td>\n",
              "      <td>2264</td>\n",
              "      <td>2021-06-22 07:26:38</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   repo_id     score    creator  ...  forks             updated sponsor\n",
              "0        2  0.289873   facebook  ...  34406 2021-06-22 06:08:58       0\n",
              "3      119  0.260128   denoland  ...   4010 2021-06-22 07:18:01       1\n",
              "4      833  0.257757    mui-org  ...  22541 2021-06-22 07:35:38       0\n",
              "1       35  0.251386  microsoft  ...  19179 2021-06-22 08:42:46       0\n",
              "2       53  0.249370   sveltejs  ...   2264 2021-06-22 07:26:38       0\n",
              "\n",
              "[5 rows x 10 columns]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "#@title Predict from repos id\n",
        "repos_id = [1, 5, 6, 8] #@param {type:\"raw\"}\n",
        "n_items = 5 #@param {type: \"integer\"}\n",
        "neighbours = 5 #@param {type: \"integer\"}\n",
        "recommendations = ibm.predict_from_repos(repos_id, n_items, neighbours)\n",
        "\n",
        "display(HTML(\"<h3>Repos previously starred</h3>\"))\n",
        "display(new_repos_df.filter(col(\"repo_id\").isin(repos_id)).toPandas())\n",
        "\n",
        "display(HTML(\"<br /><h3>Predicted repos</h3>\"))\n",
        "display(recommendations.join(new_repos_df, \"repo_id\").toPandas().sort_values(\"score\", ascending=False))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mAyXaHdQ30k5"
      },
      "source": [
        "### Matrix Factorization"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 95,
      "metadata": {
        "id": "UcEnANqOHp9F"
      },
      "outputs": [],
      "source": [
        "from pyspark.ml.recommendation import ALS, ALSModel\n",
        "\n",
        "class MatrixFactorization(RecommenderModel):\n",
        "\n",
        "  def __init__(self, users_df, repos_df, starred_repos_df):\n",
        "    super().__init__(users_df, repos_df, starred_repos_df)\n",
        "    # Load the weights previously saved in the notebook\n",
        "    self.model = ALSModel.load(\"als_model\")\n",
        "\n",
        "    # Take the repos in the train set not starred by the user\n",
        "    non_starred_train_df = new_repos_df.select(\"repo_id\").join(new_users_df.select(\"user_id\")).exceptAll(train)\n",
        "    # Build the training ratig matrix adding 1s for starred repos and 0s for non starred ones\n",
        "    self.train_matrix = train.withColumn(\"starred\", lit(1)).union(non_starred_train_df.withColumn(\"starred\", lit(0)))\n",
        "\n",
        "  def predict_from_user(self, user_id, K):\n",
        "    user_starred_repos = self.starred_repos.filter(col(\"user_id\") == user_id).select(\"repo_id\")\n",
        "    # Compute non starred repos by the user\n",
        "    non_starred_df = self.starred_repos.select(\"repo_id\").distinct().exceptAll(user_starred_repos).withColumn(\"user_id\", lit(user_id))\n",
        "\n",
        "    return self.predict_non_starred_repos(self.model, non_starred_df, K)\n",
        "\n",
        "  def predict_from_repos(self, starred_repos_list, K):\n",
        "    # Create a new user with a dummy id and add it to the rating matrix\n",
        "    user_id = 999999\n",
        "    user_starred_repos_df = spark.createDataFrame([(repo_id, user_id) for repo_id in starred_repos_list], [\"repo_id\", \"user_id\"])\n",
        "    non_starred_df = self.starred_repos.select(\"repo_id\").distinct().exceptAll(user_starred_repos_df.select(\"repo_id\")).withColumn(\"user_id\", lit(user_id))\n",
        "    new_user_ratings = user_starred_repos_df.withColumn(\"rating\", lit(1)).union(non_starred_df.withColumn(\"rating\", lit(0)))\n",
        "    new_train_matrix = self.train_matrix.union(new_user_ratings)\n",
        "\n",
        "    # Train the model again with the new user\n",
        "    als_params = {\n",
        "      \"rank\": 5,\n",
        "      \"maxIter\": 20, \n",
        "      \"userCol\": \"user_id\", \n",
        "      \"itemCol\": \"repo_id\",\n",
        "      \"ratingCol\": \"starred\", \n",
        "      \"coldStartStrategy\": \"drop\",\n",
        "      \"seed\": 42,\n",
        "      \"regParam\": 0.1,\n",
        "      \"implicitPrefs\": True,\n",
        "      \"nonnegative\": False,\n",
        "      \"alpha\": 1\n",
        "    }\n",
        "    temp_als = ALS(**als_params)\n",
        "    temp_model = temp_als.fit(new_train_matrix)\n",
        "\n",
        "    # Compute the non starred repos by the user\n",
        "    non_starred_df = self.starred_repos.select(\"repo_id\").distinct().filter(~col(\"repo_id\").isin(starred_repos_list)).withColumn(\"user_id\", lit(user_id))\n",
        "\n",
        "    return self.predict_non_starred_repos(temp_model, non_starred_df, K)\n",
        "  \n",
        "  def predict_non_starred_repos(self, model, non_starred_df, K):\n",
        "    # Compute a score for each non starred repo\n",
        "    predict = model.transform(non_starred_df)\n",
        "    # Take the top-k scoring repos\n",
        "    top_items = predict.sort(\"prediction\", ascending=False).limit(K)\n",
        "    top_items = top_items.withColumnRenamed(\"prediction\", \"score\").select(\"repo_id\", \"score\")\n",
        "    return top_items\n",
        "\n",
        "alsm = MatrixFactorization(new_users_df, new_repos_df, train)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 96,
      "metadata": {
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "CDA4nXF2hOeo",
        "outputId": "5bac4703-656a-4ce1-a11d-46c61376061a"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<h3>Repos previously starred by user 463</h3>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>repo_id</th>\n",
              "      <th>user_id</th>\n",
              "      <th>creator</th>\n",
              "      <th>name</th>\n",
              "      <th>about</th>\n",
              "      <th>language</th>\n",
              "      <th>stars</th>\n",
              "      <th>forks</th>\n",
              "      <th>updated</th>\n",
              "      <th>sponsor</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2</td>\n",
              "      <td>463</td>\n",
              "      <td>facebook</td>\n",
              "      <td>react</td>\n",
              "      <td>A declarative, efficient, and flexible JavaScr...</td>\n",
              "      <td>JavaScript</td>\n",
              "      <td>170264</td>\n",
              "      <td>34406</td>\n",
              "      <td>2021-06-22 06:08:58</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>169</td>\n",
              "      <td>463</td>\n",
              "      <td>date-fns</td>\n",
              "      <td>date-fns</td>\n",
              "      <td>⏳ Modern JavaScript date utility library ⌛️</td>\n",
              "      <td>JavaScript</td>\n",
              "      <td>26155</td>\n",
              "      <td>1260</td>\n",
              "      <td>2021-06-21 12:37:46</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>13629</td>\n",
              "      <td>463</td>\n",
              "      <td>square</td>\n",
              "      <td>okhttp</td>\n",
              "      <td>Square’s meticulous HTTP client for the JVM, A...</td>\n",
              "      <td>Java</td>\n",
              "      <td>40244</td>\n",
              "      <td>8538</td>\n",
              "      <td>2021-06-14 08:10:00</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>25974</td>\n",
              "      <td>463</td>\n",
              "      <td>square</td>\n",
              "      <td>retrofit</td>\n",
              "      <td>A type-safe HTTP client for Android and the JVM</td>\n",
              "      <td>Java</td>\n",
              "      <td>38265</td>\n",
              "      <td>6872</td>\n",
              "      <td>2021-06-24 19:20:04</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   repo_id  user_id   creator  ...  forks             updated sponsor\n",
              "0        2      463  facebook  ...  34406 2021-06-22 06:08:58       0\n",
              "1      169      463  date-fns  ...   1260 2021-06-21 12:37:46       0\n",
              "2    13629      463    square  ...   8538 2021-06-14 08:10:00       0\n",
              "3    25974      463    square  ...   6872 2021-06-24 19:20:04       0\n",
              "\n",
              "[4 rows x 10 columns]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<br /><h3>Predicted repos for user 463</h3>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>repo_id</th>\n",
              "      <th>score</th>\n",
              "      <th>creator</th>\n",
              "      <th>name</th>\n",
              "      <th>about</th>\n",
              "      <th>language</th>\n",
              "      <th>stars</th>\n",
              "      <th>forks</th>\n",
              "      <th>updated</th>\n",
              "      <th>sponsor</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>35</td>\n",
              "      <td>0.088463</td>\n",
              "      <td>microsoft</td>\n",
              "      <td>vscode</td>\n",
              "      <td>Visual Studio Code</td>\n",
              "      <td>TypeScript</td>\n",
              "      <td>117529</td>\n",
              "      <td>19179</td>\n",
              "      <td>2021-06-22 08:42:46</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>352</td>\n",
              "      <td>0.088030</td>\n",
              "      <td>vuejs</td>\n",
              "      <td>vue</td>\n",
              "      <td>🖖 Vue.js is a progressive, incrementally-adopt...</td>\n",
              "      <td>JavaScript</td>\n",
              "      <td>184750</td>\n",
              "      <td>29397</td>\n",
              "      <td>2021-06-20 19:50:52</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>120</td>\n",
              "      <td>0.073240</td>\n",
              "      <td>nodejs</td>\n",
              "      <td>node</td>\n",
              "      <td>Node.js JavaScript runtime ✨🐢🚀✨</td>\n",
              "      <td>JavaScript</td>\n",
              "      <td>79974</td>\n",
              "      <td>20533</td>\n",
              "      <td>2021-06-22 08:41:01</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>398</td>\n",
              "      <td>0.072026</td>\n",
              "      <td>microsoft</td>\n",
              "      <td>TypeScript</td>\n",
              "      <td>TypeScript is a superset of JavaScript that co...</td>\n",
              "      <td>TypeScript</td>\n",
              "      <td>72208</td>\n",
              "      <td>9438</td>\n",
              "      <td>2021-06-22 07:25:10</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>615</td>\n",
              "      <td>0.071224</td>\n",
              "      <td>electron</td>\n",
              "      <td>electron</td>\n",
              "      <td>Build cross-platform desktop apps with JavaScr...</td>\n",
              "      <td>C++</td>\n",
              "      <td>94042</td>\n",
              "      <td>12446</td>\n",
              "      <td>2021-06-22 08:39:16</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   repo_id     score    creator  ...  forks             updated sponsor\n",
              "0       35  0.088463  microsoft  ...  19179 2021-06-22 08:42:46       0\n",
              "2      352  0.088030      vuejs  ...  29397 2021-06-20 19:50:52       0\n",
              "1      120  0.073240     nodejs  ...  20533 2021-06-22 08:41:01       0\n",
              "3      398  0.072026  microsoft  ...   9438 2021-06-22 07:25:10       0\n",
              "4      615  0.071224   electron  ...  12446 2021-06-22 08:39:16       0\n",
              "\n",
              "[5 rows x 10 columns]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<br /><h3>Ground Truth for user 463</h3>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>repo_id</th>\n",
              "      <th>user_id</th>\n",
              "      <th>creator</th>\n",
              "      <th>name</th>\n",
              "      <th>about</th>\n",
              "      <th>language</th>\n",
              "      <th>stars</th>\n",
              "      <th>forks</th>\n",
              "      <th>updated</th>\n",
              "      <th>sponsor</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>120</td>\n",
              "      <td>463</td>\n",
              "      <td>nodejs</td>\n",
              "      <td>node</td>\n",
              "      <td>Node.js JavaScript runtime ✨🐢🚀✨</td>\n",
              "      <td>JavaScript</td>\n",
              "      <td>79974</td>\n",
              "      <td>20533</td>\n",
              "      <td>2021-06-22 08:41:01</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>204</td>\n",
              "      <td>463</td>\n",
              "      <td>django</td>\n",
              "      <td>django</td>\n",
              "      <td>The Web framework for perfectionists with dead...</td>\n",
              "      <td>Python</td>\n",
              "      <td>58081</td>\n",
              "      <td>24842</td>\n",
              "      <td>2021-06-22 08:28:44</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>961</td>\n",
              "      <td>463</td>\n",
              "      <td>bumptech</td>\n",
              "      <td>glide</td>\n",
              "      <td>An image loading and caching library for Andro...</td>\n",
              "      <td>Java</td>\n",
              "      <td>31342</td>\n",
              "      <td>5710</td>\n",
              "      <td>2021-06-11 23:50:48</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1279</td>\n",
              "      <td>463</td>\n",
              "      <td>ReactiveX</td>\n",
              "      <td>rxjs</td>\n",
              "      <td>A reactive programming library for JavaScript</td>\n",
              "      <td>TypeScript</td>\n",
              "      <td>25031</td>\n",
              "      <td>2575</td>\n",
              "      <td>2021-06-21 19:24:28</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1369</td>\n",
              "      <td>463</td>\n",
              "      <td>r-spacex</td>\n",
              "      <td>SpaceX-API</td>\n",
              "      <td>🚀 Open Source REST API for SpaceX launch, rock...</td>\n",
              "      <td>JavaScript</td>\n",
              "      <td>8548</td>\n",
              "      <td>686</td>\n",
              "      <td>2021-06-19 16:04:27</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>2045</td>\n",
              "      <td>463</td>\n",
              "      <td>DrKLO</td>\n",
              "      <td>Telegram</td>\n",
              "      <td>Telegram for Android source</td>\n",
              "      <td>C++</td>\n",
              "      <td>17143</td>\n",
              "      <td>5993</td>\n",
              "      <td>2021-06-03 05:06:31</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>8254</td>\n",
              "      <td>463</td>\n",
              "      <td>google</td>\n",
              "      <td>gson</td>\n",
              "      <td>A Java serialization/deserialization library t...</td>\n",
              "      <td>Java</td>\n",
              "      <td>19688</td>\n",
              "      <td>3818</td>\n",
              "      <td>2021-06-19 12:39:51</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>9684</td>\n",
              "      <td>463</td>\n",
              "      <td>jquense</td>\n",
              "      <td>yup</td>\n",
              "      <td>Dead simple Object schema validation</td>\n",
              "      <td>TypeScript</td>\n",
              "      <td>13746</td>\n",
              "      <td>616</td>\n",
              "      <td>2021-06-21 18:30:52</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   repo_id  user_id    creator  ...  forks             updated sponsor\n",
              "0      120      463     nodejs  ...  20533 2021-06-22 08:41:01       0\n",
              "1      204      463     django  ...  24842 2021-06-22 08:28:44       1\n",
              "2      961      463   bumptech  ...   5710 2021-06-11 23:50:48       0\n",
              "3     1279      463  ReactiveX  ...   2575 2021-06-21 19:24:28       0\n",
              "4     1369      463   r-spacex  ...    686 2021-06-19 16:04:27       0\n",
              "5     2045      463      DrKLO  ...   5993 2021-06-03 05:06:31       0\n",
              "6     8254      463     google  ...   3818 2021-06-19 12:39:51       0\n",
              "7     9684      463    jquense  ...    616 2021-06-21 18:30:52       0\n",
              "\n",
              "[8 rows x 10 columns]"
            ]
          },
          "execution_count": 96,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#@title Predict from user id\n",
        "user_id = 463 #@param {type: \"integer\"}\n",
        "n_items = 5 #@param {type: \"integer\"}\n",
        "recommendations = alsm.predict_from_user(user_id, n_items)\n",
        "\n",
        "display(HTML(\"<h3>Repos previously starred by user \" + str(user_id) + \"</h3>\"))\n",
        "display(train.filter(col(\"user_id\") == user_id) \\\n",
        "  .join(new_repos_df, \"repo_id\") \\\n",
        "  .toPandas())\n",
        "\n",
        "display(HTML(\"<br /><h3>Predicted repos for user \" + str(user_id) + \"</h3>\"))\n",
        "display(recommendations.join(new_repos_df, \"repo_id\").toPandas().sort_values(\"score\", ascending=False))\n",
        "\n",
        "display(HTML(\"<br /><h3>Ground Truth for user \" + str(user_id) + \"</h3>\"))\n",
        "test.filter(col(\"user_id\") == user_id) \\\n",
        "  .join(new_repos_df, \"repo_id\") \\\n",
        "  .toPandas()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 97,
      "metadata": {
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 450
        },
        "id": "8HsVOgsphOep",
        "outputId": "e9bb8410-8728-44d3-aebb-ca760c6b120b"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<h3>Repos previously starred</h3>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>repo_id</th>\n",
              "      <th>creator</th>\n",
              "      <th>name</th>\n",
              "      <th>about</th>\n",
              "      <th>language</th>\n",
              "      <th>stars</th>\n",
              "      <th>forks</th>\n",
              "      <th>updated</th>\n",
              "      <th>sponsor</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>vercel</td>\n",
              "      <td>next.js</td>\n",
              "      <td>The React Framework</td>\n",
              "      <td>JavaScript</td>\n",
              "      <td>69687</td>\n",
              "      <td>13111</td>\n",
              "      <td>2021-06-22 07:49:26</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   repo_id creator     name  ...  forks             updated  sponsor\n",
              "0        1  vercel  next.js  ...  13111 2021-06-22 07:49:26        0\n",
              "\n",
              "[1 rows x 9 columns]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<br /><h3>Predicted repos</h3>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>repo_id</th>\n",
              "      <th>score</th>\n",
              "      <th>creator</th>\n",
              "      <th>name</th>\n",
              "      <th>about</th>\n",
              "      <th>language</th>\n",
              "      <th>stars</th>\n",
              "      <th>forks</th>\n",
              "      <th>updated</th>\n",
              "      <th>sponsor</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2</td>\n",
              "      <td>0.043444</td>\n",
              "      <td>facebook</td>\n",
              "      <td>react</td>\n",
              "      <td>A declarative, efficient, and flexible JavaScr...</td>\n",
              "      <td>JavaScript</td>\n",
              "      <td>170264</td>\n",
              "      <td>34406</td>\n",
              "      <td>2021-06-22 06:08:58</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>35</td>\n",
              "      <td>0.037504</td>\n",
              "      <td>microsoft</td>\n",
              "      <td>vscode</td>\n",
              "      <td>Visual Studio Code</td>\n",
              "      <td>TypeScript</td>\n",
              "      <td>117529</td>\n",
              "      <td>19179</td>\n",
              "      <td>2021-06-22 08:42:46</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>398</td>\n",
              "      <td>0.032068</td>\n",
              "      <td>microsoft</td>\n",
              "      <td>TypeScript</td>\n",
              "      <td>TypeScript is a superset of JavaScript that co...</td>\n",
              "      <td>TypeScript</td>\n",
              "      <td>72208</td>\n",
              "      <td>9438</td>\n",
              "      <td>2021-06-22 07:25:10</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>120</td>\n",
              "      <td>0.031155</td>\n",
              "      <td>nodejs</td>\n",
              "      <td>node</td>\n",
              "      <td>Node.js JavaScript runtime ✨🐢🚀✨</td>\n",
              "      <td>JavaScript</td>\n",
              "      <td>79974</td>\n",
              "      <td>20533</td>\n",
              "      <td>2021-06-22 08:41:01</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>119</td>\n",
              "      <td>0.030806</td>\n",
              "      <td>denoland</td>\n",
              "      <td>deno</td>\n",
              "      <td>A secure JavaScript and TypeScript runtime</td>\n",
              "      <td>Rust</td>\n",
              "      <td>76219</td>\n",
              "      <td>4010</td>\n",
              "      <td>2021-06-22 07:18:01</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   repo_id     score    creator  ...  forks             updated sponsor\n",
              "0        2  0.043444   facebook  ...  34406 2021-06-22 06:08:58       0\n",
              "1       35  0.037504  microsoft  ...  19179 2021-06-22 08:42:46       0\n",
              "4      398  0.032068  microsoft  ...   9438 2021-06-22 07:25:10       0\n",
              "3      120  0.031155     nodejs  ...  20533 2021-06-22 08:41:01       0\n",
              "2      119  0.030806   denoland  ...   4010 2021-06-22 07:18:01       1\n",
              "\n",
              "[5 rows x 10 columns]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "#@title Predict from repos id\n",
        "repos_id = [1, 5, 6, 8] #@param {type:\"raw\"}\n",
        "n_items = 5 #@param {type: \"integer\"}\n",
        "recommendations = alsm.predict_from_repos(repos_id, n_items)\n",
        "\n",
        "display(HTML(\"<h3>Repos previously starred</h3>\"))\n",
        "display(new_repos_df.filter(col(\"repo_id\").isin(repos_id)).toPandas())\n",
        "\n",
        "display(HTML(\"<br /><h3>Predicted repos</h3>\"))\n",
        "display(recommendations.join(new_repos_df, \"repo_id\").toPandas().sort_values(\"score\", ascending=False))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ClwKUUpHz-PE"
      },
      "source": [
        "## WebServer"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oRnOEqUfCfkX"
      },
      "source": [
        "In the following blocks we setup a web server offering a simple REST HTTP API using Flask, which allows to interact with the models.\n",
        "\n",
        "We also hosted a simple but nice-looking front end [here](https://recommend-hub.netlify.app/).\n",
        "\n",
        "In order to make it work, we used *ngrok* which allows to access to services hosted on Colab without the need of port forwarding.\n",
        "For this reason, it is necessary to perform the following steps:\n",
        "- Copy the link ending with _ngrok.io_ from the execution of the last cell \n",
        "- Paste the link into the popup shown when loading the first front end page\n",
        "- Add an 's' to the protocol of the link obtaining something like _http**s**://xxxxxx.ngrok.io_\n",
        "\n",
        "The user interface works in the following way:\n",
        "- It is possible to generate predictions by selecting some **repos** which will be considered as starred by a new user\n",
        "  - Repos can be filtered by using a repo id, by partial strings for the creator, name and description and/or by main programming language\n",
        "  - **Note:** for the Matrix Factorization model the prediction could take a while, because we need to include the new user in the system and train again the model\n",
        "- It is possible to generate predictions starting by an already existing **user** in the dataset\n",
        "  - Users can be filtered by user id or by partial names\n",
        "- Once selected the source for the recommendations, using the \"Settings\" panel on the right column of the interface, it is required to select the model which is going to be used to generate the results and how many repos to recommend \n",
        "- Press the \"Predict\" button and wait for the magic to happen!\n",
        "- After a short time, the results page is going to be displayed and:\n",
        "  - If the source is a list of repos, it's going to contain:\n",
        "    - The list of repos selected in the previous page\n",
        "    - The list of repos recommended by the model\n",
        "  - If instead the source is a user, it's going to contain:\n",
        "    - The list of repos starred by that user in the train set\n",
        "    - The list of repos recommended by the model\n",
        "    - The list of repos really starred by the user in the test set (ground truth). Repos which are both in the ground truth and in the recommended list are highlighted by a green border.\n",
        "  - It is possible to hover on a predicted repo in order to check the score achieved by it. The score is computed based on the way which each model decides the top-k repos to recommend.\n",
        "- There is also a \"Project repo\" button that links to the GitHub repo of this project\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 98,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "brqcVx4gIDSf",
        "outputId": "4f54e92c-7065-46e9-c8da-f857949afa21"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Collecting flask-ngrok\n",
            "  Downloading flask_ngrok-0.0.25-py3-none-any.whl (3.1 kB)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from flask-ngrok) (2.23.0)\n",
            "Requirement already satisfied: Flask>=0.8 in /usr/local/lib/python3.7/dist-packages (from flask-ngrok) (1.1.4)\n",
            "Requirement already satisfied: Werkzeug<2.0,>=0.15 in /usr/local/lib/python3.7/dist-packages (from Flask>=0.8->flask-ngrok) (1.0.1)\n",
            "Requirement already satisfied: Jinja2<3.0,>=2.10.1 in /usr/local/lib/python3.7/dist-packages (from Flask>=0.8->flask-ngrok) (2.11.3)\n",
            "Requirement already satisfied: click<8.0,>=5.1 in /usr/local/lib/python3.7/dist-packages (from Flask>=0.8->flask-ngrok) (7.1.2)\n",
            "Requirement already satisfied: itsdangerous<2.0,>=0.24 in /usr/local/lib/python3.7/dist-packages (from Flask>=0.8->flask-ngrok) (1.1.0)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from Jinja2<3.0,>=2.10.1->Flask>=0.8->flask-ngrok) (2.0.1)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->flask-ngrok) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->flask-ngrok) (2021.5.30)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->flask-ngrok) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->flask-ngrok) (1.24.3)\n",
            "Installing collected packages: flask-ngrok\n",
            "Successfully installed flask-ngrok-0.0.25\n",
            "Collecting flask_cors\n",
            "  Downloading Flask_Cors-3.0.10-py2.py3-none-any.whl (14 kB)\n",
            "Requirement already satisfied: Flask>=0.9 in /usr/local/lib/python3.7/dist-packages (from flask_cors) (1.1.4)\n",
            "Requirement already satisfied: Six in /usr/local/lib/python3.7/dist-packages (from flask_cors) (1.15.0)\n",
            "Requirement already satisfied: click<8.0,>=5.1 in /usr/local/lib/python3.7/dist-packages (from Flask>=0.9->flask_cors) (7.1.2)\n",
            "Requirement already satisfied: itsdangerous<2.0,>=0.24 in /usr/local/lib/python3.7/dist-packages (from Flask>=0.9->flask_cors) (1.1.0)\n",
            "Requirement already satisfied: Werkzeug<2.0,>=0.15 in /usr/local/lib/python3.7/dist-packages (from Flask>=0.9->flask_cors) (1.0.1)\n",
            "Requirement already satisfied: Jinja2<3.0,>=2.10.1 in /usr/local/lib/python3.7/dist-packages (from Flask>=0.9->flask_cors) (2.11.3)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from Jinja2<3.0,>=2.10.1->Flask>=0.9->flask_cors) (2.0.1)\n",
            "Installing collected packages: flask-cors\n",
            "Successfully installed flask-cors-3.0.10\n"
          ]
        }
      ],
      "source": [
        "!pip install flask-ngrok\n",
        "!pip install flask_cors"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 99,
      "metadata": {
        "id": "eVDWlacZQzvP"
      },
      "outputs": [],
      "source": [
        "models = {\"CB\": cbm, \"MF\": alsm, \"IB\": ibm, \"UB\": ubm}\n",
        "\n",
        "# List of the repos in the system\n",
        "repos_dict = new_repos_df.toPandas().to_dict(\"records\")\n",
        "\n",
        "# List of the users in the system with the number of starred repos\n",
        "users_dict = new_users_df.join(new_starred_df, \"user_id\") \\\n",
        "  .groupBy(\"user_id\", \"name\").count().select(col(\"user_id\") \\\n",
        "  .alias(\"userId\"), \"name\", col(\"count\").alias(\"numStarredRepos\")) \\\n",
        "  .toPandas().to_dict(\"records\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "42yYiPFkIF4I",
        "outputId": "e8e15e13-b808-4cd0-9ded-d8a9f2a4a774"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " * Serving Flask app \"__main__\" (lazy loading)\n",
            " * Environment: production\n",
            "\u001b[31m   WARNING: This is a development server. Do not use it in a production deployment.\u001b[0m\n",
            "\u001b[2m   Use a production WSGI server instead.\u001b[0m\n",
            " * Debug mode: off\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " * Running on http://127.0.0.1:5000/ (Press CTRL+C to quit)\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " * Running on http://aebe-34-80-216-224.ngrok.io\n",
            " * Traffic stats available on http://127.0.0.1:4040\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "127.0.0.1 - - [30/Aug/2021 19:53:29] \"\u001b[33mGET / HTTP/1.1\u001b[0m\" 404 -\n",
            "127.0.0.1 - - [30/Aug/2021 19:53:29] \"\u001b[33mGET /favicon.ico HTTP/1.1\u001b[0m\" 404 -\n",
            "127.0.0.1 - - [30/Aug/2021 19:53:46] \"\u001b[37mGET //users HTTP/1.1\u001b[0m\" 200 -\n",
            "127.0.0.1 - - [30/Aug/2021 19:53:46] \"\u001b[37mGET //repos HTTP/1.1\u001b[0m\" 200 -\n"
          ]
        }
      ],
      "source": [
        "from flask import Flask, jsonify, request\n",
        "from flask_cors import CORS, cross_origin\n",
        "from flask_ngrok import run_with_ngrok\n",
        "\n",
        "app = Flask(__name__)\n",
        "CORS(app)\n",
        "run_with_ngrok(app)\n",
        "\n",
        "@app.route(\"/recommend\", methods=[\"POST\"])\n",
        "def recommend():\n",
        "  data = request.get_json()\n",
        "  print(\"Received\", data)\n",
        "\n",
        "  model = data[\"model\"]\n",
        "  params = data[\"params\"]\n",
        "\n",
        "  fromUser = \"user_id\" in params\n",
        "\n",
        "  response = {}\n",
        "\n",
        "  # Predict recommended repos\n",
        "  predictions = models[model].predict_from_user(**params) if fromUser else models[model].predict_from_repos(**params)\n",
        "  predictions = predictions.join(new_repos_df, \"repo_id\")\n",
        "\n",
        "  predictions = predictions.toPandas().to_dict(\"records\")\n",
        "  response[\"predictions\"] = predictions\n",
        "\n",
        "  # Get ground truth if the prediction is for a user\n",
        "  if fromUser:\n",
        "    gt = test.filter(col(\"user_id\") == params[\"user_id\"]).join(new_repos_df, \"repo_id\")\n",
        "    gt = gt.toPandas().to_dict(\"records\")\n",
        "    response[\"gt\"] = gt\n",
        "\n",
        "  # Get previously starred repos\n",
        "  if fromUser:\n",
        "    starred = train.filter(col(\"user_id\") == params[\"user_id\"]).select(\"repo_id\").join(new_repos_df, \"repo_id\")\n",
        "  else:\n",
        "    starred = repos_df.filter(col(\"repo_id\").isin(params[\"starred_repos_list\"]))\n",
        "  \n",
        "  starred = starred.toPandas().to_dict(\"records\")\n",
        "  response[\"starred\"] = starred\n",
        "  \n",
        "  return jsonify(response)\n",
        "\n",
        "@app.route(\"/users\")\n",
        "def get_users():\n",
        "  return jsonify(users_dict)\n",
        "\n",
        "@app.route(\"/repos\")\n",
        "def get_repos():\n",
        "  return jsonify(repos_dict)\n",
        "\n",
        "if __name__ == '__main__':\n",
        "    app.run()"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [
        "ZimRQ6bDhwxq",
        "fyNsHIX5RfDq",
        "_atfLeMguoU0",
        "VUvkdfkzR0mG",
        "C9nmqKx2bC44",
        "NyevOIbNR_MV",
        "50MrrbK8SWqr",
        "buN6VH9wSob8",
        "RMnFuercTJ-X",
        "mikzAWuPTNSt",
        "5PhVpiHxTgEj",
        "TpnWv_DUUOzK",
        "AMQAVIHskbDP",
        "tMbHdvF6XiAA",
        "KF2vRPKZXYod",
        "-vsCxoDkDqJ0",
        "xfYTQXtZDt-j",
        "yM74pKm3D54q",
        "WjKwA9yVBHvd",
        "ZgwaSQB7s_0W",
        "sYFLSFjo9uDM",
        "bUmyPtqPZIoG",
        "ivh2_D61ZbGy",
        "w7PDH4KYZkvC",
        "4Sz_1UZV9rJ0",
        "e5gl2bWi1FG7",
        "vQ_qg-oPgcsf",
        "PXy7lPy0h8IM",
        "dYjw1Hp0h0xc",
        "Iopwhnx7j3r8",
        "WI9RtuGdwp6O",
        "RvgTFkKD3_7h",
        "WfDKiOJpH34F",
        "I9hH8yH-H6xP",
        "mSegmhjq4LJJ",
        "U92b8l_ofl65"
      ],
      "machine_shape": "hm",
      "name": "GitHub Recommender System.ipynb",
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}