Logistic_Regression.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.feature_selection import RFE\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "import joblib"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocess the Data\n",
    "* Use MinMaxScaler to scale the numerical data.\n",
    "* Separate the data into training and testing data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>koi_fpflag_nt</th>\n",
       "      <th>koi_fpflag_ss</th>\n",
       "      <th>koi_fpflag_co</th>\n",
       "      <th>koi_fpflag_ec</th>\n",
       "      <th>koi_period</th>\n",
       "      <th>koi_time0bk</th>\n",
       "      <th>koi_impact</th>\n",
       "      <th>koi_duration</th>\n",
       "      <th>koi_depth</th>\n",
       "      <th>koi_prad</th>\n",
       "      <th>koi_teq</th>\n",
       "      <th>koi_insol</th>\n",
       "      <th>koi_model_snr</th>\n",
       "      <th>koi_steff</th>\n",
       "      <th>koi_slogg</th>\n",
       "      <th>koi_srad</th>\n",
       "      <th>ra</th>\n",
       "      <th>dec</th>\n",
       "      <th>koi_disposition</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9.488036</td>\n",
       "      <td>170.538750</td>\n",
       "      <td>0.146</td>\n",
       "      <td>2.95750</td>\n",
       "      <td>615.8</td>\n",
       "      <td>2.26</td>\n",
       "      <td>793.0</td>\n",
       "      <td>93.59</td>\n",
       "      <td>35.8</td>\n",
       "      <td>5455.0</td>\n",
       "      <td>4.467</td>\n",
       "      <td>0.927</td>\n",
       "      <td>291.93423</td>\n",
       "      <td>48.141651</td>\n",
       "      <td>CONFIRMED</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>54.418383</td>\n",
       "      <td>162.513840</td>\n",
       "      <td>0.586</td>\n",
       "      <td>4.50700</td>\n",
       "      <td>874.8</td>\n",
       "      <td>2.83</td>\n",
       "      <td>443.0</td>\n",
       "      <td>9.11</td>\n",
       "      <td>25.8</td>\n",
       "      <td>5455.0</td>\n",
       "      <td>4.467</td>\n",
       "      <td>0.927</td>\n",
       "      <td>291.93423</td>\n",
       "      <td>48.141651</td>\n",
       "      <td>CONFIRMED</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>19.899140</td>\n",
       "      <td>175.850252</td>\n",
       "      <td>0.969</td>\n",
       "      <td>1.78220</td>\n",
       "      <td>10829.0</td>\n",
       "      <td>14.60</td>\n",
       "      <td>638.0</td>\n",
       "      <td>39.30</td>\n",
       "      <td>76.3</td>\n",
       "      <td>5853.0</td>\n",
       "      <td>4.544</td>\n",
       "      <td>0.868</td>\n",
       "      <td>297.00482</td>\n",
       "      <td>48.134129</td>\n",
       "      <td>FALSE POSITIVE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.736952</td>\n",
       "      <td>170.307565</td>\n",
       "      <td>1.276</td>\n",
       "      <td>2.40641</td>\n",
       "      <td>8079.2</td>\n",
       "      <td>33.46</td>\n",
       "      <td>1395.0</td>\n",
       "      <td>891.96</td>\n",
       "      <td>505.6</td>\n",
       "      <td>5805.0</td>\n",
       "      <td>4.564</td>\n",
       "      <td>0.791</td>\n",
       "      <td>285.53461</td>\n",
       "      <td>48.285210</td>\n",
       "      <td>FALSE POSITIVE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2.525592</td>\n",
       "      <td>171.595550</td>\n",
       "      <td>0.701</td>\n",
       "      <td>1.65450</td>\n",
       "      <td>603.3</td>\n",
       "      <td>2.75</td>\n",
       "      <td>1406.0</td>\n",
       "      <td>926.16</td>\n",
       "      <td>40.9</td>\n",
       "      <td>6031.0</td>\n",
       "      <td>4.438</td>\n",
       "      <td>1.046</td>\n",
       "      <td>288.75488</td>\n",
       "      <td>48.226200</td>\n",
       "      <td>CONFIRMED</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   koi_fpflag_nt  koi_fpflag_ss  koi_fpflag_co  koi_fpflag_ec  koi_period  \\\n",
       "0              0              0              0              0    9.488036   \n",
       "1              0              0              0              0   54.418383   \n",
       "2              0              1              0              0   19.899140   \n",
       "3              0              1              0              0    1.736952   \n",
       "4              0              0              0              0    2.525592   \n",
       "\n",
       "   koi_time0bk  koi_impact  koi_duration  koi_depth  koi_prad  koi_teq  \\\n",
       "0   170.538750       0.146       2.95750      615.8      2.26    793.0   \n",
       "1   162.513840       0.586       4.50700      874.8      2.83    443.0   \n",
       "2   175.850252       0.969       1.78220    10829.0     14.60    638.0   \n",
       "3   170.307565       1.276       2.40641     8079.2     33.46   1395.0   \n",
       "4   171.595550       0.701       1.65450      603.3      2.75   1406.0   \n",
       "\n",
       "   koi_insol  koi_model_snr  koi_steff  koi_slogg  koi_srad         ra  \\\n",
       "0      93.59           35.8     5455.0      4.467     0.927  291.93423   \n",
       "1       9.11           25.8     5455.0      4.467     0.927  291.93423   \n",
       "2      39.30           76.3     5853.0      4.544     0.868  297.00482   \n",
       "3     891.96          505.6     5805.0      4.564     0.791  285.53461   \n",
       "4     926.16           40.9     6031.0      4.438     1.046  288.75488   \n",
       "\n",
       "         dec koi_disposition  \n",
       "0  48.141651       CONFIRMED  \n",
       "1  48.141651       CONFIRMED  \n",
       "2  48.134129  FALSE POSITIVE  \n",
       "3  48.285210  FALSE POSITIVE  \n",
       "4  48.226200       CONFIRMED  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Read the training data\n",
    "training_df = pd.read_csv(\"Data/Cleaned_Data.csv\") \n",
    "training_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_feature = \"koi_disposition\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(9201, 18) (9201,)\n"
     ]
    }
   ],
   "source": [
    "# Split the data\n",
    "X = training_df.drop(columns=[target_feature])\n",
    "y = training_df[[target_feature]].values.ravel() \n",
    "print(X.shape, y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a Train Test Split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42, stratify=y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Scale the training data to better train\n",
    "def scale_data(X, X_train, X_test):\n",
    "    '''\n",
    "        return the scaled training and testing data with the scaler\n",
    "    '''\n",
    "    X_scaler = MinMaxScaler().fit(X)\n",
    "    # apply the scale to training and testing data sets\n",
    "    X_train_scaled = X_scaler.transform(X_train)\n",
    "    X_test_scaled = X_scaler.transform(X_test)\n",
    "    #\n",
    "    return X_train_scaled, X_test_scaled, X_scaler\n",
    "\n",
    "X_train, X_test, X_scaler = scale_data(X, X_train, X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=10, max_iter=500, penalty='l1', solver='liblinear',\n",
       "                   tol=1e-06, verbose=2)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classifier = LogisticRegression(\n",
    "    penalty=\"l1\",\n",
    "    dual=False,\n",
    "    tol=0.000001,\n",
    "    C=10,\n",
    "    fit_intercept=True,\n",
    "    intercept_scaling=1,\n",
    "    solver=\"liblinear\",\n",
    "    max_iter=500,\n",
    "    multi_class=\"auto\",\n",
    "    verbose=2\n",
    ")\n",
    "classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LibLinear]"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=10, max_iter=500, penalty='l1', solver='liblinear',\n",
       "                   tol=1e-06, verbose=2)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classifier.fit(\n",
    "    X_train,\n",
    "    y_train,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Data Score: 0.8123188405797102\n",
      "Testing Data Score: 0.8096479791395046\n"
     ]
    }
   ],
   "source": [
    "print(f\"Training Data Score: {classifier.score(X_train, y_train)}\")\n",
    "print(f\"Testing Data Score: {classifier.score(X_test, y_test)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Logistic_Regression_Classifier.sav']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "joblib.dump(classifier, \"Logistic_Regression_Classifier.sav\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tune Model Parameters\n",
    "* Use GridSearch to tune model parameters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "grid = GridSearchCV(\n",
    "    classifier, \n",
    "    {\n",
    "        \"penalty\": [\"l1\", \"l2\"],\n",
    "        \"tol\": [0.000001, 0.001, 0.1],\n",
    "        \"C\": [0.1, 1, 10],\n",
    "        \"solver\":[\"lbfgs\", \"liblinear\", \"sag\", \"saga\"],\n",
    "        \"max_iter\": [100, 500, 1000]\n",
    "    }, \n",
    "    verbose=1\n",
    ")\n",
    "grid.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Grid best score: 0.8089855072463769\n",
      "Grid best penalty: l1\n",
      "Grid best tol: 1e-06\n",
      "Grid best C: 10\n",
      "Grid best solver: liblinear\n",
      "Grid best max_Iter: 500\n",
      "Training Data Score: 0.8123188405797102\n",
      "Testing Data Score: 0.8096479791395046\n"
     ]
    }
   ],
   "source": [
    "print(f\"Grid best score: {grid.best_score_}\")\n",
    "print(f\"Grid best penalty: {grid.best_estimator_.penalty}\")\n",
    "print(f\"Grid best tol: {grid.best_estimator_.tol}\")\n",
    "print(f\"Grid best C: {grid.best_estimator_.C}\")\n",
    "print(f\"Grid best solver: {grid.best_estimator_.solver}\")\n",
    "print(f\"Grid best max_Iter: {grid.best_estimator_.max_iter}\")\n",
    "\n",
    "print(f\"Training Data Score: {grid.score(X_train, y_train)}\")\n",
    "print(f\"Testing Data Score: {grid.score(X_test, y_test)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "PythonAdv",
   "language": "python",
   "name": "pythonadv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}