stock_pred_v1_5.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Length mismatch: Expected axis has 0 elements, new values have 7 elements",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<timed exec>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m~/anaconda/lib/python3.5/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36m__setattr__\u001b[0;34m(self, name, value)\u001b[0m\n\u001b[1;32m   2683\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2684\u001b[0m             \u001b[0mobject\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__getattribute__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2685\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mobject\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__setattr__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2686\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mAttributeError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2687\u001b[0m             \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/src/properties.pyx\u001b[0m in \u001b[0;36mpandas.lib.AxisProperty.__set__ (pandas/lib.c:44748)\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m~/anaconda/lib/python3.5/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36m_set_axis\u001b[0;34m(self, axis, labels)\u001b[0m\n\u001b[1;32m    426\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    427\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_set_axis\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 428\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_data\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_axis\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    429\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_clear_item_cache\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    430\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda/lib/python3.5/site-packages/pandas/core/internals.py\u001b[0m in \u001b[0;36mset_axis\u001b[0;34m(self, axis, new_labels)\u001b[0m\n\u001b[1;32m   2633\u001b[0m             raise ValueError('Length mismatch: Expected axis has %d elements, '\n\u001b[1;32m   2634\u001b[0m                              \u001b[0;34m'new values have %d elements'\u001b[0m \u001b[0;34m%\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2635\u001b[0;31m                              (old_len, new_len))\n\u001b[0m\u001b[1;32m   2636\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2637\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maxes\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnew_labels\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: Length mismatch: Expected axis has 0 elements, new values have 7 elements"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "dfname = \"tweets_AAPL_1.csv\"\n",
    "with open(dfname,'rb') as dfile:\n",
    "    dstr = str(dfile.read())[1:5]\n",
    "    \n",
    "darr = dstr.split('\\\\r')\n",
    "dmat = [rs.split(\",\") for rs in darr]\n",
    "dmat = [r[:1] + [\",\".join(r[1:-5])] + r[-5:] for r in dmat]\n",
    "dmat = [r for r in dmat if len(r) == 7]\n",
    "df = pd.DataFrame(dmat)\n",
    "df.columns = [\"datestr\",\"twt\",\"open\",\"high\",\"low\",\"close\",\"adjclose\"]\n",
    "df[\"twtarr\"] = df[\"twt\"].apply(lambda text: text.lower().split())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4.89 s, sys: 307 ms, total: 5.2 s\n",
      "Wall time: 6.55 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "df.to_csv(\"aapltwt/\"+dfname)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "i = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4.37 s, sys: 101 ms, total: 4.47 s\n",
      "Wall time: 6.11 s\n"
     ]
    }
   ],
   "source": [
    "df = df[df[\"datestr\"] != \"\"]\n",
    "%time df[\"date\"] = df[\"datestr\"].apply( \\\n",
    "    lambda date: datetime.strptime(date,\"%m/%d/%y\"))\n",
    "datedf = df[[\"date\",\"open\"]].groupby(\"date\").aggregate(lambda gp: tuple(set(gp))[0])\n",
    "pval = datedf.values.astype(float)\n",
    "up = ((pval[1:] - pval[:-1]) > 0).astype(int)\n",
    "datedf[\"openup\"] = list(up.T[0]) + [0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 15 s, sys: 254 ms, total: 15.3 s\n",
      "Wall time: 18.1 s\n"
     ]
    }
   ],
   "source": [
    "%time df[\"openup\"] = df[\"date\"].apply(lambda date: datedf[\"openup\"].loc[date])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 56.8 s, sys: 875 ms, total: 57.6 s\n",
      "Wall time: 31.8 s\n"
     ]
    }
   ],
   "source": [
    "import gensim\n",
    "%time w2vM = gensim.models.Word2Vec(df[\"twtarr\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# number of samples to aggregate\n",
    "Ns = int(1e6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 26.6 s, sys: 889 ms, total: 27.5 s\n",
      "Wall time: 34.3 s\n",
      "CPU times: user 26.6 s, sys: 893 ms, total: 27.5 s\n",
      "Wall time: 34.3 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# MEAN AGGREGATION\n",
    "# tvecs = np.array([np.array([w2vM[t] if t in w2vM\n",
    "#                                 else np.zeros((100,))\n",
    "#                             for t in twt]).mean(axis=0)\n",
    "#                  for twt in df[\"twtarr\"][:Ns]])\n",
    "# tvecs = np.array([np.array([w2vM[t]\n",
    "#                             for t in twt\n",
    "#                             if t in w2vM]).mean(axis=0)\n",
    "#                  for twt in df[\"twt\"][:Ns]])\n",
    "# SUM AGGREGATION\n",
    "# tvecs = np.array([np.array([w2vM[t] if t in w2vM\n",
    "#                                 else np.zeros((100,))\n",
    "#                             for t in twt]).sum(axis=0)\n",
    "#                  for twt in sentDf[\"SentimentText\"][:Ns]])\n",
    "# MEAN AGGREGATION\n",
    "%time df[\"twtvecs\"] = df[\"twtarr\"][:Ns].apply(\\\n",
    "    lambda twt: np.array([w2vM[t] \\\n",
    "                         if t in w2vM else np.zeros((100,)) \\\n",
    "                         for t in twt]).mean(axis=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(221343, 11)\n",
      "CPU times: user 2.27 s, sys: 153 ms, total: 2.42 s\n",
      "Wall time: 2.65 s\n",
      "CPU times: user 63.8 ms, sys: 10.6 ms, total: 74.4 ms\n",
      "Wall time: 87.2 ms\n",
      "(221343, 11)\n"
     ]
    }
   ],
   "source": [
    "print(df.shape)\n",
    "%time veclens = df[\"twtvecs\"].apply(lambda vec: int(np.prod(vec.shape)))\n",
    "df_empty = df[veclens != 100]\n",
    "%time df = df[veclens == 100]\n",
    "print(df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dategps = df[[\"date\",\"twtvecs\"]].groupby(\"date\")\n",
    "datevecs = [np.mean(gp.values[:,1], axis=0) for k,gp in dategps]\n",
    "datedf[\"datevecs\"] = datevecs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import sklearn\n",
    "from sklearn import ensemble,svm,neural_network,discriminant_analysis\n",
    "from sklearn.metrics import roc_curve,auc,precision_recall_curve\n",
    "\n",
    "from matplotlib import pyplot as plt\n",
    "% matplotlib inline\n",
    "\n",
    "def roc_auc(clf,X,y):\n",
    "    probs = clf.predict_proba(X)\n",
    "    fpr, tpr, thresholds = roc_curve(y, probs[:, 1])\n",
    "    roc_auc = auc(fpr, tpr)\n",
    "    roc_str = 'ROC (AUC Gain = %0.2f)' % (roc_auc - 0.5)\n",
    "    plt.plot(fpr, tpr, lw=1,label=roc_str)\n",
    "    plt.plot([0,1],[0,1],label=\"RAN CLF\")\n",
    "    plt.title(roc_str)\n",
    "    plt.show()\n",
    "\n",
    "def prrc_auc(clf,X,y):\n",
    "    probs = clf.predict_proba(X)\n",
    "    pr, rc, thresholds = precision_recall_curve(y, probs[:, 1])\n",
    "    roc_auc = auc(rc, pr)\n",
    "    roc_str = 'Prec vs Recall (AUC Gain = %0.2f)' % (roc_auc - np.mean(y))\n",
    "    plt.plot(rc,pr, lw=1,label=roc_str)\n",
    "    plt.plot([0,1],[np.mean(y),np.mean(y)],label=\"RAN CLF\")\n",
    "    plt.axis([0,1,0,1])\n",
    "    plt.title(roc_str)\n",
    "    plt.show()\n",
    "\n",
    "def evaluate(clf,X,y):\n",
    "    yhat = clf.predict(X)\n",
    "    accu = np.mean(yhat == y)\n",
    "    prec = np.mean(y[yhat == 1])\n",
    "    recl = np.mean(yhat[y == 1])\n",
    "    f1 = 2 * prec * recl / (prec + recl)\n",
    "    print(\"Accuracy\",accu,\"Precision\",prec,\"Recall\",recl,\"F1\",f1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Average of tweets in a day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# dvecs = np.array([vec.T for vec in avgvecs[\"vecs\"].values])\n",
    "X = np.array([vec.T for vec in datedf[\"datevecs\"].values])\n",
    "y = datedf[\"openup\"].values\n",
    "# inverse classifier\n",
    "# y = 1 - y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# generate test/train split\n",
    "ratio = 0.8\n",
    "tidx = np.random.rand(X.shape[0]) < ratio\n",
    "pidx = ~tidx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 31.9 ms, sys: 2.33 ms, total: 34.2 ms\n",
      "Wall time: 32.5 ms\n",
      "TEST\n",
      "Accuracy 0.571428571429 Precision 0.666666666667 Recall 0.56 F1 0.608695652174\n",
      "CPU times: user 3.61 ms, sys: 746 µs, total: 4.35 ms\n",
      "Wall time: 3.98 ms\n",
      "TRAIN\n",
      "Accuracy 0.953846153846 Precision 0.923728813559 Recall 1.0 F1 0.960352422907\n",
      "CPU times: user 4.62 ms, sys: 1.03 ms, total: 5.65 ms\n",
      "Wall time: 5.04 ms\n",
      "ROC AUC\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXkAAAEKCAYAAAD3tSVSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGKNJREFUeJzt3X2wXHV9x/H3BwJEIWAiLaNBqBIRpQakiii0roCSSNtY\nEHnwKTrWtIqTcUYFOlWuHTuQP9qiUrE4NBYiE4JoCT7woHJrfQjiKKAh4bkhCZgICYgoGvHbP865\nuSfL7t2ze88+nHM+r5kz2bP727O/e+7Nbz/7Pb+zRxGBmZlV027D7oCZmfWPB3kzswrzIG9mVmEe\n5M3MKsyDvJlZhXmQNzOrMA/yNlCSrpT018Pux3RIOk/SpcPuRz9I+mNJd0raY9h9sWJ4kK8hSf8n\n6deSfinpIUnLJT27qc1rJX0rbbNd0rWSXtrUZpakiyRtSNvdI+lfJc1p87ovB+ZHxOqm+xuS/iDp\nI033v07SxhbbuVnSezLrh0paJekXaV9vk/QhSWrTj33Sfj4g6Yl0f6ySdHTHnQdExAUR8b48bYsi\naU9J/ynp8fR39qEO7c9Kf64nJH1Z0nMyjy2T9GC6rQcknTvxWERsBb4NLOnfT2OD5EG+ngI4OSL2\nBY4EXgGcN/GgpNcANwBfAZ4HvBC4A/iepD9J2+xBMhi8FHhjuq3XAI8A7QbLJcAXW9z/TuDR9N9W\nfW1L0iHAGmAD8KcRMRs4DTgKmNWi/Z7AzcDhwJuAfdOfYSWwYKrXGrJPAIcALwCOBz4q6Y2tGko6\nHPgc8DbgAOA3wCWZJpcBL4uI/YDXAm+X9ObM41fiQb46IsJLzRbgAeD4zPoy4LrM+neAz7R43teB\nL6S33ws8DDyri9e9D3ht033PBn4JvBV4Cjgq89jrgAdbbOdm4D3p7RXZvufow3uBzcDMDu0uAh4E\nHgduBY7LPHY+cEV6+2DgDyRvUBuArcA/9OF3thk4IbP+CeDKNm3/GViRWX8R8Ftg7xZt55K8gX84\nc9/uwJPAC4b9t+pl+ouTfM1JOhBYCNyTrj+LJN19qUXzVcAb0tsnANdHxG9yvs6zST4R3NX00KnA\nE8DVwI3Au7r8EU5o09ep2t8QEU91aPdDYD4wmyTZXp1+CpjQ/AnjWODFwInAxyW9pNVGJZ2TlpS2\npf9mb29r85znkHyiuiNz9+0kn0ZaOTx9POloxP0kg/yhTf14AthI8kZ7Zab908C9wBFttm8l4kG+\nvv5b0i9J0uoWYCy9fw7J38XDLZ7zMLB/evu5bdq08xySgfGJpvvfCayMJEJeCZwhafcuttttP/YH\nfj6xIumIdIB9XNK6ifsj4sqIeCwi/hAR/wbsBbQcuEl+rrGI+F1E3EEywLYcICNiWUTMjog56b/Z\n2y2PZQD7pK/xeOa+X9KiHJVp/3jTfbu0T/sxi6RUd0WL9k+Q/M6s5DzI19eiSOrorwMOY3Lw3k5S\nfnhei+c8j6TmDkkNvVWbdh5L/9050KSfIl7PZIpcDTwLODld/z3QapbHHsCOHvuxS/uIuD2SOv4p\nJAP5RN8+nM4y2S5pO0ntfv9nbG3SlsztX5MMtEX5Vfrvvpn79uOZb5jZ9vs23deyfUTcTlIm+6em\nh2Yx+TuzEvMgX18CiIj/Bf4L+Jd0/dfAD0gOXjZ7K/DN9PY3gZPS8k5H6XbvI1MyIEnxAq6T9HD6\n+F5MlmweBPZvnvlDUgffkOnHqXn6kPoW8Map+i3pz4GPAG+ZSNskSbjlbJ1upNMvn0hnI2WXJ9JP\nVs8QEY+RfFrJfjo4Aljb5mXWZtumB6f3AO5u034GSd1+ov3uwDwyJR8rLw/yBslBxjekUxwBzgXe\nJensdLrhbEmfBI5hMvFdQVLPvUbSS5R4bjqItZul8nWSTw4T3klSJjqSZFA6AngLcLKk2RGxEbgF\nWCZp73Qa4UeB35HMqIHkIOhr02mBBwBImifpCknNaRbgcpIB8yuSDpe0m6S9gFdl2uxD8knh0fQ1\nP0770gh0MfhHMv1yVkTs27TMSj9ZtXMF8I+SnpNOZf1bYHmbtl8E/krSsZL2JvmdXRMRT6a/p/dN\nTKlMp41+gMk3b0hmRz2Q7n8rOQ/y9bTLQcOIeIQkzX88Xf8ecBJJQn6YZDbOEcCxEXFf2uZ3JAcZ\n1wM3kdR015DUyG9p87qfB94OIOnVwEHAZyNia2a5juQg8Jnpc04nmQZ4L7CJpLxzcvr6EwcVX0Ny\nUHdtWlq5mmRGTKvyxG/TbdwJfC3t93rgz0g+qUAyffQGkuT7AEn5ZaoBr/kgbD8u0nA+cD/JJ5hv\nAxdGxE0TD6afBI4FiIg7gb8jKYP9nKQE9oHMtv4GuDf95HA58KmI+PfM428jmYJpFaDkeNcUDaTL\ngL8EtkTE/DZtPk0yQ+NJYHFE3FZ0R60aJK0AVkXTCVE2GiT9ETAOvGLijdTKLc8gfxzJgZzLWw3y\nkhYCZ0fEyWk6+1REHNOX3pqZWVc6lmsi4rskMy7aWUTykY+IuAXYb6I2amZmw1VETX4uu9YrN6f3\nmZnZkPnAq5lZhc0oYBubSb40acKB6X3PIKkfsw7MzCovIno6TyNvkhft5wKvJv32QEnHAI9FxJY2\nbYf+ZT2jspx//vlD78OoLN4X3hdV2xfQ+3O3/GoLp151Ki+9+KWs2bgm3V7vOg7ykq4Evg8cmn4H\n9bslLZH0vnTQ/jrwgKR7gf8A3j+tHpmZ1VBEcNXPrmL+JfOZN2ceP17yY1594Kunvd2O5ZqIOCtH\nm7On3RMzs5ra+uRW3v+193PnL+7k2jOuLWRwn+ADr0PSaDSG3YWR4X0xyftiUh32Rb/Se1bHk6EK\nfTEpBvl6ZmbDIEGnoS6b3pcvWj7l4C6J6POBVzMzK8Ag0ntWEVMozcwsh37W3ttxkjcz67NBp/cs\nJ3kzsz4aRnrPcpI3M+uDYab3LCd5M7Oi7b2V064eXnrPcpI3MyvIRHrn74eb3rM8T97MrADZ2vu6\nC5YTm4ob3D1P3sxsSFrV3tk83PSe5Zq8mVmPhj1zJg8neTOzLo3KzJk8nOTNzLpQhvSe5SRvZpZD\nmdJ7lpO8mVkHZUvvWU7yZmZtlDW9ZznJm5m1UOb0nuUkb2aWUYX0nuUkb2aWqkp6z3KSN7Paq1p6\nz3KSN7Naq2J6z3KSN7NaqnJ6z3KSN7PaqXp6z3KSN7PaqEt6z3KSN7NaqFN6z3KSN7NKq2N6z3KS\nN7PKqmt6z3KSN7PKqXt6z3KSN7NKcXrflZO8mVWC03trTvJmVnpO7+05yZtZaTm9d+Ykb2al5PSe\nj5O8mZWK03t3nOTNrDSc3ruXK8lLWiBpvaS7JZ3T4vF9Ja2WdJukn0paXHhPzay2nN57p4iYuoG0\nG3A3cALwEHArcEZErM+0OQ/YNyLOk7Q/cBdwQET8vmlb0en1zMyysul9+aLlpRjcJShyqJNERKiX\n5+ZJ8kcD90TEhojYAawEFjW1CWBWensW8GjzAG9m1g2n92LkqcnPBTZm1jeRDPxZFwOrJT0E7AOc\nXkz3zKyOXHsvTlEHXk8CfhIRx0s6BLhJ0vyI+FVzw7GxsZ23G40GjUajoC6YWdlFBKvWrmLp9UtZ\nfORiVpyygpkzZg67WwM3Pj7O+Ph4IdvKU5M/BhiLiAXp+rlARMSyTJuvAhdExPfS9W8B50TEj5q2\n5Zq8mbVUxtp7O2Wryd8KzJN0sKQ9gTOA1U1tNgAnpp05ADgUuL+XDplZvbj23l8dyzUR8bSks4Eb\nSd4ULouIdZKWJA/HpcAngS9IuiN92kcjYlvfem1mleDae/91LNcU+mIu15gZz6y9jzXGKlV7H6Vy\njc94NbOBcnofLH93jZkNhGvvw+Ekb2Z95/Q+PE7yZtY3Tu/D5yRvZn1RtvQ+Zw5s317MtmbPLmY7\nRXCSN7NClTW9b9+ezIgpYtk2QhPIneTNrDBlS+914CRvZtNW1vReB07yZjYtTu+jzUnezHri9F4O\nTvJm1jWn9/Jwkjez3Jzey8dJ3sxycXovJyd5M5uS03u5OcmbWVtO7+XnJG9mz+D0Xh1O8ma2C6f3\nanGSNzPA6b2qnOTNzOm9wpzkzWrM6b36nOTNasrpvR6c5M1qxum9XpzkzWqk2/Re5NWSRt0oXc2p\nSE7yZjXQa3ov8mpJo76M0tWciuQkb1Zxrr3Xm5O8WUW59m7gJG9WSU7vNsFJ3qxCnN6tmZO8WUU4\nvVsrTvJmJef0blNxkjcrMad360QRMbgXk2KQr2c26no/2Sjg8FWwcCncthjGx+D3M4vtHMkJQlWd\nP14mkogI9fJcJ3mzIZo42agb2fS+fJHTu03NNXmzknDt3XrhJG9WAq69W69yJXlJCyStl3S3pHPa\ntGlI+omkn0m6udhumtWT07tNV8cDr5J2A+4GTgAeAm4FzoiI9Zk2+wHfB94YEZsl7R8Rj7TYlg+8\nmmVI7Wvyu9bel3twr7HpHHjNk+SPBu6JiA0RsQNYCSxqanMWcE1EbAZoNcCbWT5O71akPDX5ucDG\nzPomkoE/61Bgj7RMsw/w6Yi4opgumtWHa+9WtKJm18wAjgIWAguAj0maV9C2zSrP6d36JU+S3wwc\nlFk/ML0vaxPwSEQ8BTwl6TvAEcC9zRsbGxvbebvRaNBoNLrrsVnFOL1bs/HxccbHxwvZVp4Dr7sD\nd5EceH0Y+CFwZkSsy7Q5DPgMSYrfC7gFOD0i7mzalg+8mqUigt1evooDFi9l8ZGLGWuMMXNG8Wet\nWvn19YzXiHha0tnAjSTlncsiYp2kJcnDcWlErJd0A3AH8DRwafMAb2aTJtI7Dad36y9/d43ZAEUE\nq9auYun1SXpftnCM2OH0blObTpL3IG82IK3mvU81T95sQr/nyZvZNHjmjA2Tv7vGrI88c8aGzUne\nrA+c3m1UOMmbFczp3UaJD7zayOn9aknD1v3VmnzlJcvDV4aySunlaknD5qs12ahyTd5sGlx7t1Hn\nJG/WI9ferQyc5M265PRuZeIkb9YFp3crGyd5sxyc3q2snOTNOnB6tzJzkjdrw+ndqsBJ3qwFp3er\nCid5swynd6saJ3mzlNO7VZGTvNWe07tVmZO81ZrTu1Wdk7zVktO71YWTvNWO07vViZO81YbTu9WR\nk7zVgtO71ZWTvE3bnDkgFbfMnl1c35zere6c5G3aRvVKTk7vZk7yVkFO72aTnOStUpzezXblJG+V\n4PRu1pqTvJWe07tZe07yVlpO72adOclbKTm9m+XjJG+l4vRu1h0neSsNp3ez7jnJ28hzejfrnZO8\njTSnd7PpcZK3keT0blaMXEle0gLgIpI3hcsiYlmbdq8Cvg+cHhFfLqyXVitO72bF6ZjkJe0GXAyc\nBBwOnCnpsDbtLgRuKLqTVg9O72bFy5PkjwbuiYgNAJJWAouA9U3tPgh8CXhVoT20WnB6N+uPPDX5\nucDGzPqm9L6dJD0feHNEXAKouO5Z1Tm9m/VXUbNrLgLOyax7oLeOnN7N+i/PIL8ZOCizfmB6X9Yr\ngZWSBOwPLJS0IyJWN29sbGxs5+1Go0Gj0eiyy/U0Z05ycY5R1O2VnCKCVWtXsfT6pSw+cjErTlnB\nzBkz+9M5sxIaHx9nfHy8kG0pOlzSR9LuwF3ACcDDwA+BMyNiXZv2y4HrWs2ukRSdXs9ak0bz6kvd\nyqb35YuWO72b5SCJiOipQtKxJh8RTwNnAzcCa4GVEbFO0hJJ72v1lF46YtXm2rvZcHRM8oW+mJN8\nz8qc5J3ezaanr0nerFdO72bD5++usb7wzBmz0eAkb4VyejcbLU7yVhind7PR4yTfJ3PmJAdLi1q6\nnYs+SE7vZqPLSb5Ptm8v72yYbji9m402J3nridO7WTk4yVvXnN7NysNJ3nJzejcrHyd5y8Xp3ayc\nnORtSk7vZuXmJG9tOb2blZ+TvD2D07tZdTjJZxR5YY5RPnlpKk7vZtXiJJ8xcQJTEcu2bcP+abrj\n9G5WTU7y5vRuVmFO8jXm9G5WfU7yNeX0blYPTvI14/RuVi9O8jXi9G5WP07yNeD0blZfTvIV5/Ru\nVm9O8hXl9G5m4CRfSU7vZjbBSb5CnN7NrJmTfEU4vZtZK07yJef0bmZTcZIvMad3M+vESb6EnN7N\nLC8n+ZJxejezbjjJl4TTu5n1otRJvsgrOcHoXs3J6d3MelXqJF/klZxG8WpOTu9mNl2lTvJV5vRu\nZkUodZKvIqd3MyuSk/wIcXo3s6LlSvKSFkhaL+luSee0ePwsSbeny3clvbz4rlaX07uZ9UvHJC9p\nN+Bi4ATgIeBWSddGxPpMs/uBv4iIxyUtAD4PHNOPDleN07uZ9VOeJH80cE9EbIiIHcBKYFG2QUSs\niYjH09U1wNxiu1k9Tu9mNgh5avJzgY2Z9U0kA3877wW+MZ1OVZ3Tu5kNSqEHXiW9Hng3cFy7NmNj\nYztvNxoNGo1GkV0YaRHBqrWrWHr9UhYfuZgVp6xg5oyZw+6WmY2Y8fFxxsfHC9mWImLqBtIxwFhE\nLEjXzwUiIpY1tZsPXAMsiIj72mwrOr1eN6TkJKYyyKb35YuWO72bWW6SiAj18tw8NflbgXmSDpa0\nJ3AGsLqpAweRDPDvaDfA15Vr72Y2TB3LNRHxtKSzgRtJ3hQui4h1kpYkD8elwMeAOcBnJQnYERFT\n1e1rwbV3Mxu2juWaQl+sJuWa5tr7WGPMtXcz69l0yjU+47VgTu9mNkr83TUFce3dzEaRk3wBnN7N\nbFQ5yU+D07uZjTon+R45vZtZGTjJd8np3czKxEm+C07vZlY2TvI5OL2bWVk5yXfg9G5mZeYk34bT\nu5lVgZN8C07vZlYVTvIZTu9mVjVO8imndzOrotonead3M6uyWid5p3czq7paJnmndzOri9olead3\nM6uT2iR5p3czq6NaJHmndzOrq0onead3M6u7yiZ5p3czswomead3M7NJA0/yUnHbmj1713WndzOz\nXQ08yUcUt2zbNrFNp3czs1ZKX5N3ejcza6+0NXmndzOzzkqZ5J3ezczyKVWSd3o3M+tOaZK807uZ\nWfdGPsk7vZuZ9W6kk7zTu5nZ9Ixkknd6NzMrxsglead3M7PijEySd3o3MyveSCR5p3czs/7IleQl\nLZC0XtLdks5p0+bTku6RdJukI/Ns1+ndzKy/Og7yknYDLgZOAg4HzpR0WFObhcAhEfFiYAnwuU7b\n3frkVk67+jQ+8T+f4NozruXCEy9k5oyZPf0QZTQ+Pj7sLowM74tJ3heTvC+KkSfJHw3cExEbImIH\nsBJY1NRmEXA5QETcAuwn6YBWG3N6T/gPeJL3xSTvi0neF8XIU5OfC2zMrG8iGfinarM5vW9L88ZO\nu/o0197NzAZk4Ade582Zx4pTVtSqNGNmNiyKiKkbSMcAYxGxIF0/F4iIWJZp8zng5oi4Kl1fD7wu\nIrY0bWvqFzMzs5Yioqfr6uVJ8rcC8yQdDDwMnAGc2dRmNfAB4Kr0TeGx5gF+Op00M7PedBzkI+Jp\nSWcDN5IcqL0sItZJWpI8HJdGxNclvUnSvcCTwLv7220zM8ujY7nGzMzKqy9fa9Cvk6fKqNO+kHSW\npNvT5buSXj6Mfg5Cnr+LtN2rJO2QdMog+zdIOf+PNCT9RNLPJN086D4OSo7/I/tKWp2OFT+VtHgI\n3ew7SZdJ2iLpjinadD9uRkShC8kbx73AwcAewG3AYU1tFgJfS2+/GlhTdD9GYcm5L44B9ktvL6jz\nvsi0+xbwVeCUYfd7iH8X+wFrgbnp+v7D7vcQ98V5wAUT+wF4FJgx7L73YV8cBxwJ3NHm8Z7GzX4k\n+UJPniq5jvsiItZExOPp6hqS8wuqKM/fBcAHgS8BWwfZuQHLsy/OAq6JiM0AEfHIgPs4KHn2RQCz\n0tuzgEcj4vcD7ONARMR3ge1TNOlp3OzHIN/q5KnmgavdyVNVk2dfZL0X+EZfezQ8HfeFpOcDb46I\nS4Aqz8TK83dxKDBH0s2SbpX0joH1brDy7IuLgZdJegi4HVg6oL6Nmp7GzZH4FkoDSa8nmZV03LD7\nMkQXAdmabJUH+k5mAEcBxwN7Az+Q9IOIuHe43RqKk4CfRMTxkg4BbpI0PyJ+NeyOlUE/BvnNwEGZ\n9QPT+5rbvKBDmyrIsy+QNB+4FFgQEVN9XCuzPPvilcBKSSKpvS6UtCMiVg+oj4OSZ19sAh6JiKeA\npyR9BziCpH5dJXn2xbuBCwAi4j5JDwCHAT8aSA9HR0/jZj/KNTtPnpK0J8nJU83/SVcD74SdZ9S2\nPHmqAjruC0kHAdcA74iI+4bQx0HpuC8i4kXp8kKSuvz7KzjAQ77/I9cCx0naXdKzSQ60rRtwPwch\nz77YAJwIkNagDwXuH2gvB0e0/wTb07hZeJIPnzy1U559AXwMmAN8Nk2wOyKi+QvgSi/nvtjlKQPv\n5IDk/D+yXtINwB3A08ClEXHnELvdFzn/Lj4JfCEztfCjEbFtSF3uG0lXAg3guZIeBM4H9mSa46ZP\nhjIzq7CRucarmZkVz4O8mVmFeZA3M6swD/JmZhXmQd7MrMI8yJuZVZgHeTOzCvMgb2ZWYf8PwTyr\ndKiwUbAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x114494f60>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 245 ms, sys: 17.1 ms, total: 262 ms\n",
      "Wall time: 326 ms\n",
      "PRECISION/RECALL AUC\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXkAAAEKCAYAAAD3tSVSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XucVXW9//HXG8E7IqiZopBoJF5REdTsOKIlZkXHYyWk\nlpbSr7ylx0vlOeKlY9QxL3k7JmlqCj3UlMq7MoqFgKkgCt5DQMRQbt4QmM/vj+8aZrthZjbDnr1n\n1ryfj8d+zOy1v3utz16z572/+7tuigjMzCyfOlW7ADMzaz0OeTOzHHPIm5nlmEPezCzHHPJmZjnm\nkDczyzGHvLVLkuok9cl+v1HShU203VLSDEkbVK7C8pM0XdK/VbuO1iDpZEm/qHYdeeSQbwMk/VPS\nB5KWSJqXhdbG1a6rFFmty7LaF0h6QNLnKrDotTnA41zgxohYVjhR0k2Slkvaumj6ah8aknpnHyyd\nCqYNlzRF0lJJcyX9VdLnGytC0gBJf5b0bnabLukiSd1KeRERsVtEPF7SKy4TSf0lPSXp/ey17tlE\n2/Ul/U7SYklvSvpx0eNflfRc9l55QlK/god/C3xb0pat9Vo6Kod82xDAERGxGbA3MAA4b00NJamS\nhZVoVFZ7T+BN4IYKLLOk9SBpfeA7wK1F0zcGjgQWAceUuMxVHyySzgB+DVwMfAroBVwNfLWROg4A\nxgMTgM9FRA9gCLACaDQ4q0lSF+Bu4GZg8+znPZI6N/KUC4Adge2BwcDZkr6UzeuzpL/BSdm8/gKM\nq//QzD6A7wWOa7UX1EE55NsOAUTEPOA+YDcASeMlXZz1fN4HdpC0maTRWW9pdtYbXBV6kk6U9ELW\nY5ouqf9qC5OukfSroml3Szo9+/0cSXOyecyQdHBzLyD7R/0j8InlSTohq+cdSfdJ6lXw2K6SHswe\nmyfp3Gz6vpL+Lmlh1kv+TRPh0pRBwMKIeLNo+lHAQuBC4LtrM0NJm5EC7YcRcU9EfBgRKyPi3og4\nt5GnjQJGR8QvI+JfABExJyIuqO+dS+oj6ZHsG9Hbkm7NllW/3NclDc5+P1/SWEm/z/5Gz0nae21e\nRwlqgPUi4sqIWB4RvyG9Twc30v444MKIWBIRM4HraVi3XwImRMTEiKjL1kdP4KCC5z8GHFHm19Dh\nOeTbGEnbA18Gni6YfAzwfaAr8Abwe2AZ0AfYC/hi9jiSvgH8N3BM1rv+GvDOGhZ1O/DNguVuns3n\ndkl9gR8B+2TzOAz4Zwm1bwIMB14umDaUNFzydWArUk/29uyxTYGHSD24bYCdgEeyp64ETgd6APuT\nguWHzdWwBrsDL65h+nHAbcBYYGdJe63FPA8ANiD1cpuVfWvYH7iruabA/wCfBvoB2wEjm2j/VdJr\n6Ab8mfRNorEaphYMEy0s+nlVI0/bFZhWNG1qNr14/puT/obTmmub6UR6vbsVTJtBG/1W05455NuO\nuyW9CzxO+lp/ScFjN0XEzKwH1AM4HPhxRHwUEQuAy4Gjs7bfA34ZEU8DRMRrETG7eGERMQEISQdm\nk44CJkbEfFLArg/sJqlzRLwREa83UftZWe1LSAFY+JV7BHBJRLyU1f8LoH/2YfYVYF5EXB4RH0fE\n+xExJavv6YiYHMkbpF7hQay9zYGlhROybxIHA7dFxNvAw6zdMEEPYEH2ekrRnfS/9lZBDaOykH1P\n0k8BIuLViHgkIlZExDvAZTT9mp+IiAcinYDqFmCPxhpGxJ4R0SO7dS/6eXIjT9sUWFw0bQmps7Gm\ntlHUvrDtw8BBkv4tGwb6KdAFKNz2tJT0gWVl5JBvO4Zm/3A7RMQpRRsJC0O6N+mfY159bwy4jtRL\nhjQe+mqJyxwLDMt+Hw78AVLYkHrRI4H5km6TtE0T8/lVNsbcG/gQKNzw2hu4or4XSfpWEaSv6o3W\nKumzShsp50laBPwcaMlGuYWsHkrHAi9ExHPZ/dtJG/3Wy+6vIK3jQl2AuizY3wG2VMFG2BJqqCP1\ndAGIiHMiojvwJ6AzgKRPSbo9GyZbRBrDbuo1v1Xw+wfAhmtRUyneAzYrmtaNog/NgrYUtV/VNiJe\nJG0buZq03aYH8AIwp6B9V1b/ULF15JBvO5rakFi4J8ls4CNgi4Le2OYRsUfB4zuWuMzbgaOynu0g\n4M5VC4wYExFfIIU0pB54kyJiDunD4Uo17K74BjCiqBe5aUQ82Uyt15K+vu8YEZsDP6PEja1FpgF9\ni6YdC/TJPkDmAZcCW5CGyepr/kzRc/rQ8GE7kTRc9vVSCoiID4BJpA29Tfkf0ofBrtlrPoaWvebV\nZNtmlhTdlmY/r2nkac+z+reDPbLpnxARi4B5fHK4Zc/CthFxV0TsHhFbkToQOwBTCtr3Iw3xWBk5\n5NuZiHgLeBC4TFJXJX3UsP/0DcB/1m+Ek7Rj4YbOonk9S+qV3gDcHxFLsuf0lXSw0p4pH5N65yUN\nTUTEw8Bc0jANwP8BP5W0SzbvbpKOyh77C/BpSacq7X63qaSB2WNdgSUR8YGknYH/V9IKWt1kYPP6\nbyKS9icF9r6kENqTNG58O6mnCenD7ghJh0rqJGlb0ofM7dlrXAKcD1wtaaikjSR1ljREje/rfTZw\ngqSzJW2V1bIdKejqdSX1iJdK6gmctZavtdEPhGz3y82Kbl2zn41t66gFVko6Jfv7nEp6HzzaSPtb\ngPMkba60e+SJwI2ripP2ztbnVqTht7sj4qWC5x9E2unAyikifKvyDXgNGNzIY48CJxRN6wpcQ+pZ\nLgT+AXyz4PGTgJmkMdFpwJ5NLPs80hj8kQXTdif1PBcDC4BxwKcbef7vSHtUFE77ZlZbl+z+t7M6\nFgGzgBsK2u5CGq99l/Q1/uxs+hdIPfklpL0uRgKPFzxvJdAn+/3G4hqK6hlVMN9rgT+uoc2+pA+z\nzbP7RwBPZev3ddI3mQ2KnjOM1BNdmtX+Z2C/JurYF/hr9lrfzdbJRUD3gnXxVPaanwZ+DLyxpvcJ\n6UPm5oLHemfrpFOZ35t7ZjW9n/3co+Cx4cBzBffXB0Zn75t5wGlF85qQvbYF2ft3o4LHNszeM1tV\n+/8xbzdlK7hRkkaTNpDNj4YhgeI2V5I2Br4PfDdSD9GsTVA6wOZxYK8oOiDK2gZJJwPbReO7oFoL\nlRLyB5K+Qt68ppCXdDhwckQcIWkQcEVE7Ncq1ZqZ2Vppdkw+Ip4gfWVtzFDSkXBExCSgm4oOEzcz\ns+oox4bXnnxyF7+52TQzM6sy711jZpZjLTkXSLG5pINa6m2XTVuNpLU5c6CZmWUiokXHTJTakxeN\n74M7juyQcEn7AYsiHRq/RpXYZehnPwsuuqj6uy41dTv//POrXkNbuXldeF14XTR9WxfN9uQl3UY6\nG90Wkt4g7Z+7fsrruD4i7pX0ZUmvkHahPH6dKjIzs7JpNuQjYngJbRo7wZGZmVWRN7xWSU1NTbVL\naDO8Lhp4XTTwuiiPZg+GKuvCpKjE8s47DzbcMP00M2vvJBGtvOHVzMzaIYe8mVmOOeTNzHLMIW9m\nlmMOeTOzHHPIm5nlmEPezCzHHPJmZjnmkDczyzGHvJlZjjnkzcxyzCFvZpZjDnkzsxxzyJuZ5ZhD\n3swsxxzyZmY55pA3M8sxh7yZWY455M3Mcswhb2aWYw55M7Mcc8ibmeWYQ97MLMcc8mZmOeaQNzPL\nMYd8GxcBjz8OxxwDW28NH31U7YrMrD1xyLdRCxbApZdCv37wgx/APvvAkiWwfHm1KzOz9sQh34bU\n1cGjj8KwYbDTTjBtGtxwAzz/PPz4x9C5c7UrNLP2xrHRBsyfDzfdlAJ9o43gpJPgmmuge/dqV2Zm\n7Z1Dvkrq6uDhh+G3v00/jzwSbrkFBg0CqdrVmVleOOQrbMWK1GP/5S+hWzcYMSLd79at2pWZWR45\n5CskAv76VzjrLNhmG7j9dhg40L12M2tdDvkKeOYZOPNMeOst+N//hS9/2eFuZpXhvWta0ezZ8J3v\npFD/1rfS3jJHHOGAN7PKKSnkJQ2RNFPSS5LOWcPjm0kaJ+lZSc9J+m7ZK21HliyBn/0M+veHXr3g\npZfS2Lt3gTSzSms25CV1Aq4CDgN2BYZJ2rmo2Y+A5yOiP3AwcKmkDhdpK1bAddfB5z4Hc+fC1Klw\n0UXQtWu1KzOzjqqUIB4IvBwRswAkjQGGAjML2gRQH2VdgXciYkU5C23LCjeqbrst3Hsv7LVXtasy\nMyst5HsCswvuzyEFf6GrgHGS3gQ2Bb5VnvLavqlT09Go8+en0xAcfnj7H3N///30evr0qXYlZrau\nyjWkchjwTEQMlrQj8JCkPSLiveKGI0eOXPV7TU0NNTU1ZSqh8m69Fc44Iw3JfO977XvMva4unQjt\n5pvhjjugd2947rlqV2XWMdXW1lJbW1uWeZUSS3OBXgX3t8umFToeuAQgIl6V9DqwM/BU8cwKQ769\nqquD886DMWNg/HjYdddqV9Ryr7ySgv2WW2DTTdPeQH/6E5xySrUrM+u4ijvAF1xwQYvnVUrITwF2\nktQbmAccDQwrajMLOBT4m6Stgb7Aay2uqg177z049lh4912YPBm23LLaFa29xYvhj3+E3/8eXn45\nnRDtrrvS3kASvPBCtSs0s3JpNuQjYqWkk4EHSXvjjI6IGZJGpIfjeuBi4CZJ07KnnR0R77Za1VXy\nxhvwta/BgAEwdiysv361KyrdihXw0EMp2O+/Hw49FM45B4YMgS5dql2dmbWWkkaRI+J+4HNF0/6v\n4Pd5pHH53Jo4Ef7jP9IeNKef3n42rr74Yjo3zh/+ANtvn4ZjrrkGevSodmVmVgnteFNh5dRvYL3p\npnT0answcSKMGpV+nnBCOk/9zsVHN5hZ7jnkm1BXl45cHTu2fWxgratL++iPGpUOxjrzTLjtNth4\n42pXZmbV4pBvxHvvpeuqtocNrB9/nM5q+atfpe0EZ58NRx3VvnfpNLPycAysQf0G1n32SXuhtNUN\nrEuXpouOXHZZOpXCZZelDartZXuBmbU+n4WyyMSJsN9+cNxxaYNlWwz4+fPTMNIOO8CkSXD33enq\nUl/8ogPezD7JIV/glltg6NAU7mec0TYD85RT0gbUhQtTwI8dm75xtAXLlqXz+JhZ2+Hhmsyf/pR6\nx215A+ugQWk3yBdfhE99qtrVJCtWwAMPwI03wrhxaftF//7VrsrM6jnkgTlz4Ac/gHvuabsBD2lI\npq2YOTMF+y23pPPcHH98OkXCBx9UuzIzK9Thh2tWrkynKTj11DQWb41bvBiuvx723x8GD07THnkk\nbcc46STvqmnWFnX4nvyoUWkc+dxzq11J21RXlw6kuvHGdM78Qw9NJ2c77DDvomnWHnTof9NJk+CK\nK+Cpp2C99apdTdvy3ntw/vnpKN8ePdJwzBVXtO3jBcxsdR025JcsgeHD4dpr08ZMa9CtW9qzaNGi\ntJ3CG1LN2q8OG/I/+lEaejjyyGpX0vb07AmzZlW7CjMrhw4Z8rfemoZonlrtkiZWacuXp43fG25Y\n7UrM8qnD7V3z2mvpmqy33QabbFLtajqmiPQBe+qp6cLnZ5xR7YrM8qtDhfzy5Wkc/qc/hb32qnY1\nHc/s2XDJJbDLLnD00bDFFvDDH8KHH1a7MrP86lAhf8EF0L07nHZatSvpOJYuTVejOuSQtAF31iwY\nPTpddvD88+Ezn6l2hWb51mHG5B97LIXLs89Cpw710VZ5K1emg6Ruvhn+8hc46KDUYz/iCI+9m1Va\nhwj5d99NR7X+7new9dbVribfLr0UnnwyjbUfd1w6/fFWW1W7KrOOK/chHwEnnpiuz3r44dWuJt8O\nPTSdsOzii6Ffv2pXY2bQAUL+hhvg1VfT3jTWui68sNoVmFmxXIf8zJlpT5rHH4cNNqh2NWZmlZfb\nTZDLlsGwYfDzn3vowMw6rtyG/NVXp8vjnXhitSsxM6ue3Ib8xhuni1y3xUv4WeW9/Xbau+r111t3\nOXV18MwzcNVVvoCKtQ25HJP/9rcbjqi0juvNN9NlHe+4IwXvRhvByJEwYkR5l/P22/Dgg+kyiA8+\nCJtvDm+9BQce6DN4WvXlsiffrx/stlu1q7BqeOMNuPzyFLC77ZauGXD66TBvHnzta+VZxscfQ20t\n/OQnsPfe0Lcv3HVXWuakSekavDvsUJ5lma2rXPbkrWN57TW4887UY3/1VRg6NO1Vdcgh5dur6pVX\nUk/9gQfS0dN9+6arY115ZbrAepcu5VmOWbk55K1deumlFOp33AFz58K//3s6CKumpjyBu3w5TJgA\n48alUzO8/34K9WHD0ti+r5Bl7YVD3tqNBQtgzJh0TpzZs9NRzL/+NXzhC+W5fOOSJXD//SnY77sP\n+vRJ3wruvBP22MMb8a19cshbm7ZsGdx7bzqTZW1tOsnZRReloZhyXEh8zpwU6vfcA3//expXHzo0\nXeC9Z891n79ZtTnkrc2JgMmTU4997Ni0AfU730n3N9ts3ec9dWoK9XvugX/+M31wnHhiGvrp2rUs\nL8GszXDIW5sxa1a6NOPNN6cwPu64dAWpcp1zXkpXodpmm9Rbv/TS1HMvxzcCs7bKb2+rumefhcGD\nYdo0+OY309DMoEHlHwP/yU/g5JNh1109vm4dh0PeqqpvX9hxx7TXyle+0ronkuvdu/XmbdZWlRTy\nkoYAl5MOnhodEaPW0KYGuAzoAvwrIg4uY52WU5//fLpZy9XV+Wpn1rhmQ15SJ+Aq4BDgTWCKpHsi\nYmZBm27A1cCXImKuJO9FbNZKFi5MB2SNHw+PPgrz56dTK5itSSk9+YHAyxExC0DSGGAoMLOgzXDg\nzoiYCxARC8pdqFlHtXQpPPFECvTx49NpEw44IG3HuOaadEUuy4933oEXXki3GTPSz3VRSsj3BGYX\n3J9DCv5CfYEuksYDmwJXRsQt61aaWcf04YcwcWIK9UcfTRukBwxIoX755TBwIKy/fmq7bFl1a7WW\niUjnU6oP8fqfL7yQ/qa77JLOwbXLLvDFL8JDD7V8WeXa8NoZ2BsYDGwCTJQ0MSJeKdP8zXKrrg6e\nfjqdF+fhh2HKlHSE7cEHpwO/DjggnUHT2p+IdEbS6dPT7fnnG8K8S5cU4vWBfuSR6ee225Z3769S\nQn4u0Kvg/nbZtEJzgAUR8RHwkaTHgT2B1UJ+5MiRq36vqamhpqZm7So2y4F58xpOT/zQQ+lcOEOG\nwFlnpdM0+KCs9ueddxqCvD7Up09PG8V33z3turvPPun4j379YKutGp9XbW0ttbW1ZalLEdF0A2k9\n4EXShtd5wGRgWETMKGizM/AbYAiwATAJ+FZEvFA0r2BkWeo2M+s4RkJEtKh/32zIw6pdKK+gYRfK\nX0gaAUREXJ+1+U/geGAl8NuI+M0a5hOlLM+svevfH847L/XYH3ggXUx+l13SmSwPOyyNq5fjSNtl\ny9KpHkodm//443QU8d/+Bt/4RvmOJs6jurp02otp09Jt6tT0c+7chmtW1PfQd9sNttuu9Q6yk9S6\nIV8uDnnrKA44IF3ApD7UDz0UevQo/3KaC/lly9KFTB57LN0mTYKddoJFi+DCC+HYY8tfU3u0eDE8\n91xDoE+blu537562j9Tf9twTPvvZyp8KY11C3ke8mrWCRx9NR+9W+vQJH34ITz6ZAr22NvXa+/WD\ngw6C005L5+rp3r3jhntEOvPoM8+k02nU//zXv1JvvD7Mhw9PvfTu3atd8bpzyJu1gg03rNyyVq5M\nQ0OPPZZCa/fdU6ifc046mnhdz9zZXq1YkS4uUxzo660He+2VbsOGpdNK77hjea5J0BY55M3asS5d\n0sVTIuC//xv23x823bTaVVXeRx+l4ZV//KMh0KdPT7sj7rVX2kZy5pnp5zbbVLvaynLIm7VjnTql\nc+53JIWB/o9/pCGpF19MY+X77JNC/Zhj0vi5d0V1yJtZK1mxIoXx9OlpWKQlGyubCvQBA1Kof//7\naRzdB4ytmUPezMpiwYJ0Oob621NPpd0KZ81K1wfo27fp569cmQ4kmjw53aZM+WQPfcAAB3pLOOTN\nbK2tWJF66IWh/vbbaf///feHs89Owd6jx5rDPSLtYlof6JMnp1M7bLttet7AgXDCCWnIxYG+bhzy\nZtasxYvThc7/9rcU6FOmpEDef/+0W+ZZZ6WDvRrbQ2XhwnQah8JQh4ZAP++81FPPwy6LbY1D3sxW\n8+ab6fTGEyak2yuvwL77pl0yzzwzhfMWW5Q2rw03hEMOSUMugwalc7dcdRVsv70vw1gJDnmzDi4i\n7U8+YUJDsC9alHroBx4I110He+/dcHrjtTVhAmyyiS+YXi0+rYFZB3Tsseno2Lq6FOwbbZTOfvmF\nL6Rg79fPlxRsS3zuGjNbKzfemMbV63vrvXo1/xyrHoe8mVmOrUvI+wuZmVmOOeTNzHLMIW9mlmMO\neTOzHHPIm5nlmEPezCzHHPJmZjnmkDczyzGHvJlZjjnkzcxyzCFvZpZjDnkzsxxzyJuZ5ZhD3sws\nxxzyZmY55pA3M8sxh7yZWY455M3Mcswhb2aWYw55M7Mcc8ibmeWYQ97MLMcc8mZmOVZSyEsaImmm\npJckndNEu30lLZd0ZPlKNDOzlmo25CV1Aq4CDgN2BYZJ2rmRdr8AHih3kWZm1jKl9OQHAi9HxKyI\nWA6MAYauod0pwB3A22Wsz8zM1kEpId8TmF1wf042bRVJ2wJfj4hrAZWvPDMzWxfl2vB6OVA4Vu+g\nNzNrAzqX0GYu0Kvg/nbZtEIDgDGSBGwJHC5peUSMK57ZyJEjV/1eU1NDTU3NWpZsZpZvtbW11NbW\nlmVeioimG0jrAS8ChwDzgMnAsIiY0Uj7G4E/R8Rda3gsmluemZl9kiQiokUjJM325CNipaSTgQdJ\nwzujI2KGpBHp4bi++CktKcTMzMqv2Z58WRfmnryZ2Vpbl568j3g1M8sxh7yZWY455M3Mcswhb2aW\nYw55M7Mcc8ibmeWYQ97MLMcc8mZmOeaQNzPLMYe8mVmOOeTNzHLMIW9mlmMOeTOzHHPIm5nlmEPe\nzCzHHPJmZjnmkDczyzGHvJlZjjnkzcxyzCFvZpZjDnkzsxxzyJuZ5ZhD3swsxxzyZmY55pA3M8sx\nh7yZWY455M3Mcswhb2aWYw55M7Mcc8ibmeWYQ97MLMcc8mZmOeaQNzPLMYe8mVmOOeTNzHLMIW9m\nlmMlhbykIZJmSnpJ0jlreHy4pKnZ7QlJu5e/VDMzW1uKiKYbSJ2Al4BDgDeBKcDRETGzoM1+wIyI\nWCxpCDAyIvZbw7yiueWZmdknSSIi1JLnltKTHwi8HBGzImI5MAYYWtggIp6MiMXZ3SeBni0pxszM\nyquUkO8JzC64P4emQ/z7wH3rUpSZmZVH53LOTNLBwPHAgY21GTly5Krfa2pqqKmpKWcJZmbtXm1t\nLbW1tWWZVylj8vuRxtiHZPfPBSIiRhW12wO4ExgSEa82Mi+PyZuZraXWHpOfAuwkqbek9YGjgXFF\nBfQiBfyxjQW8mZlVXrPDNRGxUtLJwIOkD4XRETFD0oj0cFwP/BfQA7hGkoDlETGwNQs3M7PmNTtc\nU9aFebjGzGyttfZwjZmZtVMOeTOzHHPIm5nlmEPezCzHHPJmZjnmkDczyzGHvJlZjjnkzcxyzCFv\nZpZjDnkzsxxzyJuZ5ZhD3swsxxzyZmY55pA3M8sxh7yZWY455M3Mcswhb2aWYw55M7Mcc8ibmeWY\nQ97MLMcc8mZmOeaQNzPLMYe8mVmOOeTNzHLMIW9mlmMOeTOzHHPIm5nlmEPezCzHHPJmZjnmkDcz\nyzGHvJlZjjnkzcxyzCFvZpZjDnkzsxxzyJuZ5VhJIS9piKSZkl6SdE4jba6U9LKkZyX1L2+ZZmbW\nEs2GvKROwFXAYcCuwDBJOxe1ORzYMSI+C4wArmuFWnOltra22iW0GV4XDbwuGnhdlEcpPfmBwMsR\nMSsilgNjgKFFbYYCNwNExCSgm6Sty1ppzvgN3MDrooHXRQOvi/IoJeR7ArML7s/JpjXVZu4a2piZ\nWYV5w6uZWY4pIppuIO0HjIyIIdn9c4GIiFEFba4DxkfE2Oz+TOCgiJhfNK+mF2ZmZmsUEWrJ8zqX\n0GYKsJOk3sA84GhgWFGbccCPgLHZh8Ki4oBflyLNzKxlmg35iFgp6WTgQdLwzuiImCFpRHo4ro+I\neyV9WdIrwPvA8a1btpmZlaLZ4RozM2u/WmXDqw+eatDcupA0XNLU7PaEpN2rUWcllPK+yNrtK2m5\npCMrWV8llfg/UiPpGUnTJY2vdI2VUsL/yGaSxmVZ8Zyk71ahzFYnabSk+ZKmNdFm7XMzIsp6I31w\nvAL0BroAzwI7F7U5HPhr9vsg4Mly19EWbiWui/2AbtnvQzryuiho9wjwF+DIatddxfdFN+B5oGd2\nf8tq113FdfET4JL69QC8A3Sudu2tsC4OBPoD0xp5vEW52Ro9eR881aDZdRERT0bE4uzuk+T3+IJS\n3hcApwB3AG9XsrgKK2VdDAfujIi5ABGxoMI1Vkop6yKArtnvXYF3ImJFBWusiIh4AljYRJMW5WZr\nhLwPnmpQyroo9H3gvlatqHqaXReStgW+HhHXAnneE6uU90VfoIek8ZKmSDq2YtVVVinr4ipgF0lv\nAlOB0ypUW1vTotwsZRdKqwBJB5P2Sjqw2rVU0eVA4ZhsnoO+OZ2BvYHBwCbAREkTI+KV6pZVFYcB\nz0TEYEk7Ag9J2iMi3qt2Ye1Ba4T8XKBXwf3tsmnFbbZvpk0elLIukLQHcD0wJCKa+rrWnpWyLgYA\nYySJNPZ6uKTlETGuQjVWSinrYg6wICI+Aj6S9DiwJ2n8Ok9KWRfHA5cARMSrkl4HdgaeqkiFbUeL\ncrM1hmtWHTwlaX3SwVPF/6TjgONg1RG1azx4KgeaXReSegF3AsdGxKtVqLFSml0XEdEnu+1AGpf/\nYQ4DHkr7H7kHOFDSepI2Jm1om1HhOiuhlHUxCzgUIBuD7gu8VtEqK0c0/g22RblZ9p58+OCpVUpZ\nF8B/AT2Aa7Ie7PKIGFi9qltHieviE0+peJEVUuL/yExJDwDTgJXA9RHxQhXLbhUlvi8uBm4q2LXw\n7Ih4t0rR4OOGAAAAS0lEQVQltxpJtwE1wBaS3gDOB9ZnHXPTB0OZmeWYz0JpZpZjDnkzsxxzyJuZ\n5ZhD3swsxxzyZmY55pA3M8sxh7yZWY455M3Mcuz/A4i4hvHTpUqXAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x144730898>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 231 ms, sys: 11.8 ms, total: 243 ms\n",
      "Wall time: 311 ms\n"
     ]
    }
   ],
   "source": [
    "rf = sklearn.ensemble.RandomForestClassifier()\n",
    "rf.max_depth = 5\n",
    "clf = rf\n",
    "%time clf.fit(X[tidx],y[tidx])\n",
    "\n",
    "print(\"TEST\")\n",
    "%time evaluate(clf,X[pidx],y[pidx])\n",
    "\n",
    "print(\"TRAIN\")\n",
    "%time evaluate(clf,X[tidx],y[tidx])\n",
    "\n",
    "print(\"ROC AUC\")\n",
    "%time roc_auc(clf,X[pidx],y[pidx])\n",
    "\n",
    "print(\"PRECISION/RECALL AUC\")\n",
    "%time prrc_auc(clf,X[pidx],y[pidx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/aduriseti/anaconda/lib/python3.5/site-packages/sklearn/neural_network/multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 388 ms, sys: 34.1 ms, total: 422 ms\n",
      "Wall time: 468 ms\n",
      "TEST\n",
      "Accuracy 0.642857142857 Precision 0.727272727273 Recall 0.64 F1 0.68085106383\n",
      "CPU times: user 2.14 ms, sys: 1.01 ms, total: 3.15 ms\n",
      "Wall time: 2.57 ms\n",
      "TRAIN\n",
      "Accuracy 0.789743589744 Precision 0.798245614035 Recall 0.834862385321 F1 0.816143497758\n",
      "CPU times: user 2.41 ms, sys: 816 µs, total: 3.23 ms\n",
      "Wall time: 2.52 ms\n",
      "ROC AUC\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXkAAAEKCAYAAAD3tSVSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGDFJREFUeJzt3X2wXHV9x/H3BwJEJGAilioIpUQEqQGpPIi0rIAQpNNY\nEHloxeCgmaE4GWcqgU6VS8cp8kdbqoxaHIyFyIQgVYIiDyK3FjWICkQh4bkhCTQREp6hRvj2j3Nu\ncrLs3j1379mHc87nNbOT3b3nnv3dk+R33/vbs/cqIjAzs2raZtADMDOz3vEkb2ZWYZ7kzcwqzJO8\nmVmFeZI3M6swT/JmZhXmSd76StLVkv5y0OOYDEkXSLp80OPoBUl/IOl+SdsNeixWDE/yNSTpfyS9\nJOk5SU9IWihpx6ZtjpB0W7rNRknXS9q/aZtpki6VtCrd7iFJ/yJpRpvHfTcwKyKWNt3fkPSapM82\n3X+UpNUt9nO7pE9kbu8raYmk36ZjvUfSZySpzTh2Ssf5mKTn0+OxRNKhHQ8eEBEXR8Sn8mxbFEnb\nS/qGpGfTv7PPjLPtH6Z/X2vT47pni22OlfRLSS9IelzSRwAiYj3wI2Be774a6ydP8vUUwIkRsTNw\nEPAe4IKxD0p6H3Az8B3grcDewHLgJ5L+KN1mO5LJYH/guHRf7wOeAtpNlvOAb7W4/0zg6fTPVmNt\nS9I+wDJgFfAnETEdOAU4GJjWYvvtgduBA4APATunX8NiYPZ4jzVgFwH7AG8HjgbOk3Rcm21fA34A\nnESL4yfpXSR/DxeQfP0HAr/MbHI1nuSrIyJ8qdkFeAw4OnP7EuCGzO0fA19u8Xk3At9Mr58NPAm8\nYQKP+whwRNN9OwLPAR8FXgEOznzsKODxFvu5HfhEen1Rduw5xnA2sBaY2mG7S4HHgWeBu4AjMx+7\nELgqvb4XyaR6Jsk3mvXA3/fg72wtcEzm9kXA1R0+Z9t0bHs23f8t4KIOn/ci8PZB/1v1ZfIXl3zN\nSdoDOAF4KL39BuAI4NstNl8CfDC9fgxwU0S8nPNxdiR5RvBA04dOBp4HrgVuAT4+wS/hmDZjHW/7\nmyPilQ7b/RyYBUwnKdtr02cBY5oL+f3AO4Bjgc9LemernUpakC4pbUj/zF7f0OZz3kTyjGp55u57\nSZ6NdOPwZLdani7pXClp+uYvLOJV4GGSwreS8yRfX9+V9BxJra4DRtL7Z5D8u3iyxec8CeyaXn9z\nm23aeRPJxPh80/1nAosjScirgdMkbTuB/U50HLsC/zt2Q9KB6QT7rKQVY/dHxNUR8UxEvBYR/wrs\nALScuEm+rpGI+F1ELCeZgFtOkBFxSURMj4gZ6Z/Z6y1fywB2Sh/j2cx9z9FiOSqnPYC/Af6K5BvT\njsCXm7Z5nuTvzErOk3x9zYlkHf0oYD+2TN4bSZ7iv7XF57yVZM0dkjX0Vtu080z65+aJKX0W8QGS\nyR1gKfAG4MT09u+BVmd5bAds6nIcW20fEfdGso5/EslEPja2v0vPMtkoaSPJ2vWur9vbFusy118i\nmZiL8kL6586Z+3bh9d8w83oZ+EZEPBIRLwH/RPJsLmsaW/7OrMQ8ydeXACLiv4H/AP45vf0S8DOS\nFy+bfRT4YXr9h8Dx6fJOR+l+HwH2zdx9ZjqOGyQ9mX58B7Ys2TwO7Np85g/JOviqzDhOzjOG1G3A\nceONW9KfAZ8FPjJW2yTl3PJsnYlIT798Pj0bKXt5Pn1m9ToR8QzJs5Xss4MDgfu6HMby8T6YPpOa\nSfKMxErOk7xB8iLjB9NTHAHOBz4u6dz0dMPpkr5Aspb7j+k2VwGrgeskvVOJN6eTWLuzVG4keeYw\n5kySZaKDSCatA4GPACdKmh4Rq4E7gUskvTE9jfA84HckZ9RA8iLoEZIukbQbgKSZkq6SlC3fMVeS\nTJjfkXSApG0k7QAcktlmJ5JnCk+nj/l5xl8ayT35R3L65bSI2LnpMi19ZtXOVcA/SHpTeirrJ4GF\nbQeUfE1T05tT09tjFgJnSdo7/Qa6ALgh8/FDgcfS428l50m+nrZ60TAiniKp+c+nt38CHE9SyE+S\nnI1zIPD+iHgk3eZ3JC8yrgRuJVkvXkayRn5nm8f9OslaMJIOA/YEvhIR6zOXG0heBD49/ZxTgd1I\nXghcQ7K8c2L6+ETEoySnbu4N3JcurVxLckbM65YzIuL/0n3cD3w/HfdK4E9JnqlAcvrozcCD6df+\nEsk3tHaaX4TtxS9puBB4lOQZzI+AL0bErWMfTJ8JvD+z/cskzz6C5Ot7afPgIhaSfLO7k+TrexmY\nn/ncvwa+1oOvwQZAyetd42wgXQH8BbAuIma12eZLJGt6LwJzI+Keogdq1SBpEbAkmt4QZcNB0luA\nUeA9Y99IrdzyTPJHkrzwc2WrSV7SCcC5EXFiWmf/FhGH92S0ZmY2IR2XayLiDpIzLtqZQ/LUj4i4\nE9hlbG3UzMwGq4g1+d3Zer1ybXqfmZkNmF94NTOrsCkF7GMtyQ9NGrNHet/rSOrFWQdmZpUXEV29\nTyNvyYv25wIvJf3pgZIOB56JiHVtth34D+sZlsuFF1448DEMy8XHwsfCx2LLZd0L6zj5mpPZ/7L9\nWbZ6GRGTa+OOk7ykq4GfAvumP3f6LEnzJH0qnbRvBB6T9DDw78A5kxqRmVkNRQTX/OYaZn11FjNn\nzORX837FYXscNun9dlyuiYgzcmxz7qRHYmZWU+tfXM853z+H+397P9efdn0hk/sYv/A6II1GY9BD\nGBo+Flv4WGxRh2PRq3rP6vhmqEIfTIp+Pp6Z2bDK1vvCOQvHndwlET1+4dXMzArQj3rPKuIUSjMz\ny6GXa+/tuOTNzHqs3/We5TV5MyulGTNg43g/VWtYvHE9nHgOvOV++O5CWNvN5O41eTOrmY0bIWJ4\nL6+9Fiz+9TXsNjKLBZ+cycuX/opYc1hX+5oMr8mbmRVsEGvv7bjkzcwKMsi193Zc8mZmBRimes9y\nyZuZTcIw1nuWS97MrEvDWu9ZLnkzswka9nrPcsmbWV8UfV779OnF7WsiylDvWS55M+uLos9r37Ch\nv+MvU71nueTNzDooW71nueTNzNooa71nueTNzFooc71nueTNzDKqUO9ZLnkzs1RV6j3LJW9mtVe1\nes9yyZtZrVWx3rNc8mZWS1Wu9yyXvJnVTtXrPcslb2a1UZd6z3LJm1kt1Knes1zyZlZpdaz3LJe8\nmVVWXes9yyVvZpVT93rPcsmbWaW43rfmkjezSnC9t+aSN7PSc72355I3s9JyvXfmkjezUnK95+OS\nN7NScb1PjEvezErD9T5xuUpe0mxJKyU9KGlBi4/vLGmppHsk/VrS3MJHama15XrvniJi/A2kbYAH\ngWOAJ4C7gNMiYmVmmwuAnSPiAkm7Ag8Au0XE75v2FZ0ez8yqSYJu/vtn633hnIW1nNwlERHq5nPz\nlPyhwEMRsSoiNgGLgTlN2wQwLb0+DXi6eYI3M5sI13sx8qzJ7w6sztxeQzLxZ10GLJX0BLATcGox\nwzOzOvLae3GKeuH1eODuiDha0j7ArZJmRcQLzRuOjIxsvt5oNGg0GgUNwaz3ZsyAjRsHPYpymj69\n8zYRwZL7ljD/pvnMPWgui05axNQpU3s/uCEzOjrK6OhoIfvKsyZ/ODASEbPT2+cDERGXZLb5HnBx\nRPwkvX0bsCAiftG0L6/JW6l1u65snXntvb1er8nfBcyUtJek7YHTgKVN26wCjk0HsxuwL/BoNwMy\ns3rx2ntvdVyuiYhXJZ0L3ELyTeGKiFghaV7y4bgc+ALwTUnL0087LyI29GzUZlYJXnvvvY7LNYU+\nmJdrrOS8XFOM5rX3kcZILdfe85rMco3f8WpmfeV67y//7Boz6wuvvQ+GS97Mes71PjgueTPrGdf7\n4LnkrfKKfANTnjf0WML1Phxc8lZ5GzcmZ8QUcdngE4M7cr0PF5e8mRXG9T58XPJmNmmu9+Hlkjez\nSXG9DzeXvJl1xfVeDi55M5sw13t5uOTNLDfXe/m45M0sF9d7ObnkzWxcrvdyc8mbWVuu9/JzyZvZ\n67jeq8Mlb2Zbcb1Xi0vezADXe1W55M3M9V5hLnmzGnO9V59L3qymXO/14JI3qxnXe7245G3oFPmb\nnMC/zSnL9V4/LnkbOkX+Jif/NqeE672+XPJmFed6rzeXvFlFud4NXPJmleR6tzEuebMKcb1bM5e8\nWUW43q0Vl7xZybnebTwuebMSc71bJy55m7QZM0Aq7uI3L3Xmere8XPI2aWNvXrL+cL3bRLjkzUrC\n9W7dcMmblYDr3bqVq+QlzZa0UtKDkha02aYh6W5Jv5F0e7HDNKsn17tNlqLDYqqkbYAHgWOAJ4C7\ngNMiYmVmm12AnwLHRcRaSbtGxFMt9hWdHs/KR/KafC9k633hnIWe3GtMEhGhbj43T8kfCjwUEasi\nYhOwGJjTtM0ZwHURsRag1QRvZvm43q1IedbkdwdWZ26vIZn4s/YFtkuXaXYCvhQRVxUzRLP68Nq7\nFa2os2umAAcDJwCzgc9JmlnQvs0qz/VuvZKn5NcCe2Zu75Hel7UGeCoiXgFekfRj4EDg4eadjYyM\nbL7eaDRoNBoTG7FZxbjerdno6Cijo6OF7CvPC6/bAg+QvPD6JPBz4PSIWJHZZj/gyyQVvwNwJ3Bq\nRNzftC+/8FpBfuG1OxHBkvuWMP+m+cw9aC4jjRGmTpk66GHZEJrMC68dSz4iXpV0LnALyfLOFRGx\nQtK85MNxeUSslHQzsBx4Fbi8eYI3sy1c79YvHUu+0AdzyVeSSz4/17t1o6clb2bFcL3bIPhn15j1\nmM+csUFyyZv1kOvdBs0lb9YDrncbFi55s4K53m2YuOTNCuJ6t2HkkjcrgOvdhpVL3mwSXO827Fzy\nZl1yvVsZuOTNJsj1bmXikjebANe7lY1L3iwH17uVlUverAPXu5WZS96sDde7VYFLvqZmzICNG4vZ\n1/TpxexnmLjerSpc8jW1cWPyM+CLuGzYMOivpjiud6sal7xZyvVuVeSSt9pzvVuVueSt1lzvVnUu\neasl17vVhUveasf1bnXikrfacL1bHbnkrRZc71ZXLnmrNNe71Z1L3irL9W7mkrcKcr2bbeGSt0px\nvZttzSVvleB6N2vNJW+l53o3a88lb6XlejfrzCVvpeR6N8vHJW+l4no3mxiXfEkU+ZucoJy/zcn1\nbjZxLvmSKPI3OZXttzm53s2655K3oeZ6N5scl7wNJde7WTFylbyk2cClJN8UroiIS9psdwjwU+DU\niPjPwkZpteJ6NytOx5KXtA1wGXA8cABwuqT92mz3ReDmogdp9eB6NytenpI/FHgoIlYBSFoMzAFW\nNm33aeDbwCGFjtBqwfVu1ht51uR3B1Znbq9J79tM0tuAD0fEVwEVNzyrOte7WW8VdXbNpcCCzG1P\n9NaR692s9/JM8muBPTO390jvy3ovsFiSgF2BEyRtioilzTsbGRnZfL3RaNBoNCY4ZCu7iGDJfUuY\nf9N85h40l0UnLWLqlKmDHpbZ0BgdHWV0dLSQfSkixt9A2hZ4ADgGeBL4OXB6RKxos/1C4IZWZ9dI\nik6PZ61JyZuYyi5b7wvnLHS9m+UgiYjoaoWk45p8RLwKnAvcAtwHLI6IFZLmSfpUq0/pZiBWbV57\nNxuMjiVf6IO55LtW5pJ3vZtNTk9L3qxbrnezwfPPrrGe8JkzZsPBJW+Fcr2bDReXvBXG9W42fFzy\nNmmud7Ph5ZK3SXG9mw03l7x1xfVuVg4ueZsw17tZebjkLTfXu1n5uOQtF9e7WTm55G1crnezcnPJ\nW1uud7Pyc8nb67jezarDJW9bcb2bVYtL3gDXu1lVueTN9W5WYS75GnO9m1WfS76mXO9m9eCSrxnX\nu1m9uORrxPVuVj8u+RpwvZvVl0u+4lzvZvXW95KXfOnmMn36xI6z693MYAAlH9HvR6wf17uZjfGa\nfIW43s2smdfkK8L1bmatuORLzvVuZuNxyZeY693MOnHJl5Dr3czycsmXjOvdzCbCJV8Srncz64ZL\nvgRc72bWLZf8EHO9m9lkueSHlOvdzIrgkh8yrnczK5JLfoi43s2saLlKXtJsSSslPShpQYuPnyHp\n3vRyh6R3Fz/U6nK9m1mvdCx5SdsAlwHHAE8Ad0m6PiJWZjZ7FPjziHhW0mzg68DhvRhw1bjezayX\n8pT8ocBDEbEqIjYBi4E52Q0iYllEPJveXAbsXuwwq8f1bmb9kGdNfndgdeb2GpKJv52zgR9MZlBV\n53o3s34p9IVXSR8AzgKObLfNyMjI5uuNRoNGo1HkEIZaRLDkviXMv2k+cw+ay6KTFjF1ytRBD8vM\nhszo6Cijo6OF7EvR4Vc1STocGImI2ent84GIiEuatpsFXAfMjohH2uwrOj1eVWXrfeGcha53M8tN\nEhGhbj43z5r8XcBMSXtJ2h44DVjaNIA9SSb4j7Wb4OvKa+9mNkgdl2si4lVJ5wK3kHxTuCIiVkia\nl3w4Lgc+B8wAviJJwKaIGG/dvha89m5mg9ZxuabQB6vJck3z2vtIY8Rr72bWtcks1/gdrwVzvZvZ\nMPHPrimI197NbBi55AvgejezYeWSnwTXu5kNO5d8l1zvZlYGLvkJcr2bWZm45CfA9W5mZeOSz8H1\nbmZl5ZLvwPVuZmXmkm/D9W5mVeCSb8H1bmZV4ZLPcL2bWdW45FOudzOrotqXvOvdzKqs1iXvejez\nqqtlybvezawualfyrnczq5PalLzr3czqqBYl73o3s7qqdMm73s2s7ipb8q53M7MKlrzr3cxsi0qV\nvOvdzGxrlSh517uZWWulL3nXu5lZe6Utede7mVlnpSx517uZWT6lKnnXu5nZxJSm5F3vZmYTN/Ql\n73o3M+veUJe8693MbHKGsuRd72ZmxRi6kne9m5kVZ2hK3vVuZla8oSh517uZWW/kKnlJsyWtlPSg\npAVttvmSpIck3SPpoDz7db2bmfVWx0le0jbAZcDxwAHA6ZL2a9rmBGCfiHgHMA/4Wqf9rn9xPadc\newoX/ddFXH/a9Xzx2C8ydcrUrr6IMhodHR30EIaGj8UWPhZb+FgUI0/JHwo8FBGrImITsBiY07TN\nHOBKgIi4E9hF0m6tduZ6T/gf8BY+Flv4WGzhY1GMPGvyuwOrM7fXkEz8422zNr1vXfPOTrn2FK+9\nm5n1Sd9feJ05YyaLTlpUq6UZM7NBUUSMv4F0ODASEbPT2+cDERGXZLb5GnB7RFyT3l4JHBUR65r2\nNf6DmZlZSxGhbj4vT8nfBcyUtBfwJHAacHrTNkuBvwWuSb8pPNM8wU9mkGZm1p2Ok3xEvCrpXOAW\nkhdqr4iIFZLmJR+OyyPiRkkfkvQw8CJwVm+HbWZmeXRcrjEzs/LqyY816NWbp8qo07GQdIake9PL\nHZLePYhx9kOefxfpdodI2iTppH6Or59y/h9pSLpb0m8k3d7vMfZLjv8jO0tams4Vv5Y0dwDD7DlJ\nV0haJ2n5ONtMfN6MiEIvJN84Hgb2ArYD7gH2a9rmBOD76fXDgGVFj2MYLjmPxeHALun12XU+Fpnt\nbgO+B5w06HEP8N/FLsB9wO7p7V0HPe4BHosLgIvHjgPwNDBl0GPvwbE4EjgIWN7m413Nm70o+ULf\nPFVyHY9FRCyLiGfTm8tI3l9QRXn+XQB8Gvg2sL6fg+uzPMfiDOC6iFgLEBFP9XmM/ZLnWAQwLb0+\nDXg6In7fxzH2RUTcAWwcZ5Ou5s1eTPKt3jzVPHG1e/NU1eQ5FllnAz/o6YgGp+OxkPQ24MMR8VWg\nymdi5fl3sS8wQ9Ltku6S9LG+ja6/8hyLy4B3SXoCuBeY36exDZuu5s2h+CmUBpI+QHJW0pGDHssA\nXQpk12SrPNF3MgU4GDgaeCPwM0k/i4iHBzusgTgeuDsijpa0D3CrpFkR8cKgB1YGvZjk1wJ7Zm7v\nkd7XvM3bO2xTBXmOBZJmAZcDsyNivKdrZZbnWLwXWCxJJGuvJ0jaFBFL+zTGfslzLNYAT0XEK8Ar\nkn4MHEiyfl0leY7FWcDFABHxiKTHgP2AX/RlhMOjq3mzF8s1m988JWl7kjdPNf8nXQqcCZvfUdvy\nzVMV0PFYSNoTuA74WEQ8MoAx9kvHYxERf5xe9iZZlz+nghM85Ps/cj1wpKRtJe1I8kLbij6Psx/y\nHItVwLEA6Rr0vsCjfR1l/4j2z2C7mjcLL/nwm6c2y3MsgM8BM4CvpAW7KSKafwBc6eU8Flt9St8H\n2Sc5/4+slHQzsBx4Fbg8Iu4f4LB7Iue/iy8A38ycWnheRGwY0JB7RtLVQAN4s6THgQuB7ZnkvOk3\nQ5mZVdjQ/I5XMzMrnid5M7MK8yRvZlZhnuTNzCrMk7yZWYV5kjczqzBP8mZmFeZJ3syswv4fdxCS\nMU+ru1sAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1481d3e48>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 187 ms, sys: 6.17 ms, total: 193 ms\n",
      "Wall time: 202 ms\n",
      "PRECISION/RECALL AUC\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXkAAAEKCAYAAAD3tSVSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHOFJREFUeJzt3XmUXGWd//H3JwnBhZCwiI6JRBYDQsiCJgR/8LMJKkEd\ngxlkhEEEXDgqyOiZAVT8EQVFmHFEVgfJCKiAHkGJCrL3QJQlLEkgG03AQBYkgUBCFOyE7++P53a6\nUvRS3V1L58nndc49qbr1VN1v3VR/7lPPXUoRgZmZ5WlAowswM7PaccibmWXMIW9mljGHvJlZxhzy\nZmYZc8ibmWXMIW9bJEmvSdq9uP0TSd/uou3OkhZK2rZ+FVafpMck/d9G11ELkk6W9L1G15Ejh3w/\nIOnPkv4qaa2klUVovanRdVWiqPXVovbVkm6RtFcdFt2TEzzOAH4SEa+WzpR0paRWSW8tm/+6jYak\nkcWGZUDJvGMkzZa0TtJySb+X9H86K0LSeyX9VtILxfSYpLMlDa3kTUTE6Ii4u6J3XCWSxkl6UNL6\n4r2O7aLtJyT9sWh7Z9ljBxXraW0xrSvW58eLJj8G/kXSzrV8P1sjh3z/EMBHImJ7YH/gvcCZHTWU\npHoWVqHzitqHAyuAK+qwzIrWg6TBwKeBn5XNfxMwDXgROLbCZW7asEj6KvBfwDnALsCuwCXAP3ZS\nx/uAu4B7gL0iYkdgCrAB6DQ4G0nSNsBvgKuBYcW/N0oa1MlTngd+AJxb/kBEzIqIIRGxffFZ+Siw\nDvhD8firwE3AcVV/I1s5h3z/IYCIWAncDIwGkHSXpHMkzZK0HthN0vaSZkhaIemZoje4KfQkfU7S\ngqLH9Jikca9bmHSppP8om/cbSf9a3D5d0rLiNRZKOqS7N1D8of4S2Gx5kk4s6nle0s2Sdi15bF9J\ntxaPrZR0RjF/gqQ/SVpT9JIv6iJcunIAsCYiVpTNPxJYA3wbOL4nLyhpe+BbwBcj4saI+FtEbIyI\nmyLijE6edh4wIyLOj4hVABGxLCK+1dY7l7S7pDuKb0TPSfpZsay25T4laXJx+yxJv5B0VfF/9Kik\n/XvyPirQBAyMiAsjojUiLiJ9Tid31Dgi7oyIXwErK3jt44FfRcTfSub9L/CRvpVs5Rzy/YykdwAf\nBh4umX0s8FlgCPA0cBXwKrA7MB74YPE4kj4B/D/g2KLH9DFSD6vctcBRJcsdVrzOtZJGAV8C3lO8\nxmHAnyuo/c3AMUBLybyppOGSI4C3kHqy1xaPbQfcRurB/QOwJ3BH8dSNwL8COwIHkoLli93V0IH9\ngMUdzD8OuAb4BbC3pPE9eM33AduSerndKr41HAjc0F1T4LvA24B3AyOA6V20/0fSexgK/Jb0TaKz\nGuaWDBOtKfv34k6eti8wr2ze3GJ+rxXr45+AK8seWkg//VazJXPI9x+/kfQCcDfpa33pV94rI2JR\nRLxGCr3Dga9ExCsRsRq4APhk0fYzwPkR8TBARDwZEc+ULywi7gFC0kHFrCOBeyPiL6SAHQyMljQo\nIp6OiKe6qP3fi9rXkgKw9Cv3ScC5EfF4Uf/3gHHFxuyjwMqIuCAi/h4R6yNidlHfwxHxQCRPA5cD\n769gPZYbRhoW2KT4JnEIcE1EPAfcTs+GCXYEVhfvpxI7kP7Wni2p4bwiZF+W9HWAiFgSEXdExIaI\naBv66Oo9z4qIWyJdgOqnwJjOGkbE2IjYsZh2KPv35E6eth3wUtm8taTORl/8E7Cq+AyWWkfaYFkV\nOeT7j6nFH9xuEXFK2U7C0pAeCWwDrGzrjQE/IvWSAd4BLKlwmb8Aji5uHwP8HFLYkHrR04G/SLpG\n0j908Tr/UYwxjwT+BpTueB0J/LCtF0n6VhGk8ftOa5X0rmIn5UpJLwLfAXqzU24Nrw+lTwELIuLR\n4v61pJ1+A4v7G0jruNQ2wGtFsD8P7KySnbAV1PAa6dsKABFxekTsAPwaGAQgaRdJ1xbDZC+S9iN0\n9Z6fLbn9V+ANPaipEi8D25fNG0rZRrMXjiON75cbwus3KtZHDvn+o6sdiaVHkjwDvALsVNIbGxYR\nY0oe36PCZV4LHFn0bA8Art+0wIjrIuJgUkhD6oF3KSKWkTYOF6r9cMWngZPKepHbRcR93dR6Genr\n+x4RMQz4BhXubC0zDxhVNu9TwO7FBmQl8H1gJ9IwWVvN7yx7zu60b2zvJQ2XHVFJARHxV+B+0o7e\nrnyXtDHYt3jPx9K79/w6xb6ZtWVT29Eul3bytPm8/tvBmGJ+b+sYQRrr7yjk300aDrIqcshvYSLi\nWeBW4AeShijZXe3HT18B/FvbTjhJe5Tu6Cx7rTmkXukVwB8iYm3xnFGSDlE6MuXvpN55RUMTEXE7\nsJw0TAPw38DXJe1TvPZQSUcWj/0OeJukL0saLGk7SROLx4YAayPir5L2Br5Q0Qp6vQeAYW3fRCQd\nSArsCaTx37GkMeZrSUfhQNrYfUTSByQNkPR20kbm2uI9rgXOAi6RNFXSGyUNkjRFnR/rfRpwoqTT\nJL2lqGUEsFtJmyGk3vM6ScOBf+/he+10g1Acfrl92dR2tEtn+zqagY2STin+f75M+hzc2VHjYl1t\nS/rWM1DSth3sLD8O+GMnw3/vJx10YNUUEZ4aPAFPApM7eexO4MSyeUOAS0k9yzXAQ8BRJY9/HlhE\nGj+dB4ztYtlnksbgp5XM24/U83wJWA3MBN7WyfP/B/h22byjitq2Ke7/S1HHi8BS4IqStvuQxsRf\nIB1+eVox/2BST34t6aiL6cDdJc/bCOxe3P5JeQ1l9ZxX8rqXAb/soM0E0sZsWHH/I8CDxfp9ivRN\nZtuy5xwNzCYNX6wg7fyc1EUdE4DfF+/1hWKdnA3sULIuHize88PAV4CnO/qckDYyV5c8NrJYJwOq\n/NkcW9S0vvh3TMljxwCPltz/NGkjsLFk+p+y11sAHN/Bct5QfGbe0ui/x9wmFSu4U5JmkHaQ/SXa\nhwTK21xI2hm4vvgPnNPli5rVkdIJNncD46PshCjrHySdDIyIzg9BtV6qJOQPIn2FvLqjkJd0OHBy\nRHxE0gHADyNiUk2qNTOzHul2TD4iZpG+snZmKsVOlIi4HxiqstPEzcysMaqx43U4mx/it7yYZ2Zm\nDeaja8zMMtaba4GUW046qaXNiGLe60jqyZUDzcysEBG9Omei0p686PwY3JkUp4RLmgS8GOnU+A41\n+nCi/jKdddZZdVnOV74SfP/7jX+//WFdbAmT14XXRUdTX3Tbk5d0DekMtZ0kPU06Pndwyuu4PCJu\nkvRhSU+QDqE8oU8VmZlZ1XQb8hFxTAVtOrvAkZmZNZB3vDZIU1NTo0voN7wu2nldtPO6qI5uT4aq\n6sKkqOfyDL76VRgxIv1rZlsmSUSNd7yamdkWyCFvZpYxh7yZWcYc8mZmGXPIm5llzCFvZpYxh7yZ\nWcYc8mZmGXPIm5llzCFvZpYxh7yZWcYc8mZmGXPIm5llzCFvZpYxh7yZWcYc8mZmGXPIm5llzCFv\nZpYxh7yZWcYc8mZmGXPIm5llzCFvZpYxh7yZWcYc8mZmGXPIm5llzCFvZpYxh7yZWcYc8mZmGXPI\nm5llzCFvZpYxh7yZWcYc8mZmGXPIm5llzCFvZpaxikJe0hRJiyQ9Lun0Dh7fXtJMSXMkPSrp+KpX\namZmPdZtyEsaAFwMHAbsCxwtae+yZl8C5kfEOOAQ4PuSBlW7WDMz65lKevITgZaIWBoRrcB1wNSy\nNgEMKW4PAZ6PiA3VK9PMzHqjkpAfDjxTcn9ZMa/UxcA+klYAc4FTq1OemZn1RbWGVA4DHomIyZL2\nAG6TNCYiXi5vOH369E23m5qaaGpqqlIJZmZ5aG5uprm5uSqvpYjouoE0CZgeEVOK+2cAERHnlbT5\nHXBuRPyxuH8HcHpEPFj2WtHd8qy6vvpVGDEi/WtmWyZJRIR689xKhmtmA3tKGilpMPBJYGZZm6XA\nB4pi3gqMAp7sTUFmZlY93Q7XRMRGSScDt5I2CjMiYqGkk9LDcTlwDnClpHnF006LiBdqVrWZmVWk\nojH5iPgDsFfZvP8uub2SNC5vZmb9iM94NTPLmEPezCxjDnkzs4w55M3MMuaQNzPLmEPezCxjDnkz\ns4w55M3MMuaQNzPLmEPetjjPPgu33troKsy2DP71JtsiLFkCv/51mhYsgBdfhA0bYODARlfWeBGg\nXl2f0LYGDnnrlyJg3rz2YH/2WZg6Fc48EyZPhje9qdEVNsaGDTB/Ptx/f5ruuw/WrIEVKxpdmfVX\nDnnrN157De69tz3YX3sNpk2DSy6BAw+sX689AhYtSkNCt90GO+8MV15Zn2WXW7GiPczvvx8eegiG\nD4cDDoBJk+CznwX/7o51xSFvDdXaCnfemUL9N7+BXXaBj38crr8exo6t3zDE6tVw++3twT5gAHzo\nQzBxIsws//WEGvnb31KItwX6/ffD+vXtgf61r6V6dtih/Tmvvlqf2mzL5ZC3uotIYXb11XDddbDH\nHqnHPmsW7LlnfWp49VX4059SqN96KzzxBLz//SnYzzgD3vWutIF58MHahfyaNfDHP8I996T3PmcO\n7LNPCvSpU+Hcc9O68Xi79YVD3urmmWfgZz9L4d7aCscdl3qtu+9en+UvWgR/+EMK9Vmz4N3vTqF+\nwQUpWLfZprbLX748BXrb9NRTqWd+8MHw7W+nGt785totf8UKeOCBND30EHzrW2mZljeHvNXUunVw\nww0p2OfMgU98AmbMSGPste6hbtyYeuszZ6Zp/Xo4/HA48cS0sdlxx9otOwIef3zzUH/pJTjooBTq\nxx0H++9fuw3L2rXpW0hbqD/wALzyShr6mTgxrYuWltqEfGsrLFwIjzySppaWtL5Lh5msfhzyVnUb\nN8Idd6Rg/93v0jDIF74AH/0ovOENtV32+vWpp37jjfD736edlFOnwjXXpFCt5YZl6dL0vm+/Pe1n\nGDw4BfrBB8Npp6VvDgNqcGZK2/BXaaAvXQrjxqVAP+oo+M//hN12a3//TzxRnWWvW5eOgmoL9Dlz\nUsC/851p+ePHpyG5Vasc8o3ikLeqmT8frroKfv5zePvbU2/1Bz+At7yltstduRJ++9vUW7/77tRb\n/djH0nDEyJG1W+7q1XDXXe3Bvm5dOrzz0EPhO99JoVprAwemw0lPOCEF+oEHwqmnwr77Vv9bwqpV\naWNSGujLl8Po0SnQJ0yAz30O9ttv82GnH/+4unVYzzjkrU82bEjhetFFsHgxfPrT6eiUffap/bLP\nPTeFe0sLTJkCxx6bNjBDh9ZmeevXp7H8229Pwb5kSeqlH3oofPGLKexq0VPvyqBBaQdutb+hvPhi\nCvTZs9Owz+zZabjpPe9JvfOpU+Gss2CvvVIN1dDamj5Dc+emDciUKWndWt845K1Xnn8errgCLr00\nDYl8+cvpCJnBg+uz/MmT4bnn4LvfTUFby+UuXZqORX/ooRRwH/hA2qhNnFj7nbWV6GvAr18PDz/c\nHuYPPph20o4fn3rn06a1H+lTrY3YmjUpzNumOXPSjvFdd02Hzq5ald6XQ77vHPLWI3PnpoC7/vrU\nm7vhhtS7q7fbbqvPcvbcM+1PeN/70saklke/1Nsll8B558GTT6ZvIRMmwAc/CF//etp/UM2Tz266\nCX760/ZQf+EFGDMmBfqkSXDSSamGtvV7/vlpOMz6ziFv3dqwIZ2odNFFaYjiC19IX6t32aXRldXe\nsGHp8MbcnHhi+j+cMCGNodfym9CBB6Yd0ePGpeG8sWPTYbP1HtraWjnkrVOrV6edZpddlnZgnnJK\nOhu1PwxRWN8cckia6uGqq+qzHOuYt6X2OosXw2c+k876bGlJhyPec086FM8Bb7ZlcU/eNmlpSUMT\nt9ySdqS2tKSLc5nZlss9eWPJEjj++LRzca+90okyZ57pgDfLgUN+K/bkk2kH3AEHpBN3WlpSuG+/\nfaMrM7Nqcchvhf7853Rm4oQJMGJECvezzkpHkphZXhzyW5Gnn07HI7/nPfDWt7aPwfuaImb5cshv\nBZYtS6fdjx+frry4eDGcc05tr8JolqtXXkln6La0NLqSyvjomswNHJguPXDyyem08VpfLMwsFxs2\npIMSHnssTY8+mv5dujR9+x0/Pl3ptL9TRNRvYVLUc3mWrjHz2msOd9uytF3W4Pzza7+siDSU2Rbm\nbdPixelqqqNHbz6NGpUuUHfhhXDzzV2/dmtrOlpt/nxYsCD9cM5FF/X8ktuSiIheXaXIPfnM7bRT\noysw6z9efjkFeNs1dObNSz307bZLl3cYPTpdFO3UU9P1eyq9VtHf/56GbxYsSFNbqC9ZAu94R7oq\n6z77wC9/mQ5yGDGitu+zlEPezLITkY4iawvytn+XL09h23ZxtKOOSuHem87QwoXpl87mz08/5bjr\nrum1990XjjgCvvGN1Ot/4xvbn3P11VV7ixWrKOQlTQEuIO2onRER53XQpgn4AbANsCoi6nRlDDPb\nmr36auqdt/2Yydy5qXc+ZEgK8jFjUhiffXYK3Wpc/378+PRDLaNGwTe/mf6t9a+e9Va3b1fSAOBi\n4FBgBTBb0o0RsaikzVDgEuBDEbFcks+VNLOqW7cuhfgjj6Rr4D/ySPot3T32SD/vOG4cHHlkCvZa\nDlW+7W1p2GVLUMk2bSLQEhFLASRdB0wFFpW0OQa4PiKWA0SErwRtZn3y0kvpV7jawvzhh9OOy9Gj\nU0960qR02ev99tt8SMQ2V0nIDweeKbm/jBT8pUYB20i6C9gOuDAiflqdEs1sazNsGFx7bTrsd/x4\nOPzw9GMme+/tK6H2VLV2vA4C9gcmA28G7pV0b0RU6TfhzWxr8vnPp0tvVPu3a7dGlYT8cmDXkvsj\ninmllgGrI+IV4BVJdwNjgdeF/PTp0zfdbmpqoqmpqWcVm9lWYWsO+DvuaObXv25m1ar0e7d90e3J\nUJIGAotJO15XAg8AR0fEwpI2ewMXAVOAbYH7gX+OiAVlrxVM71vBZmZbnen0+mSois54LQ6h/CHt\nh1B+T9JJQETE5UWbfwNOADYCP46Iizp4HZ/xamZbrREjYNo0WLEinSxVfnx920lTe+21+c7kvpzx\n6ssamJnVydlnp2vitIX5qFGw7bbdP88hb2aWsb6EvC81bGaWMYe8mVnGHPJmZhlzyJuZZcwhb2aW\nMYe8mVnGHPJmZhlzyJuZZcwhb2aWMYe8mVnGHPJmZhlzyJuZZcwhb2aWMYe8mVnGHPJmZhlzyJuZ\nZcwhb2aWMYe8mVnGHPJmZhlzyJuZZcwhb2aWMYe8mVnGHPJmZhlzyJuZZcwhb2aWMYe8mVnGHPJm\nZhlzyJuZZcwhb2aWMYe8mVnGHPJmZhlzyJuZZcwhb2aWMYe8mVnGHPJmZhmrKOQlTZG0SNLjkk7v\not0ESa2SplWvRDMz661uQ17SAOBi4DBgX+BoSXt30u57wC3VLtLMzHqnkp78RKAlIpZGRCtwHTC1\ng3anAL8CnqtifWZm1geVhPxw4JmS+8uKeZtIejtwRERcBqh65ZmZWV9Ua8frBUDpWL2D3sysHxhU\nQZvlwK4l90cU80q9F7hOkoCdgcMltUbEzPIXmz59+qbbTU1NNDU19bBkM7O8NTc309zcXJXXUkR0\n3UAaCCwGDgVWAg8AR0fEwk7a/wT4bUTc0MFj0d3yzMxsc5KIiF6NkHTbk4+IjZJOBm4lDe/MiIiF\nkk5KD8fl5U/pTSFmZlZ93fbkq7ow9+TNzHqsLz15n/FqZpYxh7yZWcYc8mZmGXPIm5llzCFvZpYx\nh7yZWcYc8mZmGXPIm5llzCFvZpYxh7yZWcYc8mZmGXPIm5llzCFvZpYxh7yZWcYc8mZmGXPIm5ll\nzCFvZpYxh7yZWcYc8mZmGXPIm5llzCFvZpYxh7yZWcYc8mZmGXPIm5llzCFvZpYxh7yZWcYc8mZm\nGXPIm5llzCFvZpYxh7yZWcYc8mZmGXPIm5llzCFvZpYxh7yZWcYc8mZmGXPIm5llrKKQlzRF0iJJ\nj0s6vYPHj5E0t5hmSdqv+qWamVlPKSK6biANAB4HDgVWALOBT0bEopI2k4CFEfGSpCnA9IiY1MFr\nRXfLMzOzzUkiItSb51bSk58ItETE0ohoBa4DppY2iIj7IuKl4u59wPDeFGNmZtVVScgPB54pub+M\nrkP8s8DNfSnKzMyqY1A1X0zSIcAJwEGdtZk+ffqm201NTTQ1NVWzBDOzLV5zczPNzc1Vea1KxuQn\nkcbYpxT3zwAiIs4razcGuB6YEhFLOnktj8mbmfVQrcfkZwN7ShopaTDwSWBmWQG7kgL+U50FvJmZ\n1V+3wzURsVHSycCtpI3CjIhYKOmk9HBcDnwT2BG4VJKA1oiYWMvCzcyse90O11R1YR6uMTPrsVoP\n15iZ2RbKIW9mljGHvJlZxhzyZmYZc8ibmWXMIW9mljGHvJlZxhzyZmYZc8ibmWXMIW9mljGHvJlZ\nxhzyZmYZc8ibmWXMIW9mljGHvJlZxhzyZmYZc8ibmWXMIW9mljGHvJlZxhzyZmYZc8ibmWXMIW9m\nljGHvJlZxhzyZmYZc8ibmWXMIW9mljGHvJlZxhzyZmYZc8ibmWXMIW9mljGHvJlZxhzyZmYZc8ib\nmWXMIW9mljGHvJlZxioKeUlTJC2S9Lik0ztpc6GkFklzJI2rbplmZtYb3Ya8pAHAxcBhwL7A0ZL2\nLmtzOLBHRLwLOAn4UQ1qzUpzc3OjS+g3vC7aeV2087qojkp68hOBlohYGhGtwHXA1LI2U4GrASLi\nfmCopLdWtdLM+APczuuinddFO6+L6qgk5IcDz5TcX1bM66rN8g7amJlZnXnHq5lZxhQRXTeQJgHT\nI2JKcf8MICLivJI2PwLuiohfFPcXAe+PiL+UvVbXCzMzsw5FhHrzvEEVtJkN7ClpJLAS+CRwdFmb\nmcCXgF8UG4UXywO+L0WamVnvdBvyEbFR0snAraThnRkRsVDSSenhuDwibpL0YUlPAOuBE2pbtpmZ\nVaLb4RozM9ty1WTHq0+eatfdupB0jKS5xTRL0n6NqLMeKvlcFO0mSGqVNK2e9dVThX8jTZIekfSY\npLvqXWO9VPA3sr2kmUVWPCrp+AaUWXOSZkj6i6R5XbTpeW5GRFUn0objCWAksA0wB9i7rM3hwO+L\n2wcA91W7jv4wVbguJgFDi9tTtuZ1UdLuDuB3wLRG193Az8VQYD4wvLi/c6PrbuC6+Bpwbtt6AJ4H\nBjW69hqsi4OAccC8Th7vVW7Woifvk6fadbsuIuK+iHipuHsf+Z5fUMnnAuAU4FfAc/Usrs4qWRfH\nANdHxHKAiFhd5xrrpZJ1EcCQ4vYQ4PmI2FDHGusiImYBa7po0qvcrEXI++SpdpWsi1KfBW6uaUWN\n0+26kPR24IiIuAzI+UisSj4Xo4AdJd0labakT9WtuvqqZF1cDOwjaQUwFzi1TrX1N73KzUoOobQ6\nkHQI6aikgxpdSwNdAJSOyeYc9N0ZBOwPTAbeDNwr6d6IeKKxZTXEYcAjETFZ0h7AbZLGRMTLjS5s\nS1CLkF8O7Fpyf0Qxr7zNO7ppk4NK1gWSxgCXA1Mioquva1uyStbFe4HrJIk09nq4pNaImFmnGuul\nknWxDFgdEa8Ar0i6GxhLGr/OSSXr4gTgXICIWCLpKWBv4MG6VNh/9Co3azFcs+nkKUmDSSdPlf+R\nzgSOg01n1HZ48lQGul0XknYFrgc+FRFLGlBjvXS7LiJi92LajTQu/8UMAx4q+xu5EThI0kBJbyLt\naFtY5zrroZJ1sRT4AEAxBj0KeLKuVdaP6PwbbK9ys+o9+fDJU5tUsi6AbwI7ApcWPdjWiJjYuKpr\no8J1sdlT6l5knVT4N7JI0i3APGAjcHlELGhg2TVR4efiHODKkkMLT4uIFxpUcs1IugZoAnaS9DRw\nFjCYPuamT4YyM8uYr0JpZpYxh7yZWcYc8mZmGXPIm5llzCFvZpYxh7yZWcYc8mZmGXPIm5ll7P8D\nKUKJRvlAVN0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1481d3400>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 215 ms, sys: 11.9 ms, total: 226 ms\n",
      "Wall time: 227 ms\n"
     ]
    }
   ],
   "source": [
    "mlp = sklearn.neural_network.MLPClassifier()\n",
    "clf = mlp\n",
    "%time clf.fit(X[tidx],y[tidx])\n",
    "\n",
    "print(\"TEST\")\n",
    "%time evaluate(clf,X[pidx],y[pidx])\n",
    "\n",
    "print(\"TRAIN\")\n",
    "%time evaluate(clf,X[tidx],y[tidx])\n",
    "\n",
    "print(\"ROC AUC\")\n",
    "%time roc_auc(clf,X[pidx],y[pidx])\n",
    "\n",
    "print(\"PRECISION/RECALL AUC\")\n",
    "%time prrc_auc(clf,X[pidx],y[pidx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 14.1 ms, sys: 10.4 ms, total: 24.5 ms\n",
      "Wall time: 44 ms\n",
      "TEST\n",
      "Accuracy 0.595238095238 Precision 0.595238095238 Recall 1.0 F1 0.746268656716\n",
      "CPU times: user 1.8 ms, sys: 587 µs, total: 2.39 ms\n",
      "Wall time: 1.93 ms\n",
      "TRAIN\n",
      "Accuracy 1.0 Precision 1.0 Recall 1.0 F1 1.0\n",
      "CPU times: user 3.1 ms, sys: 1.03 ms, total: 4.13 ms\n",
      "Wall time: 3.06 ms\n",
      "ROC AUC\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/aduriseti/anaconda/lib/python3.5/site-packages/sklearn/discriminant_analysis.py:682: UserWarning: Variables are collinear\n",
      "  warnings.warn(\"Variables are collinear\")\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXkAAAEKCAYAAAD3tSVSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAF3dJREFUeJzt3X+wpXV92PH3BwRxWWBBWsaA0ARCsFQwNouotF7BuIu0\nJaMxAq1kTW3pJGQYZowsnUauGTrKH22t2UFm7RYrhMFVkrImBjCGO1bLIpkKBNgVFnCBhSwBAVFD\nXNlP/3ieZZ89nHPPc+89P5/n/Zq5s+ec+73P+e7D3e99n+855xKZiSSpmfYb9wQkScPjIi9JDeYi\nL0kN5iIvSQ3mIi9JDeYiL0kN5iKvkYqIGyLiX417HksREZdHxPpxz2MYIuIfRsQDEXHAuOeiwXCR\nb6GI+H5E/CQifhgRT0bEtRGxrGPMOyLiG+WY5yLi5oh4U8eYQyLiMxGxvRz3UET814g4osf9vhk4\nJTM3ddw+ExG7I+L3Om5/V0Q83uU4t0fEb1WunxgRGyPib8u53h0Rl0ZE9JjH8nKej0bEi+X52BgR\np/U9eUBmfioz/32dsYMSEQdGxP+MiBfK/2aX9hl/Qfn3ejEi/jgiVtQ5VmY+DfwlcNHw/jYaJRf5\ndkrgnMw8FHgL8MvA5Xs+GRFvB24F/gR4A/DzwL3AtyPiH5VjDqBYDN4EvLc81tuBZ4Bei+VFwB91\nuf1C4Nnyz25z7Skijgc2A9uBf5KZhwMfBN4KHNJl/IHA7cDJwPuAQ8u/w43A6vnua8w+CRwPvBE4\nE/h4RLy328CIOBm4BvjXwFHA3wGfW8CxbsBFvjky04+WfQCPAmdWrl8FfLVy/ZvAH3b5uq8BXygv\nfxR4CnjdAu73YeAdHbctA34I/AbwEvDWyufeBTzW5Ti3A79VXr6+Ovcac/gosAM4qM+4zwCPAS8A\ndwFnVD53BXBdefk4YDfFD6jtwNPAfxzCf7MdwFmV658Ebugx9j8D11eu/wLw98DBdY4F7A/8GHjj\nuL9X/Vj6hyXfchFxDHA28FB5/XXAO4CvdBm+EfjV8vJZwC2Z+Xc172cZxSOC73V86gPAi8CXgduA\n31zgX+GsHnOdb/ytmflSn3HfAU4BDqco2y+XjwL26HyE8U7gF4H3AJ+IiF/qdtCIuKzcUvpB+Wf1\n8g96fM0KikdU91Zuvofi0Ug3J5efLyaa+QjFIn9inWNl5svANuDUHsfXFHGRb6//HRE/pKjVncBs\nefsRFN8XT3X5mqeAI8vLr+8xppcVFAvjix23XwjcmEVC3gCcFxH7L+C4C53HkcDf7LkSEaeWC+wL\nEbFlz+2ZeUNmPp+ZuzPzvwGvBbou3BR/r9nM/Glm3kuxaHZdIDPzqsw8PDOPKP+sXu76XAawvLyP\nFyq3/ZAu21GV8S903LZnfN1jvUjx30xTzkW+vc7NYh/9XcBJ7F28n6PYfnhDl695A8WeOxR76N3G\n9PJ8+ecri0n5KOLdFIs7wCbgdcA55fWfAd1e5XEAsGuR89hnfGbek8U+/vspFvI9c/tY+SqT5yLi\nOYq9+yNfdbS9dlYu/4RiMR2UH5V/Hlq57TBe/QOzOv7Qjtv2jK97rEPY+99MU8xFvr0CIDP/D/C/\ngP9SXv8JcAfFk5edfgP4i/LyXwCryu2dvsrjPgycWLn5wnIeX42Ip8rPv5a9WzaPAUd2vvKHYh98\ne2UeH6gzh9I3gPfON++I+GfA7wG/vqe2KWq366t1FqJ8+eWL5auRqh8vlo+sXiUzn6d4tFJ9dHAq\ncH+Pu7m/OrZ8cvoA4ME6xyofSZ1AZctH08tFXlA8yfir5UscAdYCvxkRF5cvNzw8Iq4ETgf+oBxz\nHfA4cFNE/FIUXl8uYr1epfI1ikcOe1xIsU30FoqF5lTg14FzIuLwzHwcuBO4KiIOLl/693HgpxSv\nqIHiSdB3RMRVEXEUQEScEBHXRURnzQJ8kWKR+5OIODki9ouI1wIrK2OWUzxSeLa8z0/Qe2sEFrD4\nZ/Hyy0My89COj0PKR1a9XAf8p4hYUb6U9d8B1/YY+0fAv4yId0bEwRT/zW7KzB/XPNZpwKPl+deU\nc5Fvp32eNMzMZyhq/hPl9W8DqygK+SmKV+OcCrwzMx8ux/yU4knGrcDXKfZ4N1Pskd/Z434/D/wb\ngIh4G3AscHVmPl35+CrFk8Dnl1/zIYqXAW4DnqDY3jmnvP89Tyq+neJJ3fvLrZUvU7wi5lXbGZn5\n9+UxHgD+rJz3VuCfUjxSgeLlo7cCD5Z/959Q/EDrpfNJ2GH8TxquAB6heATzl8CnM/Prez5ZPhJ4\nJ0BmPgD8B4ptsL+h2AL7nbrHonjp5TVD+DtoDKJ4vmueAREbgH8B7MzMU3qM+SzFKzR+DKzJzLsH\nPVE1Q0RcD2zMjjdEaTJExD8A5oBf3vODVNOtziJ/BsWTNV/stshHxNnAxZl5Tlln/z0zTx/KbCVJ\nC9J3uyYzv0XxiotezqXY5yQz7wQO27M3Kkkar0HsyR/NvvuVO8rbJElj5hOvktRgrxnAMXZQ/KKj\nPY4pb3uViBjGqw4kqfEyc1Hv06hb8kHv1wJvovztgRFxOvB8Zu7sMXbsv6xnUj6uuOKKsc9hUj48\nF54Lz8Xej/se3cnRl36AAy99E//jls1kLq2N+5Z8RNwAzACvj4jHKF5je2CxXuf6zPxaRLwvIrZR\nvITyI0uakSS10O7dyaUbNrJu2yWsXLaG+9Zez4rlBy35uH0X+cy8oMaYi5c8E0lqqfu//zSrPvvb\n/C0PsH7VzfzbVW8b2LF94nVMZmZmxj2FieG52MtzsVcbzsXu3ckln/8Sp3zuFI5ZdgI7/+D/DXSB\nhxpvhhronUXkKO9PkiZVtd6vXnXtvIt7RJBDfuJVkjQAo6j3qkG8hFKSVMMw9957seQlachGXe9V\nlrwkDdE46r3KkpekIRhnvVdZ8pI0YOOu9ypLXpIGZFLqvcqSl6QBmKR6r7LkJWkJJrHeqyx5SVqk\nSa33KktekhZo0uu9ypKXpAWYhnqvsuQlqYZpqvcqS16S+pi2eq+y5CWph2mt9ypLXpK6mOZ6r7Lk\nJamiCfVeZclLUqkp9V5lyUtqvabVe5UlL6nVmljvVZa8pFZqcr1XWfKSWqfp9V5lyUtqjbbUe5Ul\nL6kV2lTvVZa8pEZrY71XWfKSGqut9V5lyUtqnLbXe5UlL6lRrPd9WfKSGsF6786SlzT1rPfeLHlJ\nU8t678+SlzSVrPd6LHlJU8V6XxhLXtLUsN4XrlbJR8TqiNgaEQ9GxGVdPn9oRGyKiLsj4q8jYs3A\nZyqptaz3xYvMnH9AxH7Ag8BZwJPAXcB5mbm1MuZy4NDMvDwijgS+BxyVmT/rOFb2uz9JqqrW+9Wr\nrm3l4h4RZGYs5mvrlPxpwEOZuT0zdwE3Aud2jEngkPLyIcCznQu8JC2E9T4YdfbkjwYer1x/gmLh\nr1oHbIqIJ4HlwIcGMz1JbeTe++AM6onXVcB3M/PMiDge+HpEnJKZP+ocODs7+8rlmZkZZmZmBjQF\nSdNu9+7k0g0bWbftElYuW8N9a69nxfKDxj2tkZubm2Nubm4gx6qzJ386MJuZq8vra4HMzKsqY/4U\n+FRmfru8/g3gssz8q45juScvqSv33nsb9p78XcAJEXFcRBwInAds6hizHXhPOZmjgBOBRxYzIUnt\n4t77cPXdrsnMlyPiYuA2ih8KGzJzS0RcVHw61wNXAl+IiHvLL/t4Zv5gaLOW1AjuvQ9f3+2agd6Z\n2zWS6Nh7P2ANt6ydbeXee11L2a7xHa+SRsp6Hy1/d42kkXDvfTwseUlDZ72PjyUvaWis9/Gz5CUN\nhfU+GSx5SQNlvU8WS17SwFjvk8eSl7Rk1vvksuQlLYn1PtkseUmLYr1PB0te0oJZ79PDkpdUm/U+\nfSx5SbVY79PJkpc0L+t9ulnyknqy3qefJS/pVaz35rDkJe3Dem8WS14SYL03lSUvyXpvMEteajHr\nvfkseamlrPd2sOSllrHe28WSl1rEem8fS15qAeu9vSx5qeGs93az5KWGst4FlrzUSNa79rDkpQax\n3tXJkpcawnpXN5a8NOWsd83HkpemmPWufix5aQpZ76rLkpemjPWuhbDkpSlhvWsxLHlpCljvWqxa\nJR8RqyNia0Q8GBGX9RgzExHfjYj7IuL2wU5TaifrXUsVmTn/gIj9gAeBs4AngbuA8zJza2XMYcD/\nBd6bmTsi4sjMfKbLsbLf/UkqVOv96lXXuri3WESQmbGYr61T8qcBD2Xm9szcBdwInNsx5gLgpszc\nAdBtgZdUj/WuQaqzJ3808Hjl+hMUC3/VicAB5TbNcuCzmXndYKYotYd77xq0QT3x+hrgrcCZwMHA\nHRFxR2ZuG9DxpUbbvTu5dMNG1m27hJXL1nDf2utZsfygcU9LDVBnkd8BHFu5fkx5W9UTwDOZ+RLw\nUkR8EzgVeNUiPzs7+8rlmZkZZmZmFjZjqWGsd3Wam5tjbm5uIMeq88Tr/sD3KJ54fQr4DnB+Zm6p\njDkJ+ENgNfBa4E7gQ5n5QMexfOJVKu1T7wes4Za1s9a7ulrKE699Sz4zX46Ii4HbKJ6o3ZCZWyLi\nouLTuT4zt0bErcC9wMvA+s4FXtJe1rtGpW/JD/TOLHm1nPWuxRhqyUsaDOtd4+DvrpGGzNe9a5ws\neWmIrHeNmyUvDYH1rklhyUsDZr1rkljy0oBY75pElrw0ANa7JpUlLy2B9a5JZ8lLi2S9axpY8tIC\nWe+aJpa8tADWu6aNJS/VYL1rWlnyUh/Wu6aZJS/1YL2rCSx5qQvrXU1hyUsV1ruaxpKXSta7msiS\nV+tZ72oyS16tZr2r6Sx5tZL1rraw5NU61rvaxJJXa1jvaiNLXq1gvautLHk1mvWutrPk1VjWu2TJ\nq4Gsd2kvS16NYr1L+7Lk1QjWu9SdJa+pZ71LvVnymlrWu9SfJa+pZL1L9VjymirWu7QwlrymhvUu\nLZwlr4lnvUuLZ8lrolnv0tJY8ppI1rs0GLUW+YhYHRFbI+LBiLhsnnErI2JXRLx/cFNU29z//ac5\n9mMf5Jotn2T9mTez+cpPs2L5QeOeljSV+i7yEbEfsA5YBZwMnB8RJ/UY92ng1kFPUu1gvUuDV2dP\n/jTgoczcDhARNwLnAls7xv0u8BVg5UBnqFZw710ajjrbNUcDj1euP1He9oqI+Dng1zLzc0AMbnpq\nOutdGq5BvbrmM0B1r96FXn1Z79Lw1VnkdwDHVq4fU95W9SvAjRERwJHA2RGxKzM3dR5sdnb2lcsz\nMzPMzMwscMqadrt3J5du2Mi6bZewctka7lt7vU+sShVzc3PMzc0N5FiRmfMPiNgf+B5wFvAU8B3g\n/Mzc0mP8tcBXM/OPu3wu+92fmq1a71evutZ6l2qICDJzUTskfUs+M1+OiIuB2yj28Ddk5paIuKj4\ndK7v/JLFTETNZr1L49G35Ad6Z5Z8K1nv0tIspeR9x6uGxlfOSOPn767RUPjKGWkyWPIaKOtdmiyW\nvAbGepcmjyWvJbPepcllyWtJrHdpslnyWhTrXZoOlrwWzHqXpoclr9qsd2n6WPKqxXqXppMlr3lZ\n79J0s+TVk/UuTT9LXq9ivUvNYclrH9a71CyWvADrXWoqS17Wu9RglnyLWe9S81nyLWW9S+1gybeM\n9S61iyXfIta71D6WfAtY71J7WfINZ71L7WbJN5T1Lgks+Uay3iXtYck3iPUuqZMl3xDWu6RuLPkp\nZ71Lmo8lP8Wsd0n9WPJTyHqXVJclP2Wsd0kLYclPCetd0mJY8lPAepe0WJb8BLPeJS2VJT+hrHdJ\ng2DJTxjrXdIgWfITxHqXNGi1Sj4iVkfE1oh4MCIu6/L5CyLinvLjWxHx5sFPtbmsd0nD0rfkI2I/\nYB1wFvAkcFdE3JyZWyvDHgH+eWa+EBGrgc8Dpw9jwk1jvUsapjolfxrwUGZuz8xdwI3AudUBmbk5\nM18or24Gjh7sNJvHepc0CnX25I8GHq9cf4Ji4e/lo8CfL2VSTWe9SxqVgT7xGhHvBj4CnNFrzOzs\n7CuXZ2ZmmJmZGeQUJtru3cmlGzaybtslrFy2hvvWXs+K5QeNe1qSJszc3Bxzc3MDOVZk5vwDIk4H\nZjNzdXl9LZCZeVXHuFOAm4DVmflwj2Nlv/trqmq9X73qWutdUm0RQWbGYr62zp78XcAJEXFcRBwI\nnAds6pjAsRQL/Id7LfBt5d67pHHqu12TmS9HxMXAbRQ/FDZk5paIuKj4dK4Hfh84Arg6IgLYlZnz\n7du3gnvvksat73bNQO+sJds1++y9H7CGW9bOuvcuadGWsl3jO14HzHqXNEn83TUD4t67pElkyQ+A\n9S5pUlnyS2C9S5p0lvwiWe+SpoElv0DWu6RpYskvgPUuadpY8jVY75KmlSXfh/UuaZpZ8j1Y75Ka\nwJLvwnqX1BSWfIX1LqlpLPmS9S6piVpf8ta7pCZrdclb75KarpUlb71LaovWlbz1LqlNWlPy1ruk\nNmpFyVvvktqq0SVvvUtqu8aWvPUuSQ0seetdkvZqVMlb75K0r0aUvPUuSd1Nfclb75LU29SWvPUu\nSf1NZclb75JUz1SVvPUuSQszNSVvvUvSwk18yVvvkrR4E13y1rskLc1Elrz1LkmDMXElb71L0uBM\nTMlb75I0eBNR8ta7JA1HrZKPiNURsTUiHoyIy3qM+WxEPBQRd0fEW+oc13qXpOHqu8hHxH7AOmAV\ncDJwfkSc1DHmbOD4zPxF4CLgmn7Hvf/7T3Psxz7INVs+yfozb2bzlZ9mxfKDFvWXmEZzc3PjnsLE\n8Fzs5bnYy3MxGHVK/jTgoczcnpm7gBuBczvGnAt8ESAz7wQOi4ijuh3Mei/4DbyX52Ivz8VenovB\nqLMnfzTweOX6ExQL/3xjdpS37ew82LEf+6B775I0IiN/4vWYZSdw39rrW7U1I0njEpk5/4CI04HZ\nzFxdXl8LZGZeVRlzDXB7Zn6pvL4VeFdm7uw41vx3JknqKjNjMV9Xp+TvAk6IiOOAp4DzgPM7xmwC\nfgf4UvlD4fnOBX4pk5QkLU7fRT4zX46Ii4HbKJ6o3ZCZWyLiouLTuT4zvxYR74uIbcCPgY8Md9qS\npDr6btdIkqbXUH6twbDePDWN+p2LiLggIu4pP74VEW8exzxHoc73RTluZUTsioj3j3J+o1Tz38hM\nRHw3Iu6LiNtHPcdRqfFv5NCI2FSuFX8dEWvGMM2hi4gNEbEzIu6dZ8zC183MHOgHxQ+ObcBxwAHA\n3cBJHWPOBv6svPw2YPOg5zEJHzXPxenAYeXl1W0+F5Vx3wD+FHj/uOc9xu+Lw4D7gaPL60eOe95j\nPBeXA5/acx6AZ4HXjHvuQzgXZwBvAe7t8flFrZvDKPmBvnlqyvU9F5m5OTNfKK9upnh/QRPV+b4A\n+F3gK8DTo5zciNU5FxcAN2XmDoDMfGbEcxyVOucigUPKy4cAz2bmz0Y4x5HIzG8Bz80zZFHr5jAW\n+W5vnupcuHq9eapp6pyLqo8Cfz7UGY1P33MRET8H/Fpmfg5o8iux6nxfnAgcERG3R8RdEfHhkc1u\ntOqci3XAP46IJ4F7gEtGNLdJs6h1cyJ+C6UgIt5N8aqkM8Y9lzH6DFDdk23yQt/Pa4C3AmcCBwN3\nRMQdmbltvNMai1XAdzPzzIg4Hvh6RJySmT8a98SmwTAW+R3AsZXrx5S3dY55Y58xTVDnXBARpwDr\ngdWZOd/DtWlW51z8CnBjRATF3uvZEbErMzeNaI6jUudcPAE8k5kvAS9FxDeBUyn2r5ukzrn4CPAp\ngMx8OCIeBU4C/mokM5wci1o3h7Fd88qbpyLiQIo3T3X+I90EXAivvKO265unGqDvuYiIY4GbgA9n\n5sNjmOOo9D0XmfkL5cfPU+zL/3YDF3io92/kZuCMiNg/IpZRPNG2ZcTzHIU652I78B6Acg/6ROCR\nkc5ydILej2AXtW4OvOTTN0+9os65AH4fOAK4uizYXZnZ+Qvgpl7Nc7HPl4x8kiNS89/I1oi4FbgX\neBlYn5kPjHHaQ1Hz++JK4AuVlxZ+PDN/MKYpD01E3ADMAK+PiMeAK4ADWeK66ZuhJKnBJub/8SpJ\nGjwXeUlqMBd5SWowF3lJajAXeUlqMBd5SWowF3lJajAXeUlqsP8PBJLIDu4Ee/EAAAAASUVORK5C\nYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x148265dd8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 221 ms, sys: 11.9 ms, total: 233 ms\n",
      "Wall time: 236 ms\n",
      "PRECISION/RECALL AUC\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXkAAAEKCAYAAAD3tSVSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGbJJREFUeJzt3Xu0XGV9//H3JwmhArkYoYCJ4WaBEgxgSQiaykkIJMiv\njeVnETCAiC2rFi1trcTWlqPSAnW1IirSSH6m1hLoqv1pvKCJkQMiYEJBwJKYi0ouhMRALgSMhuTb\nP/Y+mWFyzpl9zpnbec7ntdaszOx5Zu9nduZ89p5n7z1fRQRmZpamIc3ugJmZ1Y9D3swsYQ55M7OE\nOeTNzBLmkDczS5hD3swsYQ55G5Ak7ZN0fH7/i5I+3kPbwyWtkHRw43pYe5J+LOltze5HPUi6VtLN\nze5HihzyLUDSzyW9LGmnpE15aB3S7H4Vkff1V3nft0r6jqSTGrDo3lzgMRf4YkT8qnyipAWS9kg6\nsmL6ARsNScfkG5YhZdMuk7Rc0ouSNkr6pqS3dtcJSWdK+rqkF/LbjyV9QtKoIm8iIk6NiAcKveMa\nkXS6pEclvZS/19N6aPtJSask7ZD0tKTLezGvLwDvlnR4vd7LYOWQbw0BXBgRI4E3A2cCH+2qoSQ1\nsmMF3ZL3fSzwLHBnA5ZZaD1IGg5cCXy5YvohwEXAdmBOwWXu37BI+gvgn4Ebgd8ExgOfA36vm368\nBbgP+D5wUkSMAWYBrwDdBmczSToI+CrwJWB0/u/XJA3r5iW7yD7Ho4D3AJ+WNKXIvPIN8LeAK+r2\nhgYph3zrEEBEbALuBU4FkHSfpBslPSjpJeA4SSMlzZf0rKT1+d7g/tCT9Ef5ntTOfG/x9AMWJt0u\n6ZMV074q6br8/vWSNuTzWCFpWrU3kP+h/gfwquVJem/en+cl3StpfNlzEyQtzp/bJGluPn2SpIck\nbcv3kj/TQ7j05CxgW0Q8WzH9ncA24ONkgVSYpJHAx4D3R8TXIuKXEbE3Ir4VEXO7edktwPyI+MeI\n+AVARGyIiI917p1LOl7S0vwb0RZJX86X1bncn0mant+/QdI9kv41/z96StKbe/M+CmgDhkbEbRGx\nJyI+Q/Y5nd5V4/y9rM7vLyPboJ2dPz2twLzuBy6s8XsY9BzyLUbSG4C3A4+VTZ4DvA8YAawD/hX4\nFXA8cAZwXv48kv4Q+DtgTr53/fvA810saiFwcdlyR+fzWSjpROBPgd/J5zET+HmBvh8KXAasLps2\nm2y45B3AEWR/+Avz5w4DlpDtwR0NvBFYmr90L3AdMIYsKKYD76/Why68CfhJF9OvAO4C7gFOlnRG\nL+b5FuBgsj3TqvJvDWcD/1WtKfAPwFHAbwPjgPYe2v8e2XsYBXyd7JtEd314omyYaFvFv5/t5mUT\ngCcrpj2RT+/5jUivASYBP84nnVJgXito0W81A5lDvnV8VdILwANkX+tvKntuQUSsjIh9ZKF3AfDn\nEbE7IrYCtwKX5G2vBv4xIh4DiIifRsT6yoVFxPeBkDQ1n/RO4OGI2EwWsMOBUyUNi4h1EfGzHvr+\nV3nfd5IFYPlX7muAmyJiVd7/m4HT843Z/wE2RcStEfHriHgpIpbn/XssIpZFZh0wDzinwHqsNBp4\nsXxC/k1iGnBXRGwBvkvvhgnGAFvz91PEa8n+1p4r68MtecjukvTXABGxNiKWRsQrEfE88Cl6fs8P\nRsR3IvsBqn8DJnbXMCJOi4gx+e21Ff9e283LDgN2VEzbSbazUc0dwOMRsaQX83qRbINlNeSQbx2z\n8z+44yLiAxUHCctD+hjgIGBT594Y2R/UEfnzbwDWFlzmPcCl+f3LgH+HLGzI9qLbgc2S7pJ0dA/z\n+WQ+xnwM8Eug/MDrMWRjsy/kG4Lnyca2x/bUV0m/lR+k3CRpO/D3QF8Oym3jwFC6HHg6Ip7KHy8k\nO+g3NH/8Ctk6LncQsC8P9ueBw1V2ELZAH/aRfVsBICKuj4jXAv8fGAYg6TclLcyHybaTHUfo6T0/\nV3b/ZeA3etGnInYBIyumjaJio1kpHwY8BXhXL+c1ggM3BNZPDvnW0dOBxPIzSdYDu4HXle2NjY6I\niWXPn1BwmQuBd+Z7tmcBX9m/wIi7I+J3yUIasj3wHkXEBrKNw20qna64DrimYi/ysIh4pEpfP0/2\n9f2EiBgN/A0FD7ZWeBI4sWLa5cDx+QZkE/BPwOvIhsk6+3xsxWuOp7SxfZhsuOwdRToQES8DPyQ7\n0NuTfyDbGEzI3/Mc+vaeD5Afm9lZcXsx//f2bl72Pxz47WBiPr275XyMbHjvvIjY1ct5/TbZEI7V\nkEN+gImI54DFwKckjVDmeJXOn74T+FDnQThJJ5Qf6KyY14/I9krvBL4dETvz15woaZqyM1N+TbZ3\nXmhoIiK+C2wkG6YB+BfgryWdks97lKR35s99AzhK0gclDZd0mKTJ+XMjgJ0R8bKkk4E/KbSCDrQM\nGN35TUTS2WSBPYls/Pc0snHhhWRn4UC2sbtQ0gxJQyS9nmwjszB/jzuBG4DPSZot6TWShkmape7P\n9f4w8F5JH5Z0RN6XccBxZW1GkO3xvihpLPBXvXyv3W4Q8tMvR1bcRuT/dnesowPYK+kD+f/PB8k+\nB9/rcuHSR8i+Gc6IiO19mNc5ZCcdWC1FhG9NvgE/BaZ389z3gPdWTBsB3E62Z7kN+G/g4rLn/xhY\nSTbm+SRwWg/L/ijZGPxFZdPeRLbnuQPYCiwCjurm9f8P+HjFtIvzvh2UP3533o/twDPAnWVtTyEb\nE3+B7PTLD+fTf5dsT34n2VkX7cADZa/bCxyf3/9iZR8q+nNL2Xw/D/xHF20mkW3MRuePLwQezdfv\nz8i+yRxc8ZpLgeVkQw7Pkh38nNJDPyYB38zf6wv5OvkE8NqydfFo/p4fA/4cWNfV54RsI/OlsueO\nydfJkBp/Nk/L+/RS/u/EsucuA54qe7wvX4c783WyE5hbcF6/kX9mjmj232NqN+UruFuS5pMdINsc\npSGByja3kR0MfAl4T2R7iGYtQdkFNg8AZ0TFBVHWGiRdC4yL7k9BtT4qEvJTyb5CfqmrkJd0AXBt\nRFwo6Szg0xExpS69NTOzXqk6Jh8RD5J9Ze3ObLKr14iIHwKjVHGZuJmZNUctDryO5dWn+G3Mp5mZ\nWZP57Bozs4T15bdAKm0ku6il07h82gEk9eaXA83MLBcRfbpmouievOj+HNxF5JeEK/vFue2RXRrf\npc7Tel56Kfj2t4O//Mtg4sRg9OjgD/4guP32YPXqYN++5p96VM/bDTfc0PQ+tMrN68Lrwuui51t/\nVN2Tl3QX2a/RvU7SOrLzc4dneR3zIuJbkt4uaQ3ZKZRXFVnwIYfAzJnZDWDzZvjud2HJErjxRhg+\nHM47L7udey6MGdO3N2hmNphVDfmIuKxAm+5+4KiwI4+Ed787u0XA009ngb9gAVx9NZx0Uin03/IW\nOHhA1/gxM2uMWozJ15wEEyZkt+uug1//Gh5+OAv9uXNhxQp461vh/POz0J8wIXvNQNLW1tbsLrQM\nr4sSr4sSr4vaqHoxVE0XJkUtlrdtG3zve1noL1kCv/wlzJiRBf6MGXB0T7+XaGY2wEgi+njgdUCG\nfKW1a0uBf999MHZsaWjnbW+DQw+t+SLNzBpm0Id8ub174dFHS6H/2GNw5plZ4J9/PpxxBgwdWn0+\nZmatwiHfg1274P77S6G/eTNMn17a0z/22IZ2x8ys1xzyvbBxY3aq5uLF2b8jR5YCf9o0GD26qd0z\nMzuAQ76P9u2Dp54q7eU/9BCcemop9KdMgYMqi8CZmTWYQ75Gdu+GH/ygFPpr1sA555RC/6STBt6p\nmmY28Dnk62TrVli6tBT6+/aVAn/GDDjiiOrzMDPrL4d8A0TAqlWlwL//fjjuuFLoT50Kr3lNs3tp\nZilyyDfBnj2wbFkp9J98MhvD7zxVc+JEGOIfcjazGnDIt4AdO6CjoxT627aVrsI97zwYN67ZPTSz\ngcoh34KeeaYU+EuXZuP3nYHf1gYjRjS7h2Y2UDjkW9y+ffD446XQX7YMTj+9FPqTJsGwlvypODNr\nBQ75Aebll+H73y+F/rp12YVYnaF/wgk+VdPMShzyA9xzz5VO1Vy8OPutfBdMMbNODvmElBdMWbIk\n2+N3wRSzwc0hn7DygilLlqRRMMXMeschP4i4YIrZ4OOQH8RcMMUsfQ55A1wwxSxVDnnrkgummKXB\nIW+FuGCK2cDkkLdec8EUs4HDIW/95oIpZq3LIW8154IpZq3DIW915YIpZs3lkLeGcsEUs8ZyyFtT\nuWCKWX055K2luGCKWW055K1luWCKWf855G3AcMEUs95zyNuA5YIpZtU55C0JLphi1jWHvCXJBVPM\nMg55GxRcMMUGq7qHvKRZwK3AEGB+RNxS8fxI4MvAeGAo8E8RsaCL+TjkrWZcMMUGi7qGvKQhwCrg\nXOBZYDlwSUSsLGvzEWBkRHxE0uHAT4AjI+KVink55K0uXDDFUtafkC9y8flkYHVEPBMRe4C7gdkV\nbQLovMRlBPB8ZcCb1dPQoXDWWfDRj2a/rbNpE3zoQ7BlC1x5JRx5JFx8MXzhC/Dznze7t2aNU2RP\n/v8CMyPij/PHc4DJEfHBsjaHAYuAk4HDgHdFxL1dzMt78tYULphiA1l/9uRrda3hTODxiJgu6QRg\niaSJEbGrsmF7e/v++21tbbS1tdWoC2bdGzs226O/8spXF0y54w644goXTLHW0tHRQUdHR03mVWRP\nfgrQHhGz8sdzgSg/+CrpG8BNEfGD/PFS4PqIeLRiXt6Tt5bjginW6up94HUo2YHUc4FNwDLg0ohY\nUdbmc8CWiPiYpCOBR4HTIuKFink55K3luWCKtZpGnUL5aUqnUN4s6RqyPfp5ko4GFgCdZyrfFBEL\nu5iPQ94GFBdMsVbgi6HMGsQFU6wZHPJmTeKCKdYIDnmzFuGCKVYPDnmzFuSCKVYrDnmzAcAFU6yv\nHPJmA1BnwZTFi7PQd8EU645D3myAc8EU64lD3iwxLphi5RzyZonrLJjSObSze7cLpgwmDnmzQcYF\nUwYXh7zZIOaCKelzyJvZfrt2Zb+x0xn6mzfD9OmlPf1jj212D623HPJm1q2NG0uB74IpA5ND3swK\nKS+YsmQJPPSQC6YMBA55M+uTyoIpa9dmB25dMKW1OOTNrCY6C6Z0nqoZ4YIprcAhb2Y154IprcMh\nb2Z154IpzeOQN7OGKy+YsngxbN/ugin14pA3s6ZzwZT6ccibWUtxwZTacsibWUtzwZT+ccib2YDy\n3HPZ1bedoe+CKT1zyJvZgOWCKdU55M0sGV0VTJk6tRT6g7FgikPezJLlgikOeTMbRAZjwRSHvJkN\nSt0VTOmshZtKwRSHvJkZry6YsngxbNmSRsEUh7yZWRc2bCidqjmQC6Y45M3MqhjIBVMc8mZmvVRZ\nMGXNGjjnnNYsmOKQNzPrp1/8Ivthtc7Qb6WCKQ55M7MaarWCKQ55M7M66qpgytlnl0K/3gVTHPJm\nZg20Y0d2IVZn6Ne7YErdQ17SLOBWYAgwPyJu6aJNG/Ap4CDgFxExrYs2DnkzS069C6bUNeQlDQFW\nAecCzwLLgUsiYmVZm1HAQ8D5EbFR0uERsbWLeTnkzSxp3RVM6bwK98wze18wpd4hPwW4ISIuyB/P\nBaJ8b17SnwBHR8TfVZmXQ97MBpXOgimdP7C2fn3vC6b0J+SLbE/GAuvLHm8AJle0ORE4SNJ9wGHA\nbRHxb33pkJlZSg45BGbOzG7w6oIpn/hE/Qum1KrK4jDgzcB04FDgYUkPR8SaGs3fzCwJRx0Fc+Zk\nt/KCKQsWwNVX175gSpGQ3wiML3s8Lp9WbgOwNSJ2A7slPQCcBhwQ8u3t7fvvt7W10dbW1rsem5kl\nQsqKoEyYANddVyqYcuedHcyZ08HWrTB+fPX59LiMAmPyQ4GfkB143QQsAy6NiBVlbU4GPgPMAg4G\nfgi8KyKerphX0N6/DpuZDTrtNOQUyk9TOoXyZknXkB2AnZe3+RBwFbAX+EJEfKaL+fjAq5lZL/li\nKDOzhPUn5Ot4Ia6ZmTWbQ97MLGEOeTOzhDnkzcwS5pA3M0uYQ97MLGEOeTOzhDnkzcwS5pA3M0uY\nQ97MLGEOeTOzhDnkzcwS5pA3M0uYQ97MLGEOeTOzhDnkzcwS5pA3M0uYQ97MLGEOeTOzhDnkzcwS\n5pA3M0uYQ97MLGEOeTOzhDnkzcwS5pA3M0uYQ97MLGEOeTOzhDnkzcwS5pA3M0uYQ97MLGEOeTOz\nhDnkzcwS5pA3M0uYQ97MLGEOeTOzhDnkzcwSVijkJc2StFLSKknX99BukqQ9ki6qXRfNzKyvqoa8\npCHAZ4GZwATgUkknd9PuZuA7te6kmZn1TZE9+cnA6oh4JiL2AHcDs7to9wHgP4EtNeyfmZn1Q5GQ\nHwusL3u8IZ+2n6TXA++IiM8Dql33zMysP2p14PVWoHys3kFvZtYChhVosxEYX/Z4XD6t3JnA3ZIE\nHA5cIGlPRCyqnFl7e/v++21tbbS1tfWyy2Zmaevo6KCjo6Mm81JE9NxAGgr8BDgX2AQsAy6NiBXd\ntP8i8PWI+K8unotqyzMzs1eTRET0aYSk6p58ROyVdC2wmGx4Z35ErJB0TfZ0zKt8SV86YmZmtVd1\nT76mC/OevJlZr/VnT95XvJqZJcwhb2aWMIe8mVnCHPJmZglzyJuZJcwhb2aWMIe8mVnCHPJmZglz\nyJuZJcwhb2aWMIe8mVnCHPJmZglzyJuZJcwhb2aWMIe8mVnCHPJmZglzyJuZJcwhb2aWMIe8mVnC\nHPJmZglzyJuZJcwhb2aWMIe8mVnCHPJmZglzyJuZJcwhb2aWMIe8mVnCHPJmZglzyJuZJcwhb2aW\nMIe8mVnCHPJmZglzyJuZJcwhb2aWMIe8mVnCHPJmZgkrFPKSZklaKWmVpOu7eP4ySU/ktwclvan2\nXTUzs95SRPTcQBoCrALOBZ4FlgOXRMTKsjZTgBURsUPSLKA9IqZ0Ma+otjwzM3s1SUSE+vLaInvy\nk4HVEfFMROwB7gZmlzeIiEciYkf+8BFgbF86Y2ZmtVUk5McC68seb6DnEH8fcG9/OmVmZrUxrJYz\nkzQNuAqY2l2b9vb2/ffb2tpoa2urZRfMzAa8jo4OOjo6ajKvImPyU8jG2Gflj+cCERG3VLSbCHwF\nmBURa7uZl8fkzcx6qd5j8suBN0o6RtJw4BJgUUUHxpMF/OXdBbyZmTVe1eGaiNgr6VpgMdlGYX5E\nrJB0TfZ0zAP+FhgD3C5JwJ6ImFzPjpuZWXVVh2tqujAP15iZ9Vq9h2vMzGyAcsibmSXMIW9mljCH\nvJlZwhzyZmYJc8ibmSXMIW9mljCHvJlZwhzyZmYJc8ibmSXMIW9mljCHvJlZwhzyZmYJc8ibmSXM\nIW9mljCHvJlZwhzyZmYJc8ibmSXMIW9mljCHvJlZwhzyZmYJc8ibmSXMIW9mljCHvJlZwhzyZmYJ\nc8ibmSXMIW9mljCHvJlZwhzyZmYJc8ibmSXMIW9mljCHvJlZwhzyZmYJc8ibmSXMIW9mlrBCIS9p\nlqSVklZJur6bNrdJWi3pR5JOr203zcysL6qGvKQhwGeBmcAE4FJJJ1e0uQA4ISJ+C7gGuKMOfU1K\nR0dHs7vQMrwuSrwuSrwuaqPInvxkYHVEPBMRe4C7gdkVbWYDXwKIiB8CoyQdWdOeJsYf4BKvixKv\nixKvi9ooEvJjgfVljzfk03pqs7GLNmZm1mA+8GpmljBFRM8NpClAe0TMyh/PBSIibilrcwdwX0Tc\nkz9eCZwTEZsr5tXzwszMrEsRob68bliBNsuBN0o6BtgEXAJcWtFmEfCnwD35RmF7ZcD3p5NmZtY3\nVUM+IvZKuhZYTDa8Mz8iVki6Jns65kXEtyS9XdIa4CXgqvp228zMiqg6XGNmZgNXXQ68+uKpkmrr\nQtJlkp7Ibw9KelMz+tkIRT4XebtJkvZIuqiR/Wukgn8jbZIel/RjSfc1uo+NUuBvZKSkRXlWPCXp\nPU3oZt1Jmi9ps6Qne2jT+9yMiJreyDYca4BjgIOAHwEnV7S5APhmfv8s4JFa96MVbgXXxRRgVH5/\n1mBeF2XtlgLfAC5qdr+b+LkYBfwPMDZ/fHiz+93EdfER4KbO9QA8Dwxrdt/rsC6mAqcDT3bzfJ9y\nsx578r54qqTquoiIRyJiR/7wEdK9vqDI5wLgA8B/Alsa2bkGK7IuLgO+EhEbASJia4P72ChF1kUA\nI/L7I4DnI+KVBvaxISLiQWBbD036lJv1CHlfPFVSZF2Uex9wb1171DxV14Wk1wPviIjPAymfiVXk\nc3EiMEbSfZKWS7q8Yb1rrCLr4rPAKZKeBZ4A/qxBfWs1fcrNIqdQWgNImkZ2VtLUZveliW4Fysdk\nUw76aoYBbwamA4cCD0t6OCLWNLdbTTETeDwipks6AVgiaWJE7Gp2xwaCeoT8RmB82eNx+bTKNm+o\n0iYFRdYFkiYC84BZEdHT17WBrMi6OBO4W5LIxl4vkLQnIhY1qI+NUmRdbAC2RsRuYLekB4DTyMav\nU1JkXVwF3AQQEWsl/Qw4GXi0IT1sHX3KzXoM1+y/eErScLKLpyr/SBcBV8D+K2q7vHgqAVXXhaTx\nwFeAyyNibRP62ChV10VEHJ/fjiMbl39/ggEPxf5GvgZMlTRU0iFkB9pWNLifjVBkXTwDzADIx6BP\nBH7a0F42juj+G2yfcrPme/Lhi6f2K7IugL8FxgC353uweyJicvN6XR8F18WrXtLwTjZIwb+RlZK+\nAzwJ7AXmRcTTTex2XRT8XNwILCg7tfDDEfFCk7pcN5LuAtqA10laB9wADKefuemLoczMEuZfoTQz\nS5hD3swsYQ55M7OEOeTNzBLmkDczS5hD3swsYQ55M7OEOeTNzBL2vwrfLAAwBrNNAAAAAElFTkSu\nQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1485a4b00>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 194 ms, sys: 7.58 ms, total: 202 ms\n",
      "Wall time: 205 ms\n"
     ]
    }
   ],
   "source": [
    "qda = sklearn.discriminant_analysis.QuadraticDiscriminantAnalysis()\n",
    "clf = qda\n",
    "%time clf.fit(X[tidx],y[tidx])\n",
    "\n",
    "print(\"TEST\")\n",
    "%time evaluate(clf,X[pidx],y[pidx])\n",
    "\n",
    "print(\"TRAIN\")\n",
    "%time evaluate(clf,X[tidx],y[tidx])\n",
    "\n",
    "print(\"ROC AUC\")\n",
    "%time roc_auc(clf,X[pidx],y[pidx])\n",
    "\n",
    "print(\"PRECISION/RECALL AUC\")\n",
    "%time prrc_auc(clf,X[pidx],y[pidx])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Single tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# number of samples to train on\n",
    "N = int(1e6)\n",
    "X = np.array([x.T for x in df[\"twtvecs\"][:N].values])\n",
    "y = df[\"openup\"][:N].values\n",
    "# inverse classifier\n",
    "# y = 1 - y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# generate test/train split\n",
    "ratio = 0.8\n",
    "tidx = np.random.rand(min(X.shape[0],N)) < ratio\n",
    "pidx = ~tidx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 14.4 s, sys: 432 ms, total: 14.8 s\n",
      "Wall time: 19.3 s\n",
      "TEST\n",
      "Accuracy 0.576429722796 Precision 0.572461577809 Recall 0.989192446766 F1 0.725224230106\n",
      "CPU times: user 79.2 ms, sys: 35.4 ms, total: 115 ms\n",
      "Wall time: 129 ms\n",
      "TRAIN\n",
      "Accuracy 0.577599043408 Precision 0.572439797358 Recall 0.990414440375 F1 0.725534893608\n",
      "CPU times: user 305 ms, sys: 112 ms, total: 418 ms\n",
      "Wall time: 576 ms\n",
      "ROC AUC\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXkAAAEKCAYAAAD3tSVSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmYFNW5x/Hvi7KoQVFRk7glikiCIprEBY0OkAioCQYl\noomIXgMuGOQGBWOUMTeuV+KGS3CB6KAsGgXccB0EZFOCC4sIGEDgQjAsAoIs7/3jFNKOM0zPTHdV\nd8/v8zz9MN19pupQDO/8+tSpU+buiIhIYaqTdAdERCR7VORFRAqYiryISAFTkRcRKWAq8iIiBUxF\nXkSkgKnIS6zM7Ekz+2XS/agJM7vOzAYl3Y9sMLP9zWyWmdVNui+SGSrytZCZ/cvMNpjZWjNbamaD\nzWz3Mm1amdnrUZtVZjbKzH5Qpk1DM7vbzBZG7T42s7+a2T4V7PdooIW7jy7zepGZbTOza8q8fpqZ\nLS5nO2+a2SUpz5ua2Qgz+3fU1xlm1tvMrIJ+fCvq5ydm9nl0PEaY2fGVHjzA3W919+7ptM0UM6tn\nZo+Z2Zro36x3Je0viP5en5vZP8ysUcp73zWz58zsMzNbZGY9tr/n7iuAN4Ae5W1X8o+KfO3kwJnu\nvifQEjgWuG77m2Z2EjAWeBb4DvB94H1gopl9L2pTl1AMfgCcHm3rJGAlUFGx7AEMLef1rsBn0Z/l\n9bVCZnY4MBlYCBzl7nsDnYHjgIbltK8HvAk0B84A9oz+DsOA9jvbV8JuAg4HDgbaANea2enlNTSz\n5sBDwG+AA4AvgAdTmpQA84H9gLOAW8zstJT3n0RFvnC4ux617AF8ArRJeX47MCbl+VvAfeV834vA\nkOjrS4FlwG5V2O98oFWZ13YH1gK/BjYCx6W8dxqwqJztvAlcEn1dktr3NPpwKbAEaFBJu7uBRcAa\nYBpwSsp7/YEnoq8PBbYRfkEtBFYAf8zCv9kSoG3K85uAJytoezNQkvL8MGATsEf02Absm/L+34C/\npzzfBVgPHJz0z6oeNX8oyddyZnYQ0AH4OHq+G9AKeLqc5iOAn0dftwVedvcv0tzP7oRPBB+Veesc\n4HNgJPAKcFEV/wptK+jrztqPdfeNlbSbCrQA9iYk25HRp4Dtyn7COBk4AvgZcKOZHVneRs2sbzSk\n9J/oz9Sv/1PB9zQifKJ6P+Xl9wifRsrTPHo/dNR9AaHINwUs6nvqUJYBR6W03wrMA46pYPuSR1Tk\na6/nzGwtIa0uB4qj1/ch/FwsK+d7lgGNo6/3raBNRRoRisvnZV7vCgzzECGfBLqY2S5V2G5V+9EY\n+L/tT8zsmKjArjGz2dtfd/cn3X21u29z97uA+kC5hZvw9yp29y/d/X1CgS23QLr77e6+t7vvE/2Z\n+nW55zKAb0X7WJPy2lrKGY5Kab+mzGtrgYbuvg6YCNxgZvXN7DjCL9rdy7T/nPBvJnlORb726uhh\nHP00oBk7ivcqwsf575TzPd8hjLlDGEMvr01FVkd/flWYok8RrQnFHWA0sBtwZvR8C1DeLI+6wOZq\n9uNr7d39PQ/j+J0IhXx73/pEs0xWmdkqwth9429sbYflKV9vIBTaTFkX/blnymt78c1fmKnt9yzz\nWmr73xCGcBYB9wNPAJ+Wad+QHf9mksdU5GsvA3D38cDfgQHR8w3AJMLJy7J+DbwWff0a0C4a3qlU\ntN35hCGD7bpG/RhjZsui9+uzY8hmEdC47Mwfwjj4wpR+nJNOHyKvA6fvrN9m9lPgGuDc7WmbkITL\nna1TFdH0y8+j2Uipj8+jT1bf4O6rCZ9WUj8dHAPMrGA3M1PbRien6wJzo+0tdvdfuPsB7n4S4QTs\n1JT2uwBNSBnykTyW9EkBPeJ/8M0Tr40J6e/o6PnJhNTXk5BI9wb+AvwHODxqUw+YQjgZeyShAO5L\nmKXTvoL93g30S3k+B7gB2D/l8QvCCdi9ozYTgPsIJwzrAdcSfhnUi94/jPDp4nbggOi1JoR0umc5\nfagPvAO8TBi7rhO91g9YELXpQEi2B0T7vJHwyaFN9H5/4PHo6+0nXuuk7OOrE8MZ/De7NdpuI8Js\noGXAzyto+0NCCj85Om5DgaEp7zeL/l3rAr8lnCxOPRF7EvBh0j+nemTmoSRfO33tpKG7rySk+Ruj\n5xOBdoSEvIzwS+EY4GR3nx+1+ZJwknEO8CphDHgyodBPqWC/DxOKCmZ2AnAI8IC7r0h5jCGcBD4/\n+p7zCMV2HqHwtiZM//wy6scCQlH6PjAzGloZSZgR843hDHffFG1jFvBC1O85wI8In1QgTB8dS0i+\nnxCGX74xXz91s5U8z4T+wALCJ5g3gNvc/dXtb0afBE4GcPdZwGWEYbD/IwyBXZmyrXbRtv4DdAfa\nuftnKe//hjAFUwqAue/859HMHiXMpV3u7i0qaHMvIf2sB7q5+4xMd1QKg5mVACO8zAVRkhvMbD+g\nFDh2+y9SyW/pFPlTCB/lHy+vyJtZB6Cnu58ZpbN73P3ErPRWRESqpNLhGnefQJhxUZGOwONR2ynA\nXmZ2QGa6JyIiNZGJMfkD+fp45ZLoNRERSZhOvIqIFLBdM7CNJYRFk7Y7KHrtG8wsG7MOREQKnrtX\n6zqNdJO8UfGFIKOJVg80sxOB1e6+vIK2ic8ZzZVH//79E+9Drjx0LHQsCv1Y/OtfzvDhTs+ejplz\n6KFO167OgAHOuHHOhg072i5ft5xzhp/DDwb+gMmLJ+Nes2xcaZI3syeBImBfM1tEmK9bL9RrH+Tu\nL5rZGWY2jzCF8uIa9UhEJM+tXw9TpsBrr8Hw4bB0KXToAE2bwtSp8OMff/N73J0RM0fQ6+VedGvZ\njZJOJTTYtUGN+1JpkXf3C9Jo07PGPRERyWP//jeMHQvPPw8vvww//CGcdBI8/TQccwzU2cm4yYr1\nK7jihSuY9e9ZjOoyihMOOiFj/dKJ14QUFRUl3YWcoWOxg47FDvlwLLZtg4kToXNn2H9/GDoUiopg\n7lx4+20YMACOPbbiAu/uDP9wOC0ebEGTfZowvcf0jBZ4SONiqIzuzMzj3J+ISDYsXQp33w1DhsCe\ne0KvXqHQf/vb6W8jNb0P7jh4p8XdzPAsn3gVEanVtm6FadOgd29o3hxWr4YXX4R58+Cqq9Iv8HGk\n91SZmEIpIlKwvvgCbr8dbr4ZDj44JPYPP4QDq3HJZzbH3iuiJC8iUo516+D662G//WDGjFDYFywI\nBb+qBT7u9J5KSV5EJLJxIzz+eBhrf+89+OUvYfJkOOqoSr+1Qkmk91RK8iJS661YAX/4A+y2G9x1\nF/ToAStXwlNPVb/AJ5neUynJi0it9eGHcOut4YKlSy6BJUvgu9+t+XaTTu+plORFpNbZtg3uvRdO\nPRWOPhoWL4ZBg2pe4HMlvadSkheRWmPFCrj6anjjjXDx0jvvwGGHZWjbOZTeUynJi0jBW7sW/vu/\nQ2rfbz94/fVwYjUTBT4X03sqJXkRKVju8MADcMMN0L49TJgARxyRue3nanpPpSIvIgXniy/gzjvh\nkUegUSMYNy6k+EzJ1oqR2aAiLyIF48svYeBA6NcvLBT27LNhgTCr1qov5cuH9J5KRV5ECsI//wnd\nukGDBmEFyPLWbK+JfErvqVTkRSSvbd4MI0dCz55w441hRchMJnfIv/SeSrNrRCQvrVkTxt0PPzzM\ncf/HP8L0yEwW+FyfOZMOJXkRySsLF8Kf/xyK+hlnhHH3H/0o8/vJ5/SeSkleRPLCBx/Ab38LTZrA\nHnvA7NnhTkyZLvCFkN5TKcmLSE6bPh369g3rzPTsGe6l2qhRdvZVKOk9lZK8iOSkRYugQ4ew3O/P\nfw7z54f13bNR4AstvadSkheRnLJ0KTz0ULhStVcvGDUK6tXL3v4KMb2nUpIXkZywaFFY0/2oo8JC\nYpMnh+UIslXgCzm9p1KSF5FETZ8OAwbAmDFw0UUwaRIceWR291no6T2VkryIJGLmTOjYEc48E5o1\nC0n+vvuyW+BrS3pPpSQvIrH7+9+hT59wIvWpp2D33bO/z9qU3lOpyItIbGbOhJtvhilT4JVXwuJh\n2Zava85kioq8iGTdBx/AbbfBa69B797wt79Bw4bZ329tTe+pNCYvIlmxZQu8/DKcfjq0aQPNm8NH\nH4VlgLNd4Gvj2HtFlORFJONmzgwzZdatg9/9LsycqV8/nn0rvX+dkryIZMz69XDHHdC6NXTuDLNm\nhbnvcRR4pffyKcmLSEa89RZcfHG4mGnSpLAEcFyU3iumJC8iNTJpUlhbpls3uOWWsAxBXAVe6b1y\nSvIiUi2rVsHll8OECdC/fyjydevGt3+l9/QoyYtIlaxZE+7AdOihsM8+MHduOLkaV4FXeq8aJXkR\nSdvYsdC9O7RqFdZ3P+SQePev9F51aSV5M2tvZnPMbK6Z9S3n/T3NbLSZzTCzD8ysW8Z7KiKJ+eST\ncKu9Hj3C/VSfeireAq/0Xn2VJnkzqwMMBNoCS4FpZjbK3eekNLsSmOnuvzSzxsBHZlbi7luy0msR\nicWCBeFk6rPPhrXdn3kGdtst3j4ovddMOkn+eOBjd1/o7puBYUDHMm0c2H4NW0PgMxV4kfz1xRdw\n441hRchddw33U73xxngLvNJ7ZqQzJn8gsDjl+aeEwp9qIDDazJYC3wLOy0z3RCRub78drlY95piQ\n5A8+OP4+KL1nTqZOvLYD/unubczscOBVM2vh7uvKNiwuLv7q66KiIoqKijLUBRGpiXXrwiJijzwC\nDz4Iv/pV/H2o7StGbldaWkppaWlGtmXuvvMGZicCxe7ePnreD3B3vz2lzfPAre4+MXr+OtDX3d8p\nsy2vbH8iEr/x46FrVzjhBLjzTjjooPj7kJreB3ccrPSewsxwd6vO96YzJj8NaGJmh5pZPaALMLpM\nm4XAz6LOHAA0BRZUp0MiEp9Nm+Cee6BTp1Dchw2Lv8Br7D27Kh2ucfetZtYTeIXwS+FRd59tZj3C\n2z4I+AswxMzej77tWnf/T9Z6LSI19umncOGF8NlnMHEiNG0afx809p59lQ7XZHRnGq4RSdyWLWHc\n/Y9/hJ49wy344loGeLuyY+/FRcW1cuw9XTUZrtEVryK1yNSpobCbweuvx3P7vbKU3uOltWtEaoHP\nPgsLiHXsGBYVmzgx/gKvsfdkKMmLFLiSEujbN5xc/egj2HPP+Pug9J4cFXmRAuUebpo9ZgyMGAEn\nn5xEHzTvPWkq8iIFaMIE+NOfwgVOU6fCvvvG3wel99ygMXmRArJiBfzmN+HRtStMnhx/gdfYe25R\nkhcpAO4wZAj06xfWnZk1C/bYI/5+KL3nHhV5kTz38cdwxRXhdnwvv5zMtEiNvecuFXmRPLU9vV9z\nTZg907t3WBY4bkrvuU1FXiQPzZwZLmpatQpKS+Goo+Lvg9J7flCRF8kjmzbBgAFw111QXBxux6f0\nLjuj2TUieeKDD+C00+Ctt+Cdd+DKK+Mv8Jo5k3+U5EVy3JIlcMMNMHw4/PWvcOmlsMsu8fdD6T0/\nKcmL5Ch3ePhhaNkyzHVftCgMz8Rd4JXe85uSvEgOWrECfv1rWL0axo2DH/4woX4ovec9JXmRHPPS\nS9CqVVhr5t13kynwSu+FQ0leJEesWAGXXRZOsN57L5xxRkL9UHovKEryIglbtSpc0NSsGRx+OHz4\nYTIFXum9MCnJiyRkyxYYPBj+8hdo0wbefz/+m2hvp/ReuFTkRRIwezZ07x7muZeUwE9/mkw/dNVq\n4VORF4nR1q1w661hzP3qq+Haa5O5YhWU3msLFXmRmCxbBh06wN57w/TpyQ3NKL3XLiryIlnmDk88\nEU6uduwI998Pdesm0xel99pHRV4kixYuDFepLl8e5r8fd1wy/VB6r71U5EWyYO1a+POf4ZFHwlrv\nffoovUsyNE9eJIO+/DKcVD3iCFi5EqZMgeuuS6bAa967gJK8SMa89FK4efZPfgKvvgotWiTXF6V3\n2U5JXqSGliyB884L91l9/HF48cXkCrzSu5SlJC9SA489Fua6X355uHp1992T64vSu5RHRV6kGtat\ngxtvhKFD4Y03kh2a0cwZ2RkVeZEqevNN+K//glNOCYuJ7bdfcn1RepfKaExeJE3r1oX7qnbtCvfd\nF8bfkyrwGnuXdCnJi1TCPQzLXH89tG0b1ntv1Ci5/ii9S1WoyIvsxNtvh+UINm0Khf6UU5Lri8be\npTpU5EXKsWJFOLFaUhJWjbzySqiT4OCm0rtUV1o/tmbW3szmmNlcM+tbQZsiM/unmX1oZm9mtpsi\n8ZkyBY46Kly9+q9/wVVXJVfgNfYuNWXuvvMGZnWAuUBbYCkwDeji7nNS2uwFvA2c7u5LzKyxu68s\nZ1te2f5EkrJ6NfzpTzByJDz0EPzqV8n2JzW9D+44WMW9FjMz3N2q873p5JPjgY/dfaG7bwaGAR3L\ntLkAeMbdlwCUV+BFctn06XD00bB5M8yalWyBV3qXTEpnTP5AYHHK808JhT9VU6BuNEzzLeBed38i\nM10UyZ5t2+DOO+GOO2DQIOjUKdn+aOxdMi1TJ153BY4D2gB7AJPMbJK7z8vQ9kUybunSMOd90yZ4\n91049NDk+qKZM5It6RT5JcAhKc8Pil5L9Smw0t03AhvN7C3gGOAbRb64uPirr4uKiigqKqpaj0Uy\nYPTocCPtK68MSwEndZ9VUHqXbyotLaW0tDQj20rnxOsuwEeEE6/LgKnA+e4+O6VNM+A+oD1QH5gC\nnOfus8psSydeJVEzZkBxMbz3Xpj33qpVcn0pm96Li4qV3qVcNTnxWml+cfetZtYTeIVwovZRd59t\nZj3C2z7I3eeY2VjgfWArMKhsgRdJ0sqV0L9/mDlz/fXw1FOw227J9UfpXeJSaZLP6M6U5CVmq1fD\nPfeER+vW8PDDsM8+yfVH6V2qI6tJXiQfbdsGf/1ruFr1l78MFzgdcUSyfVJ6lySoyEvBmTcPzj47\nJPYpU6BJk2T7o5kzkiQVeSkYW7bAXXfBbbeFdWd+/3uwan3AzRyld0ma1pOXgjBjBpxwArzyCkyb\nBr16JVvgddWq5Aoleclr7mGdmf794fbboVs3pXeRVCrykreWL4dLLgkrRY4fD0cemWx/NPYuuUhF\nXvLS88+H1P6734W577vvnmx/lN4lV2lMXvLK1q1wyy2huI8ZE6ZIJlngNfYuuU5JXvLGmDHQuzcc\ndFA4uXrQQcn2R+ld8oGSvOS8uXPD+u59+sCDD0JpabIFXuld8omSvOSsTZtgwIBw5WqvXjBsGNSv\nn2yflN4l3yjJS8758ssw1t64MbzxBrzzDtxwQ7IFXuld8pWSvOSUt9+Gyy6Db387TIts2TLpHim9\nS35TkpecsGkTXHstnHtuWAp47NjkC7zSuxQCJXlJ3Pjx0LMnfO978P77YZgmaUrvUiiU5CUx//43\nXHwxdOkSbsH37LPJF3ildyk0SvKSiJUr4ayz4JBDQnrfd9+ke6T0LoVJSV5itWBBuFq1SRNo2xaG\nD0++wCu9SyFTkpdYrFsHd94Z5rz37g1z5oQZNElTepdCpyQvWbVtGzzySLj13rx5Yd33m25KvsAr\nvUttoSQvWbNgAZx3Xljf/fnn4Uc/SrpHgdK71CZK8pJxW7fCE09Aq1Zw/vkweXJuFHild6mNlOQl\no5YvDzfRBhgxAk49Ndn+bKf0LrWVkrxkzJNPQosWcNJJMHFibhR4pXep7ZTkpcZWrQpXrE6bBi+9\nBMcdl3SPAqV3ESV5qaFJk+Cww2C33cJqkblQ4JXeRXZQkpdqmT8fBg6EkpJwkvWss5LuUaD0LvJ1\nSvJSJUuWwIUXwgknQIMGIcnnQoFXehcpn5K8pK2kBC69FK6+Gj75BBo2TLpHgdK7SMWU5KVS69fD\n/feHlSInToTbbsuNAq/0LlI5JXmp0LZtMGRIuInHkUfCiy/C0Ucn3atA6V0kPSryUq45c6BjR9hr\nL3jhhdyYNQMhvY+YOYJeL/eiW8tulHQqocGuDZLulkjOUpGXr/n8c/if/4GHHw634+vXL6w9kwuU\n3kWqTkVevrJ2LZxxBuy3H8yaBd/5TtI9CpTeRapPRV4AePVV+P3voXVruOceqFs36R4FSu8iNaPZ\nNbXcqlVw7rlw2WVw883wwAO5UeA1c0YkM9Iq8mbW3szmmNlcM+u7k3Y/MbPNZtYpc12UbJk+PcyW\nOfBAmDkTOuXIv9qK9SvoPLIzN427iVFdRnHbz27T8IxINVVa5M2sDjAQaAc0B843s2YVtLsNGJvp\nTkpmuYfE3q4d3HtvGJ5pkAM1VOldJPPSGZM/HvjY3RcCmNkwoCMwp0y7q4CngZ9ktIeSUevWwVVX\nhcXExo+HZt/4dZ0Mjb2LZEc6wzUHAotTnn8avfYVM/sucLa7PwjkyIQ7Kev558NFTevXw4QJuVHg\nld5FsitTs2vuBlLH6lXoc8j69dC/Pzz2GAweHC5yygVK7yLZl06RXwIckvL8oOi1VD8GhpmZAY2B\nDma22d1Hl91YcXHxV18XFRVRVFRUxS5LVYwdC5dfHu63OmcO7L9/0j3SvHeRypSWllJaWpqRbZm7\n77yB2S7AR0BbYBkwFTjf3WdX0H4wMMbd/1HOe17Z/iQz1qyBXr3gmWdg5Eho3z7pHgWp6X1wx8FK\n7yJpMDPcvVojJJWOybv7VqAn8AowExjm7rPNrIeZdS/vW6rTEckMdxg3Do49FnbZBRYsyI0Cr7F3\nkWRUmuQzujMl+axaswY6dIDly+GOO+Ccc5LuUaD0LlIzWU3ykh9WroRf/xqaNoWPP86NAq/0LpI8\nrV2T57ZsgQcfhGuugSuvDDf0qJMDv7o1c0YkN+RAOZDq+OKLcCPtZs3g2Wdh8mQYMCD5dWeU3kVy\ni5J8Hpo4EX73O2jUKNy56eSTc2PNd6V3kdyjJJ9nnnoKzj4b+vQJxf6UU5Iv8ErvIrlLST5PLFkS\n5r1/8EG41+pPcmSFIKV3kdymJJ8HnnoKWraEH/wAZszIjQKv9C6SH5Tkc9iWLdC5M7z7briZ9vHH\nJ92jQOldJH8oyeeo9euhS5ewNPBHH+VGgVd6F8k/SvI5ZsMGeOihMB3y5z+HoUOhfv2ke6X0LpKv\nlORzyPz5YbXI0aNh1KgwPTLpAq/0LpLflORzxLBhcP75cNddYRZN0tMiQeldpBAoySdswwa4+upQ\n2MePD18nXeCV3kUKh5J8gj75BC6+GOrVC/Pfc+GGHkrvIoVFST4BW7eGW/Edeyy0bh3uvZp0gVd6\nFylMSvIxW7EiLEtQp06Y+37yyUn3SOldpJApycdo/nw47riw3sz48ckXeKV3kcKnJB+Td98Nt+G7\n6Sa44oqke6P0LlJbKMnHYOhQOPFEeOCB5Au80rtI7aIkn0UbNsB558HcuTBlShiqSZLSu0jtoySf\nJc89By1aQMOGMG1asgVe6V2k9lKSz7AtW6B37zBz5sEHoV27ZPuj9C5SuynJZ9DSpdCmTbiwafr0\nZAu80ruIgJJ8xgwbBlddBd27Q//+4SrWpCi9i8h2KvI1tHFjWHfmzTdh7Njkx95HzBxBr5d70a1l\nN0o6ldBg1wbJdUhEEqciXwOLF4eVI/fbD6ZOhUaNkuuL0ruIlEdj8tU0Zgw0bx7G3Z95JrkCr7F3\nEdkZJflqKCmBa64JC4udempy/VB6F5HKKMlX0dNPwx/+AK+9llyBV3oXkXQpyadpzZowc2bSpHCh\nU/PmyfRD6V1EqkJJPg0bNsC55+64ucdJJ8XfB6V3EakOJflKfPABnHlmuMhp0KBk5r8rvYtIdSnJ\n78TEidC2Ldx+OwwZEn+BV3oXkZpSkq/AmDFwySXwxBNhHfi4Kb2LSCYoyZexeTNccEFYouC55+Iv\n8ErvIpJJSvIppk6Fiy8ON9qeOjX+m2srvYtIpqWV5M2svZnNMbO5Zta3nPcvMLP3oscEMzs6813N\nns2b4eab4ayzoF8/mD073gKv9C4i2VJpkjezOsBAoC2wFJhmZqPcfU5KswXAqe6+xszaAw8DJ2aj\nw5m2di1ceiksWgRvvQXNmsW7f6V3EcmmdJL88cDH7r7Q3TcDw4COqQ3cfbK7r4meTgYOzGw3s2PE\niFDUGzWCV16Jt8ArvYtIHNIZkz8QWJzy/FNC4a/IpcBLNelUHIYMCevPjB4d/8VNSu8iEpeMnng1\ns9bAxcApFbUpLi7+6uuioiKKiooy2YVKrVkDd90FAwfCG2+E+7DGReu9i0g6SktLKS0tzci2zN13\n3sDsRKDY3dtHz/sB7u63l2nXAngGaO/u8yvYlle2v2yaNAk6dYJWreCOO+Dww+Pbd2p6H9xxsNK7\niKTNzHB3q873pjMmPw1oYmaHmlk9oAswukwHDiEU+AsrKvBJu//+sGrk3XeH9d/jKvAaexeRJFU6\nXOPuW82sJ/AK4ZfCo+4+28x6hLd9EHADsA/wgJkZsNnddzZuH6tx4+Cmm2DaNGjZMr79auxdRJJW\n6XBNRneWwHDN3/4G118PDz0UVpKMQ9mx9+KiYo29i0i11WS4pqCveL3nnnCR0wsvwAkxhWildxHJ\nJQW7ds3gwXDrrTB+fDwFXmPvIpKLCi7Jb90K//u/IcW/9BIceWT296n0LiK5qqCS/Nq14eYeo0eH\nBH/ssdndn9K7iOS6gknya9aEAn/00fDII7Brlv9mSu8ikg8KIslv3AitW8Nhh8HDD2e3wCu9i0g+\nyfskv3Ej/OIX0LQpDB0Ku+ySvX0pvYtIvsnrJL94MZx+OuyxB5SUZK/AK72LSL7K2yQ/YQKccw5c\nfTVce232CrzSu4jks7xM8pMnh4XGHn4YrrsuOwVe6V1ECkHeJfmnn4bLL4fHHgtj8dmg9C4ihSKv\nkvzrr8Nvfxvu4pSNAq/0LiKFJm+S/LJlcNFFIcln4yInpXcRKUR5keS//BLat4fu3eGsszK7baV3\nESlkOb/U8Guvwfnnw09/GlJ8nQz+WtLdmkQkH2T7zlCJmT0bzjwTbrkl3M0pUwVe6V1EaoucTfKf\nfx5ust27N/z+95nrg9K7iOSbgkzy/fqFIZpMFXildxGpjXJyds24cfDsszBzZma2p5kzIlJb5VyS\nX7IEunROnuk3AAAFhUlEQVQJN/3Ye++abUvpXURqu5xK8mvXhiGaiy6Czp1rti2ldxGRHEry7qG4\nt2oV7s1a/e0ovYuIbJczSb5v33BV67hxYNU6h6z0LiJSVk4k+bfeCjfffvRRqF+/6t+v9C4iUr7E\nk/zSpWHZ4CefhObNq/79Su8iIhVLNMlv2QJFRXDGGWHpgqpQehcRqVyiSf6552DTJhgypGrfp/Qu\nIpKexJL85s1w4YUwcGD6a9IovYuIVE1iSf7++6Fx4/Rv/qH0LiJSdYkk+UWLoE+f9IZplN5FRKov\nkVUou3eHVatg5Midt9eKkSIiebYK5dat4YRrnz4Vt1F6FxHJjNjH5N99N/x5QgU1W2PvIiKZE3uS\nv/pqOPvsb76u9C4iknmxj8nvsYezYAHsv/+O1zX2LiJSsayPyZtZezObY2ZzzaxvBW3uNbOPzWyG\nmbWsaFtnnbWjwCu9i4hkV6VF3szqAAOBdkBz4Hwza1amTQfgcHc/AugBPFTR9s47L/y5Yv0KOo/s\nzE3jbmJUl1Hc9rPbaLBrg2r/RfJNaWlp0l3IGToWO+hY7KBjkRnpJPnjgY/dfaG7bwaGAR3LtOkI\nPA7g7lOAvczsgPI21q6d0jvoBziVjsUOOhY76FhkRjqzaw4EFqc8/5RQ+HfWZkn02vKyG+v6fGfN\nnBERiUnsUyib7NOEkk4ltWpoRkQkKZXOrjGzE4Fid28fPe8HuLvfntLmIeBNdx8ePZ8DnObuy8ts\nK76pPCIiBaS6s2vSSfLTgCZmdiiwDOgClF39fTRwJTA8+qWwumyBr0knRUSkeiot8u6+1cx6Aq8Q\nTtQ+6u6zzaxHeNsHufuLZnaGmc0D1gMXZ7fbIiKSjlgvhhIRkXhlZVmDTF48le8qOxZmdoGZvRc9\nJpjZ0Un0Mw7p/FxE7X5iZpvNrFOc/YtTmv9Hiszsn2b2oZm9GXcf45LG/5E9zWx0VCs+MLNuCXQz\n68zsUTNbbmbv76RN1eumu2f0QfjFMQ84FKgLzACalWnTAXgh+voEYHKm+5ELjzSPxYnAXtHX7Wvz\nsUhp9zrwPNAp6X4n+HOxFzATODB63jjpfid4LK4Dbt1+HIDPgF2T7nsWjsUpQEvg/Qrer1bdzEaS\nz+jFU3mu0mPh7pPdfU30dDLh+oJClM7PBcBVwNPAijg7F7N0jsUFwDPuvgTA3VfG3Me4pHMsHGgY\nfd0Q+Mzdt8TYx1i4+wRg1U6aVKtuZqPIl3fxVNnCVdHFU4UmnWOR6lLgpaz2KDmVHgsz+y5wtrs/\nCBTyTKx0fi6aAvuY2ZtmNs3MLoytd/FK51gMBH5oZkuB94BeMfUt11SrbiZ2j1f5OjNrTZiVdErS\nfUnQ3UDqmGwhF/rK7AocB7QB9gAmmdkkd5+XbLcS0Q74p7u3MbPDgVfNrIW7r0u6Y/kgG0V+CXBI\nyvODotfKtjm4kjaFIJ1jgZm1AAYB7d19Zx/X8lk6x+LHwDAzM8LYawcz2+zuo2PqY1zSORafAivd\nfSOw0czeAo4hjF8XknSOxcXArQDuPt/MPgGaAe/E0sPcUa26mY3hmq8unjKzeoSLp8r+Jx0NdIWv\nrqgt9+KpAlDpsTCzQ4BngAvdfX4CfYxLpcfC3Q+LHt8njMtfUYAFHtL7PzIKOMXMdjGz3Qkn2mbH\n3M84pHMsFgI/A4jGoJsCC2LtZXyMij/BVqtuZjzJuy6e+ko6xwK4AdgHeCBKsJvdvewCcHkvzWPx\ntW+JvZMxSfP/yBwzGwu8D2wFBrn7rAS7nRVp/lz8BRiSMrXwWnf/T0JdzhozexIoAvY1s0VAf6Ae\nNaybuhhKRKSAxX6PVxERiY+KvIhIAVORFxEpYCryIiIFTEVeRKSAqciLiBQwFXkRkQKmIi8iUsD+\nH4ThOEN8p4OjAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x148198208>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 302 ms, sys: 32.1 ms, total: 334 ms\n",
      "Wall time: 428 ms\n",
      "PRECISION/RECALL AUC\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXkAAAEKCAYAAAD3tSVSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAH49JREFUeJzt3XmYXFW57/HvLyOBDE2ABJKYSIIBGRJkDEeONgElgAgP\nDyogKHI9RBGneziCAxI9XpHj41VERTlGUJHhCApBwAGkLzIHEYKQkEBCZgIJJCEhc977x9qdrlR6\nqO5UV1Xv/n2eZz9Ve9eqvd/aXf3utddae5ciAjMzy6ce1Q7AzMw6j5O8mVmOOcmbmeWYk7yZWY45\nyZuZ5ZiTvJlZjjnJW5ckaauk0dnz6yV9s5Wye0qaKalv5SIsP0n/lPSeasfRGSRdLOk71Y4jj5zk\na4CklyW9JWm1pKVZ0tq12nGVIot1Qxb7ckl/krR/BTbdngs8LgOuj4gNhQsl3SBpk6ShRct3OGhI\nGpUdWHoULDtH0nRJb0paLOluSe9uKQhJR0i6S9Lr2fRPSf8paVApHyIiDo6IB0v6xGUi6VBJT0pa\nm33W8a2U7SPpF5JWSVoi6YtFr58q6dnsu/KQpHcWvPzfwEcl7dlZn6W7cpKvDQGcEhEDgcOAI4Cv\nNVdQkioZWImuymIfDiwBfl6BbZa0HyT1AT4O3Fi0fFfgDGAlcG6J29x2YJH0v4H/C3wLGAKMBH4M\nnNpCHP8CPAD8Ddg/IgYDk4DNQIuJs5ok9QbuAH4F1GWPd0rq1cJbvgGMAd4GTAS+JOn92breQfob\nXJit6w/AtMaDZnYAvgf4WKd9oG7KSb52CCAilgL3AgcDSHpA0reyms9aYF9JAyVNzWpLC7Pa4Lak\nJ+nfJD2f1Zj+KenQHTYm/UTSd4uW3SHpC9nzSyUtytYxU9JxbX2A7B/1f4DttifpgiyeFZLulTSy\n4LWDJP05e22ppMuy5UdKekTSG1kt+ZpWkktrjgbeiIglRcvPBN4Avgmc354VShpISmgXRcSdEbEu\nIrZExD0RcVkLb7sKmBoR/xURrwFExKKI+EZj7VzSaEn3Z2dEr0q6MdtW43bnSZqYPb9C0q2Sfpn9\njZ6VdFh7PkcJ6oGeEfHDiNgUEdeQvqcTWyj/MeCbEbE6ImYB19G0b98P/C0iHo2Irdn+GA68t+D9\n/w84pcyfodtzkq8xkt4GnAw8VbD4XOCTwABgAfBLYAMwGngX8L7sdSR9CPg6cG5Wu/4gsKKZTd0M\nfLhgu3XZem6WNBb4DHB4to4TgZdLiH034BxgTsGy00jNJacDe5Fqsjdnr/UH/kKqwe0D7Afcn711\nC/AFYDBwDCmxXNRWDM04BHihmeUfA24CbgUOkPSudqzzX4C+pFpum7KzhmOA37VVFPg2sDfwTmAE\nMKWV8qeSPsMg4C7SmURLMTxT0Ez0RtHjj1p420HAjKJlz2TLi9dfR/obzmirbKYH6fMeXLBsJjV6\nVtOVOcnXjjskvQ48SDqtv7LgtRsiYlZWAxoMnAR8MSLWR8Ry4AfAWVnZ/wX8V0Q8BRARcyNiYfHG\nIuJvQEg6Nlt0JvBoRCwjJdg+wMGSekXEgoiY10rs/5HFvpqUAAtPuScDV0bE7Cz+7wCHZgezDwBL\nI+IHEbExItZGxPQsvqci4olIFpBqhe+l/eqANwsXZGcSxwE3RcSrwH20r5lgMLA8+zyl2J30v/ZK\nQQxXZUl2jaSvAETESxFxf0RsjogVwPdp/TM/FBF/inQDql8D41oqGBHjI2JwNu1e9HhxC2/rD6wq\nWraaVNlormwUlS8sex/wXknvyZqBvgL0Bgr7nt4kHbCsjJzka8dp2T/cvhHx2aJOwsIkPYr0z7G0\nsTYG/JRUS4bUHvpSidu8FTg7e34O8BtIyYZUi54CLJN0k6R9WlnPd7M25lHAOqCw43UUcHVjLZJ0\nVhGkU/UWY5X0DqVOyqWSVgL/B+hIp9wb7JiUzgOej4hns/mbSZ1+PbP5zaR9XKg3sDVL7CuAPVXQ\nCVtCDFtJNV0AIuLSiNgd+D3QC0DSEEk3Z81kK0lt2K195lcKnr8F7NKOmEqxBhhYtGwQRQfNgrIU\nld9WNiJeIPWN/JjUbzMYeB5YVFB+ADseVGwnOcnXjtY6EgtHkiwE1gN7FNTG6iJiXMHrY0rc5s3A\nmVnN9mjg9m0bjLglIv6VlKQh1cBbFRGLSAeHH6ppuOICYHJRLbJ/RDzWRqzXkk7fx0REHfBVSuxs\nLTIDGFu07DxgdHYAWQp8D9iD1EzWGPPbi94zmqaD7aOk5rLTSwkgIt4CHid19Lbm26SDwUHZZz6X\njn3mHWR9M6uLpjezx5+08Lbn2PHsYFy2fDsRsRJYyvbNLeMLy0bE7yLikIjYi1SB2BeYXlD+naQm\nHisjJ/kuJiJeAf4MfF/SACWj1TR++ufAJY2dcJLGFHZ0Fq3raVKt9OfAHyNidfaesZKOUxqZspFU\nOy+paSIi7gMWk5ppAH4GfEXSgdm6B0k6M3vtD8Dekj6nNPyuv6SjstcGAKsj4i1JBwCfLmkH7egJ\noK7xTETSMaSEfSQpCY0ntRvfTKppQjrYnSLpBEk9JA0jHWRuzj7jauAK4MeSTpPUT1IvSZPU8ljv\nLwEXSPqSpL2yWEaQEl2jAaQa8ZuShgP/0c7P2uIBIRt+ObBoGpA9ttTX0QBskfTZ7O/zOdL34K8t\nlP818DVJdUrDI/8NuH5bcNJh2f7ci9T8dkdEzC54/3tJgw6snCLCU5UnYC4wsYXX/gpcULRsAPAT\nUs3yDeDvwIcLXr8QmEVqE50BjG9l218jtcGfUbDsEFLNcxWwHJgG7N3C+39BGlFRuOzDWWy9s/mP\nZnGsBOYDPy8oeyCpvfZ10mn8l7Ll/0qqya8mjbqYAjxY8L4twOjs+fXFMRTFc1XBeq8F/qeZMkeS\nDmZ12fwpwJPZ/p1HOpPpW/Ses0k10Tez2O8CJrQSx5HA3dlnfT3bJ/8J7F6wL57MPvNTwBeBBc19\nT0gHmV8VvDYq2yc9yvzdHJ/FtDZ7HFfw2jnAswXzfYCp2fdmKfD5onX9Lftsy7Pvb7+C13bJvjN7\nVfv/MW+Tsh3cIklTSR1ky6KpSaC4zA9JnYFrgfMj1RDNaoLSBTYPAu+KoguirDZIuhgYES0PQbUO\nKiXJH0s6hfxVc0le0knAxRFxiqSjgasjYkKnRGtmZu3SZpt8RDxEOmVtyWmkK+GIiMeBQSq6TNzM\nzKqjHB2vw9l+iN/ibJmZmVWZR9eYmeVYR+4FUmwx6aKWRiOyZTuQ1J47B5qZWSYiOnTNRKk1edHy\nGNxpZJeES5oArIx0aXwLgksuqf6wompPV1xxRdVjqJXJ+8L7wvui9WlntJnkJd0EPAKMlbRA0ick\nTZZ0IUBE3APMk/Qi6cKXNm8idUdJt3UyM7Od1WZzTUScU0KZlm5w1KwXX4TDD4e//7097zIzs/aq\neMfrhRfCsGGweXOlt1xb6uvrqx1CzfC+aOJ90cT7ojzavBiqrBuT4uGHg9Wr4aKLYO7cim3azKzL\nkkR0csdr2QwZAmvWwLzW7k5uZmZlUfEk368fjB4Nh+7wg3RmZlZuFU/yfftC796waVOlt2xm1v1U\nPMnvsgv06QMbN1Z6y2Zm3U9VavI9e8KcObB+faW3bmbWvVQ8yffqBXvvnZ7feWelt25m1r1UPMlL\n0L9/en7TTZXeuplZ91K1u1B+/ONQwSH6ZmbdUsUvhmrcnrJh/atXw4ABFQvBzKzL6VIXQzW67rr0\nePvt1YrAzCz/qlaTX78+XRgFbrYxM2tNl6zJ77JL0/Of/axaUZiZ5VtVf/7vpz9Nj5/6VDWjMDPL\nr6om+cmT4XvfS89feaWakZiZ5VPVf8j7Ix9Jj3fdVd04zMzyqGodr4UmTICRI2H2bHj66YqFY2bW\nJexMx2tNJHkVhL516/bzZmbdXZccXdMSj5s3MyufmqjJP/tsGis/fjwccADMnFmxkMzMat7O1OR7\nlTuYjjjkkPR4ySX+MREzs3Kqqeaat96Cq6/2FbBmZuVSU0m+ri499ugBjz2WRt1s3VrdmMzMurKa\naJPfvsz28x/5CHz5yzBunEfdmFn3lKvRNQsWND0/7ji49VY49ND0aGZm7VNzNfmtW+GEE+Duu+Hx\nx1OiBxg2DBYvrkCQZmY1pstfDNWaCDj6aJg+Hf74RzjxxE4KzsysRuU6yQOsWdP061EeeWNm3U2u\n2uSb078/3Hhjej5kCFx+eXXjMTPrKrpETR5SDb5HwSHp/vth4sQyBWZmVsNyX5OHNHzyoYfg5JPT\n/PHHp2VTp1Y3LjOzWtZlavKFXn4Z9t23aX7Llu1r+WZmedItavKF3v721Hzzxhtp/rzzqhqOmVnN\n6pI1+UJ9+8LGjbBu3fY/Dm5mlhfdriZfaP369NivX7rBmZmZNSkpyUuaJGmWpNmSLm3m9YGSpkl6\nWtKzks4ve6QtxpYulALYbTf47W8rtWUzs9rXZnONpB7AbOB4YAkwHTgrImYVlPkyMDAivixpT+AF\nYGhEbC5aV9mbaxo98EDTkMrp0+GIIzplM2ZmFdfZzTVHAXMiYn5EbAJuAU4rKhNAdk0qA4AVxQm+\nsx13XGqbHzgQjjwSXnqpkls3M6tNpST54cDCgvlF2bJCPwIOlLQEeAb4fHnCa5/evWHVqvR8v/3g\ngguqEYWZWe0o18//nQj8IyImShoD/EXSuIhYU1xwypQp257X19dTX19fphCarF+fRtpcf32afL8b\nM+tKGhoaaGhoKMu6SmmTnwBMiYhJ2fxlQETEVQVl/gBcGREPZ/P3A5dGxJNF6+q0NvliEXDvvXDK\nKWl+40bo1cs/PGJmXU9nt8lPB/aTNEpSH+AsYFpRmfnACVkwQ4GxwNyOBFQuUroFwr//e5rv0ydd\nFfue92z/wyRmZnlW0sVQkiYBV5MOClMj4juSJpNq9NdJ2ge4Adgne8uVEXFzM+upWE2+0NNPpxE3\nF164/fKNG1M7vplZLcv9/eTL6aWX4Kab4OtfT/PXXAMXX1zVkMzMWuUk3wEPPwzHHpueP/IIHHNM\ndeMxM2uJk/xOaOyIrbGwzMy26db3rtlZd96ZHkePhq9+FTZscMI3s/zo9kn+gx9MTTcHHwzf/nYa\nX9+jB1x0EWyu6DW7Zmbl1+2bawotXQqbNsHpp6cfIpkxA+bO3f4HSszMKs1t8p2g8DdlP/rRdOWs\nh1uaWTW4Tb4TSCnRf//7qTmnTx949tlqR2Vm1j6uyZegsFZ/wQVw7bUp6ZuZVYJr8p2ssVb/oQ/B\nL36RfnJQghtvrHZkZmatc02+ndauhfPPh9tua1p24IHw3HNVC8nMcs41+Qpq/InBCHjhBfjUp+D5\n51PN/uqrqx2dmdn2XJMvgw0b0vh6gEMOSR20F10El18Oe+9d3djMrOvzEMoasWhRuvnZL3+ZxtvP\nmQOjRsFZZ6XXr7zS97M3s/Zzkq9BW7ak+9mPGQP9+8N3v5uW/+xnO97y2MysNU7yXcCGDfDjH6cf\nMbnwQvjc52DECNh1V19kZWatc5LvQn77W/jMZ+C115qWvfUW9OtXvZjMrLY5yXdBEbB1a/rd2UZ3\n3AGnntp04ZWZGXgIZZckQc+eKdlPnQqHH55ujNazJ3zlK6lN38xsZ7kmX0M2boQzzoC7707zX/hC\nenzf+2D//dM97z06x6z7cXNNzmzdCg88kK6qveceGDQojb0/7LB02+P9908duIMHVztSM6sEJ/lu\nYO3aNDrnrbfgH/+AadNgyJA0Mmf//dN07rlw9NGpycfM8sNJvht65RX4+99Tm/6cOald/+WXYf16\nmDgx/Uj5UUfBhAlQV1ftaM1sZzjJ2zarV8N998E110BDQ1q2//7w9rfDpZfCccdVMzoz6wgneWvR\nokXw9NNwww1w++1p2d57w0knpZr+unWwzz4wfDgceWTTPXjMrHY4yVtJNmyAhQvTbZHvuw9efTVd\nlPXYYynZQ/qpw3e/G972NjjmGNhjj+rGbGZO8lYmr7wCt94KDz0EzzyT2vobnXkmnHBC+uEUj+ox\nqywneesUEbBiBTz1FMyfD7/5DTz6aBrPP3AgjBuXbsB2+OFw2mkwdGj6WUSP5TcrLyd5q5iNG+HJ\nJ9OInTvuSLdUnjYN5s2DVatg2LB00dYll6R2/8LbNphZxzjJW02IgJdeShdwfetbsGYN1NfDgAHp\nIHD22fCud6XRPkOH+h49ZqVykreaNH9+ulL3qafgne9MNf6//hWWLEmjeA48MN2yYbfd0sieMWPS\n3Tj32isdAHwLZrPESd66lIjUtPPEE/Dww+kiriVLYMaMNOKnUN++cPzx6SAA6Qxg5Eg4+OD06PZ/\n6w6c5C1XtmyBzZvTUM+FC2Hp0nQQ2Lw5jfpZuRIeeSTd0+fUU1PNf9dd0319Dj0UDjrI4/0tX5zk\nrduJSIn/ttvS+P+1a2Hu3HQQmDs31fJHjUqjgIYOhbFj04Fgr73SQWCfffxDLdZ1OMmbFXj55dT0\ns3lzur+PBMuXp/v6zJmTzgL69k19AcOHpwu/Bg+GyZNhv/3SmcLIkemgYFYLnOTN2qnxGoDFi1Pf\nwHPPpds7z5iRhoeuXJnK7bVXOiCMH58OBqNHpyahMWPSvDuHrRI6PclLmgT8gPRLUlMj4qpmytQD\n3wd6A69FxA63wnKSt65iy5bUObxiRZpefTX1Dzz/fJrmzoUFC1JT0N57p+afPn3SAWHw4NRM1KdP\nOhsYPDgdLBrLDhvmswRrn05N8pJ6ALOB44ElwHTgrIiYVVBmEPAI8P6IWCxpz4hY3sy6nOQtNzZv\nTvf+KewYXrcuLVu/vqmv4JVXYNmydNBYtiyVHTYs9Q3suWda18iR6XqCAQNSh/KgQelAsGlTOmsY\nMCCdYfi3Arqnzk7yE4ArIuKkbP4yIApr85I+DewTEV9vY11O8tbtbdoEs2alC8fmzUsHhzVr4M03\n07RyZbpl9GuvpWakoUPT66tXpzOEwYPTjePq6tLBoK4uHSz23DN1Ju+6a9M0aFBqYho61AeIrmxn\nknwpF50PBxYWzC8CjioqMxboLekBoD/ww4j4dUcCMsu73r3hkEPS1B5btqQDQGMT0qpVaX7lytSx\nvHBhOpNYty79gthrr6VpxYp0BtG/fzo49O+fOp2LH/v1gxEj0plDXV1aVjztuquvVO5qynVnkV7A\nYcBEYDfgUUmPRsSLZVq/WbfXs2dK0h25/fOGDemgsHZtOitYu3bH5wsWpAPFo482lS2e1q1rGpnU\n2tSjR+p/GDgwjW7q2TP1UfTrl+aHDGnquO7VKz327t207h49Unmffey8UpL8YmBkwfyIbFmhRcDy\niFgPrJf0IDAe2CHJT5kyZdvz+vp66uvr2xexmbVb374pse6srVtTf0NzB4DCacWKVO7ll1NzVP/+\nqZlqxYp0drFhQ2p+2rw5Ld+8ualPY3lBb17//k3NUv36pQ7uurqm/osBA9KBpG/fdAYycGCaGpux\nuupFcQ0NDTQ0/rTbTiqlTb4n8AKp43Up8ARwdkTMLChzAHANMAnoCzwOfCQini9al9vkzawkW7em\nM43GJqlly5r6LQqnhQvTmceaNelx9er0uGpVWk/jqKZevVLy37w5ze+xR7pZ3vDh6fnuu6eziREj\nau9CuUoNobyapiGU35E0mdQBe11W5hLgE8AW4L8j4ppm1uMkb2YVs25dOjgsW5aao9asSYl+yRJ4\n/fV0ncTixfDCC00HByl1bg8bls4cGofHjh6d+jqGDm0aAdU4FZ5tdAZfDGVmViabNqVhr41DY+fP\nhxdfTM1Iu+/e1L9ROL3xRmqGAnjHO1JT1ahRTaOa9t03/cjOQQelZrMhQ9rX3+Akb2ZWZRGpWWne\nvJTAV61KfRK//336sZ0tW9ItNd58M5U/7LDUjzB2bDowjB2bbsldV7fjup3kzcy6kMWLU1/CvHnp\nfkqzZ6cDwLx5qfb/wQ/CySfDBz6Q+hKc5M3McmD9erj//qZpxgz49Kfh2mud5M3Mcmf+fDjnHHjk\nkc694rWs9A3/lI+ZWcneT7ozWAe5Jm9mVuN2pk3ed6EwM8sxJ3kzsxxzkjczyzEneTOzHHOSNzPL\nMSd5M7Mcc5I3M8sxJ3kzsxxzkjczyzEneTOzHHOSNzPLMSd5M7Mcc5I3M8sxJ3kzsxxzkjczyzEn\neTOzHHOSNzPLMSd5M7Mcc5I3M8sxJ3kzsxxzkjczyzEneTOzHHOSNzPLMSd5M7Mcc5I3M8sxJ3kz\nsxxzkjczyzEneTOzHHOSNzPLMSd5M7McKynJS5okaZak2ZIubaXckZI2STqjfCGamVlHtZnkJfUA\nfgScCBwEnC3pgBbKfQf4U7mDNDOzjimlJn8UMCci5kfEJuAW4LRmyn0WuA14tYzxmZnZTiglyQ8H\nFhbML8qWbSNpGHB6RFwLqHzhmZnZzihXx+sPgMK2eid6M7Ma0KuEMouBkQXzI7JlhY4AbpEkYE/g\nJEmbImJa8cqmTJmy7Xl9fT319fXtDNnMLN8aGhpoaGgoy7oUEa0XkHoCLwDHA0uBJ4CzI2JmC+Wv\nB+6KiN8181q0tT0zM9ueJCKiQy0kbdbkI2KLpIuBP5Oad6ZGxExJk9PLcV3xWzoSiJmZlV+bNfmy\nbsw1eTOzdtuZmryveDUzyzEneTOzHHOSNzPLMSd5M7Mcc5I3M8sxJ3kzsxxzkjczyzEneTOzHHOS\nNzPLMSd5M7Mcc5I3M8sxJ3kzsxxzkjczyzEneTOzHHOSNzPLMSd5M7Mcc5I3M8sxJ3kzsxxzkjcz\nyzEneTOzHHOSNzPLMSd5M7Mcc5I3M8sxJ3kzsxxzkjczyzEneTOzHHOSNzPLMSd5M7Mcc5I3M8sx\nJ3kzsxxzkjczyzEneTOzHHOSNzPLMSd5M7Mcc5I3M8sxJ3kzsxwrKclLmiRplqTZki5t5vVzJD2T\nTQ9JOqT8oZqZWXspIlovIPUAZgPHA0uA6cBZETGroMwEYGZErJI0CZgSEROaWVe0tT0zM9ueJCJC\nHXlvKTX5o4A5ETE/IjYBtwCnFRaIiMciYlU2+xgwvCPBmJlZeZWS5IcDCwvmF9F6Ev8kcO/OBGVm\nZuXRq5wrk3Qc8Ang2JbKTJkyZdvz+vp66uvryxmCmVmX19DQQENDQ1nWVUqb/ARSG/ukbP4yICLi\nqqJy44DbgUkR8VIL63KbvJlZO3V2m/x0YD9JoyT1Ac4CphUFMJKU4M9rKcGbmVnltdlcExFbJF0M\n/Jl0UJgaETMlTU4vx3XA5cBg4CeSBGyKiKM6M3AzM2tbm801Zd2Ym2vMzNqts5trzMysi3KSNzPL\nMSd5M7Mcc5I3M8sxJ3kzsxxzkjczyzEneTOzHHOSNzPLMSd5M7Mcc5I3M8sxJ3kzsxxzkjczyzEn\neTOzHHOSNzPLMSd5M7Mcc5I3M8sxJ3kzsxxzkjczyzEneTOzHHOSNzPLMSd5M7Mcc5I3M8sxJ3kz\nsxxzkjczyzEneTOzHHOSNzPLMSd5M7Mcc5I3M8sxJ3kzsxxzkjczyzEneTOzHHOSNzPLMSd5M7Mc\nc5I3M8sxJ3kzsxwrKclLmiRplqTZki5tocwPJc2R9LSkQ8sbppmZdUSbSV5SD+BHwInAQcDZkg4o\nKnMSMCYi3gFMBn7aCbHmSkNDQ7VDqBneF028L5p4X5RHKTX5o4A5ETE/IjYBtwCnFZU5DfgVQEQ8\nDgySNLSskeaMv8BNvC+aeF808b4oj1KS/HBgYcH8omxZa2UWN1PGzMwqzB2vZmY5pohovYA0AZgS\nEZOy+cuAiIirCsr8FHggIm7N5mcB742IZUXran1jZmbWrIhQR97Xq4Qy04H9JI0ClgJnAWcXlZkG\nfAa4NTsorCxO8DsTpJmZdUybST4itki6GPgzqXlnakTMlDQ5vRzXRcQ9kk6W9CKwFvhE54ZtZmal\naLO5xszMuq5O6Xj1xVNN2toXks6R9Ew2PSTpkGrEWQmlfC+yckdK2iTpjErGV0kl/o/US/qHpH9K\neqDSMVZKCf8jAyVNy3LFs5LOr0KYnU7SVEnLJM1opUz782ZElHUiHTheBEYBvYGngQOKypwE3J09\nPxp4rNxx1MJU4r6YAAzKnk/qzvuioNz9wB+AM6oddxW/F4OA54Dh2fye1Y67ivviy8CVjfsBWAH0\nqnbsnbAvjgUOBWa08HqH8mZn1OR98VSTNvdFRDwWEauy2cfI7/UFpXwvAD4L3Aa8WsngKqyUfXEO\ncHtELAaIiOUVjrFSStkXAQzIng8AVkTE5grGWBER8RDwRitFOpQ3OyPJ++KpJqXsi0KfBO7t1Iiq\np819IWkYcHpEXAvkeSRWKd+LscBgSQ9Imi7pvIpFV1ml7IsfAQdKWgI8A3y+QrHVmg7lzVKGUFoF\nSDqONCrp2GrHUkU/AArbZPOc6NvSCzgMmAjsBjwq6dGIeLG6YVXFicA/ImKipDHAXySNi4g11Q6s\nK+iMJL8YGFkwPyJbVlzmbW2UyYNS9gWSxgHXAZMiorXTta6slH1xBHCLJJHaXk+StCkiplUoxkop\nZV8sApZHxHpgvaQHgfGk9us8KWVffAK4EiAiXpI0DzgAeLIiEdaODuXNzmiu2XbxlKQ+pIuniv9J\npwEfg21X1DZ78VQOtLkvJI0EbgfOi4iXqhBjpbS5LyJidDbtS2qXvyiHCR5K+x+5EzhWUk9Ju5I6\n2mZWOM5KKGVfzAdOAMjaoMcCcysaZeWIls9gO5Q3y16TD188tU0p+wK4HBgM/CSrwW6KiKOqF3Xn\nKHFfbPeWigdZISX+j8yS9CdgBrAFuC4inq9i2J2ixO/Ft4AbCoYWfikiXq9SyJ1G0k1APbCHpAXA\nFUAfdjJv+mIoM7Mc810ozcxyzEnezCzHnOTNzHLMSd7MLMec5M3McsxJ3swsx5zkzcxyzEnezCzH\n/j9l9Rn9zmlFfgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x13a317048>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 312 ms, sys: 26.9 ms, total: 339 ms\n",
      "Wall time: 404 ms\n"
     ]
    }
   ],
   "source": [
    "rf = sklearn.ensemble.RandomForestClassifier()\n",
    "rf.max_depth = 5\n",
    "clf = rf\n",
    "%time clf.fit(X[tidx],y[tidx])\n",
    "\n",
    "print(\"TEST\")\n",
    "%time evaluate(clf,X[pidx],y[pidx])\n",
    "\n",
    "print(\"TRAIN\")\n",
    "%time evaluate(clf,X[tidx],y[tidx])\n",
    "\n",
    "print(\"ROC AUC\")\n",
    "%time roc_auc(clf,X[pidx],y[pidx])\n",
    "\n",
    "print(\"PRECISION/RECALL AUC\")\n",
    "%time prrc_auc(clf,X[pidx],y[pidx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2min 39s, sys: 18 s, total: 2min 57s\n",
      "Wall time: 3min 10s\n",
      "TEST\n",
      "Accuracy 0.700501736781 Precision 0.736763277202 Recall 0.731257533146 F1 0.734000080655\n",
      "CPU times: user 169 ms, sys: 182 ms, total: 351 ms\n",
      "Wall time: 316 ms\n",
      "TRAIN\n",
      "Accuracy 0.744906822489 Precision 0.774825455824 Recall 0.771747613616 F1 0.773283472106\n",
      "CPU times: user 587 ms, sys: 350 ms, total: 937 ms\n",
      "Wall time: 1.17 s\n",
      "ROC AUC\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXkAAAEKCAYAAAD3tSVSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3X28VXP6//HXhZKbMkK+hPiW1KBM7kI/Tm5SbibjZuR2\nmDHyIMyYMRgmx4xBbgaDhoYRpa+KMYoojOMuVKLohhSpNKhImKhcvz8+K+22c9r7nLP3Xnuv9X4+\nHufR2Xuvs/Z1Vudc570/67M/y9wdERFJpvXiLkBERIpHTV5EJMHU5EVEEkxNXkQkwdTkRUQSTE1e\nRCTB1OSlpMxsmJn9OO46GsPMLjOzQXHXUQxm1srMpptZk7hrkcJQk08hM3vfzL4ys8/N7EMzu9fM\nNs7aZn8zeyba5lMze9TMOmZt09zMbjGzudF2s8zsL2bWso7n3R3o5O6jsu6vMrNvzezirPsPMrN5\nteznWTP7ecbt9mY2wsw+iWp9w8x+bWZWRx2bRnW+Z2bLouMxwsz2yXnwAHe/1t3PzmfbQjGzpmb2\nDzNbGv2f/Xod2x5hZi9Ex+JDMxtkZptmPL6tmf3LzBab2Qdm1nf1Y+7+MfBvoG9t+5bKoyafTg4c\n6e4tgD2AHwGXrX7QzPYDxgKPANsAOwFTgZfMbMdomyaEZtAR6BHtaz9gEVBXs+wLPFDL/acDi6N/\na6u1TmbWFngFmAvs5u6bAycAXYDmtWzfFHgW2BU4AmgRfQ8PAj3X9VwxuwpoC2wPHAz8zsx61LFt\nC+BPhP+7jsB2wPUZjw8FZgNbAUcB15jZQRmPD0NNPjncXR8p+wDeAw7OuD0AGJ1x+3ngtlq+bgww\nOPr8LGAhsFE9nnc2sH/WfRsDnwM/BZYDXTIeOwj4oJb9PAv8PPp8aGbtedRwFrAAaJZju1uAD4Cl\nwESgW8ZjVwJDos/bAN8S/kDNBT4Gfl+E/7MFwCEZt68ChuX5tT8BpkSfbxLVu0XG43cB92XcXh/4\nEtg+7p9VfTT+Q0k+5cxsO6AXMCu6vRGwP/BQLZuPAA6LPj8EeNLd/5vn82xMeEXwdtZDxwHLgJHA\nOOBn9fwWDqmj1nVtP9bdl+fYbgLQCdickGxHRq8CVst+hXEAsDNwKNDfzHapbadmdkk0jLIk+jfz\n8yV1fM0PCKl8asbdUwivRvJxEDBt9e6i2jOHsgzY7btvzH0V8C7QOc/9SxlTk0+vf5nZ54S0+hFQ\nHd3fkvBzsbCWr1kIbBl9vkUd29TlB4Tmsizr/tOBBz1EyGFAHzNbvx77rW8dWwL/WX3DzDpHDXap\nmc1Yfb+7D3P3z9z9W3e/GdgQqLVxE76vanf/xt2nEhpwrQ3S3Qe4++bu3jL6N/PzWs9lAJtGz7E0\n477PqWU4KpuZHQacBvwhev4vgJeAP5jZhmbWhfCHduOsL11G+D+TCqcmn169PYyjHwR0YE3z/pTw\ncn6bWr5mG8KYO4Qx9Nq2qctn0b/fNaboVUR3QnMHGAVsBBwZ3V4J1DbLowmwooF1rLW9u0/xMI5/\nLKGRr67tt9Esk0/N7FPCOPeW39vbGh9lfP4VoTEXyhfRvy0y7tuM7//BXIuZdSWcAznO3WdnPHQK\n8L+EP/B3AEOA+Vlf3pw1/2dSwdTk08sA3P0F4D7gpuj2V8DLhJOX2X4KPB19/jRweDS8k1O039lA\n+4y7T4/qGG1mC6PHN2TNkM0HwJbZM38I4+BzM+o4Lp8aIs8APdZVt5n9P+Bi4PjVaZuQnGudrVMf\n0fTLZdFspMyPZdErq+9x988Ir1YyXx10Zs0QTG3P8yPgX8AZ7l6Ttb957n60u2/t7vsRTsBOyPja\n9YF2hFckUuHU5AXCScbDoimOAJcCPzOzftF0w83N7GqgK/DHaJshwDzgYTPbxYItoiZW1yyVMYRX\nDqudThgm2oPQtDoDxwNHmtnm7j4PeBUYYGabRNMIfwd8Q5hRA+Ek6P5mNsDMtgYws3ZmNsTMMpPv\navcTGuYjZrarma1nZhsCe2dssynhlcLi6Dn7s+6hkbybv4fpl83dvUXWR/PolVVdhgBXmNkPoqms\nvwTurbUYs92AJ4Dz3X1MLY93iP5fm5jZqYTzLH/J2GQf4L3o+EuFU5NPp7VOGrr7IkKa7x/dfgk4\nnJCQFxJm43QGDlj9st/dvyGcZJwJPEUYL36FMEb+ah3P+3fgVAAz2xfYARjo7h9nfIwmnAQ+Kfqa\nE4GtCScC5xOGd46Mnh93n0OYurkTMC0aWhlJmBHzveEMd/862sd04PGo7pnAnoRXKhCmj44F3om+\n968If9Dqkn0SthgXabgSmEN4BfNv4Dp3f2r1g9ErgQOimxcRhpbuie5fZmZvZuzr8GhfS4CzgcPd\nfXHG46cAdxbhe5AYWDjftY4NzO4hzKX9yN071bHNXwkzNL4kvDx8o9CFSjKY2VBghGe9IUrKg5lt\nBdQAP1r9h1QqWz5NvhvhxM/9tTV5M+sF9HP3I6N0dqu7dy1KtSIiUi85h2vc/UXCjIu69CaMc+Lu\nrwKbrR4bFRGReBViTL41a49XLojuExGRmOnEq4hIgm1QgH0sICyatNp20X3fY2bFmHUgIpJ47t6g\n92nk2+SNuucCjwLOA4ZH77D7zN0/qmNbcp3oTYvq6mqqq6vjLqMs6FiskaZj8cUX8O67MH8+LFoE\nH38Mn3wCn34aPiZPrmbDDatZtAiWLIGmTeEHP4AttoDmzWGzzcJHs2bho2XL8HirVuHxjTYK97uH\n7Zo2DR9NmkCLFmHbDTfMXWepffzlx5z7+LlM/2Q69/a+l32325c6Vs3OS84mb2bDgCpgCzP7gDBf\ntyng7j7I3cdE61e/S5hCeWaDqxGRRPn6a1iwAF5+Gd55B2bNgunTYc4cWLYsNOYuXeB//id8bLkl\n7LJLaMDNmsGZZ8IOO8A228Cmm0Ijel3Zc3dGTBvBhU9eyBl7nMHQY4fSbINmjd5vzibv7ifnsU2/\nRlciIhVpxQqYPTs08sWL4c03w+0JE2DlSmjdGjp3hjZtoGdP6Ns3NO42bWC9dZwVfOstOPTQ0n0f\nccpM74/2eZR9t9u3YPsuxJi8NEBVVVXcJZQNHYs1yvVYuMN//gMzZoRkPn06vPhi+HfZMth+e+jQ\nITTvffYJCbxtW9h2W1i/PmuKZijXY1FIxUrvmXK+GaqgT2bmGpMXKW/ffBNS9JQpIZ2/8EJo8GbQ\nrl0YStlvP9h7b9h559DMN1BcrLfaxt7rYmYNPvGqJi+SYv/9L7z6Krz2GkycCP/+dzj52bo1HHgg\n7LsvdOwYhltatUr2mHipZKf36qrqnOldTV5E8vL11/DSS2G8/LXXYPRo6NQJunYNzXznnaFbt3DS\nUwqvPuk9k5q8iNRp8WJ45BEYNQqeegq22gqOOgr22gt69w5TEqW4GpLeM6nJi8haZsyAMWPg8cdh\n0iQ47DA47jg4+OAwVVFKp6HpPVNjmrxOl4gkwNKlMGxYSOpjx4YZLT17wq9+Bd27hzcHSWmVYuZM\nPpTkRSrUokXw7LMweHD499BDoU+fcJK0Q4eGT12UxitEes+kJC+SEt98A0OHwqBBYUima9fQ2EeM\ngE02ibs6KZf0nklNXqQCTJ4MN90E//oX/PCHUF0NhxyiWTDlpJjvWm0MLTUsUqaWL4chQ8KY+j77\nhDVd5s4N89mPPFINvly4O8PfGk6nv3WiXct2TO47uWwaPCjJi5SVVatCan/4YbjvPth117DWy5gx\nYVVFKS/lmt4zKcmLlIFFi+C226B9ezj11PCmpWeegaefDmPuavDlpdzTeyYleZGYzJsXEvrgwWFp\ngSOOgIED4fDD465M1qUS0nsmNXmREnr/ffi//4MnnoBp08KblC6+GHr1Ulovd+U4cyYfmicvUmTv\nvx+mPI4aBR99FJYUOO64cEJV0x4rQ6HnvddXY+bJa0xepAg++wz+9rewPsyee4bL1w0YAAsXwr33\nhkavBl/+KmnsvS5K8iIF8skn8NxzYWbM6NFhjP3nPw/vRNV665Un7vSeSUleJEZvvQWnnBKW6f3H\nP0J6f/fd8C7Unj3V4CtNEtJ7Jv34iTTQ66/DrbfC8OFwxRVhZsxmm8VdlTRGpc2cyYeSvEg9rFoF\nDz0UhmB69IAdd4T33oPLL1eDr2RJS++ZlORF8rB8eVjK9/LLw0U3LrssXHBj443jrkwaK4npPZOS\nvEgd3GHqVPjNb8KFNoYPDzNjpkyBk05Sg690SU7vmZTkRbIsXAg33hhmyUAYmnn++XAtVEmGpKf3\nTEryIoSx9nHj4PjjwyyZ+fPhscfCePvdd6vBJ0Va0nsmzZOX1Bs5MoyxN28eVnw8+mho3TruqqTQ\nymnee33pylAiDbBsWRhvHzs2LBJWVQXWoF8jKWeVuuZMoajJS+osWgTXXw8PPBDWj5k4EVq1irsq\nKYY0jb3XRWPykhqffw5//jN07AiLF4dlfocOVYNPojSOvddFSV4S74034I47whTIgw+GmppwxSVJ\nJqX3tSnJS2LNnh0WCOvVC7beGt55J1wIWw0+mZTea6ckL4mzYAHccEOY+ti3b7g4R8uWcVclxaT0\nXjcleUmMTz4JV1lq1y7Me58yBW66SQ0+yZTec1OSl4r39ddw551w9dVw7LHw9tuwww5xVyXFpvSe\nHyV5qVjucP/94R2qjz0W5rvfdZcafNIpvdePkrxUnM8+C7Nl7rsvXELvH/8I68tI8im9119eSd7M\neprZTDN7x8wuqeXxFmY2yszeMLM3zeyMglcqqbd4cRhj32knePPNcA3VyZPV4NNA6b3hciZ5M1sP\nuB04BPgQmGhmj7r7zIzNzgOmufuPzWxL4G0zG+ruK4tStaTKrFlhvP2BB8IFsEePhm7d4q5KSkXp\nvXHySfL7ALPcfa67rwAeBHpnbeNA8+jz5sBiNXhprFWr4I9/hC5doE0beP/9MM9dDT4dlN4LI58x\n+dbAvIzb8wmNP9PtwCgz+xDYFDixMOVJGi1fHmbL/OUv0L49zJgB220Xd1VSSkrvhVOoE6+HA6+7\n+8Fm1hZ4ysw6ufsX2RtWV1d/93lVVRVVVVUFKkEq3bffhtky118frsT00EOwT3ackERL+4qRq9XU\n1FBTU1OQfeVcT97MugLV7t4zun0p4O4+IGObx4Br3f2l6PYzwCXuPilrX1pPXmo1cSKcd174/E9/\nChfJ1rK/6VLJ670XW2PWk89nTH4i0M7M2phZU6APMCprm7nAoVExWwPtgTkNKUjSZeFCOOssOOyw\nsATBK6/A4YerwaeJxt6LK+dwjbuvMrN+wDjCH4V73H2GmfUND/sg4GpgsJlNjb7sd+6+pGhVS8X7\n6iu45JIwPPOLX4QZNFttFXdVUmoaey8+Xf5PSmrlSrjmGrj5ZjjwQBg4UJfaS6PssffqqupUjr3n\nS5f/k7LnHi7S8ZvfhIt0vPFGmBYp6aP0Xlpau0aK7o034KCD4KKLwpuanntODT6NNPYeDyV5KZo5\nc6C6OrxT9aaboF8/2EA/camk9B4fJXkpiqlTwxz37bcPa8786ldq8Gmk9B4//dpJQX3+eXgz08CB\ncO21YVqkpJPSe3lQkpeCeeIJ2GWXsMbMyy+rwaeV0nt5UZKXRvvvf+GCC8KFO4YNg+7d465I4qL0\nXn6U5KXB3GHcONh773B91alT1eDTSum9fCnJS4N88AGccw68806YQXPyybCeIkMqKb2XN/1aSr2s\nWBGWAe7SBfbaC6ZNg1NPVYNPI6X3yqAkL3mbPh3OPz+s9z5mjJYBTjOl98qh/CU5ffEFXHkl7Lcf\nVFVBTY0afFopvVceJXmp0+qLeFx4IXTtClOmwI47xl2VxEXpvTIpycv3uMPjj8Mee8CAAWEGzdix\navBppfRe2ZTk5TsrVsB994WlgNdbLwzRnHqqLuCRZkrvlU9JXoBwIrV9e7j77nD5vbffhtNOU4NP\nK6X35FCST7nly0Nyv+MOGDIEjjgi7ookbkrvyaIkn2KTJsGee4Z1ZiZPVoNPO6X3ZFKST6FVq+DW\nW8OwzIAB8Mtfalgm7ZTek0tNPmVmzgxNfelSGD8eOnaMuyKJU/a1VoceO1TXWk0YNfkUueeecI3V\nK68M71zVRTzSTek9HfRrngKLFoXFxCZNghdegN13j7siiZPSe7qoySfYokVhzH3gQDj77DB7ZqON\n4q5K4qT0nj6aXZNA7qGhd+gAS5aEOe8336wGn2aaOZNeSvIJ8+WXcOKJobE/+WRYDljSTek93ZTk\nE+Tdd6Fbt5DYX39dDT7tlN4FlOQTYdWqMPZ+441w+eXw61/rIh5pp/Quq6nJV7hPPw2X3luyJMx7\n79Ah7ookTpo5I9nU5CvYlClw9NFw1FEwahQ0aRJ3RRInpXepjV7UVyB3uOEG6N4drr46TJFUg08v\njb3LuijJV5gFC8Ia70uWwPPPw267xV2RxEnpXXJRkq8QK1fC4MHQpUuYQTNxohp8mim9S76U5CvA\nK6/ABReElSJHj9ZFtNNO6V3qQ0m+jC1ZAqecAscdB2eeGdZ9V4NPL6V3aQgl+TI1fjyccUZo6jNn\nQvPmcVckcVJ6l4bKK8mbWU8zm2lm75jZJXVsU2Vmr5vZW2b2bGHLTI8VK+Dcc8PUyMsvh/vvV4NP\nM6V3aaycSd7M1gNuBw4BPgQmmtmj7j4zY5vNgDuAHu6+wMy2LFbBSTZ/frh49kYbwYwZ0KpV3BVJ\nnJTepRDySfL7ALPcfa67rwAeBHpnbXMy8LC7LwBw90WFLTP55syBAw6ATp3CyVU1+PRSepdCymdM\nvjUwL+P2fELjz9QeaBIN02wK/NXdhxSmxOQbPx769AnDNJdeGnc1Eieldym0Qs2u2QDoAvQCegJ/\nMLN2Bdp3oj32GPToES6srQafXkrvUiz5JPkFwA4Zt7eL7ss0H1jk7suB5Wb2PNAZeDd7Z9XV1d99\nXlVVRVVVVf0qTohVq+D22+G668K67926xV2RxEXpXbLV1NRQU1NTkH2Zu697A7P1gbcJJ14XAhOA\nk9x9RsY2HYDbCCl+Q+BV4ER3n561L8/1fGnwwQdwzDHh84cfhp12irceiUf2ipHVVdVaMVJqZWa4\nuzXka3MmeXdfZWb9gHGE4Z173H2GmfUND/sgd59pZmOBqcAqYFB2g5dgzhw45BD42c+gf3+t+55W\nSu9SKjmTfEGfLOVJfsKEMP/9iivg/PPjrkbioPQuDVHUJC+FMW5cWD3yrrvgJz+JuxqJg9K7xEGD\nBUW2cmU4wXr88TB8uBp8GmnmjMRJSb6IPvkEjjwSNtkEXnoJdt897oqk1JTeJW5K8kUyYwbsuScc\ndBA8/bQafNoovUu5UJIvgnHj4PTT4Y9/hLPPjrsaKTWldyknSvIF9OWX4eTqL34BQ4aowaeN0ruU\nIyX5Apk6FU4+OQzLzJwZxuElPZTepVwpyRfASy9B9+7Qrx8MG6YGnyZK71LulOQbqX9/uOUWeOCB\n8EYnSQ+ld6kESvIN5A4XXwwjR8Lbb6vBp4nSu1QSJfkGWLwYTjoJFi0KSwVvs03cFUmpKL1LpVGS\nr6d588Lc9x12CGvRtG0bd0VSCkrvUqmU5Oth2jTo2RPOOQd+/3uwBi0XJJVG6V0qmZJ8nqZNCzNo\n+veHyy9Xg08DpXdJAiX5PEyYAD/+Mdx4Y3gnqySf0rskhZJ8DmPGhEXG/v53Nfg0UHqXpFGSX4cn\nngjLFIwcGa7mJMmm9C5JpCRfh+HD4YQTwjtY1eCTTeldkkxJvhZDh8JFF0FNDey1V9zVSDEpvUvS\nKclneeGFcP3Vp55Sg08ypXdJCyX5DM89B336wH33QefOcVcjxaL0LmmiJB+5445w/dW77grTJSV5\nlN4ljVKf5L/9FgYMgNtuC0sGd+wYd0VSDErvklapT/LV1TBiBLz6qhp8Eim9S9qZu5fuycy8lM+X\ny4QJcMQR4apO224bdzVSaJnp/d7e96q5S8UyM9y9QYuppDbJT54cGvxtt6nBJ43Su8gaqRyTnz4d\nevSAG24I68JLcmjsXWRtqUvyH38cEvxVV8GZZ8ZdjRSK0rtI7VKV5CdNCtMjTz0Vzjsv7mqkUJTe\nReqWmiQ/axbsvXcYorn++rirkUJQehfJLRVJfuXKcKHtW2+FU06JuxopBKV3kfykIsnffDO0agUX\nXBB3JdJYSu8i9ZP4JD98OPz5z/Daa3FXIo2l9C5Sf4lO8o8+Gi66/cQT0LZt3NVIQym9izRcYpP8\nX/8K11wDo0fDfvvFXY00lNK7SOMkMsnfeSfcdBOMHw/dusVdjTSE0rtIYeS1do2Z9QRuIfxRuMfd\nB9Sx3d7AeOBEd/9nLY8Xfe2a558Pb3aaPBnaty/qU0mRaM0ZkbUVde0aM1sPuB04HNgVOMnMOtSx\n3XXA2IYUUghjxsBRR8Hdd6vBVyKld5HCy2dMfh9glrvPBTCzB4HewMys7c4HHgL2LmiFeZo1K7yT\n9cEHQ5KXyqKxd5HiyGdMvjUwL+P2/Oi+75jZtsAx7v43oEEvKRrjm2/Cm5wuukgNvtIovYsUV6Fm\n19wCXJJxu2SN/ttv4YQTYOut4fLLS/WsUghK7yLFl0+TXwDskHF7u+i+THsBD5qZAVsCvcxshbuP\nyt5ZdXX1d59XVVVRVVVVz5LXNnAgzJ4NL74IVvLXENIQ7s6IaSO48MkLOWOPMxh67FCabdAs7rJE\nykZNTQ01NTUF2VfO2TVmtj7wNnAIsBCYAJzk7jPq2P5eYHQpZtfMnQtdusDYsbDXXgXbrRSRZs6I\n1F9RZ9e4+yqgHzAOmAY86O4zzKyvmZ1d25c0pJCGuOyysCa8Gnz509i7SDwq9hqvd90F114Lb74J\nzZsXZJdSJErvIo2Tumu8PvUU9O8PTz6pBl/OlN5F4ldxSf7rr2GbbeCBB6BXrwIVJgWn9C5SOKlK\n8jffDLvvrgZfrpTeRcpLRa1CuWABDBgAjz8edyVSG817Fyk/FZXkf/tbOO002H//uCuRTErvIuWr\nYpL8W2/BuHEwaVLclUgmpXeR8lYxSf6226BfP9hpp7grEVB6F6kUFZHkp02DESNCmpf4Kb2LVI6K\nSPL9+4cVJlu3zr2tFI/Su0jlKft58i+/HC4EMmcObLZZkQqTnDTvXSQ+iZ4nf801cO65avBxUXoX\nqWxlneRnz4Yf/QgWLoRNNiliYVIrpXeR8pDYJH/bbXDOOWrwpab0LpIcZTu7ZtUqGDYMnnkm7krS\nRTNnRJKlbJP84MFhTvzuu8ddSToovYskU1kmeXe44oqwZrwUn9K7SHKVZZJ/8UXYeGM4+ui4K0k2\npXeR5CvLJD9sGJx8si7MXUxK7yLpUHZJ3h3GjIE+feKuJJmU3kXSpeyS/EsvhZk1HTvGXUnyKL2L\npE/ZJfnbb4ef/hTWK7vKKpfSu0h6lVWSX7oUxo6FiRPjriQ5lN5F0q2s8vJNN8ERR0C7dnFXUvmU\n3kUEymjtms8+gx13DNMnd9utZCUlktacEUmWRKxd88gjcOCBavCNofQuItnKZkz+n/+EY46Ju4rK\npbF3EalNWST5WbNg/Hg48cS4K6k8Su8isi5lkeQffjg0+E03jbuSyqL0LiK5xJ7k3WHIEDj++Lgr\nqRxK7yKSr9hn1yxYAG3bwldf6Q1Q+dDMGZH0qejZNePHw847q8HnovQuIg0R+5j8hAnQo0fcVZQ3\njb2LSEPFnp9XLyss36f0LiKNFeuY/Ndfh4t0L18OG8T+mqK8aOxdRFar2DH5kSNh773V4DMpvYtI\nIcXaXh99FHr1irOC8qKxdxEptLySvJn1NLOZZvaOmV1Sy+Mnm9mU6ONFM9s9n/0+/bSu4wpK7yJS\nPDnH5M1sPeAd4BDgQ2Ai0MfdZ2Zs0xWY4e5LzawnUO3uXWvZ13dj8itWQNOmYTx+ww0L9v1UHI29\ni0guxR6T3weY5e5z3X0F8CDQO3MDd3/F3ZdGN18BWufa6VtvQZs26W3wSu8iUgr5jMm3BuZl3J5P\naPx1OQt4ItdOn3wSDjooj2dPII29i0ipFPTEq5l1B84EutW1TXV1NRAWJTvyyCqgqpAllDV3Z8S0\nEVz45IWcsccZDD12KM02aBZ3WSJSZmpqaqipqSnIvvIZk+9KGGPvGd2+FHB3H5C1XSfgYaCnu8+u\nY1/fjckfcABcfTV07974b6ISaOxdRBqq2GPyE4F2ZtbGzJoCfYBRWQXsQGjwp9XV4DO5hzXk27Zt\nSMmVRWPvIhKnnMM17r7KzPoB4wh/FO5x9xlm1jc87IOAPwAtgYFmZsAKd69z3P7tt6FJE9h++8J8\nE+VKY+8iErdYljW46y54+WUYPLhkT11S2WPv1VXVGnsXkQZrzHBNLO94nTEDdt01jmcuPqV3ESkn\nsaxdk8TxeI29i0g5iiXJT5kCW2wRxzMXh9K7iJSrWJL8+usn46Sr0ruIlLuSn3j95hunWTP48kto\nVsHnIjXvXURKpaLWk58XLZBQqQ1e6V1EKknJx+Tfew9a51y+rDxp7F1EKk3Jk/ySJdC5c6mftXGU\n3kWkUpU8yU+YADvvXOpnbTildxGpZCVP8p9+CttuW+pnrT+ldxFJgpIn+dmz4fjjS/2s9aP0LiJJ\nUfIkP3cuNG9e6mfNj9K7iCRNyZO8GbRqVepnzU3pXUSSqORJfs4caNmy1M9aN6V3EUmykif5Fi3K\nZ90apXcRSbqSJ/mtty71M36f0ruIpEXJk/yWW5b6Gdem9C4iaVLyJL/RRqV+xkDpXUTSqORJPo7h\nGqV3EUmrkif5Uq4+qfQuImlX8iS/alVpnkfpXUQkhiS/447F3b/Su4jIGiVP8sV8I5TSu4jI2kqe\n5DffvPD7VHoXEaldxc+uUXoXEalbxc6uUXoXEcmt5Em+adPG70PpXUQkPyVP8o1ZS17pXUSkfkqe\n5Bt64lXpXUSk/kqe5Fu0qN/2Su8iIg1X8iRfnwXKlN5FRBqn5El+gzz+rCi9i4gURsmTfC5K7yIi\nhVPyJF8XpXcRkcIriySv9C4iUhx5JXkz62lmM83sHTO7pI5t/mpms8zsDTPbI5/9Kr2LiBRXziZv\nZusBtwPfe+fwAAAEIUlEQVSHA7sCJ5lZh6xtegFt3X1noC9wZ679fvzlx5ww8gSueu4qHu3zKNcd\neh3NNijhFUViVlNTE3cJZUPHYg0dizV0LAojnyS/DzDL3ee6+wrgQaB31ja9gfsB3P1VYDMzq3Up\nMqX3QD/Aa+hYrKFjsYaORWHkMybfGpiXcXs+ofGva5sF0X0fZe/shJEnaOxdRKRESn7itV3Ldgw9\ndmiqhmZEROJi7r7uDcy6AtXu3jO6fSng7j4gY5s7gWfdfXh0eyZwkLt/lLWvdT+ZiIjUyt2tIV+X\nT5KfCLQzszbAQqAPcFLWNqOA84Dh0R+Fz7IbfGOKFBGRhsnZ5N19lZn1A8YRTtTe4+4zzKxveNgH\nufsYMzvCzN4FvgTOLG7ZIiKSj5zDNSIiUrmKsqxBsd48VYlyHQszO9nMpkQfL5rZ7nHUWQr5/FxE\n2+1tZivM7NhS1ldKef6OVJnZ62b2lpk9W+oaSyWP35EWZjYq6hVvmtkZMZRZdGZ2j5l9ZGZT17FN\n/fumuxf0g/CH412gDdAEeAPokLVNL+Dx6PN9gVcKXUc5fOR5LLoCm0Wf90zzscjY7hngMeDYuOuO\n8ediM2Aa0Dq6vWXcdcd4LC4Drl19HIDFwAZx116EY9EN2AOYWsfjDeqbxUjyBX3zVIXLeSzc/RV3\nXxrdfIXw/oIkyufnAuB84CHg41IWV2L5HIuTgYfdfQGAuy8qcY2lks+xcGD1hUObA4vdfWUJaywJ\nd38R+HQdmzSobxajydf25qnsxlXXm6eSJp9jkeks4ImiVhSfnMfCzLYFjnH3vwFJnomVz89Fe6Cl\nmT1rZhPN7LSSVVda+RyL24EfmtmHwBTgwhLVVm4a1DfLYhVKATPrTpiV1C3uWmJ0C5A5JpvkRp/L\nBkAX4GBgE+BlM3vZ3d+Nt6xYHA687u4Hm1lb4Ckz6+TuX8RdWCUoRpNfAOyQcXu76L7sbbbPsU0S\n5HMsMLNOwCCgp7uv6+VaJcvnWOwFPGhmRhh77WVmK9x9VIlqLJV8jsV8YJG7LweWm9nzQGfC+HWS\n5HMszgSuBXD32Wb2HtABmFSSCstHg/pmMYZrvnvzlJk1Jbx5KvuXdBRwOnz3jtpa3zyVADmPhZnt\nADwMnObus2OosVRyHgt3/9/oYyfCuPy5CWzwkN/vyKNANzNb38w2Jpxom1HiOkshn2MxFzgUIBqD\nbg/MKWmVpWPU/Qq2QX2z4Ene9eap7+RzLIA/AC2BgVGCXeHu2QvAVbw8j8VaX1LyIkskz9+RmWY2\nFpgKrAIGufv0GMsuijx/Lq4GBmdMLfyduy+JqeSiMbNhQBWwhZl9AFwJNKWRfVNvhhIRSbCyucar\niIgUnpq8iEiCqcmLiCSYmryISIKpyYuIJJiavIhIgqnJi4gkmJq8iEiC/X9aKRLVG9VmIQAAAABJ\nRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x117336e48>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 355 ms, sys: 116 ms, total: 472 ms\n",
      "Wall time: 474 ms\n",
      "PRECISION/RECALL AUC\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXkAAAEKCAYAAAD3tSVSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmYFOXZ7/HvzeIOKGpcMGBQCYqyGDYVdZBEFuWgxqgQ\nNWoWkrgkbxbRnDfHUXNel1w5xiVqUNQ3i6CvGkVwwW2iGEGMCC4giwqyo4CAOIBwnz/ugmnbWZqZ\nnu6Zmt/nuvqiu6q66umi51dPP/XUU+buiIhIOjUrdgFERKT+KORFRFJMIS8ikmIKeRGRFFPIi4ik\nmEJeRCTFFPLSKJnZVjPrmDy/18yuqWbZfcxslpntXLgS5p+ZvWVmJxS7HPXBzC4xs+uLXY40Usg3\nAGb2gZltMLO1ZrY0Ca3dil2uXCRl3ZiU/SMze9rMvl6ATe/IBR5XAPe6+8bMiWZ2n5ltNrP9sqZ/\n6aBhZh2SA0uzjGkjzGyama0zs8VmNtHMjquqEGbW08weN7NVyeMtM7vWzNrk8iHc/Uh3fzGnT5wn\nZtbdzF4zs0+Tz9qtmmV/b2ZzzOwTM3vHzM7LmNcv2U9rk8e6ZH+enixyF/BdM9unvj9TU6OQbxgc\nOMXdWwNHAz2B/6xsQTOzQhYsRzckZW8HLAHuLsA2c9oPZrYT8D3gb1nTdwPOANYA5+a4ze0HFjP7\nBfD/gN8BXwHaA38ChlZRjmOBF4CXgK+7e1tgEPA5UGVwFpOZtQQeBf4C7Jn8+5iZtajiLeuJ73Eb\n4ALgZjPrC+Duk929lbu3Tr4rpwLrgKeS+RuBJ4Dz6/EjNUkK+YbDANx9KfAkcCSAmb1gZr8zs8lm\n9inwNTNrbWZjzGyJmX2Y1Aa3h56Z/TCpSa1Naovdv7Qxs9vN7PdZ0x41s58nz0eZ2aJkHbPMrH9N\nHyD5Q30Q+ML2zOyipDwfm9mTZtY+Y14XM5uUzFtqZlck03uZ2b/MbHVSS761mnCpTh9gtbsvyZp+\nJrAauIYIpJyZWWvgauCn7v6Yu3/m7lvc/Ql3v6KKt90AjHH3G919JYC7L3L3q7fVzs2so5k9l/wi\nWmFmf0u2tW2775vZScnzq8zsATP77+T/6E0zO3pHPkcOSoDm7n6Lu29291uJ7+lJlS2cfJa5yfNX\niQPaMVWs+wLgIXf/LGPaP4FT8lR2SSjkGxgz+yowBHg9Y/K5wA+AVsBC4L+BjUBHoAfwrWQ+ZvYd\n4P8A5yY1pv8FfFzJpsYCZ2Vsd89kPWPNrBNwMfCNZB0DgQ9yKPvuwAhgbsa0YURzyWnAvsQf/thk\n3h7AM0QN7gDgUOC55K1bgJ8DbYmgOAn4aU1lqMRRwLuVTD8fuB94AOhsZj12YJ3HAjsTtdwaJb8a\njgEeqWlR4L+A/YHDgYOA0mqWH0p8hjbA48QviarKMCOjmWh11r+3VfG2LsDMrGkzkunVfxCzXYFe\nwNuVzNsN+DZwX9asWTTQXzWNmUK+4XjUzFYBLxI/66/LmHefu892961E6A0G/sPdy939I+CPwDnJ\nst8HbnT31wHc/T13/zB7Y+7+EuBm1i+ZdCbwirsvJwJ2J+BIM2vh7gvd/f1qyv7rpOxriQDM/Mk9\nErjO3eck5b8e6J4czE4Flrr7H919k7t/6u7TkvK97u6velgIjAZOzGE/ZtuTaBbYLvkl0R+4391X\nAM+yY80EbYGPks+Ti72Iv7VlGWW4IQnZ9Wb2GwB3n+/uz7n75+7+MXAT1X/mye7+tMcAVH8Fula1\noLt3c/e2yWOvrH8vqeJtewCfZE1bS1Q2anInMN3dJ1Uy79vAyuQ7mGkdccCSPFLINxzDkj+4r7n7\npVknCTNDugPQEli6rTZG/EHtm8z/KjA/x20+AAxPno8A/g4RNkQtuhRYbmb3m9kB1azn90kbcwfg\nMyDzxGsHom12VXIg+Jho225XXVnN7LDkJOVSM1sD/F+gNiflVvPlUDoPeMfd30xejyVO+jVPXn9O\n7ONMLYGtSbB/DOxjGSdhcyjDVuLXCgDuPsrd9wL+AbQAMLOvmNnYpJlsDXEeobrPvCzj+QZglx0o\nUy7WA62zprUh66CZLWkGPAI4u4pFzifa97O14ssHFakjhXzDUd2JxMyeJB8C5cDeGbWxPd29a8b8\nQ3Lc5ljgzKRm2wd4ePsG3ce5+/FESEPUwKvl7ouIg8MtVtFdcSEwMqsWuYe7T6mhrHcQP98Pcfc9\ngf9Njidbs8wEOmVNOw/omBxAlgJ/APYmmsm2lfngrPd0pOJg+wrRXHZaLgVw9w3AVOJEb3X+izgY\ndEk+87nU7jN/SXJuZm3WY1tvl9ureNvbfPnXQVcqaYLJ2M7VRPPet9x9fSXzDyLa+isL+cOJ5iDJ\nI4V8I+Puy4BJwE1m1spCR6voP3038KttJ+HM7JDME51Z63qDqJXeDTzl7muT93Qys/4WPVM2EbXz\nnJom3P1ZYDHRTAPwZ+A3ZnZEsu42ZnZmMm8CsL+ZXWZmO5nZHmbWO5nXCljr7hvMrDPwk5x20Je9\nCuy57ZeImR1DBHYvov23G9HGPJbohQNxsDvFzL5pZs3M7EDiIDM2+YxrgauAP5nZMDPb1cxamNkg\nq7qv9+XARWZ2uZntm5TlIOBrGcu0ImrP68ysHfDrHfysVR4Qku6XrbMe23q7VHWuowzYYmaXJv8/\nlxHfg+cr3bjZlcQvw2+6+5oq1nk+8HIVzX8nEp0OJJ/cXY8iP4D3gJOqmPc8cFHWtFbA7UTNcjXw\nb+CsjPk/AmYT7aczgW7VbPs/iTb4MzKmHUXUPD8BPgLGA/tX8f57gGuypp2VlK1l8vq7STnWAAuA\nuzOWPYJoE19FdL+8PJl+PFGTX0v0uigFXsx43xagY/L83uwyZJXnhoz13gE8WMkyvYiD2Z7J61OA\n15L9+z7xS2bnrPcMB6YRzRdLiJOffaspRy9gYvJZVyX75Fpgr4x98VrymV8H/gNYWNn3hDjI/CVj\nXodknzTL83ezW1KmT5N/u2bMGwG8mfF6a7IP1yb7ZC1wRdb63gEuqGQ7uyTfmX2L/feYtoclO7hK\nZjaGOEG23CuaBLKXuYU4Gfhp8h/4RrUrFSkgiwtsXgR6eNYFUdIwmNklwEFedRdUqaVcQr4f8RPy\nL5WFvJkNBi5x91PMrA9ws7v3rZfSiojIDqmxTd7dJxM/WasyjOQkirtPBdpY1mXiIiJSHPk48dqO\nL3bxW5xMExGRIlPvGhGRFKvNWCDZFhMXtWxzUDLtS8xsR0YOFBGRhLvX6pqJXGvyRtV9cMeTXBJu\nMeLcGo9L4yvVoYOzdWt07Vm2zPngA2fjxuJ3Myr046qrrip6GRrKQ/tC+0L7ovpHXdQY8mZ2P/Av\noJOZLTSzC81spJn9CMDdnwDeN7N5xIUv1Q4itWABbN4M110H++8PBx8MRx8NI0fC/OQC95kz4dln\nv/i+Tz6B6dPh/ffh88+hjp9bRKRJqLG5xt1H5LBMVQMcVequu+C++2DMGJg2De68E95+G0aPjvm7\n7ALl5fG8Z0947TXYYw9Yn3WRdJ8+cOCB8Oqr0Lo1zJsH3/hGTLv6ajjyyMwyxr/r18e6GuSo7CIi\neZaPNvkd9rOfwdSpEcgXXQR33AGffQa33w5HHAEDB8KcOfCrX8E550C7dnDssdCsGTRvDuvWQWkp\nLF4MvXpBt27QsWOsY9Mm+Pvf4aijIsgPOyzWVZlevWDPPeO9p54Kxx0Xr83i10bL7CGq8qikpKT+\nVt7IaF9U0L6ooH2RHzVeDJXXjZk5OIceCnPn1rx8XSxZAlOmQKdOsHEjtG8Pu+8Ou+0WB4lVq+CR\nR2DFChg7NpqRKspZUfPfeec4CJx/PpxyCnTuDC1a6JeAiBSOmeG1PPFalJC/7DK4+eaCbTYn7hHc\nCxdGc86mTfDOO/Dvf8M998Svga1ZQ3T16AF77w19+8bybdrEgeToo+NcQ7t28etDRKQuGlXIt27t\nPPMM9O5d8/INzbZdtWkTPPlkvH777TgXsG5dNPGsXQtLl1Y0EXXpAhdeCGefHecKFPoisqMaVcgX\ncnvFtno1XHopjB8fB4FMRxwBw4fDkCFw+OGw667FKaOINHwK+UZi5UpYsyZq+m+9BX/4A7z3XsX8\n3XaL9v+f/xyGDYN9anMfJBFJHYV8I1deHgeA1aujO+lTT8X1AO3aRe+gvfeG734XTjwR2rYtdmlF\npNAU8im0ciU8+GA8nzcvrhWYPDl6+3TqBK1aRRfUSy6JA4F6+4ikl0K+idi8OXr8fP55NPP84x/R\n/dMsriP45S+jjX/nnWtel4g0Hgr5JmzLFli2DG68EW67Lbp57rdfNO1cfnnU9kWkcVPIy3bLlsVF\nYGPGwKRJ0ZTz4x/Dd74T4S8ijY9CXipVXh7NOXfeGeP7HH541OwHD4YDDogTu506FbuUIlIThbzU\naNMmePnleEydGgPDLV8eJ3APOywC/9xzYcCA6M0jIg2HQl5qxT26aj7xRDTxzJoFr78eJ3EPPxyO\nOQZOOCEOAiJSPAp5yZtFi2LIhkWLoifPQw/F9BNPhJNPjuEZDjmkuGUUaWoU8lJvtmyJ7pr33huj\ndr77bgzCNmRIjMnTu7ead0Tqm0JeCubzz+MOXg8/HME/b17c5OX006F//2jT79ix2KUUSReFvBTN\n1q1xu8bHH4/gnzEj7sg1ZEj04unVK2r+IlJ7CnlpMDZsiFE3n3gC3ngD3nwTTjsNrr32i7djFJHc\nKeSlwVqzBu6+G0aNiuGVzzorbrV4+OHRzCMiNVPIS4O3cWM05zz2WNTuV66MsB8wAIYOjbtqiUjl\nFPLS6Lz9djTpjB4dF2V9+9vRW+e44+Jm7SJSoS4hr5vRSVF06QK//nXc0H3ChDg5e8EFcZP0UaOi\n5i8idaeavDQY7tE759xzo6Y/dGg06Zx5pm6WIk2bavKSCmbQvXvcGnHxYujbF+65J26DeOON8Omn\nxS6hSOOjmrw0eDNmxE3P162LWv0Pfxi9c3Q3LGkqVJOXVOvWLZpv/v736Iffrx80awYDB8bdsJ56\nKi7KEpEvU01eGh13mD49Ttr+618xiNqGDfCb38Bll+n2h5I+6kIpTd4jj8BvfxsjZx58MIwYAb/4\nhQZPk3RQyIskFi6MWv5NN8E//wl9+kRzT8+eMbzCvvsWu4QiO04hL1KJ5cvjLljz58PEifDcc9Et\n87zzIvBbtix2CUVyo5AXycGiRXD//XDXXTGmzsUXw6GHxglc1fClIVPIi+ygZ5+N4ZFffx0mT44m\nnbPPjguxvvrVYpdO5IsU8iJ1sG5ddMMcOxYefRRat44TtyefHGPiq7eOFJtCXiRPPvssavd/+lN0\nz1y9OgZNGzAAzjkH2rUrdgmlKVLIi9ST996LIZInTIAXX4SjjoJhw+DHP1bgS+HUe8ib2SDgj8QV\nsmPc/Yas+a2BvwHtgebAH9z9vkrWo5CXRmvjRnj66Qj9hx+OkTR79ozeOn37Frt0kmb1GvJm1gyY\nAwwAlgDTgHPcfXbGMlcCrd39SjPbB3gX2M/dP89al0JeUmH1avj3v+NWh7feCvvvH234F1wAJ54Y\nwy6I5Et9j13TG5jr7gvcfTMwDhiWtYwDrZLnrYCPswNeJE322gu++U245ZYYN+e556B9+6jVt2wJ\n11wDs2bFEAwixZRLyLcDPsx4vSiZluk24AgzWwLMAH6Wn+KJNHxmcf/aa6+FDz+Mtvtly6CkBPbb\nD773vbjtoQZRk2Jokaf1DASmu/tJZnYI8IyZdXX39dkLlpaWbn9eUlJCSUlJnoogUnxm0RvnuOOi\nh87s2fD881BaGk05ffrEMMmDB0P//rrqVipXVlZGWVlZXtaVS5t8X6DU3Qclr68APPPkq5lNAK5z\n95eT188Bo9z9tax1qU1emqxly+CZZ2DmTPjb3+L1kUfCKafARRdBp07FLqE0VPV94rU5cSJ1ALAU\neBUY7u6zMpb5E7DC3a82s/2A14Bu7r4qa10KeZHEkiUwZ05chPXXv0KrVnHlbY8ecOyx8dBwCwKF\n60J5MxVdKK83s5FEjX60mR0A3AcckLzlOncfW8l6FPIilXCP9vy33oLXXosLsaZMiV47PXpE885Z\nZ8EuuxS7pFIMuhhKJIW2bIE33oD/+R8oK4shlI8/Hk46CX7wA/jKV4pdQikUhbxIE/D++3Hl7QMP\nwMsvw6mnxoBqgwfHeDuSXgp5kSZmwYIYRfPOO+P+t126RC3/O9+B3r1hjz2KXULJJ4W8SBO2fn3U\n8MePjz76K1ZEU84xx8Dpp8dFW2raadwU8iKyXXl5NOc8/3xciTt1Kuy2W9TwjzsOfvITDa7W2Cjk\nRaRKW7fCBx9EV81Zs+LuWCefDCecEEMod+mi5p2GTiEvIjlbty6C/vHH4ZVXYNWqGEWzZ8/4t39/\nOPDAYpdSMinkRaTWVq+ONv2yMpg7F156CQ44IMbO/9a3YjiGffYpdimbNoW8iOTNxo1xMdbcuXEi\nd8KEOHl72mlwxhnRvi+FpZAXkXqzZg1cfz38859xFW6XLjHeTr9+MfTC3nsXu4Tpp5AXkYLYsAH+\n/GeYNi167bz3Hnz96xH8/frFfXD33z9G45T8UciLSFGUl8MLL8ATT8CiRdFXf+vWGF3z6KOha9cI\n/j32iKtyFf61o5AXkQZj5cq4NeL06dGL5623YnrLlnEid8iQ6L555JEK/Vwp5EWkQXOHd96BiRPj\nZujPPx9DKx97bFygdfrpcXct3Ru3cgp5EWlU3KM9/6mn4NVXo8ln+fJo3jn22LgZekkJtG1b7JI2\nDAp5EWn03n0XJk+OcfUnTIgbqrRrB927x8ndn/606Y7Bo5AXkdT57LOo6b/0UgzHMGkS9OoV7flD\nh8bzpnITFYW8iKTe2rXRdXPCBLjnnnh9yCEx8Nrxx0cXzrSezFXIi0iTs359nMCdOjXG1H/ssZje\ns2dFn/2ePaF58+KWMx8U8iLS5G27T+6ECTB/Pjz6aJzcPfVU6NSpYrTNE06IC7YaE4W8iEglli2L\n4RhmzozAnzcvbpTet2/cK/f734eOHYtdypop5EVEclReDv/4RwyzfOut0T//5JOjXX/gQNh992KX\n8MsU8iIitbBuXVygNW9e3EWrrAwOOyyadLp1gxEjGsYAbAp5EZE8WL8++urPnBldN8vK4uKsoUPh\n7LNh332LUy6FvIhIPVi/PnrtjB0b4d+3LwwfDmeeWdhmHYW8iEg9W7cOxo2Du++Oi7MGDIhx9YcM\nqf/bJSrkRUQKaOXKuBp34sQYZrltWxg8OGr6AwbAQQfld3sKeRGRIikvj546M2bEvw8+GDX7oUPj\ngqzjj6/7BVkKeRGRBsI9TtrecEOEfps2MbjauefGDdJroy4hr9GbRUTyyCy6YE6cCB9/DNdeG0Mp\nH3ggdO4cJ3E3by5geVSTFxGpfxs2xJ2y7rgDli6F666D88/PbUA1NdeIiDQiZWXwy1/Cpk3w0EMx\nXn511FwjItKIlJTE6JlnnhknZh96qP62pZq8iEgRTZ0a97g94wy46aa44Xk21eRFRBqpPn1gypQY\nE797d1iwIL/rV8iLiBRZ+/ZxUVX//tCjR/S5z5ecQt7MBpnZbDObY2ajqlimxMymm9lbZvZC/ooo\nIpJ+u+4Kt90GV14Jp50WF1nlQ41t8mbWDJgDDACWANOAc9x9dsYybYB/ASe7+2Iz28fdP6pkXWqT\nFxGpwbBh0LVr9LGH+m+T7w3MdfcF7r4ZGAcMy1pmBPCwuy8GqCzgRUQkN7fcErX6FSvqvq5cQr4d\n8GHG60XJtEydgLZm9oKZTTOz8+peNBGRpqlDBzjrLLj55rqvK18nXlsARwODgUHAb83s0DytW0Sk\nybnwwhjLvq5a5LDMYqB9xuuDkmmZFgEfuXs5UG5mLwLdgHnZKystLd3+vKSkhJKSkh0rsYhIypWV\nlfHss2XMnw8XX1y3deVy4rU58C5x4nUp8Cow3N1nZSzTGbiVqMXvDEwFznb3d7LWpROvIiI5GjUK\nVq+Gu+6qxxOv7r4FuASYBLwNjHP3WWY20sx+lCwzG3gamAlMAUZnB7yIiOyYwYPjQqm60LAGIiIN\nVHk5tG4NmzdrWAMRkdTZZZe4GrYuFPIiIg3YwQfX7f0KeRGRBmzXXev2foW8iEgD1rVr3d6fSz/5\nvLKra3XuQESkadqpbm9X7xoRkQZONw0REZFKKeRFRFJMIS8ikmIKeRGRFFPIi4ikmEJeRCTFFPIi\nIimmkBcRSTGFvIhIiinkRURSTCEvIpJiCnkRkRRTyIuIpJhCXkQkxRTyIiIpppAXEUkxhbyISIop\n5EVEUkwhLyKSYgp5EZEUU8iLiKSYQl5EJMUU8iIiKaaQFxFJMYW8iEiKKeRFRFJMIS8ikmIKeRGR\nFFPIi4ikmEJeRCTFcgp5MxtkZrPNbI6ZjapmuV5mttnMzshfEUVEpLZqDHkzawbcBgwEugDDzaxz\nFctdDzyd70KKiEjt5FKT7w3MdfcF7r4ZGAcMq2S5S4GHgBV5LJ+IiNRBLiHfDvgw4/WiZNp2ZnYg\ncJq73wFY/oonIiJ1ka8Tr38EMtvqFfQiIg1AixyWWQy0z3h9UDItU09gnJkZsA8w2Mw2u/v47JWV\nlpZuf15SUkJJSckOFllEJN3KysooKyvLy7rM3atfwKw58C4wAFgKvAoMd/dZVSx/L/C4uz9SyTyv\naXsiIvJFZoa716qFpMaavLtvMbNLgElE884Yd59lZiNjto/OfkttCiIiIvlXY00+rxtTTV5EZIfV\npSavK15FRFJMIS8ikmIKeRGRFFPIi4ikmEJeRCTFFPIiIimmkBcRSTGFvIhIiinkRURSTCEvIpJi\nCnkRkRRTyIuIpJhCXkQkxRTyIiIpppAXEUkxhbyISIop5EVEUkwhLyKSYgp5EZEUU8iLiKSYQl5E\nJMUU8iIiKaaQFxFJMYW8iEiKKeRFRFJMIS8ikmIKeRGRFFPIi4ikmEJeRCTFFPIiIimmkBcRSTGF\nvIhIiinkRURSTCEvIpJiCnkRkRRTyIuIpFhOIW9mg8xstpnNMbNRlcwfYWYzksdkMzsq/0UVEZEd\nZe5e/QJmzYA5wABgCTANOMfdZ2cs0xeY5e6fmNkgoNTd+1ayLq9peyIi8kVmhrtbbd6bS02+NzDX\n3Re4+2ZgHDAscwF3n+LunyQvpwDtalMYERHJr1xCvh3wYcbrRVQf4j8AnqxLoUREJD9a5HNlZtYf\nuBDoV9UypaWl25+XlJRQUlKSzyKIiDR6ZWVllJWV5WVdubTJ9yXa2Aclr68A3N1vyFquK/AwMMjd\n51exLrXJi4jsoPpuk58GHGpmHcxsJ+AcYHxWAdoTAX9eVQEvIiKFV2NzjbtvMbNLgEnEQWGMu88y\ns5Ex20cDvwXaArebmQGb3b13fRZcRERqVmNzTV43puYaEZEdVt/NNSIi0kgp5EVEUkwhLyKSYgp5\nEZEUU8iLiKSYQl5EJMUU8iIiKaaQFxFJMYW8iEiKKeRFRFJMIS8ikmIKeRGRFFPIi4ikmEJeRCTF\nFPIiIimmkBcRSTGFvIhIiinkRURSTCEvIpJiCnkRkRRTyIuIpJhCXkQkxRTyIiIpppAXEUkxhbyI\nSIop5EVEUkwhLyKSYgp5EZEUU8iLiKSYQl5EJMUU8iIiKaaQFxFJMYW8iEiKKeRFRFJMIS8ikmI5\nhbyZDTKz2WY2x8xGVbHMLWY218zeMLPu+S2miIjURo0hb2bNgNuAgUAXYLiZdc5aZjBwiLsfBowE\n7qyHsqZKWVlZsYvQYGhfVNC+qKB9kR+51OR7A3PdfYG7bwbGAcOylhkG/AXA3acCbcxsv7yWNGX0\nBa6gfVFB+6KC9kV+5BLy7YAPM14vSqZVt8ziSpYREZEC04lXEZEUM3evfgGzvkCpuw9KXl8BuLvf\nkLHMncAL7v5A8no2cKK7L89aV/UbExGRSrm71eZ9LXJYZhpwqJl1AJYC5wDDs5YZD1wMPJAcFNZk\nB3xdCikiIrVTY8i7+xYzuwSYRDTvjHH3WWY2Mmb7aHd/wsyGmNk84FPgwvottoiI5KLG5hoREWm8\n6uXEqy6eqlDTvjCzEWY2I3lMNrOjilHOQsjle5Es18vMNpvZGYUsXyHl+DdSYmbTzewtM3uh0GUs\nlBz+Rlqb2fgkK940swuKUMx6Z2ZjzGy5mc2sZpkdz013z+uDOHDMAzoALYE3gM5ZywwGJibP+wBT\n8l2OhvDIcV/0Bdokzwc15X2RsdxzwATgjGKXu4jfizbA20C75PU+xS53EffFlcB12/YD8DHQothl\nr4d90Q/oDsysYn6tcrM+avK6eKpCjfvC3ae4+yfJyymk9/qCXL4XAJcCDwErClm4AstlX4wAHnb3\nxQDu/lGBy1gouewLB1olz1sBH7v75wUsY0G4+2RgdTWL1Co36yPkdfFUhVz2RaYfAE/Wa4mKp8Z9\nYWYHAqe5+x1Amnti5fK96AS0NbMXzGyamZ1XsNIVVi774jbgCDNbAswAflagsjU0tcrNXLpQSgGY\nWX+iV1K/YpeliP4IZLbJpjnoa9ICOBo4CdgdeMXMXnH3ecUtVlEMBKa7+0lmdgjwjJl1dff1xS5Y\nY1AfIb8YaJ/x+qBkWvYyX61hmTTIZV9gZl2B0cAgd6/u51pjlsu+6AmMMzMj2l4Hm9lmdx9foDIW\nSi77YhHwkbuXA+Vm9iLQjWi/TpNc9sWFwHUA7j7fzN4HOgOvFaSEDUetcrM+mmu2XzxlZjsRF09l\n/5GOB86H7VfUVnrxVArUuC/MrD3wMHCeu88vQhkLpcZ94e4dk8fXiHb5n6Yw4CG3v5HHgH5m1tzM\ndiNOtM0qcDkLIZd9sQD4JkDSBt0JeK+gpSwco+pfsLXKzbzX5F0XT22Xy74Afgu0BW5ParCb3b13\n8UpdP3LcF194S8ELWSA5/o3MNrOngZnAFmC0u79TxGLXixy/F78D7svoWni5u68qUpHrjZndD5QA\ne5vZQuAqYCfqmJu6GEpEJMU0CqWISIop5EVEUkwhLyKSYgp5EZEUU8iLiKSYQl5EJMUU8iIiKaaQ\nFxFJsf+41/t3AAAABUlEQVQPcH7qtn0fJgEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10645a908>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 378 ms, sys: 64.5 ms, total: 442 ms\n",
      "Wall time: 452 ms\n"
     ]
    }
   ],
   "source": [
    "mlp = sklearn.neural_network.MLPClassifier()\n",
    "clf = mlp\n",
    "%time clf.fit(X[tidx],y[tidx])\n",
    "\n",
    "print(\"TEST\")\n",
    "%time evaluate(clf,X[pidx],y[pidx])\n",
    "\n",
    "print(\"TRAIN\")\n",
    "%time evaluate(clf,X[tidx],y[tidx])\n",
    "\n",
    "print(\"ROC AUC\")\n",
    "%time roc_auc(clf,X[pidx],y[pidx])\n",
    "\n",
    "print(\"PRECISION/RECALL AUC\")\n",
    "%time prrc_auc(clf,X[pidx],y[pidx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 5.95 s, sys: 595 ms, total: 6.55 s\n",
      "Wall time: 6 s\n",
      "TEST\n",
      "Accuracy 0.601221422571 Precision 0.659579103307 Recall 0.608196062676 F1 0.63284630338\n",
      "CPU times: user 240 ms, sys: 142 ms, total: 382 ms\n",
      "Wall time: 288 ms\n",
      "TRAIN\n",
      "Accuracy 0.611000812201 Precision 0.668219933959 Recall 0.61554701727 F1 0.640802891576\n",
      "CPU times: user 1.1 s, sys: 867 ms, total: 1.97 s\n",
      "Wall time: 2.03 s\n",
      "ROC AUC\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXkAAAEKCAYAAAD3tSVSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xu8lXP6//HX5ZCG0jCNYRC+yjiM5NwobHIojEzGmWTG\nTIaGMfN1GAabmUFfxkQxo5Ec0iQlhVEp7SEpeWAilQ6mk34lJZUiu+v3x+dOy7YPa++91rrXuu/3\n8/FYj70O917r2ne7a7/XZ33uz23ujoiIJNMWcRcgIiL5oyYvIpJgavIiIgmmJi8ikmBq8iIiCaYm\nLyKSYGryUlBmNtjMTo+7jsYws9+bWf+468gHM9vJzN4zs63jrkVyQ00+hczsv2b2mZl9amYfmtlA\nM9u2yjZHmdn4aJuVZjbSzParsk1zM+tjZvOj7Wab2T1mtmMNr3sg0NbdR1W5v8zMNprZNVXuP9bM\nFlbzPBPM7GcZt/cxs6Fm9lFU69tmdrWZWQ11NIvq/MDMVkf7Y6iZHVHnzgPc/Q53/2U22+aKmTUx\ns4fNbFX0b3Z1LdvuHP17LY72a6sqjw80s8+jf7PV0VcDcPdlwEtAz/z+RFIoavLp5MCp7r490A44\nGPj9pgfN7EfAGGAEsAuwFzANeNXM9oy22ZrQDPYDToqe60fAcqCmZtkTeKKa+7sDH0dfq6u1Rma2\nNzAZmA/80N13AM4CDgGaV7N9E2ACcABwCrB99DMMATrX9loxuxXYG9gdOB641sxOqmHbjcALQDdq\n3n+93X17d28efc3cbjBq8snh7rqk7AJ8AByfcbs38GzG7ZeBvtV837+AR6LrlwJLgG/V43XnAkdV\nuW9b4FPgbGA9cEjGY8cCC6p5ngnAz6LrgzJrz6KGS4HFQNM6tusDLABWAVOBjhmP3QI8Hl3fg9BU\nuxP+0CwDbsjDv9lioFPG7VuBwXV8z5ZRba2q3D8QuK2O71sL7B7376oujb8oyaecme0GdAFmR7e/\nBRwFDKtm86HAidH1TsBod1+X5etsS3hHMKvKQ2cCq4GngLHAxfX8ETrVUGtt249x9/V1bPc60BbY\ngZBsn4reBWxSNSF3ANoAJwA3m9kPqntSM7suGlJaEX3NvL6ihu/5NuEd1bSMu/9DeDfSUJeb2XIz\nm2pm3TIfcPdKYA5wUCOeX4qEmnx6PWNmnxLS6lKgPLp/R8LvxZJqvmcJ0DK6/p0atqnJtwmNcXWV\n+7sDQzxEyMHAuWa2ZT2et751tAT+36YbZnZQ1GBXmdmMTfe7+2B3/8TdN7r7X4FtgGobN+HnKnf3\nL9x9GqEBV9sg3b23u+/g7jtGXzOvV/tZBtAseo1VGfd9SjXDUVm6l/AHaSfgZuCRaIgu02rCv5mU\nODX59OrqYRz9WGBfNjfvlYS3+LtU8z27EMbcIYyhV7dNTT6Jvn7VmKJ3EccRmjvAKOBbwKnR7S+B\n6mZ5bA1saGAdX9ve3f/jYRy/G6GRb6rtf6NZJivNbCVh7L7lN55ts6UZ1z8jNOZcWRN93T7jvhZ8\n8w9mVtz9bXdfGf0Be4HwOUm3Kps1Z/O/mZQwNfn02jSb4hXgUeAv0e3PgNcIH15WdTYwLro+Djg5\nGt6pU/S8c4F9Mu7uHtXxrJktiR7fhs1DNguAllVn/hDGwedn1HFmNjVExgMn1Va3mR0NXAP8dFPa\nJiTnamfr1Ec0/XLTjJbMy+rondU3uPsnhHcrme8ODgKmN7aeTS9Bxs8WvZNqTXhHIiVOTV4gfMh4\nYjTFEeB64GIz6xVNN9zBzP4EtAdui7Z5HFgIDDezH1jwnaiJ1TRL5V+Edw6bdCcME7UjNK2DgJ8C\np5rZDu6+EJgC9Daz7aJphNcCXxBm1ED4EPQoM+ttZt8DMLPWZva4mWUm300eIzTMEWZ2gJltYWbb\nAIdnbNOM8E7h4+g1b6b2oZGsm7+H6ZebZrRkXppH76xq8jjwBzP7djSV9ReED1CrLyj8TE2jm02j\n25seOzPanxbN0LkAGJnx7UcAH0T7X0qcmnw6fe1DQ3dfTkjzN0e3XwVOJiTkJYTZOAcBHdx9brTN\nF4QPGWcCLxLGiycTxsin1PC6/wAuBDCzI4FWwAPuvizj8izhQ+Dzou85B/ge4YPARYThnVOj18fd\n5xGmbu4FTI+GVp4izIj5xnCGu38ePcd7wPNR3TOBQwnvVCBMHx0DvB/97J8R/qDVpOqHsPk4ScMt\nwDzCO5iXgDvd/cVND0bvBDpkbL+O8O7DCT/fZxmPXUXYlysJM6sujd7RbXIB8Pc8/AwSAwufd9Wy\ngdkA4DRgqbu3rWGb+wgzNNYCPdz97VwXKslgZoOAoV7lgCgpDmb2XaACOHjTH1Ipbdk0+Y6ED34e\nq67Jm1kXoJe7nxqls3vdvX1eqhURkXqpc7jG3ScS3tbVpCthnBN3nwK02DQ2KiIi8crFmPyufH28\ncnF0n4iIxEwfvIqIJNhWOXiOxYRFkzbZLbrvG8wsH7MOREQSz90bdJxGtkneqHku8Cii1QPNrD3w\nibsvrWHb2BfrKZbLLbfcEnsNxXLRvtC+SMO+WLfOeeUV54EHnKuvdk491dluO2fbbZ3u3Z2RI51p\n05z/frSUM588k/367cfkhZNxb1w2rjPJm9lgoAz4jpktIMzXbRL6tfd393+Z2SlmNocwhfKSRlUk\nIlLCli2DadNg+XKYMQOmTIHXX4c1a2CPPcLl2GPhxz+Gfv1gzz3D97k7Q6cP5apHrqJHux4M6jaI\npls1rfW1slFnk3f387PYplejKxERKTHuMG4cvPEGLF0Kzz8Pc+bAfvtB69ZwwAHw05/CnXfCgQfC\nljUsvbds7TIuf/5y3vvoPUaeO5IjdzsyZzXmYkxeGqCsrCzuEoqG9sVm2hebFeO+WLkyJPOXX4b3\n3w8NvkUL6NABDjoIHn8cDj+85mZe1VfpfXRu03umOg+GyumLmXkhX09EpDEWLIDRo2HCBJg0Kdw+\n9ljYZ5/wtW3bkNa3aMA8xcz0PrDrwFrTu5nhDfzgVU1eRCSyfHkYcnnjDZg4EWbNgi5dQko/5RT4\nwQ+geUNX8Y9UTe/lZeV1pnc1eRGRBlq/HoYPh/vvh9deg9NOg44dw7DLUUdB0xyOntQnvWdqTJPX\nmLyIpIp7mP0yejQMGxZS+1FHwRVXwHPPwY41nZ+rUa+Z/7H3mijJi0jiLVwIL7wA//53SOtr18IZ\nZ4QhmC5doEmTup+joRqa3jNpuEZEpIo5c2DQIBg1Ct56C048Ec47Dw4+OIyxW6PP81W7hoy910RN\nXkSEMK3x0Udh5Ej473+hWze44AI45BD47ncLV0cu0nsmjcmLSCpt3AivvBLS+pgx8NFHIbHfe2/4\n8HSbbep+jlyKc+y9JkryIlJy3n8f/vY3GDoUPvsMrrwyNPf27WGrmKJrrtN7JiV5EUk097AOzOjR\n8Pe/w6efwtlnh9kw7drlf3y99tqKL71nUpMXkaK0cSOMHRsa+7/+BevWwXHHwYABYSgmzsa+ST7X\nnMkVNXkRKSrTp8OTT8Jjj4UDlc4/P1w/8sjiaOxQ/Ok9k5q8iMRuw4ZwYNK994apj+ecE8bbDz+8\neBr7JqWQ3jOpyYtIbNauDWPsAwbAdtvBVVeFuezZruJYSKWU3jOpyYtIwX38MQwZAnfdFT44vfde\n6NSpYas5FkKppfdMRbpLRSRpKivDWPsZZ8Bee4Xhmfvvh2eeCdMfi7HBuztPvvskbf/WltY7tubN\nnm+WVIMHJXkRybM1a8Kc9rvuglatwnox/frBbrvFXVntSjm9ZyrCv50ikgRr18Jf/hLWYJ84MST2\nN96A224r7gafhPSeSUleRHJqxQro3x/69g1HoI4YAUccEXdV2UlKes+kJC8ijbZxY0jqp54K3/lO\nWPVx2LBwMo5SaPBJS++ZlORFpMGWL4c+feDhh2HXXeHii8OHq82axV1Z9pKY3jMpyYtIvb3/Plxy\nCbRuHVZ+fOEFmDoVevUqnQaf5PSeSUleRLK2bBncdFNYs/1XvwqLhu2yS9xV1V/S03smJXkRqdOr\nr0L37tCmTZjPPn06/PWvpdfg05LeMynJi0i1Vq+GwYPhiSdg9my48MJwftT994+7soZJU3rPpCQv\nIl+zYAFccQXsvns429KVV8IHH4SDmUqxwacxvWdSkhcRIIy33313aOa/+hVMmlSaTT1TWtN7JiV5\nkZRbsCA09TZtwhmXFi+GBx4o7Qaf9vSeSUleJKXGjQuzZIYPD0v8zp4NO+0Ud1WNp/T+dUryIikz\nZUpY3vfUU2HffWHWLLjjjtJv8Erv1VOSF0mJZcvgmmvC+VJ//3v4+c+hRYu4q8oNpfeaKcmLJNyc\nOWG2zP/8DzRvDnPnwm9/m4wGr/ReNyV5kYSaNw+uvx7Gj4cLLoCZM4t7id/6UnrPjpK8SMLMmhXm\nth96KBxwQPhA9b77ktPgld7rR0leJCHefjvMc3/2WbjsMnj33bAyZJIovddfVknezDqb2Uwze9/M\nrqvm8e3NbJSZvW1m75hZj5xXKiLVqqwMBzAdd1xYFfK//4XevZPV4JXeG67OJG9mWwD9gE7Ah8BU\nMxvp7jMzNrsCmO7up5tZS2CWmQ1y9y/zUrWIsHo1jBwZzsC0YQO8/no4oClplN4bJ5skfwQw293n\nu/sGYAjQtco2DjSPrjcHPlaDF8mP+fPhD3+APfeEe++Fq68Oc9+T1uCV3nMjmzH5XYGFGbcXERp/\npn7AKDP7EGgGnJOb8kRkk4oK+N3vwhTIc8+F0aPh8MPjrio/lN5zJ1cfvJ4MvOXux5vZ3sCLZtbW\n3ddU3bC8vPyr62VlZZSVleWoBJFkWrw4HMQ0dmwYmjnzTGjSJO6q8sPdGTp9KFeNvooe7XowqNsg\nmm7VNO6yCq6iooKKioqcPJe5e+0bmLUHyt29c3T7esDdvXfGNs8Bd7j7q9Ht8cB17v5Glefyul5P\nRILVq8MHqPffD2efDbffHk6SnVSZ6X1g14FK7xnMDHe3hnxvNmPyU4HWZraHmTUBzgVGVdlmPnBC\nVMz3gH2AeQ0pSCTt3MPCYXvsAdOmhXOnPvhgchu8xt7zq87hGnevNLNewFjCH4UB7j7DzHqGh70/\n8CfgETObFn3bte6+Im9ViyTU+PFw222wdGkYcz+i6qdfCaOx9/yrc7gmpy+m4RqRak2bBrfeGmbJ\n3H47nHMObLNN3FXlT9Wx9/Ky8lSOvWerMcM1OuJVJEbjx8Of/wwzZkDPnvDYY7DddnFXlV9K74Wl\ntWtEYvDBB9C1a5gpc+GF4exM5eXJbvAae4+HkrxIAX32GQweDP/7v/CLX8CgQWH536RTeo+PkrxI\nAaxfH45O3WefMCTzxBNhvZmkN3il9/gpyYvkkTu88AL06hWWHXjySejQIe6qCkPpvTgoyYvkyaRJ\ncPTR4STZ//d/MGZMOhq80ntxUZIXybHFi8Pp9qZMgT/9Cc4/H771rbirKgyl9+KjJC+SI+4wYkQ4\ngKlNmzCD5uc/T0eDV3ovXkryIjmwYgVcemk4G9OAAdC5c9wVFY7Se3FTkhdpBHfo0yck9512Cqfg\nS0uDV3ovDUryIg00d26YNfPhh/Dqq7DvvnFXVDhK76VDSV6kntxh6FA49NBwef319DR4pffSoyQv\nUg9TpsBFF4Xrzz+fjimRmyi9lyYleZEsvPgi/PjHcMopcOONMHNmehq80ntpU5IXqcWSJZtPlH3T\nTfDII8k9eUd1lN5Ln5K8SDW+/DI09datoVmzMDXyZz9LT4NXek8OJXmRKiZNgksuCQ393Xdhr73i\nrqiwlN6TRUleJLJ4MZxxRhh3v/ZaeOWVdDV4pfdkUpKX1Nu4ER5+GK6/Hrp3h8cfT/4SwFUpvSeX\nkryk1saNYQmC3XeH/v1h1Ci45550NXil9+RTkpdUmjULLr4YNmyARx+FTp3AGnSa5NKl9J4OSvKS\nOq++GtZ5P+ssmDoVTjghXQ1e6T1dlOQlNT76CK67DoYMgYED4Zxz4q6o8JTe00dJXhKvsjKcU/Xg\ng6FJkzCLJm0NXuk9vZTkJdGeeSasFLnLLjB4MBxzTNwVFZ7Se7opyUsivfsunH46/PrXYebMlCnp\na/BK7wJK8pIwM2aE+e5jxsANN4T03qxZ3FUVntK7bKIkL4nw8cdhbZkDDoDDDgun47v55vQ1eKV3\nqUpJXkraunVw773wl7/AySeHszTtvHPcVcVD6V2qoyQvJeu55+DAA8MaMxMmwKBB6WzwSu9SGyV5\nKTkLF4Y13t9+G+66C37yk7grio/Su9RFSV5KxurVUF4ObdqEZYCnTUtvg1d6l2wpyUtJGDcurBDZ\noQO8805o9Gml9C71oSQvRa2yEn75SzjxxLCQ2FNPpbfBK71LQyjJS9FasAAuvDAsRTB/PrRqFXdF\n8VF6l4bKKsmbWWczm2lm75vZdTVsU2Zmb5nZu2Y2IbdlSpqsXQt9+sAhh0DHjvDCC+lt8Erv0lh1\nJnkz2wLoB3QCPgSmmtlId5+ZsU0L4H7gJHdfbGYt81WwJNvEiXDuueGApvHj4aCD4q4oPkrvkgvZ\nJPkjgNnuPt/dNwBDgK5VtjkfGO7uiwHcfXluy5Skq6wMZ2U680zo3TssLJbWBq/0LrmUzZj8rsDC\njNuLCI0/0z7A1tEwTTPgPnd/PDclStLNmwcXXBCuv/IK7LNPvPXESeldci1Xs2u2Ag4BugCdgZvM\nrHWOnlsS7IEHQmI//fQwVJPWBq/0LvmSTZJfDGR+7LVbdF+mRcByd18PrDezl4GDgDlVn6y8vPyr\n62VlZZSVldWvYkmEDz4IUyM//BAmTw4Li6WV0rtUVVFRQUVFRU6ey9y99g3MtgRmET54XQK8Dpzn\n7jMyttkX6EtI8dsAU4Bz3P29Ks/ldb2eJN/48WHs/Xe/C8sCb7113BXFw90ZOn0oV42+ih7telBe\nVk7TrZrGXZYUITPD3Rt0JuI6k7y7V5pZL2AsYXhngLvPMLOe4WHv7+4zzWwMMA2oBPpXbfAi69bB\nHXfAH/8Y1ns/6aS4K4qP0rsUSp1JPqcvpiSfWuPGhfXe27aFO++EH/4w7oriofQuDZHXJC/SGF9+\nCcOGwXnnwdNPp3dBMVB6l3ho7RrJm3Hjwgeqf/4zvPhiehu8Zs5InJTkJedWrYLLLoNXXw0N/sIL\nwRr0RrP0Kb1L3JTkJadeeQVatw7nVp0+HS66KJ0NXuldioWSvOTEqlVhSuSTT4Ylgbt1i7ui+Ci9\nSzFRkpdGmzcPjjgC1q8Pp+ZLa4NXepdipCQvDeYekvuVV8I114RLWim9S7FSk5cGWbkyjLfPnAlD\nh0JaV6eoOu99ULdBmvcuRUVNXurliy/g9tuhb1/o3BmmToUddoi7qngovUspUJOXrK1cCeefD4sW\nhemR++4bd0XxUHqXUqImL3XasAH++tdwSr6TTgrDM82bx11VPJTepdSoyUutVqwIs2XM4PHHoVOn\nuCuKh9K7lCo1eanRM8/Ab34DP/5xSPJbpfS3ReldSpnmycs3fPZZmA7ZvTs89FD4kDWNDV7z3iUJ\nUvhfV2ozZw6cdhrst1+YHvn978ddUTyU3iUplOQFCOm9b9+w3vtvfxuWBU5jg1d6l6RRkheWLYOO\nHaFNG5g0Cdq1i7uieCi9SxIpyafcyJHhLE3dusHzz6ezwSu9S5IpyafUp59Cz54wcWI4c9Mxx8Rd\nUTyU3iXplORTZtOiYvvvD9tuC++8k84Gr/QuaaEknyLTpoU13z/6CIYMCePwaaT0LmmiJJ8Sd9wB\nxx8fFhV74410Nnild0kjJfmEW7QoHLX6zjuhue+5Z9wVxUPpXdJKST7Bnn0WDjssTI188810Nnil\nd0k7JfkEqqwMa81ccw289BIcd1zcFcVD6V1EST5xPvggzJZ5+ml4//10Nnild5HNlOQTZNassBTw\nuefCnXemc1ExpXeRr1OST4AlS+CWW6BDh/D17rvT1+CV3kWql7JWkDwDB8Kvfx2WJXjppbDAWNoo\nvYvUTE2+RK1fH6ZGjh4NEybA4YfHXVHh6WxNInVTky9Bn38O550XlgeeNg223z7uigpP6V0kOxqT\nLzGLFsGRR8Lq1TB8ePoavMbeRepHSb6EPPkkXH45XHFF+IB1yy3jrqiwlN5F6k9JvgSsWxfG3y+/\nHEaMgNtuS1eDV3oXaTgl+SI3cyaccgoccgjMmAE77RR3RYWl9C7SOEryRey55+Doo6FXL3jqqXQ1\neKV3kdzIKsmbWWegD+GPwgB3713DdocDk4Bz3P3pnFWZMmvWwI03hsY+YkT6lgVWehfJnTqTvJlt\nAfQDTgYOAM4zs31r2O5OYEyui0yTzz+H00+HOXPgrbfS1eCV3kVyL5skfwQw293nA5jZEKArMLPK\ndr8GhgEpPCwnN2bPhh49wrDM0KGw9dZxV1Q4Su8i+ZHNmPyuwMKM24ui+75iZt8HznD3vwGWu/LS\n4403wlGr++8fTqydlgav9C6SX7maXdMHuC7jthp9lj7/HG66Cf7xD+jbFy66KO6KCkfpXST/smny\ni4FWGbd3i+7LdBgwxMwMaAl0MbMN7j6q6pOVl5d/db2srIyysrJ6lpwcn3wCP/oRtGoVlglOy+wZ\nrTkjUruKigoqKipy8lzm7rVvYLYlMAvoBCwBXgfOc/cZNWw/EHi2utk1ZuZ1vV5afPghdOkC7drB\nI4+ApeS9T2Z6H9h1oNK7SBbMDHdvUJeoc0ze3SuBXsBYYDowxN1nmFlPM/tldd/SkELS5PXXw7lX\nTzwxLBWchgavsXeReNSZ5HP6YkryPPAAlJfD7bfDpZfGXU1hKL2LNE5ek7zkxsaN8Nvfwp//DC+/\nnI4Gr/QuEj+tXVMAa9bAL34RPlydNAn22CPuivJPM2dEioOSfJ598gkcdxysWAEvvpj8Bq/0LlJc\nlOTzZN06ePDBMPZ++ulw//2wzTZxV5VfSu8ixUdJPg/++U/4wQ9gzJhweeihZDd4pXeR4qUkn2O3\n3hqOXr3/fujaNe5q8k/pXaS4Kcnn0EMPhcu//538Bq/0LlIalORzpG/fMP99zBjYe++4q8kvpXeR\n0qEk30ju4aTaffrA5MnhSNakUnoXKT1K8o1QWQm//GWY+/7SS8meHqn0LlKalOQb6PXXoXPncHLt\niROT2+CV3kVKm5J8A8yYEVaQvPhi6N07uSf4UHoXKX1K8vU0ZQq0bw9//CPcc08yG7zSu0hyKMnX\nw7Bh4cxN998PP/tZ3NXkh9K7SLIoyWdh40a48Ubo2TPMgU9ig1d6F0kmJfk6uIcVJN99F958M5kf\nsCq9iySXknwtNm6Eyy6D996D8eOT1+CV3kWST0m+Bp9/DmeeCYsWhSGaZs3irii3lN5F0kFJvhqf\nfALHHAMrV4YE36JF3BXljtK7SLooyVexYkVo8HvvHWbTJGmKpNK7SPooyWeYPz8c5HTwwWFN+KQ0\neKV3kfRSko8sWAAdO4aZNDfeCFtuGXdFuaH0LpJuSvKEIZqjjw4N/uabk9Hgld5FBMDcvXAvZuaF\nfL1srF8fhmjatIH+/eOuJjcy0/vArgPV3EVKnJnh7taQ7011kp8zJ5yLtWVL6Ncv7moaT+ldRKpK\n7Zj8kiVw2mmbx+CtQX8ji4fG3kWkOqlM8l98AaefDp06wQ03lHaDV3oXkdqkMsnffDPsvHMYoinl\nBq/0LiJ1SV2SHzUKHn0UBgwo3Qav9C4i2UpVkp87Fy68EB57DHbaKe5qGkbpXUTqIzVJfunScLDT\n7bfDGWfEXU39Kb2LSEOkYp78p5+GmTRt2oRhmlKjee8i6aZ58rVYtSok95YtS+9gJ6V3EWmsRI/J\nL1wIhx0GnTvDP/5RWssVaOxdRHIhsUl+40b4+c/DiT8efRSaNIm7ouwovYtILiUyyVdWwuWXw2ef\nQZ8+cVeTPaV3Ecm1rJK8mXU2s5lm9r6ZXVfN4+eb2X+iy0QzOzD3pWbHHc4+G95+O8yJL4UEr/Qu\nIvlSZ5I3sy2AfkAn4ENgqpmNdPeZGZvNA45x91Vm1hn4B9A+HwXX5cYbw1j8hAmw3XZxVFA/Su8i\nkk/ZJPkjgNnuPt/dNwBDgK6ZG7j7ZHdfFd2cDOya2zKz069fOGXfyJHF3+CV3kWkELIZk98VWJhx\nexGh8dfkUuCFxhTVEJMmhTVpxo6FXXYp9KvXj9K7iBRKTj94NbPjgEuAjjVtU15e/tX1srIyysrK\nGv2669fDZZeFD1kPO6zRT5c37s7Q6UO5avRV9GjXg0HdBtF0q6ZxlyUiRaaiooKKioqcPFedR7ya\nWXug3N07R7evB9zde1fZri0wHOjs7nNreK6cH/G6fn1YE37pUhgzpngXHdNRqyLSUPk+4nUq0NrM\n9jCzJsC5wKgqBbQiNPiLamrw+dK9OyxbBv/8Z3E2eI29i0ic6hyucfdKM+sFjCX8URjg7jPMrGd4\n2PsDNwE7Ag+YmQEb3L22cfuceOwxeOklmDcPtt8+369Wfxp7F5G4lewCZRMnhrM7jR1bfOPwVcfe\ny8vKNfYuIg3WmOGakjzi9fnn4YIL4OGHi6/BK72LSDEpubVr1qyBiy6Cp5+Gbt3irmYzjb2LSDEq\nuSR/993QoQMcf3zclWym9C4ixaqkkvzw4XDnndC3b9yVBErvIlLsSibJv/YaXHopPPEE7Lln3NUo\nvYtIaSiJJL9kSTjxx8CBYX34OCm9i0gpKfokX1kJJ58Mv/td/CfgVnoXkVJT9En+hhugRQv4wx/i\nq0HpXURKVVEn+YoKGDAA3nkHtojpz5HSu4iUsqJN8l98ERYeu/vueJYOVnoXkSQo2iTfrx+0agUX\nX1z411Z6F5GkKMokv2gRXHcd3HNPYVeWVHoXkaQpygXKLr4YdtoJ7rqrAEVFtN67iBSrfK8nX1Bj\nxsBTTxVumEbpXUSSrKjG5KdPh7POglGj4Ic/zP/raexdRJKuqJL8fffBlVfCCSfk93WU3kUkLYom\nyX/0UTjT0yuv5Pd1lN5FJE2KJsk/+CCcfXb+TgKi9C4iaVQUSd4dHnoIhg3Lz/MrvYtIWhVFkh84\nELbZBg7xFILQAAAFbklEQVQ9NLfPq/QuImkXe5Jfvx5+85uQ4nN54JPSu4hIEST5a6+Fjh3hpJNy\n83xK7yIim8V6xOvateHI1rlzYeedG//8OmpVRJKoZI94ffDBcELuxjZ4pXcRkerFluS//BK23hqm\nTm3ctEmldxFJupJM8iNGwPe+1/AGr/QuIlK32GbXDBkCV1/dsO/VzBkRkezEkuS//BLGjYPu3ev3\nfUrvIiL1E0uSnzwZdt+9fqf1U3oXEam/WJL8xIlw3HHZbav0LiLScLEk+TffzG45YaV3EZHGKXiS\n//jjcOann/yk5m2U3kVEcqPgSX7ECDj1VPjud6t/XOldRCR3Cp7kb7kFbrrpm/crvYuI5F7Bk7wZ\nHFmldyu9i4jkR1ZJ3sw6m9lMM3vfzK6rYZv7zGy2mb1tZu1qeq4f/WjzdaV3EZH8qrPJm9kWQD/g\nZOAA4Dwz27fKNl2Avd29DdAT+HtNz7epyS9bu4yznjqLW/99KyPPHcmdJ9xJ062aNvgHKTUVFRVx\nl1A0tC82077YTPsiN7JJ8kcAs919vrtvAIYAXats0xV4DMDdpwAtzOx71T1Z+/ZK76Bf4EzaF5tp\nX2ymfZEb2YzJ7woszLi9iND4a9tmcXTf0qpPdveCs5j5scbeRUQKoeAfvO7TsjWDfzooVUMzIiJx\nqXM9eTNrD5S7e+fo9vWAu3vvjG3+Dkxw9yej2zOBY919aZXnKtzi9SIiCdLQ9eSzSfJTgdZmtgew\nBDgXOK/KNqOAK4Anoz8Kn1Rt8I0pUkREGqbOJu/ulWbWCxhL+KB2gLvPMLOe4WHv7+7/MrNTzGwO\nsBa4JL9li4hINgp6+j8RESmsvCxrkMuDp0pdXfvCzM43s/9El4lmdmAcdRZCNr8X0XaHm9kGM+tW\nyPoKKcv/I2Vm9paZvWtmEwpdY6Fk8X9kezMbFfWKd8ysRwxl5p2ZDTCzpWY2rZZt6t833T2nF8If\njjnAHsDWwNvAvlW26QI8H10/Epic6zqK4ZLlvmgPtIiud07zvsjYbjzwHNAt7rpj/L1oAUwHdo1u\nt4y77hj3xe+BOzbtB+BjYKu4a8/DvugItAOm1fB4g/pmPpJ8Tg+eKnF17gt3n+zuq6KbkwnHFyRR\nNr8XAL8GhgHLCllcgWWzL84Hhrv7YgB3X17gGgslm33hQPPoenPgY3f/soA1FoS7TwRW1rJJg/pm\nPpp8dQdPVW1cNR08lTTZ7ItMlwIv5LWi+NS5L8zs+8AZ7v43IMkzsbL5vdgH2NHMJpjZVDO7qGDV\nFVY2+6IfsL+ZfQj8B7iqQLUVmwb1zVjODCXfZGbHEWYldYy7lhj1ATLHZJPc6OuyFXAIcDywHfCa\nmb3m7nPiLSsWJwNvufvxZrY38KKZtXX3NXEXVgry0eQXA60ybu8W3Vd1m93r2CYJstkXmFlboD/Q\n2d1re7tWyrLZF4cBQ8zMCGOvXcxsg7uPKlCNhZLNvlgELHf39cB6M3sZOIgwfp0k2eyLS4A7ANx9\nrpl9AOwLvFGQCotHg/pmPoZrvjp4ysyaEA6eqvqfdBTQHb46orbag6cSoM59YWatgOHARe4+N4Ya\nC6XOfeHu/xNd9iKMy1+ewAYP2f0fGQl0NLMtzWxbwgdtMwpcZyFksy/mAycARGPQ+wDzClpl4Rg1\nv4NtUN/MeZJ3HTz1lWz2BXATsCPwQJRgN7h71QXgSl6W++Jr31LwIgsky/8jM81sDDANqAT6u/t7\nMZadF1n+XvwJeCRjauG17r4ippLzxswGA2XAd8xsAXAL0IRG9k0dDCUikmAFP8eriIgUjpq8iEiC\nqcmLiCSYmryISIKpyYuIJJiavIhIgqnJi4gkmJq8iEiC/X9ebjLhgqw8PwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x148276588>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 452 ms, sys: 159 ms, total: 611 ms\n",
      "Wall time: 592 ms\n",
      "PRECISION/RECALL AUC\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXkAAAEKCAYAAAD3tSVSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHxdJREFUeJzt3XucVXW9//HXm6tXEERQRElUEi8oiniJdMRKzAwrT6k/\nLe1X+Si1c/JUaueXUqeOWnlM8xbmo+yUaKkZ5q00J9RQ8XhXEO8gDCp3FMQRPr8/vmuYzWYue4Y9\ne8+seT8fj/1g77W/s9Z3L2be67u+67u+WxGBmZnlU49qV8DMzDqOQ97MLMcc8mZmOeaQNzPLMYe8\nmVmOOeTNzHLMIW9dkqR1kkZkz38t6YctlB0kaZakvpWrYflJelbSYdWuR0eQdKaki6pdjzxyyHcC\nkl6TtErSCkl1WWhtUe16lSKr65qs7osk3SPpwxXYdFtu8DgX+HVErClcKOk3kuolDSlavtFBQ9Lw\n7MDSo2DZSZJmSlopab6kOyR9pLlKSBor6XZJS7LHs5L+U1L/Uj5EROwdEdNL+sRlImk/SY9Jejf7\nrPu2UPZfJD2Ulf17C+W+mO3LLxcsvhb4P5IGlbP+5pDvLAI4JiL6AfsDY4H/11RBSapkxUp0cVb3\nHYEFwK8qsM2S9oOkPsCXgN8VLd8C+CywDDi5xG2uP7BIOhv4b+BHwGBgZ+BK4Nhm6nEocD/wAPDh\niBgITAQ+AJoNzmqS1Bu4DfgtsE32758l9WrmRxYDlwIXtrDObYDzgGcLl2cH4DuBL256za2QQ77z\nEEBE1AF3AXsDSLpf0o8kPSjpXWAXSf0kXSdpgaR5WWtwfehJ+qqk57PW9bOS9ttoY9JVkn5atOw2\nSf+WPT9H0hvZOmZJOqK1D5D9of4B2GB7kr6c1WexpLsk7Vzw3l6S/pq9Vyfp3Gz5gZL+KWlp1kr+\nRQvh0pKDgKURsaBo+fHAUuCHwKltWaGkfsAPgG9ExJ8jYnVErI2IOyPi3GZ+7GLguoj4SUS8DRAR\nb0TEDxpa55JGSLovOyN6S9Lvsm01bPdVSROy5xdIuknS9dn/0TOS9m/L5yhBDdAzIi6PiPqI+AXp\n93RCU4Uj4u8RcTNQ18I6LwQuIx0Qiv0DOGbTqmzFHPKdjKSdgE8CjxcsPhn4CrA1MBe4HlgDjADG\nAB/P3kfSvwDnAydnretP0/Qf1FTg8wXb3SZbz1RJI4EzgAOydRwFvFZC3bcETgJeLFg2idRdchyw\nHaklOzV7byvgb6QW3A7AbsB92Y+uBf4NGAgcQgqWb7RWhybsA7zQxPIvAjcANwF7SBrThnUeCvQl\ntXJblZ01HALc2lpR4L+A7YFRwDBgcgvljyV9hv7A7aQziebq8FRBN9HSon+vaObH9gKeLlr2VLa8\nzSSNI/1OXdNMkVl00rOarswh33ncJmkJMJ10Wl94yvubiJgdEetIoXc08K2IeC8iFgE/B07Iyv5f\n4CcR8ThARLwSEfOKNxYRDwAhaXy26HhgRkS8SQrYPsDeknpFxNyIeLWFun8nq/sKUgAWnnKfDlwY\nEXOy+l8E7JcdzD4F1EXEzyPi/Yh4NyJmZvV7PCIejWQuMAU4vIT9WGwbYGXhguxM4gjghoh4C7iX\ntnUTDAQWZZ+nFANIf2sLC+pwcRay70j6HkBEvBwR90XEBxHR0PXR0md+MCLuiTQB1f8Ao5srGBH7\nRsTA7DGg6N8zm/mxrYDlRctWkBobbZJdy7iS1HhozkrSAcvKyCHfeUzK/uB2iYizii4SFob0cKA3\nUNfQGgOuIbWSAXYCXi5xmzcBJ2bPTwJ+DylsSK3oycCbkm6QtEML6/lp1sc8HFgNFF54HQ5c1tCK\nJJ1VBKn/vtm6Sto9u0hZJ2kZ8GOgPRfllrJxKJ0CPB8Rz2Svp5Iu+vXMXn9A2seFegPrsmBfDAxS\nwUXYEuqwjnS2AkBEnBMRA4A/Ab0AJA2WNDXrJltGuo7Q0mdeWPB8FbBZG+pUineAfkXL+lN00CzR\nGcBTDQfxZmzNxgcV20QO+c6jpQuJhSNJ5gHvAdsWtMa2iYjRBe/vWuI2pwLHZy3bg4Bb1m8w4saI\n+CgppCG1wFsUEW+QDg6Xq3G44lzg9KJW5FYR8XArdb2adPq+a0RsA/wHJV5sLfI0MLJo2SnAiOwA\nUgdcAmxL6iZrqPOHin5mBI0H2xmk7rLjSqlARKwCHiFd6G3Jf5EOBntln/lk2veZN5Jdm1lR9FiZ\n/XtVMz/2HBufHYzOlrfVBOAzBfv8UOASSZcXlBlF6g6yMnLIdzERsRD4K3CppK2VjFDj+OlfAd9u\nuAgnadfCC51F63qS1Cr9FXB3RKzIfmakpCOURqa8T2qdl9Q1ERH3AvNJ3TQAvwS+J2nPbN39JR2f\nvfcXYHtJ35TUR9JWWb8tpFbdiohYJWkP4Osl7aCNPQps03AmIukQUmAfSOr/3ZfUxzyVNAoH0sHu\nGEkfk9RD0lDSQWZq9hlXABcAV0qaJGlzSb0kTVTzY72/C3xZ0nclbZfVZRiwS0GZrUmt55WSdgS+\n08bP2uwBIRt+2a/osXX2b3PXOmqBtZLOyv5/vkn6PWhyeGS2r/qSznp6SupbcLH8S6QQb9jnj5Eu\nXv9HwSoOJw06sDJyyHcOLY35buq9L5L6zJ8HlgB/JF2sIxvd8GPgBkkrSN0BA1pY/w3AkWRdNZm+\npJb726QhkduRhr2VWr+fkfrpe0fEbdm6bsy6IJ4mDR0kIt4hXez9NKnrYQ5pRAfAt0ldKCtIB4ob\nS9juxpWLqAd+Q2q9Q9p3t0XE8xHxVsODNOLjGEnbRMTzpG6si0gHwYdIrfcfFqz3v4GzSUNd3yK1\n/s+gmYuxEfEQqTV7OPBC1nV1J+n6yy+yYj8ADiAN67ydgjOrEj9zWb8cItt3x5ECeilp302KiA9g\n/X0CzxT8yCmkBsGVwHhSF9KUbF0rivb3GtJBfGW2rs1IZ1LXl/MzGCha+dIQSdeRLpC9WdAlUFzm\nctLFwHeBU7MWolmnoHSDzXRgTNG1DuskJJ0JDGthCKq1UykhP550CvnbpkJe0tHAmRFxjKSDgMsi\n4uAOqa2ZmbVJq901EfEg6VStOZNId8IREY8A/VV0m7iZmVVHOfrkd2TDIX7zs2VmZlZlvvBqZpZj\n7ZkLpNh80k0tDYZlyzYiqaxX/83MuouIaNc9E6W25EXzY3Cnkd0SLulgYFl2a3yTIsKPCC644IKq\n16GzPLwvvC+8L1p+bIpWW/KSbiCNXd5W0lzSTSB9Ul7HlIi4U9InJb1EGkJ52ibVyMzMyqbVkI+I\nk0oo09wER2ZmVkW+8FolNTU11a5Cp+F90cj7opH3RXm0ejNUWTcmRSW3Z2aWB5KIDr7wamZmXZBD\n3swsxxzyZmY55pA3M8sxh7yZWY455M3McqziIe8RlGZmlVPxkP/gg0pv0cys+6p4yNfXV3qLZmbd\nV8VDfuhQWL680ls1M+ueKh7yy5fDwoWV3qqZWfdUldE1zz9fja2amXU/VQn5Hh64aWZWEVWJ2/vv\nr8ZWzcy6n6qEfN++jc//+EeYPx923x2WLYM1a6pRIzOzfKr4fPKQtnfHHfD223DqqRuX8w1TZmaN\nNmU++aqFfEuefBL23bcCFTIz6wK69JeGXHstPPQQrFsHq1fDCSfAE09Uu1ZmZvlQ8ZA/44zG59Om\nwVe+AoceChJsthnsv79D3sysXHpVeoNXXAH9+sGSJfCpT238/pgxcPvtla6VmVk+dbov8l68GAYN\nanz9zW+m1v6ll8Ijj2x4I9VPfgLf+U4HVdbMrJPoUhdeS9me2vhRamrggAPgZz9rX73MzDqzLn3h\ntSkR6TFnTnp90EHw3nvwzjtp+bp16fH5z6f3a2vhkksc8mZmxTplS77Qr38NRx8N22/f9Ptr1kDv\n3jBkCCxalJb17w8vvJCWmZl1dbnrrmmvz3wGbrut8XV9PfSq+KVlM7Pyyl13TXv96U+pO2fevPS6\nd2+45prq1snMrJpyFfINhg1rbNF//etw1FEp/NeurW69zMwqLVfdNU35whfgD3/YePmiRTBjBnzi\nE9CnT0WrZGbWJu6Tb8Vzz8HSpfDoo3D55fD66xu+P3JkulBrZtYZOeTbqL4ebrkFPvc5+OUv4ayz\n4JVXYJddql0zM7ONOeQ30QEHwOOPw7//O2y+OZx3HmyxRbVrZWaWOOQ30dy5MHz4xssvvRS+9jUH\nvplVl4dQbqKdd268y3bdOrj33rT8W9+CLbeEww+HBx+sbh3NzNrDLfkWrFkDp5ySvqKwwe23wzHH\ntH1+HTOz9urwlrykiZJmS5oj6Zwm3u8naZqkJyU9I+nU9lSms+nbNw2/jIC6utSqP/ZY6NEDbr65\n2rUzM2tdqy15ST2AOcCRwAJgJnBCRMwuKHMe0C8izpM0CHgBGBIRHxStq0u15JtSV5dG5cyYkfrq\nZ8+GnXaqdq3MLM86uiU/DngxIl6PiHrgRmBSUZkAts6ebw0sLg74vNhhB/jnP9Nj1arUn//pT6d5\n8M3MOptSQn5HYF7B6zeyZYWuAPaUtAB4CvjX8lSv8zrkEFi5Ej760dRPP2hQ6qeXYNQoWLGi2jU0\nMyvf1/8dBTwRERMk7Qr8TdLoiHinuODkyZPXP6+pqaGmpqZMVai8rbaC6dPTnDhTp6Y7aZctg8su\nS9Md//KXaQimmVlb1NbWUltbW5Z1ldInfzAwOSImZq/PBSIiLi4o8xfgwoh4KHt9H3BORDxWtK4u\n3ydfinXroGfPxtcDB6bhmA1DMs3M2qKj++RnArtJGi6pD3ACMK2ozOvAx7LKDAFGAq+0p0J50KNH\nGpHz9tvwu9/B0KHwox+llv/VV6dunmXLql1LM+sOShonL2kicBnpoHBdRFwk6XRSi36KpB2A3wA7\nZD9yYURMbWI93aIl35T3308t+nffbVy2+eZpGuTvfQ8OPLB6dTOzzs3TGnQxEfDqq7Dffql1X1eX\nlo8YAf/4R5oP38ysgUO+i1u9Ok2O9qtfpRkyR41KwzLHj4cjjnA/vll355DPiYg0BfLNN6c58J99\nNi3/y19Sd87gwdWtn5lVh0M+pyLg+OPh1lvT64EDUx/+YYfBhAnpy07MLP8c8t3A73+f7rL9+9/T\nVAqFttoqBf/o0WnKhdGj/ZWGZnnikO+G3n8flixJN2PtsAPcdx9cdVUatglwxhnwmc/AkCGw116e\nNdOsK3PI2wYuuQRuu23DOfAPOyyF/sknpykYzKzrcMhbi267DaZMgbvuSq8//GE47rjUrXPiiW7l\nm3V2Dnkr2cyZqX//1lthXjbt3LBhaaK1YcPg/PNTH7+ZdR4OeWuXCHjkEfj+99OduLNmpekWxo1L\ns2yed17q0zez6nLIW9k8/DBccw1cf/2Gy3/8Yzj9dNh22+rUy6w7c8hbh1izBhYuhJ/+FK68snH5\ndtvB2WfD7runu3OHDHH4m3Ukh7xVxNq1aa78ZcvgpptgzpwN3z/zzDTbZv/+1amfWV455K2q3n03\ntfTvvhueeSZ9UcpJJ8GHPuR5d8zKwSFvncb06WlY5uLFqbunRw848sg0emfcOKipgb59q11Ls67F\nIW+d0iuvpNE7990Hb70FDz2U7tIFGDMG9tknhf+kSamf38ya5pC3LmPJErjjDnjhhTRk89lnU9/+\n1lunVv6BB6aLuKNGpamWe/eudo3Nqs8hb13awoUp7KdNS907S5bA44+n6ZaHDoWxY9MBYMKEFP6e\nfM26G4e85dLy5amLZ84ceOCBximX99gjPYYPT4/tt4eDDoJddvEUDZZPDnnrNlavTiN45s2D11+H\n55+H++9P/f8A3/hGult35MjU7+/uHssDh7x1exHpLt0pU2DGjPQl6atXp/cOPzxNyrbffukAsM8+\n0LNndetr1hYOebMmrFmThnQ+8wysWJGma/jggxT+o0alL04fOzZ9+5a7eqwzc8ibtcHKlenC7j33\nwNSp8Nprafmhh6aw798/PR89GnbbLZ0VmFWTQ95sE0TAq6/C00/D/Plw773w0kuNX6S+666w446w\n//6pm2fixPR84MDq1tu6D4e8WQdZubLxQu8TT6RW//z5G37r1gknpAu9e++dRvoMHpymdPCdvVYu\nDnmzCquvh1Wr4NFH0xesv/JKGur58stpLp9Vq1K5ffeFPfeEnXeGI45Id/hutlm6H8CsVA55s06m\nvh7+8Y/U6n/ppXQAmD493fjVYOzY9G1c22yTDgLbbptG/owcmc4IPALIGjjkzbqI995LrfhXXkmh\n/9pr6ZrAggXpoLBuXePXMkI6Exg6NF0AHjIkjQoaOhQGDEgHBl8U7h4c8mY58847MHdumuLh7rvT\nGcCf/5xa+S++CIsWpYMDpIvA++yT7gXYbbd0X8DgwdWtv5WXQ96sG1qzJs33s2xZGh10993poFBX\nl+70HTEi3fU7fjzssEOa+mHQIN8P0BU55M1svYjUHXT33akbaPr0NNVzXV0K+AED0r0A/funrp/+\n/dPsn4cckrqEBgzw9YDOxiFvZiVZuhSefDLdE7ByJbz9dgr/hQvTsuXLU7lBg1L4jxiRuoAapoXY\ne+90f4DPBirLIW9mZbN0Kcycmb7d66WX0rDQV19N9wssXZrKDB2aRgD1759GCG21VZoCetdd0/MP\nfajx4QPCpnPIm1lFNIwEevPNFPirV8NTTzXeJDZgQFr+8suNX/S+7bbpDGD33dMNYmPGpGGjhx6a\nrhVY6xzyZtYpLV6cRgk991yaLqJfv3R94H//N50lNBg7Nh0Ettwytf6HDk3dQhMmpG8N6+4c8mbW\n5USkM4Knn04jhOrq0tnA3/6W7hdYvDg9dtst3S+w+eape2jwYPjIR9JBYbvtukd3UIeHvKSJwM+B\nHsB1EXFxE2VqgEuB3sDbEXFEE2Uc8mZWstWrU6v/jjvSReLNN4fHHktdRnPnpjJ9+sBhh6XrAZtv\nnq4V7LFHGjI6eHA+ppDo0JCX1AOYAxwJLABmAidExOyCMv2BfwKfiIj5kgZFxKIm1uWQN7OyWbUq\nzR305puN3xU8b146I5idJdSIEWm+oN13h4MPhnHj0r9bbFHdurdFR4f8wcAFEXF09vpcIApb85K+\nDuwQEee3si6HvJlVzMqVMGtWGh308MNphNCjj6blkM4C9tsvXQcYMyZNILfvvmmEUGfS0SH/OeCo\niPha9vpkYFxEfLOgTEM3zV7AVsDlEfE/TazLIW9mVbd2bboGsGRJmkL6gQfg/ffTxeGGm8ZGjUpn\nAHvume4TOPxw2GmnNGS0V6/K1ndTQr5cVe0F7A9MALYEZkiaEREvtfxjZmaV17NnCuthw9I3gH3p\nS43vrVuX+v/nz083js2aBffdB1demQ4ES5ak0T+77JL6/ceMgU9+svN+iUwpIT8f2Lng9bBsWaE3\ngEUR8R7wnqTpwL7ARiE/efLk9c9ramqoqalpW43NzDpQjx5peochQ9Lkb8Xefz/dF/DYY+lx7bVw\nyinpou/HPw4HHADHHpsOHu2dHqK2tpba2tpN+RjrldJd0xN4gXThtQ54FDgxImYVlNkD+AUwEegL\nPAJ8ISKeL1qXu2vMLHdWrUpdPQ89lGYIvfnm9OXx48enxz77wBe+0P7Qr9QQystoHEJ5kaTTSRdg\np2Rlvg2cBqwFro2IXzSxHoe8mXULixbBjBlw551p7P/LL8PHPpb69w84AE48MfX5l8I3Q5mZdXIL\nF8Lvfw9/+lMa7bNgQRrOedppcOqpLQe+Q97MrIt59VW4/vp0UffBB+Fzn4OvfhWOOmrjsg55M7Mu\nbNYsuOEGuOSSNI/PRRelCdwaOOTNzHJg5Uq4+mo4//w0vPPzn4ezz4bttnPIm5nlRn19uknr2mtT\nP/7q1V0o5Jlcsc2ZmeXDZLpOyLslb2bWNpvSJ5+DSTjNzKw5DnkzsxxzyJuZ5ZhD3swsxxzyZmY5\n5pA3M8sxh7yZWY455M3Mcswhb2aWYw55M7Mcc8ibmeWYQ97MLMcc8mZmOeaQNzPLMYe8mVmOOeTN\nzHLMIW9mlmMOeTOzHHPIm5nlmEPezCzHHPJmZjnmkDczyzGHvJlZjjnkzcxyzCFvZpZjDnkzsxxz\nyJuZ5ZhD3swsxxzyZmY55pA3M8uxkkJe0kRJsyXNkXROC+UOlFQv6bPlq6KZmbVXqyEvqQdwBXAU\nsBdwoqQ9mil3EXBPuStpZmbtU0pLfhzwYkS8HhH1wI3ApCbKnQXcDLxVxvqZmdkmKCXkdwTmFbx+\nI1u2nqShwHERcTWg8lXPzMw2RbkuvP4cKOyrd9CbmXUCvUooMx/YueD1sGxZobHAjZIEDAKOllQf\nEdOKVzZ58uT1z2tqaqipqWljlc3M8q22tpba2tqyrEsR0XIBqSfwAnAkUAc8CpwYEbOaKf9r4PaI\nuLWJ96K17ZmZ2YYkERHt6iFptSUfEWslnQn8ldS9c11EzJJ0eno7phT/SHsqYmZm5ddqS76sG3NL\n3syszTalJe87Xs3Mcswhb2aWYw55M7Mcc8ibmeWYQ97MLMcc8mZmOeaQNzPLMYe8mVmOOeTNzHLM\nIW9mlmMOeTOzHHPIm5nlmEPezCzHHPJmZjnmkDczyzGHvJlZjjnkzcxyzCFvZpZjDnkzsxxzyJuZ\n5ZhD3swsxxzyZmY55pA3M8sxh7yZWY455M3Mcswhb2aWYw55M7Mcc8ibmeWYQ97MLMcc8mZmOeaQ\nNzPLMYe8mVmOOeTNzHLMIW9mlmMOeTOzHHPIm5nlWEkhL2mipNmS5kg6p4n3T5L0VPZ4UNI+5a+q\nmZm1lSKi5QJSD2AOcCSwAJgJnBARswvKHAzMiojlkiYCkyPi4CbWFa1tz8zMNiSJiFB7fraUlvw4\n4MWIeD0i6oEbgUmFBSLi4YhYnr18GNixPZUxM7PyKiXkdwTmFbx+g5ZD/CvAXZtSKTMzK49e5VyZ\npCOA04DxzZWZPHny+uc1NTXU1NSUswpmZl1ebW0ttbW1ZVlXKX3yB5P62Cdmr88FIiIuLio3GrgF\nmBgRLzezLvfJm5m1UUf3yc8EdpM0XFIf4ARgWlEFdiYF/CnNBbyZmVVeq901EbFW0pnAX0kHhesi\nYpak09PbMQX4PjAQuEqSgPqIGNeRFTczs9a12l1T1o25u8bMrM06urvGzMy6KIe8mVmOOeTNzHLM\nIW9mlmMOeTOzHHPIm5nlmEPezCzHHPJmZjnmkDczyzGHvJlZjjnkzcxyzCFvZpZjDnkzsxxzyJuZ\n5ZhD3swsxxzyZmY55pA3M8sxh7yZWY455M3Mcswhb2aWYw55M7Mcc8ibmeWYQ97MLMcc8mZmOeaQ\nNzPLMYe8mVmOOeTNzHLMIW9mlmMOeTOzHHPIm5nlmEPezCzHHPJmZjnmkDczyzGHvJlZjjnkzcxy\nrKSQlzRR0mxJcySd00yZyyW9KOlJSfuVt5pmZtYerYa8pB7AFcBRwF7AiZL2KCpzNLBrROwOnA5c\n0wF1zZXa2tpqV6HT8L5o5H3RyPuiPEppyY8DXoyI1yOiHrgRmFRUZhLwW4CIeAToL2lIWWuaM/4F\nbuR90cj7opH3RXmUEvI7AvMKXr+RLWupzPwmypiZWYX5wquZWY4pIlouIB0MTI6Iidnrc4GIiIsL\nylwD3B8RN2WvZwOHR8SbRetqeWNmZtakiFB7fq5XCWVmArtJGg7UAScAJxaVmQacAdyUHRSWFQf8\nplTSzMzap9WQj4i1ks4E/krq3rkuImZJOj29HVMi4k5Jn5T0EvAucFrHVtvMzErRaneNmZl1XR1y\n4dU3TzVqbV9IOknSU9njQUn7VKOelVDK70VW7kBJ9ZI+W8n6VVKJfyM1kp6Q9Kyk+ytdx0op4W+k\nn6RpWVY8I+nUKlSzw0m6TtKbkp5uoUzbczMiyvogHTheAoYDvYEngT2KyhwN3JE9Pwh4uNz16AyP\nEvfFwUD/7PnE7rwvCsrdB/wF+Gy1613F34v+wHPAjtnrQdWudxX3xXnAhQ37AVgM9Kp23TtgX4wH\n9gOebub9duVmR7TkffNUo1b3RUQ8HBHLs5cPk9/7C0r5vQA4C7gZeKuSlauwUvbFScAtETEfICIW\nVbiOlVLKvghg6+z51sDiiPiggnWsiIh4EFjaQpF25WZHhLxvnmpUyr4o9BXgrg6tUfW0ui8kDQWO\ni4irgTyPxCrl92IkMFDS/ZJmSjqlYrWrrFL2xRXAnpIWAE8B/1qhunU27crNUoZQWgVIOoI0Kml8\ntetSRT8HCvtk8xz0rekF7A9MALYEZkiaEREvVbdaVXEU8ERETJC0K/A3SaMj4p1qV6wr6IiQnw/s\nXPB6WLasuMxOrZTJg1L2BZJGA1OAiRHR0ulaV1bKvhgL3ChJpL7XoyXVR8S0CtWxUkrZF28AiyLi\nPeA9SdOBfUn913lSyr44DbgQICJelvQqsAfwWEVq2Hm0Kzc7ortm/c1TkvqQbp4q/iOdBnwR1t9R\n2+TNUznQ6r6QtDNwC3BKRLxchTpWSqv7IiJGZI9dSP3y38hhwENpfyN/BsZL6ilpC9KFtlkVrmcl\nlLIvXgc+BpD1QY8EXqloLStHNH8G267cLHtLPnzz1Hql7Avg+8BA4KqsBVsfEeOqV+uOUeK+2OBH\nKl7JCinxb2S2pHuAp4G1wJSIeL6K1e4QJf5e/Aj4TcHQwu9GxJIqVbnDSLoBqAG2lTQXuADowybm\npm+GMjPLMc9CaWaWYw55M7Mcc8ibmeWYQ97MLMcc8mZmOeaQNzPLMYe8mVmOOeTNzHLs/wOx8Tda\n+aS0iQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1481d92b0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 457 ms, sys: 103 ms, total: 560 ms\n",
      "Wall time: 497 ms\n"
     ]
    }
   ],
   "source": [
    "qda = sklearn.discriminant_analysis.QuadraticDiscriminantAnalysis()\n",
    "clf = qda\n",
    "%time clf.fit(X[tidx],y[tidx])\n",
    "\n",
    "print(\"TEST\")\n",
    "%time evaluate(clf,X[pidx],y[pidx])\n",
    "\n",
    "print(\"TRAIN\")\n",
    "%time evaluate(clf,X[tidx],y[tidx])\n",
    "\n",
    "print(\"ROC AUC\")\n",
    "%time roc_auc(clf,X[pidx],y[pidx])\n",
    "\n",
    "print(\"PRECISION/RECALL AUC\")\n",
    "%time prrc_auc(clf,X[pidx],y[pidx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}