nlptask__2nb__.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "86347320-d3b7-4a87-a628-bf972b4ac706",
   "metadata": {},
   "source": [
    "## Install required packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "faf3db48-c6b0-489a-b9f0-b705519cd92d",
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip3 install -U spacy\n",
    "#!python -m spacy download en_core_web_sm\n",
    "\n",
    "# !pip3 install -U 'transformers[torch]'\n",
    "# !pip install -U optimum\n",
    "# !pip3 install -U dataset\n",
    "# !pip3 install -U evaluate\n",
    "\n",
    "# !pip3 install -U numpy\n",
    "# !pip install -U scikit-learn"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1f736e3b-beae-47a4-9652-80dcece9fb2d",
   "metadata": {},
   "source": [
    "## Prepare training data for Spacy NER\n",
    "The data is annotated using [Label Studio](https://labelstud.io/), includes the first 232 items in sample.txt  \n",
    "Data is transformed to Spacy format, saved to \"./ner/train.spacy\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "142121cc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of samples:  232\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "\n",
    "with open('ner/ner_train.json', mode='r', encoding='utf-8') as fd:\n",
    "    ner_train_data = json.loads(fd.read());\n",
    "\n",
    "print('Number of samples: ', len(ner_train_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "496a044d-3ad1-4d64-a91d-4ae2eb97d600",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'id': 6, 'annotations': [{'id': 6, 'completed_by': 1, 'result': [{'value': {'start': 0, 'end': 4, 'text': 'Gold', 'labels': ['COM']}, 'id': 'RBQlNv0cLe', 'from_name': 'label', 'to_name': 'text', 'type': 'labels', 'origin': 'manual'}], 'was_cancelled': False, 'ground_truth': False, 'created_at': '2024-03-02T11:01:07.440534Z', 'updated_at': '2024-03-02T11:01:07.440550Z', 'draft_created_at': None, 'lead_time': 50.531, 'prediction': {}, 'result_count': 0, 'unique_id': '1d691744-a08c-46a3-91ad-4cb85d778ccc', 'import_id': None, 'last_action': None, 'task': 6, 'project': 2, 'updated_by': 1, 'parent_prediction': None, 'parent_annotation': None, 'last_created_by': None}], 'file_upload': '70091d08-sample.txt', 'drafts': [], 'predictions': [], 'data': {'text': 'Gold edges lower as firmer US yields pinch appeal https://t.co/P0HPkmXvBZ'}, 'meta': {}, 'created_at': '2024-03-02T10:50:00.501559Z', 'updated_at': '2024-03-02T11:01:07.460181Z', 'inner_id': 6, 'total_annotations': 1, 'cancelled_annotations': 0, 'total_predictions': 0, 'comment_count': 0, 'unresolved_comment_count': 0, 'last_comment_updated_at': None, 'project': 2, 'updated_by': 1, 'comment_authors': []}\n"
     ]
    }
   ],
   "source": [
    "print(ner_train_data[5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "bb391fcc-d22d-4224-97be-723ab6126840",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'id': 17, 'annotations': [{'id': 17, 'completed_by': 1, 'result': [{'value': {'start': 0, 'end': 4, 'text': 'Gold', 'labels': ['COM']}, 'id': 'j7D1UyehxE', 'from_name': 'label', 'to_name': 'text', 'type': 'labels', 'origin': 'manual'}], 'was_cancelled': False, 'ground_truth': False, 'created_at': '2024-03-02T11:09:57.820429Z', 'updated_at': '2024-03-03T02:00:26.895744Z', 'draft_created_at': '2024-03-02T11:09:50.008467Z', 'lead_time': 188.464, 'prediction': {}, 'result_count': 0, 'unique_id': 'a9387a03-9297-4734-831a-574ddb42bcc6', 'import_id': None, 'last_action': None, 'task': 17, 'project': 2, 'updated_by': 1, 'parent_prediction': None, 'parent_annotation': None, 'last_created_by': None}], 'file_upload': '70091d08-sample.txt', 'drafts': [], 'predictions': [], 'data': {'text': 'Gold Price News and Forecast: XAU/USD is trapped in daily support and resistance. Posted by: EUR Editor  in EUR  1 min ago  XAU/USD struggles below $1,800 amid risk-off mood. Gold steps back from intraday high while flashing $1,772 as a quote amid Friday’s Asian session. In doing … Read Full Story at source (may require registration)     Latest posts by EUR Editor ( see all )'}, 'meta': {}, 'created_at': '2024-03-02T10:50:00.501984Z', 'updated_at': '2024-03-03T02:00:26.927460Z', 'inner_id': 17, 'total_annotations': 1, 'cancelled_annotations': 0, 'total_predictions': 0, 'comment_count': 0, 'unresolved_comment_count': 0, 'last_comment_updated_at': None, 'project': 2, 'updated_by': 1, 'comment_authors': []}\n"
     ]
    }
   ],
   "source": [
    "print(ner_train_data[16])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "7ce0de8f",
   "metadata": {
    "lines_to_next_cell": 0
   },
   "outputs": [],
   "source": [
    "import spacy\n",
    "from spacy.tokens import DocBin\n",
    "\n",
    "nlp = spacy.load(\"en_core_web_sm\")\n",
    "\n",
    "db = DocBin()\n",
    "for item in ner_train_data:\n",
    "    text = item['data']['text']\n",
    "    annotations = item['annotations']\n",
    "\n",
    "    if len(text) < 20:\n",
    "        continue\n",
    "\n",
    "    doc = nlp(text)\n",
    "    ents = []\n",
    "\n",
    "    # [E1010] Unable to set entity information for token 0 which is included in more than one span in entities, blocked, missing or outside.\n",
    "    # Eg:\n",
    "    #   text: Gold Price News and Forecast: XAU/USD is trapped in daily support and resistance. Posted by: EUR Editor  in EUR  1 min ago  XAU/USD struggles below $1,800 amid risk-off mood. Gold steps back from intraday high while flashing $1,772 as a quote amid Friday’s Asian session. In doing … Read Full Story at source (may require registration)     Latest posts by EUR Editor ( see all )\n",
    "    #   ents: [Gold Price News, XAU/USD, daily, 1 min ago, XAU/USD, 1,800, 1,772, Friday, Asian, … Read Full Story, EUR Editor, Gold]\n",
    "    #   span as text[0:16] and span as text[0:4] are overlaping each other -> token 0 which is included in more than one span\n",
    "\n",
    "    annotated = annotations[0]\n",
    "\n",
    "    for r in annotated['result']:\n",
    "        value = r['value']\n",
    "        span = doc.char_span(\n",
    "            value['start'],\n",
    "            value['end'],\n",
    "            label=value['labels'][0]\n",
    "            )\n",
    "        if span is None:\n",
    "            print('[result] ', r);\n",
    "            print('[span] ', span);\n",
    "            print('[text] ', item['data']['text']);\n",
    "            raise Error('The entity is marked inproperly that makes span to None');\n",
    "        else:\n",
    "            ents.append(span)\n",
    "\n",
    "    for ent in doc.ents:\n",
    "        span = doc.char_span(ent.start_char, ent.end_char, label=ent.label_)\n",
    "        is_span_overlaping = False\n",
    "        for included_span in ents:\n",
    "            if ((included_span.start <= span.start and span.start <= included_span.end)\n",
    "                or (included_span.start <= span.end and span.end <= included_span.end)\n",
    "                or (span.start <= included_span.start and included_span.start <= span.end)\n",
    "                or (span.start <= included_span.end and included_span.end <= span.end)):\n",
    "                    is_span_overlaping = True\n",
    "\n",
    "        if is_span_overlaping:\n",
    "            continue\n",
    "\n",
    "        ents.append(span)\n",
    "\n",
    "    if len(ents) == 0:\n",
    "        continue\n",
    "    doc.ents = ents\n",
    "    db.add(doc)\n",
    "\n",
    "db.to_disk(\"./ner/train.spacy\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "77df884b-225b-4eea-861f-830abc77e455",
   "metadata": {},
   "source": [
    "## Prepare training data for Spacy TextCategorizer\n",
    "The data is annotated using Label Studio, includes 45 out of the first 232 items in sample.txt  \n",
    "Data is transformed to Spacy format, saved to \"./sentiment_analysis_spacy/train.spacy\"  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "226c3d85-bffd-40c5-b0e6-ab09d4e537fc",
   "metadata": {
    "lines_to_next_cell": 0
   },
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "with open('sentiment_analysis_spacy/sentiment_train.json', mode='r', encoding='utf-8') as fd:\n",
    "    sent_train_data = json.loads(fd.read());"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "a8db8812-a637-454b-a85f-84efd8f49166",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'id': 2108, 'annotations': [{'id': 470, 'completed_by': 1, 'result': [{'value': {'choices': ['Negative']}, 'id': '8CdQ4W1i46', 'from_name': 'sentiment', 'to_name': 'text', 'type': 'choices', 'origin': 'manual'}], 'was_cancelled': False, 'ground_truth': False, 'created_at': '2024-03-03T08:17:06.304621Z', 'updated_at': '2024-03-03T08:24:43.405397Z', 'draft_created_at': None, 'lead_time': 97.557, 'prediction': {}, 'result_count': 0, 'unique_id': '14b4b0a3-db9d-4c72-936c-f9c8df8be723', 'import_id': 238, 'last_action': None, 'task': 2108, 'project': 6, 'updated_by': 1, 'parent_prediction': None, 'parent_annotation': None, 'last_created_by': None}], 'file_upload': '9a09ceda-sentiment_train.json', 'drafts': [], 'predictions': [], 'data': {'text': 'Gold edges lower as firmer US yields pinch appeal https://t.co/P0HPkmXvBZ'}, 'meta': {}, 'created_at': '2024-03-03T08:17:06.279707Z', 'updated_at': '2024-03-03T08:24:43.432821Z', 'inner_id': 6, 'total_annotations': 1, 'cancelled_annotations': 0, 'total_predictions': 0, 'comment_count': 0, 'unresolved_comment_count': 0, 'last_comment_updated_at': None, 'project': 6, 'updated_by': 1, 'comment_authors': []}\n"
     ]
    }
   ],
   "source": [
    "print(sent_train_data[5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "7a23ada7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "categories:  ['Positive', 'Negative', 'Neutral']\n",
      "Number of samples:  45\n"
     ]
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "import spacy\n",
    "from spacy.tokens import DocBin\n",
    "\n",
    "def read_categories():\n",
    "    return Path('sentiment_analysis_spacy/categories.txt').open().read().strip().split(\"\\n\")\n",
    "categories = read_categories();\n",
    "print('categories: ', categories)\n",
    "\n",
    "nlp = spacy.blank(\"en\")\n",
    "db = DocBin()\n",
    "for item in sent_train_data:\n",
    "    text = item['data']['text']\n",
    "    annotated = item['annotations'][0]\n",
    "\n",
    "    if len(text) < 20:\n",
    "        continue\n",
    "\n",
    "    result = annotated['result']\n",
    "    if len(result) == 0:\n",
    "        continue;\n",
    "    label = result[0]['value']['choices'][0]\n",
    "\n",
    "    doc = nlp.make_doc(text)\n",
    "    doc.cats = {category: 0 for category in categories}\n",
    "    # True labels get value 1\n",
    "    doc.cats[label] = 1\n",
    "\n",
    "    db.add(doc)\n",
    "\n",
    "print('Number of samples: ', len(db))\n",
    "\n",
    "db.to_disk(\"./sentiment_analysis_spacy/train.spacy\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "49851f6e-a0ef-4a09-89f5-bb27c7a980f6",
   "metadata": {},
   "source": [
    "## Train Spacy NER model\n",
    "The training set up is placed at directory './ner', two important files are config.cfg and train.spacy.  \n",
    "Note: To avoid spending too much time in annotation, training data also is used as validation data. In actual practice validation data must differ from training data to evaluate model's generisation.  \n",
    "Trained model is stored in \"./ner/output\" for later use."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "10a260b6-081c-470a-accb-00116481fac7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;5;4mℹ Saving to output directory: ner/output\u001b[0m\n",
      "\u001b[38;5;4mℹ Using CPU\u001b[0m\n",
      "\u001b[1m\n",
      "=========================== Initializing pipeline ===========================\u001b[0m\n",
      "[2024-03-04 22:26:41,122] [WARNING] [W112] The model specified to use for initial vectors (en_core_web_sm) has no vectors. This is almost certainly a mistake.\n",
      "\u001b[38;5;2m✔ Initialized pipeline\u001b[0m\n",
      "\u001b[1m\n",
      "============================= Training pipeline =============================\u001b[0m\n",
      "\u001b[38;5;4mℹ Pipeline: ['tok2vec', 'ner']\u001b[0m\n",
      "\u001b[38;5;4mℹ Frozen components: ['tok2vec']\u001b[0m\n",
      "\u001b[38;5;4mℹ Initial learn rate: 0.001\u001b[0m\n",
      "E    #       LOSS NER  ENTS_F  ENTS_P  ENTS_R  SCORE \n",
      "---  ------  --------  ------  ------  ------  ------\n",
      "  0       0      6.30   92.44   88.87   96.32    0.92\n",
      "  1     200   2312.57   89.89   89.43   90.35    0.90\n",
      "  3     400   2034.95   93.04   93.16   92.91    0.93\n",
      "  5     600   1955.29   96.01   96.16   95.87    0.96\n",
      "  7     800   1244.66   96.71   96.77   96.65    0.97\n",
      "  9    1000    734.19   96.35   96.77   95.93    0.96\n",
      " 11    1200   1279.18   97.92   98.39   97.47    0.98\n",
      " 14    1400    944.25   98.16   98.78   97.56    0.98\n",
      " 16    1600    733.11   98.04   97.84   98.25    0.98\n",
      " 18    1800    881.16   98.41   98.61   98.22    0.98\n",
      " 21    2000    621.95   98.17   98.22   98.13    0.98\n",
      "\u001b[38;5;2m✔ Saved pipeline to output directory\u001b[0m\n",
      "ner/output/model-last\n"
     ]
    }
   ],
   "source": [
    "!python3 -m spacy train ner/config.cfg --output ./ner/output --paths.train ./ner/train.spacy --paths.dev ./ner/train.spacy"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0497b96f-47cb-4eb7-b074-e1e0d3cfffbb",
   "metadata": {},
   "source": [
    "## Train Spacy TextCategorizer model\n",
    "The training set up is placed at directory './sentiment_analysis_spacy', two important files are config.cfg and train.spacy.  \n",
    "Note: To avoid spending too much time in annotation, training data also is used as validation data. In actual practice validation data must differ from training data to evaluate model's generisation.  \n",
    "Trained model is stored in \"./sentiment_analysis_spacy/output\" for later use."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "f78adb7b-e0bf-4236-bebd-2b1f50aa8e88",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;5;4mℹ Saving to output directory: sentiment_analysis_spacy/output\u001b[0m\n",
      "\u001b[38;5;4mℹ Using CPU\u001b[0m\n",
      "\u001b[1m\n",
      "=========================== Initializing pipeline ===========================\u001b[0m\n",
      "\u001b[38;5;2m✔ Initialized pipeline\u001b[0m\n",
      "\u001b[1m\n",
      "============================= Training pipeline =============================\u001b[0m\n",
      "\u001b[38;5;4mℹ Pipeline: ['tok2vec', 'textcat']\u001b[0m\n",
      "\u001b[38;5;4mℹ Initial learn rate: 0.001\u001b[0m\n",
      "E    #       LOSS TOK2VEC  LOSS TEXTCAT  CATS_SCORE  SCORE \n",
      "---  ------  ------------  ------------  ----------  ------\n",
      "  0       0          0.00          0.22       16.17    0.16\n",
      "  4     200         35.61         41.96       56.67    0.57\n",
      "  8     400         64.90         28.88       87.16    0.87\n",
      " 13     600         90.67         18.46       80.50    0.81\n",
      " 17     800        165.87         10.96       94.91    0.95\n",
      " 22    1000        180.13          3.98      100.00    1.00\n",
      " 26    1200        307.44          3.18      100.00    1.00\n",
      " 31    1400        347.69          3.67      100.00    1.00\n",
      " 35    1600        115.13          3.29      100.00    1.00\n",
      " 40    1800       5961.54         12.00       88.28    0.88\n",
      " 45    2000       1124.46          5.50      100.00    1.00\n",
      " 50    2200          0.00          0.00      100.00    1.00\n",
      " 55    2400          0.00          0.00      100.00    1.00\n",
      " 61    2600          0.00          0.00      100.00    1.00\n",
      "\u001b[38;5;2m✔ Saved pipeline to output directory\u001b[0m\n",
      "sentiment_analysis_spacy/output/model-last\n"
     ]
    }
   ],
   "source": [
    "!python3 -m spacy train sentiment_analysis_spacy/config.cfg --output ./sentiment_analysis_spacy/output --paths.train ./sentiment_analysis_spacy/train.spacy --paths.dev ./sentiment_analysis_spacy/train.spacy"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eeab4f40-2c38-4a00-b5d6-fecfe12cdc0c",
   "metadata": {},
   "source": [
    "## Evaluate Sentiment Analysis on gold and silver commodities related content\n",
    "Two models are used to recognise sentiment, the results are then compared against each other.  \n",
    "One model is Spacy model, another is [FinancialBERT](https://huggingface.co/ahmedrachid/FinancialBERT-Sentiment-Analysis) model from Huggingface  \n",
    "Note: one disadvantage of FinancialBERT model is that it can not process text that has more than 512 tokens."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e116ef23-535c-4463-93db-044f326da0d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# to exclude first 232 rows that have been used for training\n",
    "padding = 231\n",
    "\n",
    "with open('sample.txt', mode='r', encoding='utf-8') as fd:\n",
    "    samples = fd.readlines()\n",
    "samples = samples[padding:]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a568a57e-4cf3-4a70-97eb-0e10c21e750d",
   "metadata": {},
   "source": [
    "#### Load Spacy model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "877b0e34",
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "ner = spacy.load(\"ner/output/model-last\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4f05b55a-edd1-4020-92b7-acb51d4b6a39",
   "metadata": {},
   "source": [
    "#### Load Huggingface model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "e3ba00b1",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The BetterTransformer implementation does not support padding during training, as the fused kernels do not support attention masks. Beware that passing padded batched data during training may result in unexpected outputs. Please refer to https://huggingface.co/docs/optimum/bettertransformer/overview for more details.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'label': 'positive', 'score': 0.9998133778572083}, {'label': 'neutral', 'score': 0.9997822642326355}, {'label': 'negative', 'score': 0.9877365231513977}]\n"
     ]
    }
   ],
   "source": [
    "from transformers import BertTokenizer, BertForSequenceClassification\n",
    "from transformers import pipeline\n",
    "from datasets import Dataset, DatasetDict\n",
    "\n",
    "pretrainedFinancialBERT = BertForSequenceClassification.from_pretrained(\"ahmedrachid/FinancialBERT-Sentiment-Analysis\",num_labels=3)\n",
    "pretrainedFinancialBERT.to_bettertransformer()\n",
    "tokenizer = BertTokenizer.from_pretrained(\"ahmedrachid/FinancialBERT-Sentiment-Analysis\")\n",
    "\n",
    "bert_sent = pipeline(\"sentiment-analysis\", model=pretrainedFinancialBERT, tokenizer=tokenizer)\n",
    "\n",
    "sentences = [\"Operating profit rose to EUR 13.1 mn from EUR 8.7 mn in the corresponding period in 2007 representing 7.7 % of net sales.\",  \n",
    "             \"Bids or offers include at least 1,000 shares and the value of the shares must correspond to at least EUR 4,000.\", \n",
    "             \"Raute reported a loss per share of EUR 0.86 for the first half of 2009 , against EPS of EUR 0.74 in the corresponding period of 2008.\", \n",
    "             ]\n",
    "\n",
    "results = bert_sent(sentences)\n",
    "print(results)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "34ba713a-ec93-457f-925d-7eb42bcf4469",
   "metadata": {},
   "source": [
    "#### Filter gold and silver commodities related content\n",
    "Use Spacy model that has been trained before to perform NER"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "1773bd69",
   "metadata": {},
   "outputs": [],
   "source": [
    "gold_silver_com_sample = []\n",
    "com_label = 'COM'\n",
    "for sample in samples:\n",
    "    doc = ner(sample)\n",
    "    for ent in doc.ents:\n",
    "        if ent.label_ == com_label and ent.text.lower() in ['gold', 'silver']:\n",
    "            # print(sample);\n",
    "            # print(ent.text, ent.start_char, ent.end_char, ent.label_)\n",
    "            gold_silver_com_sample.append(sample);"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "40ecbe7a-64c5-446b-ae4e-53c3dd7a749b",
   "metadata": {},
   "source": [
    "#### Evaluate and collect data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "cf4faa11-7edb-4e09-ada5-ad2a1dd52323",
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "sent = spacy.load(\"sentiment_analysis_spacy/output/model-last\")\n",
    "\n",
    "text_length = 1000\n",
    "data = []\n",
    "for sample in gold_silver_com_sample:\n",
    "\n",
    "    doc = sent(sample)\n",
    "    spacy_label = max(doc.cats.items(), key=lambda x: x[1])[0]\n",
    "    spacy_label = spacy_label.lower()\n",
    "\n",
    "    # model's max_seq_length is 512, FinancialBERT is not able to process too long documents\n",
    "    if len(sample.split()) < 320:\n",
    "        bert_predicted = bert_sent(sample)\n",
    "        bert_label = bert_predicted[0]['label'].lower()\n",
    "        unmatched = 'X' if spacy_label != bert_label else 'O'\n",
    "    else:\n",
    "        bert_label = 'n/a'\n",
    "        unmatched = ''\n",
    "\n",
    "\n",
    "    data.append([sample[:text_length], spacy_label, bert_label, unmatched])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b94c7d6a-895e-443d-9209-fe8df943ca2d",
   "metadata": {},
   "source": [
    "#### Display data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "b3bcda08-9c8d-4a89-8dc4-71e025f6b55a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     Sample                                                         Spacy     FinancialBERT    Comparison\n",
      "---  ------------------------------------------------------------  --------  ---------------  ------------\n",
      " 0   Silver Futures Discussions. silver 25.90                      negative      neutral           X\n",
      " 1   Gold 💫 bagged, learned so much here, and nice people too     neutral       neutral           O\n",
      "     https://t.co/g9qQ8sbDue\n",
      " 2   Gold Futures Discussions. Bears game...                       negative      neutral           X\n",
      " 3   Gold Rate Today, 30 April 2021: Gold, Silver fall, know –     negative      neutral           X\n",
      "     what are the 10 grams gold rate today\n",
      "     https://t.co/kSVOTVuXw5\n",
      " 4   @BTC_Archive @PeterSchiff Gold price is suppressed by the     negative      neutral           X\n",
      "     government. Buy Bitcoin\n",
      " 5   Silver Futures Discussions. usd/inr at support so the         negative      neutral           X\n",
      "     advantage to run by double engine ( plunging zi price and\n",
      "     plunging usd/inr ) is gone so now the fall will depend\n",
      "     mostly on zi  so the rate of fall will not be much\n",
      " 6   Gold Futures Discussions. gold loses either way. if the       neutral       neutral           O\n",
      "     dollar rises, gold falls. if the dollar falls, the banks add\n",
      "     more shorts so gold doesn't look good\n",
      " 7   Gold Futures Discussions. gold loses either way. if the       neutral       neutral           O\n",
      "     dollar rises, gold falls. if the dollar falls, the banks add\n",
      "     more shorts so gold doesn't look good\n",
      " 8   The Edge Singapore | Rotate back to technology stocks,        negative      neutral           X\n",
      "     gold,..... The Edge Singapore | Rotate back to technology\n",
      "     stocks, gold, says DBS\n",
      "     https://www.theedgesingapore.com/capital/investing-\n",
      "     ideas/rotate-back-technology-stocks-gold-says-dbs\n",
      " 9   Gold Ranges for GBX - #GC1! chart https://t.co/bPruM2bunq     negative      neutral           X\n",
      "10   #goldprice  Upward pressure in gold prices, price charts      negative      neutral           X\n",
      "     show Gold is trying to break resistance level of past 6 mo…\n",
      "     https://t.co/dTklA1O9P1\n",
      "11   Gold Futures Discussions. Dis wik is crazy, wiped me out      neutral       neutral           O\n",
      "12   RT @JJK_Times: 🇯🇵 Jungkook has now two projects officially    neutral       neutral           O\n",
      "     certified by Recording Industry Association of Japan:  Gold\n",
      "     📀 \"Your eyes tell\" (…\n",
      "13   Can Utah Republicans survive an emerging sea of blue in the   neutral         n/a\n",
      "     West?. That’s what’s at stake and why the race for Utah’s\n",
      "     next Republican leader matters, outgoing Utah GOP chairman\n",
      "     says Share this story Share All sharing options for: As\n",
      "     surrounding states in the West continue to trend Democratic,\n",
      "     will Utah remain a red stronghold? Or will it eventually\n",
      "     become overtaken in an emerging sea of blue? That’s what’s\n",
      "     at stake for Utah Republicans in the race for the Utah GOP’s\n",
      "     next chairperson, a position largely off the radar of a\n",
      "     public consumed by COVID-19 updates and the promise of\n",
      "     better weather. Yet when delegates make their selection from\n",
      "     five candidates at this weekend’s state convention, it will\n",
      "     help determine the future of their party and its influence\n",
      "     on Utah politics, said Derek Brown, the state GOP’s outgoing\n",
      "     chairman, who announced last month he wouldn’t be seeking\n",
      "     another term to spend more time with his family. “Anyone who\n",
      "     is paying attention to the trends and to the numbers who i\n",
      "14   XAU/USD - Gold Spot US Dollar. Common bull, close this month  negative      neutral           X\n",
      "     for 1800\n",
      "15   Gold Futures Discussions. Enjoy the 0.000001 profit           negative      neutral           X\n",
      "16   RT @dazyle_: ⎆ You're gonna love this ⎆  10K White Gold 3 mm  neutral       neutral           O\n",
      "     Diamond Cut Hoop Earrings 0.6\" Diameter  Get your now ➣\n",
      "     https://t.co/MyrpRVXqe…\n",
      "17   Gold Futures Discussions. Fake spike, and many bulls trapped  negative      neutral           X\n",
      "     like $1781 past 24 hours... then rapid dive -$25 and many\n",
      "     RIP\n",
      "18   Gold Futures Discussions. kelattu pa y ur confusing           negative     negative           O\n",
      "19   Gold Futures Discussions. I heard that fridays are always     neutral       neutral           O\n",
      "     green, is that true? :)\n",
      "20   Recovery hopes drive shares, US Treasury yields. World share  neutral         n/a\n",
      "     indices extended gains and Treasury yields rose on Thursday,\n",
      "     after strong U.S. economic data and the Federal Reserve's\n",
      "     commitment to continue supporting the economy fueled\n",
      "     confidence in a recovery.U.S. economic growth accelerated in\n",
      "     the first quarter, fueled by massive government aid to\n",
      "     households and businesses, charting the course for what is\n",
      "     expected will be the strongest annual performance in nearly\n",
      "     four decades.\"Assuming that COVID variants remain contained,\n",
      "     the second quarter is set for a further acceleration in\n",
      "     growth as the re-openings continue,\" said Katherine Judge,\n",
      "     senior economist at CIBC Capital Markets.New York City aims\n",
      "     to \"fully reopen\" on July 1 after more than a year of\n",
      "     closures and capacity restrictions, Mayor Bill de Blasio\n",
      "     said.MSCI's gauge of stocks across the globe gained 0.33%,\n",
      "     closing at an all-time high and remaining on course for its\n",
      "     best month since November.The S&P 500 also closed at a\n",
      "     record high\n",
      "21   Check #gold on #ebay &gt;&gt; https://t.co/a8cDMHn2q1 || Is   negative      neutral           X\n",
      "     Kinross Gold Stock A Good Bet After Dropping 16% In Six\n",
      "     Month… https://t.co/T9XipeyJdH\n",
      "22   Gold Futures Discussions. 1776 maximum                        negative      neutral           X\n",
      "23   Silver Futures Discussions. Bullion banks are about to feel   neutral       neutral           O\n",
      "     the squeeze... Retail investors are coming for you... And we\n",
      "     will not stop until true price Discovery is resolved...\n",
      "     Fundamentals say that it should not be this way.. Keep\n",
      "     stacking.\n",
      "24   Gold Price News and Forecast: XAU/USD retreats on US data.    negative     positive           X\n",
      "     Posted by: EUR Editor  in EUR  1 min ago  After witnessing\n",
      "     good two-way businesses on Thursday, Gold (XAU/USD) is\n",
      "     holding on to recent recovery gains amid a broadly subdued\n",
      "     US dollar and higher Treasury yields. The returns on the US\n",
      "     debt … Read Full Story at source (may require registration)\n",
      "     Latest posts by EUR Editor ( see all )\n",
      "25   Silver Futures Discussions.    If I able to acquire supreme   neutral       neutral           O\n",
      "     power, my first job will be hand over this country to\n",
      "     British Emperor again for next 200 years or handover to a\n",
      "     international criminal .\n",
      "26   Sunshine Oil Sands (02012) Financial                          neutral       neutral           O\n",
      "     Statement/Environmental, Social and Governance\n",
      "     Information-[Annual Report]. Today&#39;s Hong Kong Economic\n",
      "     Journal | Today&#39;s Hong Kong Economic Journal |\n",
      "     Today&#39;s Economic Journal | Today&#39;s Economic Journal\n",
      "     | Today&#39;s Economic Journal | Wealth Management\n",
      "     International Finance | Today&#39;s 04:44 Important Notice |\n",
      "     Today&#39;s 08:22 Hong Kong Stocks Direct Attack |\n",
      "     Today&#39;s 10:31 International Finance | Today 07:59 real\n",
      "     estate news | Today 09:50 Hong Kong stocks direct attack |\n",
      "     Today 13:38 current affairs pulse | Today 11:15 current\n",
      "     affairs pulse | Today 09:45 Hong Kong stocks direct attack |\n",
      "     Today 09:28 immediate market review | Today 13:44 real-time\n",
      "     news analysis Comment on Today’s Hong Kong Economic Journal\n",
      "     Series Stock Financial Management Real Estate\n",
      "     Life/Innovation and Technology Subscribe to Popular Economic\n",
      "     News Brief Introduction | Terms of Service | Privacy Policy\n",
      "     | Disclaimer | Advertising Enquiry | Join the Hong Kong Econ\n",
      "27   Gold Futures Discussions.   I don't daytrade...time you guys  neutral       neutral           O\n",
      "     start zooming out a little\n",
      "28   Silver Futures Discussions. convey my wishes to short         negative      neutral           X\n",
      "     employee for yesterday work\n",
      "29   silver long. XAGUSD                                           negative      neutral           X\n",
      "30   XAU/USD - Gold Spot US Dollar. crazy crazy moment....         neutral       neutral           O\n",
      "31   Gold futures ease below Rs 46,900, silver firm: How to trade  negative      neutral           X\n",
      "     at current juncture #CommodityNews #Outlook\n",
      "     https://t.co/lvSeMxmqki\n",
      "32   Shandong Gold Mining : OVERSEAS REGULATORY ANNOUNCEMENT- SD   positive        n/a\n",
      "     GOLD 2020 SOCIAL RESPONSIBILITY REPORT. (a joint stock\n",
      "     company incorporated in the People's Republic of China with\n",
      "     limited liability) (Stock Code: 1787) OVERSEAS REGULATORY\n",
      "     ANNOUNCEMENT This announcement is made by Shandong Gold\n",
      "     Mining Co., Ltd. (the \"Company\") pursuant to Rule 13.10B of\n",
      "     the Rules Governing the Listing of Securities on The Stock\n",
      "     Exchange of Hong Kong Limited. Reference is made to the\n",
      "     overseas regulatory announcement of the Company dated 30\n",
      "     March 2021 in relation to the contents headlined \"2020\n",
      "     Social Responsibility Report (2020年度社會 責任報告)\" (the\n",
      "     \"Report\") published on the website of the Shanghai Stock\n",
      "     Exchange. The English translation of the Report is attached\n",
      "     to this announcement for the easy reference of international\n",
      "     investors only. Shandong Gold Mining Co., Ltd. Li Guohong\n",
      "     Chairman As at the date of this announcement, the executive\n",
      "     directors of the Company are Mr. Liu Qin, Mr. Wang Shuhai\n",
      "     and Mr. Tang Qi; th\n",
      "33   Shandong Gold Mining : OVERSEAS REGULATORY ANNOUNCEMENT- SD   positive        n/a\n",
      "     GOLD 2020 SOCIAL RESPONSIBILITY REPORT. (a joint stock\n",
      "     company incorporated in the People's Republic of China with\n",
      "     limited liability) (Stock Code: 1787) OVERSEAS REGULATORY\n",
      "     ANNOUNCEMENT This announcement is made by Shandong Gold\n",
      "     Mining Co., Ltd. (the \"Company\") pursuant to Rule 13.10B of\n",
      "     the Rules Governing the Listing of Securities on The Stock\n",
      "     Exchange of Hong Kong Limited. Reference is made to the\n",
      "     overseas regulatory announcement of the Company dated 30\n",
      "     March 2021 in relation to the contents headlined \"2020\n",
      "     Social Responsibility Report (2020年度社會 責任報告)\" (the\n",
      "     \"Report\") published on the website of the Shanghai Stock\n",
      "     Exchange. The English translation of the Report is attached\n",
      "     to this announcement for the easy reference of international\n",
      "     investors only. Shandong Gold Mining Co., Ltd. Li Guohong\n",
      "     Chairman As at the date of this announcement, the executive\n",
      "     directors of the Company are Mr. Liu Qin, Mr. Wang Shuhai\n",
      "     and Mr. Tang Qi; th\n",
      "34   XAU/USD - Gold Spot US Dollar. time to go to 1758.            neutral       neutral           O\n",
      "35   XAU/USD - Gold Spot US Dollar. Mahadhir Atan  bad for gold,   neutral      negative           X\n",
      "     usualy.\n",
      "36   Gold Futures. TP TRACKER  Around 1720-40ish is my buy order.  neutral      positive           X\n",
      "37   Zhejiang United Investment (08366) Change of registered       neutral         n/a\n",
      "     address or office, registered place of Hong Kong business,\n",
      "     or Hong Kong representative for receiving legal process\n",
      "     documents. Zhejiang United Investment (08366) Change of Hong\n",
      "     Kong head office and principal place of business (223KB,\n",
      "     pdf) Previous post: Today&#39;s Hong Kong Economic Journal |\n",
      "     Today&#39;s Hong Kong Economic Journal | Today&#39;s\n",
      "     Economic Journal | Today&#39;s Economic Journal |\n",
      "     Today&#39;s Economic Journal Important Notice | Today 08:22\n",
      "     International Finance | Today 07:59 Hong Kong Stocks Direct\n",
      "     Attack | Today 10:31 Current Affairs Pulse | Today 09:45\n",
      "     Hong Kong Stocks Direct Attack | Today 13:38 Current Affairs\n",
      "     Pulse | Today 11:15 Real Estate News | Today 09:50 That Is\n",
      "     Market Review｜ Today’s 13:44 Current Events｜ Today’s 12:42\n",
      "     Hong Kong Stocks Direct Attack｜ Today’s 09:28 Real-time\n",
      "     News Analysis and Commentary Today’s Economic Journal Series\n",
      "     Stock Financial Management Real Estate Life/Innovation and\n",
      "     Technology Subs\n",
      "38   @JagexTyran hello, my account got hit with RWT goldfarming    neutral       neutral           O\n",
      "     ban while i've never sold or bought any gold. i tried al…\n",
      "     https://t.co/8reI1kqhXq\n",
      "39   Silver Futures Discussions. At last, rates of silver is       negative      neutral           X\n",
      "     live....\n",
      "40   Most currencies have an asset to back them up. Most           neutral       neutral           O\n",
      "     countries use gold. USD used to be backed in gold. Somewhere\n",
      "     around the line (fellow ape told me was Nixon in 1970-71)\n",
      "41   Most currencies have an asset to back them up. Most           neutral       neutral           O\n",
      "     countries use gold. USD used to be backed in gold. Somewhere\n",
      "     around the line (fellow ape told me was Nixon in 1970-71)\n",
      "42   I sold the last of mine at 2.1, gold stocks have not been     negative      neutral           X\n",
      "     kind to me lately, still hold BRB , NXM and WML but sick of\n",
      "     going nowhere while the market rallies..\n",
      "43   Gold Futures Discussions. Double top formed. Sell             negative      neutral           X\n",
      "44   Zhongliang Holdings (02772) Notice of Annual General Meeting  neutral         n/a\n",
      "     / Suspension of Registration Procedures or Change of Date of\n",
      "     Suspension of Registration. Zhongliang Holdings (02772)\n",
      "     Notice of Annual General Meeting (339KB, pdf) Previous post:\n",
      "     Today&#39;s Hong Kong Economic Journal | Today&#39;s Hong\n",
      "     Kong Economic Journal | Today&#39;s Economic Journal |\n",
      "     Today&#39;s Economic Journal | Today&#39;s Economic Journal\n",
      "     | Dr. Yuen Kangjue | Sino Health | Wealth Management\n",
      "     International Finance | Today 04 :44 Important Notice |\n",
      "     Today 08:22 Hong Kong Stocks Direct Attack | Today 10:31\n",
      "     International Finance | Today 07:59 Real Estate News | Today\n",
      "     09:50 Hong Kong Stocks Direct Attack | Today 13:38 Current\n",
      "     Affairs Pulse | Today 11:15 Current Affairs Pulse | Today\n",
      "     09: 45 Hong Kong Stocks Direct Attack | Today&#39;s 09:28\n",
      "     Instant Market Comment | Today&#39;s 13:44 Real-time News\n",
      "     Analysis and Commentary Today&#39;s Economic Journal Series\n",
      "     Stock Wealth Management Real Estate Life/Innovation and\n",
      "     Technology S\n",
      "45   Gold Futures Discussions. how can you be so sure? here gold   positive      neutral           X\n",
      "     moved from 1765 to 1772 and still in mcx gold is at 46750\n",
      "     strange! absolutely strange!\n",
      "46   $MINE  today we buy gold plated lambos...                     neutral       neutral           O\n",
      "47   Silver too if it isn’t tarnished. Copper is just cheaper.     neutral       neutral           O\n",
      "48   Silver Futures.   if USDINR  goes up when ll see a good       neutral      positive           X\n",
      "     bounce,,\n",
      "49   Gold Bangles Design Making  #goldsmithjack                    neutral       neutral           O\n",
      "     #goldbanglesdesign #goldbanglesmaking #jewelrymaking\n",
      "     https://t.co/9BlxbiUpWj\n",
      "50   China Internet Investment (00810) Others-the latest           neutral       neutral           O\n",
      "     situation of business development. China Internet Investment\n",
      "     (00810) Voluntary Announcement-Memorandum of Understanding\n",
      "     (204KB, pdf) on a potential investment in a live e-commerce\n",
      "     marketing platform company Today&#39;s Hong Kong Economic\n",
      "     Journal, Dr. Yuen Kangjiu | Shine Health Ye Yongkeng |\n",
      "     Important Wealth Management Notice | Today 08:22 Hong Kong\n",
      "     Stocks Direct Attack | Today 10:31 International Finance |\n",
      "     Today 07:59 Current Events | Today 09:45 Hong Kong Stocks\n",
      "     Direct Attack | Today 13:38 Current Affairs Pulse | Today\n",
      "     11:15 Real Estate News | Today 09:50 Current Events Pulse |\n",
      "     Today 12:42 Instant Market Review | Today 13:44 Hong Kong\n",
      "     Stocks Direct Attack | Today 09:28 Real-time News Analysis\n",
      "     and Commentary Today&#39;s Economic Journal Series Stock\n",
      "     Wealth Management Real Estate Life/Innovation Technology\n",
      "     Subscription to the popular Hong Kong Economic Journal |\n",
      "     Terms of Service | Privacy Policy | Disclaimer | Advertising\n",
      "     Inquiry | Jo\n",
      "51   RT @SilverChartist: 4 #Silver Stocks (+an ETF) Testing 200MA  neutral       neutral           O\n",
      "     Support:  1⃣ Mag Silver, $MAG 2⃣ Alexco Resources, $AXU 3⃣\n",
      "     Southern Silver, $S…\n",
      "52   Gold Prices May Fall as US PCE Data Sends Yields, Dollar      negative      neutral           X\n",
      "     Higher https://t.co/wfeag21tSe https://t.co/ftYevFoSm1\n",
      "53   RT @eBargainsToday: Gold, Stocks, and Bitcoin: Weekly         negative      neutral           X\n",
      "     Overview — April 29: This week’s price movements for Bitcoin\n",
      "     (BTC), gold, and our sto…\n",
      "54   RT @eBargainsToday: Gold, Stocks, and Bitcoin: Weekly         negative      neutral           X\n",
      "     Overview — April 29: This week’s price movements for Bitcoin\n",
      "     (BTC), gold, and our sto…\n",
      "55   Silver Futures Discussions. According to google india and     neutral       neutral           O\n",
      "     china are by far the biggest consumers of silver. Do you see\n",
      "     the situation impacting this? Are factories etc shutdown?\n",
      "56   Gold Futures Discussions. With US high spending bills         neutral       neutral           O\n",
      "     (trillions), this will fly as soon the inflation will show\n",
      "     up. Just wait...\n",
      "57   XAU/USD - Gold Spot US Dollar. gold is acting like my gf      neutral       neutral           O\n",
      "     🤣🤣\n",
      "58   Gold Futures Discussions. The vaccine I still catch covid     neutral       neutral           O\n",
      "     and pass it on to granny like you and we both die...so not\n",
      "     sure what people are thinking....sounding like fake\n",
      "     stuff...lolz\n",
      "59   Originally posted by tobiasroy: ↑ Gold stocks getting         neutral       neutral           O\n",
      "     hammered today as well,I reckon you will be able to get back\n",
      "     in at 3.3c.AP must be busy putting his spiel together and\n",
      "     records show that when things are delivered on time means\n",
      "     they’re are good but anything late is always bad. Expand\n",
      "     Where did you get the 3.3??... pull it out of your\n",
      "     bottom??????\n",
      "60   Gold Futures Discussions. Look at silver reaction             negative      neutral           X\n",
      "61   Gold Futures Discussions. Zyielda loves cup and handle .      negative      neutral           X\n",
      "     Zyielda put in sweet and low,\n",
      "62   FUTURES: Crude Oil        $ 64.64         -0.57% 🥵 Natural   negative      neutral           X\n",
      "     Gas    $ 2.9010       -0.34% 🥵 Gold                $ 176…\n",
      "     https://t.co/c0vmEs243Q\n",
      "63   Gold Futures Discussions. Yess waterfall comming              negative      neutral           X\n",
      "64   Gold Futures Discussions. NO CLOSE beneath 1763.8 is          positive      neutral           X\n",
      "     important they had an LTS bar on yesterday the target 1811.2\n",
      "     from the 10th is still there also they made another at\n",
      "     1786.8\n",
      "65   Simor UBS Yijia Niu E(53255) Bonds and Structured             neutral       neutral           O\n",
      "     Products-[Maturity Announcement-CBBC]. Smarter UBS Yijia Niu\n",
      "     E(53255) Mandatory Redemption Event and Early Expiry Notice\n",
      "     for Callable Bull/Bear Contracts issued by UBS (94KB, pdf)\n",
      "     Hong Kong Economic Journal Today, Hong Kong Journalist, Dr.\n",
      "     Yuen Kangjou | Xin Health Ye Yongkeng | Wealth Management\n",
      "     International Finance | Today 04:44 Important Notice |\n",
      "     Yesterday 23:26 Important Notice | Yesterday 23:29 Important\n",
      "     Notice | Yesterday 23:10 Important Notice | Yesterday 23: 11\n",
      "     International Finance | Today 04:10 International Finance |\n",
      "     Today 04:09 Current Affairs Pulse | Today 00:36 Hong Kong\n",
      "     Finance | Yesterday 22:19 Important Notices | Today 08:22\n",
      "     Real-time News Analysis and Comments Introduction to the\n",
      "     popular Hong Kong Economic Journal | Terms of Service |\n",
      "     Privacy Policy | Disclaimer | Advertising Inquiry | Join the\n",
      "     Hong Kong Economic Journal | Contact the Hong Kong Economic\n",
      "     Journal Stock and index information is provided by Financial\n",
      "     Inte\n",
      "66   Old pic but at this time it is like Gold @sidharth_shukla     neutral       neutral           O\n",
      "     #SidharthShukla https://t.co/F9OPVwFcVp\n",
      "     https://t.co/LHcTn8gsJO\n",
      "67   “ Tch. A fish that can talk and wears jewelries? I'd eat you  neutral       neutral           O\n",
      "     then use your claws to pick my teeth. Gold fishhhh~”\n",
      "     https://t.co/7z905FkmTp\n",
      "68   Silver Futures Discussions. smoking marijuana is great        negative      neutral           X\n",
      "     experience when i was young i used to smoke and sit idle for\n",
      "     hours and imagination flew high in bright colors think today\n",
      "     silver smoke marijuana  and dreaming\n",
      "69   Shipping Containers Are Falling Overboard at a Rapid Rate.    negative     negative           O\n",
      "     This ship [0] went down offshore of Cornwall in 1641, with\n",
      "     60 men, 100,000 pounds of gold and 500,000 pieces-of-eight\n",
      "     aboard. Cornwall is not known to be friendly to wrecks. And\n",
      "     yet, nearly 400 years later, she's not been found. They\n",
      "     say.[0] https://en.wikipedia.org/wiki/Merchant_Royal\n",
      "70   $AGC $AGC.AX Australia Copper and Gold Ltd  - Aussie #Copper  negative      neutral           X\n",
      "     and #gold  - Assays if three (out of 7) drilling campa…\n",
      "     https://t.co/vnDnBLrfaa\n",
      "71   Gold Futures Discussions. Stochs is only used in a bearish    neutral       neutral           O\n",
      "     trend...it can run way hot in a bullish trend...101 dude\n",
      "72   Silver Futures Discussions. JAY BHOLENATH= 68130<<            negative      neutral           X\n",
      "73   Gold Price Today: Yellow metal trades higher post Fed meet;   negative     positive           X\n",
      "     Silver May futures rally over 1% https://t.co/7e8mVN64Of\n",
      "74   Gold Price Today: Yellow metal trades higher post Fed meet;   negative     positive           X\n",
      "     Silver May futures rally over 1% https://t.co/7e8mVN64Of\n",
      "75   Gold Futures Discussions. Let me guess this isnt              neutral       neutral           O\n",
      "     accumulation it's a flying saicer\n",
      "76   Gold https://t.co/VnL69bGYH0                                  negative      neutral           X\n",
      "77   Gold Futures Discussions. Hmm, still one way dive... ready    negative      neutral           X\n",
      "     to hedge\n",
      "78   Gold price at $2,300 if not for bitcoin - Bloomberg           negative     positive           X\n",
      "     Intelligence. (Kitco News) - Technological innovation is\n",
      "     benefiting many metals but not gold, said Bloomberg\n",
      "     Intelligence, advising to pair gold with bitcoin in a\n",
      "     portfolio. Read more on \"Kitco\"\n",
      "79   U.S .Stocks May Give Back Ground In Early Trading. (RTTNews)  positive        n/a\n",
      "     - After ending the previous session mostly higher, stocks\n",
      "     may move back to the downside in early trading on Friday.\n",
      "     The major index futures are currently pointing to a lower\n",
      "     open for the markets, with the Dow futures down by 172\n",
      "     points. Profit taking may contribute to initial weakness on\n",
      "     Wall Street after the upward move seen on Thursday lifted\n",
      "     the S&P 500 to a new record closing high. The tech-heavy\n",
      "     Nasdaq also reached a record intraday high during\n",
      "     yesterday's trading but pulled back and ended the session\n",
      "     only modestly higher. A steep drop by shares of Twitter\n",
      "     (TWTR) may weigh on the markets, with the social media giant\n",
      "     plunging by 13.4 percent in pre-market trading. The nosedive\n",
      "     by Twitter comes after the company reported better than\n",
      "     expected first quarter results but provided disappointing\n",
      "     guidance. On the other hand, shares of Amazon (AMZN) are\n",
      "     likely to see initial strength after the online retail giant\n",
      "     reported first qu\n",
      "80   $MRZ   Low EV Low mcap Low SOI and interesting individuals    neutral       neutral           O\n",
      "     on the top 20 register...   Bags potential?  #gold…\n",
      "     https://t.co/v04YQJ1Eh0\n",
      "81   Silver Futures Discussions. are you playing p.p. rub game?    neutral       neutral           O\n",
      "82   @f1bridget White and Gold                                     negative      neutral           X\n",
      "83   Gold Futures Discussions. Weak bulls...                       negative      neutral           X\n",
      "84   Silver? I heard the world had run outta silver,,,,            neutral       neutral           O\n",
      "85   Silver Futures Discussions. 25 to 35 is very easy             negative      neutral           X\n",
      "86   Gold Futures Discussions.     yes that's followed  but  NPA   neutral       neutral           O\n",
      "     : )   after all .... may be it's a trick\n",
      "87   Gold prices flat as international prices fall on strong US    negative      neutral           X\n",
      "     data; where is MCX gold headed?  Gold was trading flat t…\n",
      "     https://t.co/Kk0M8o3RN4\n",
      "88   Gold prices flat as international prices fall on strong US    negative      neutral           X\n",
      "     data; where is MCX gold headed?  Gold was trading flat t…\n",
      "     https://t.co/Kk0M8o3RN4\n",
      "89   Silver Futures Discussions. buy physical. drain the vault!    negative      neutral           X\n",
      "90   Gold Futures Discussions. Nice $3 from $1766, by dips again   neutral       neutral           O\n",
      "91   4 #Silver Stocks (+an ETF) Testing 200MA Support:  1⃣ Mag      neutral       neutral           O\n",
      "     Silver, $MAG 2⃣ Alexco Resources, $AXU 3⃣ Southern Silver…\n",
      "     https://t.co/8sq4OkuPyl\n",
      "92   Gold Futures Discussions. Have u sold ur past dips?           negative      neutral           X\n",
      "93   RT @healfromwithin: Check out this listing I just found on    neutral       neutral           O\n",
      "     #Poshmark: Christian Dior Vintage Gold Compact Mirror.\n",
      "     https://t.co/WkrN8XA12g #…\n",
      "94   Gold Futures Discussions. US Market will start with the       neutral       neutral           O\n",
      "     opening dump as usual this week. However I don't think the\n",
      "     support will be able to hold the line today on rising dollar\n",
      "95   XAU/USD - Gold Spot US Dollar. let s break 73, step by step   neutral       neutral           O\n",
      "96   RT @Shivfreespirit: @PeterSchiff One would expect ? Yes,That  neutral       neutral           O\n",
      "     one person is You.  No body else is expecting is anything\n",
      "     from Gold or Silver…\n",
      "97   RT @TigerWitAfrica: Amid Rising Inflation Expectation, what   neutral       neutral           O\n",
      "     is next for Gold and Dollar? https://t.co/o89lTuKpKI\n",
      "     https://t.co/olpfc0V1ZC\n",
      "98   Re: Bitcoin or Gold?. Quote from: BIN-BIN on Today at         neutral       neutral           O\n",
      "     07:45:23 AMSince gold is more stable for long-term holding,\n",
      "     investors will prefer to go for gold against bitcoin.Bitcoin\n",
      "     is new, just a decade-old currency, but gold has been around\n",
      "     for so many years over 100 years. So the two can not be\n",
      "     compared, but again bitcoin has grown magnificently, so it\n",
      "     offers quick returns.In my opinion, not all investors choose\n",
      "     Gold for long-term investments, because if we really want\n",
      "     quick returns, we should prefer Bitcoin.As we all know the\n",
      "     volatile Bitcoin price can generate a much greater profit\n",
      "     than Gold, but indeed the risk is also much higher. So for\n",
      "     investors who are looking for it to be safe, they definitely\n",
      "     prefer Gold over Bitcoin. So in the end back to each of us,\n",
      "     because I prefer quick returns I prefer investing in\n",
      "     Bitcoin. And it is proven that I managed to get a large\n",
      "     profit from Bitcoin this year, because the increase in\n",
      "     Bitcoin this year is very significant when compared to Gold.\n",
      "99   JD UBS Yijia Niu O (52617) Bonds and Structured               neutral       neutral           O\n",
      "     Products-[Maturity Announcement-CBBC]. JD UBS Yijia Niu O\n",
      "     (52617) Mandatory Redemption Event and Early Expiry Notice\n",
      "     of Callable Bull/Bear Contracts issued by UBS Bank (91KB,\n",
      "     pdf) Report to Today&#39;s Hong Kong Economic Journal, Dr.\n",
      "     Yuen Kangjue | XinHealth Ye Yongkeng | Wealth Management\n",
      "     International Finance | Today 04:44 Important Notice |\n",
      "     Yesterday 23:26 Important Notice | Yesterday 23:29 Important\n",
      "     Notice | Yesterday 23:10 International Finance | Today 04:10\n",
      "     Important Notice | Current Events at 23:11 Yesterday | Today\n",
      "     00:36 International Finance | Today 04:09 Important Notices\n",
      "     | Today 08:22 International Finance | Today 07:59 Real-time\n",
      "     News Analysis and Commentary Today&#39;s Newsletter Series\n",
      "     Stock Financial Management Real Estate Life/Innovation\n",
      "     Technology Subscription to the popular Hong Kong Economic\n",
      "     Journal | Terms of Service | Privacy Policy | Disclaimer |\n",
      "     Advertising Inquiry | Join the Hong Kong Economic Journal |\n",
      "     Contact The\n",
      "100  Gold rush I vote #Louies for #BestFanArmy at the              neutral       neutral           O\n",
      "     #iHeartAwards https://t.co/XmyXZUgo9T\n",
      "101  Gold Futures Discussions. There accumulation...               negative      neutral           X\n",
      "102  Silver Futures Discussions. don't short this. It will be      neutral      negative           X\n",
      "     scary up in may.\n",
      "103  Silver Futures Discussions. feels like watching paint dry     negative      neutral           X\n",
      "     today...\n",
      "104  Silver Futures Discussions. Do not exept shares and           neutral       neutral           O\n",
      "     properties Will Come down,,, they Will go higher as long we\n",
      "     have fiatmoney....\n",
      "105  Ghost of cap’n kidd. Related Posts November 14, 2001          neutral       neutral           O\n",
      "     silver moon highlightsred fur on a soft shouldertogether in\n",
      "     dreams September 8, 2013   I’m watching Portlandia“Season\n",
      "     3!”        34 others are […] April 27, 2017  April 27, 2017\n",
      "     at 09:07AM  No more boogers, Sasquatch!  via Instagram\n",
      "     http://bit.ly/2qbuSmt December 16, 2001  Bu, lbh rabezbhf\n",
      "     trrx. Abg bayl qvq lbh erpbtavmr ebg13, ohg lbh obgurerq gb\n",
      "     qrpelcg vg. Lbh’er fb phgr jura lbh’ir fbyirq n chmmyr.  The\n",
      "     Three Laws of Robotics are:  A robot may not injure a human\n",
      "     being, or, through […] September 9, 2010  From Twitter\n",
      "     09-08-2010  From Twitter 09-08-201003:20:35: From Twitter\n",
      "     09-07-2010 http://j.mp/9COuqG04:12:32: […]\n",
      "106  $MAG.ax $MAG @MagmaticRes   Gold and Copper.  17.5c eaten     negative     positive           X\n",
      "     for afternoon tea👌  +9.4%\n",
      "107  Silver Futures Discussions. Sold 68502                        positive      neutral           X\n",
      "108  Gold Futures Discussions.  buy time you say buy dips and      positive      neutral           X\n",
      "     sell time you say dive ...\n",
      "109  Check out this listing I just added to my #Poshmark closet:   neutral       neutral           O\n",
      "     Gold Coach Crossbody Bag. https://t.co/NNAQ2Wlpwi\n",
      "     #shopmycloset @poshmarkapp\n",
      "110  Shout out to @BlaqBonez !!! That album is forever Gold bruh   neutral       neutral           O\n",
      "     SEX OVER LOVE\n",
      "111  PRECIOUS-Under-supplied palladium races past $3,000/oz for    negative        n/a\n",
      "     first time. * Graphic: 2021 asset performance\n",
      "     http://tmsnrt.rs/2yaDPgn* Spot gold falls after strong U.S.\n",
      "     economic data on Thursday (Adds comments, details and\n",
      "     updates prices)By Sethuraman N R and Peter HobsonApril 30\n",
      "     (Reuters) - Palladium pierced the $3,000 an ounce high for\n",
      "     the first time on Friday, powered by persistent concerns\n",
      "     about short supply of the metal used primarily for\n",
      "     emissions-controlling devices in automobiles.Palladium XPD=\n",
      "     was last up 1% to $2,984.68 per ounce, after hitting an all-\n",
      "     time high of $3,007.73. It was on track to post a monthly\n",
      "     gain as well.\"Palladium is in a structural deficit since 10\n",
      "     years. We have seen above ground inventories falling to very\n",
      "     low levels, and now the price gives a signal to investors\n",
      "     still holding it,\" UBS analyst Giovanni Staunovo said.\"The\n",
      "     commodity ... comes with very low trading volume so it can\n",
      "     be very illiquid and this is making the moves even more\n",
      "     crazy. This rally can con\n",
      "112  PRECIOUS-Under-supplied palladium races past $3,000/oz for    negative        n/a\n",
      "     first time. * Graphic: 2021 asset performance\n",
      "     http://tmsnrt.rs/2yaDPgn* Spot gold falls after strong U.S.\n",
      "     economic data on Thursday (Adds comments, details and\n",
      "     updates prices)By Sethuraman N R and Peter HobsonApril 30\n",
      "     (Reuters) - Palladium pierced the $3,000 an ounce high for\n",
      "     the first time on Friday, powered by persistent concerns\n",
      "     about short supply of the metal used primarily for\n",
      "     emissions-controlling devices in automobiles.Palladium XPD=\n",
      "     was last up 1% to $2,984.68 per ounce, after hitting an all-\n",
      "     time high of $3,007.73. It was on track to post a monthly\n",
      "     gain as well.\"Palladium is in a structural deficit since 10\n",
      "     years. We have seen above ground inventories falling to very\n",
      "     low levels, and now the price gives a signal to investors\n",
      "     still holding it,\" UBS analyst Giovanni Staunovo said.\"The\n",
      "     commodity ... comes with very low trading volume so it can\n",
      "     be very illiquid and this is making the moves even more\n",
      "     crazy. This rally can con\n",
      "113  PRECIOUS-Under-supplied palladium races past $3,000/oz for    negative        n/a\n",
      "     first time. * Graphic: 2021 asset performance\n",
      "     http://tmsnrt.rs/2yaDPgn* Spot gold falls after strong U.S.\n",
      "     economic data on Thursday (Adds comments, details and\n",
      "     updates prices)By Sethuraman N R and Peter HobsonApril 30\n",
      "     (Reuters) - Palladium pierced the $3,000 an ounce high for\n",
      "     the first time on Friday, powered by persistent concerns\n",
      "     about short supply of the metal used primarily for\n",
      "     emissions-controlling devices in automobiles.Palladium XPD=\n",
      "     was last up 1% to $2,984.68 per ounce, after hitting an all-\n",
      "     time high of $3,007.73. It was on track to post a monthly\n",
      "     gain as well.\"Palladium is in a structural deficit since 10\n",
      "     years. We have seen above ground inventories falling to very\n",
      "     low levels, and now the price gives a signal to investors\n",
      "     still holding it,\" UBS analyst Giovanni Staunovo said.\"The\n",
      "     commodity ... comes with very low trading volume so it can\n",
      "     be very illiquid and this is making the moves even more\n",
      "     crazy. This rally can con\n",
      "114  Gold Futures Discussions. Caught between still                negative      neutral           X\n",
      "115  Silver Futures Discussions.  90% scary up in may and the      positive      neutral           X\n",
      "     next few month. 10% go down. For me, I will not short this\n",
      "     until 13 may\n",
      "116  RT @aguila_gold: Great graphics as always from @VisualCap on  negative      neutral           X\n",
      "     the #gold industry.   Gold price now gradually testing 2\n",
      "     month high  https://t…\n",
      "117  European Equities: It’s A Busy Economic Calendar With GDP     neutral         n/a\n",
      "     Figures in Focus. French GDP (QoQ) (Q1)German GDP (QoQ)\n",
      "     (Q1)German GDP (YoY) (Q1)Spanish GDP (QoQ) (Q1)Italian CPI\n",
      "     (MoM) (Apr) PrelimEurozone CPI (YoY) (Apr) PrelimEurozone\n",
      "     Unemployment Rate (Mar)Eurozone GDP (QoQ) (Q1)Eurozone GDP\n",
      "     (YoY) (Q1)The Majors It was a relatively bearish day for the\n",
      "     European majors on Thursday. The DAX30 fell by 0.90%, with\n",
      "     the CAC40 and the EuroStoxx600 seeing losses of 0.07% and\n",
      "     0.26% respectively.Economic data from the U.S failed to\n",
      "     prevent a pullback from early gains, with stats from Germany\n",
      "     disappointing.The downside came amidst positive earnings\n",
      "     results from both side of the Atlantic. From the Eurozone,\n",
      "     Airbus returned to profit, with Total SE beating\n",
      "     estimates.The Stats It was a busier day on the economic\n",
      "     calendar on Thursday. The German economy was back in focus,\n",
      "     with unemployment and inflation figures the key stats of the\n",
      "     day.Prelim April inflation figures from Spain and finalized\n",
      "     consumer conf\n",
      "118  Gold Futures Discussions. ameros funding should get bent and  negative      neutral           X\n",
      "     impaled by a horse\n",
      "119  Gold Futures Discussions. Ouchhh... pain in the A/s/ss        negative      neutral           X\n",
      "120  Gold is about to break the long time descending channel.      neutral       neutral           O\n",
      "     Gold has been on a decline path within 6 months or so, now\n",
      "     by getting support of Ichimuko and gathering strength from\n",
      "     the previous bearish leg and also getting provided by enough\n",
      "     support from lower boundary of the channel, IS READY to make\n",
      "     a move and find its way up. So keep an eye on it and hope\n",
      "     we'll see its drought come to an end. Trade Safe ...\n",
      "121  Silver Futures Discussions.  in one hour                      negative      neutral           X\n",
      "122  I've got no Reddit gold, but gold's decadent.\"                neutral       neutral           O\n",
      "123  Gold Futures Discussions. too many buyers r there in mcx,,    neutral       neutral           O\n",
      "     bcoz of covid situation n lockdown.. like last year,, so I\n",
      "     think it's testing the patience of mcx buyers\n",
      "124  $SPY they pumped every commodity x3 x4 but short PMs. Thats   negative      neutral           X\n",
      "     the life blood of printers. Printer wud jam if gold takes\n",
      "     off.\n",
      "125  Silver Price Analysis: XAG/USD to resume the core uptrend     negative     positive           X\n",
      "     above $30.72 – Credit Suisse. Strategists at Credit Suisse\n",
      "     continue to view consolidation in XAG/USD as temporary with\n",
      "     a long-term base still seen in place here. What’s more,\n",
      "     silver is expected to outperform gold. See – Silver Price\n",
      "     Analysis: XAG/USD to nosedive towards $21.50 by year-end –\n",
      "     CE  Silver maintains a long-term base  “Silver stays trapped\n",
      "     in a turgid range below the 50% retracement of the 2011/2020\n",
      "     bear market at $30.72. With a long-term base still in place\n",
      "     though our base remains to view this a correction within a\n",
      "     broader uptrend.”  “Above $30.72 remains needed to suggest\n",
      "     the core uptrend has indeed resumed for a move to our\n",
      "     objective still at $35.23/35.365 – the 61.8% retracement and\n",
      "     key high from October 2012.” “Silver is still expected to on\n",
      "     balance outperform Gold with the trend for the Gold/Silver\n",
      "     ratio still seen lower, despite the recent loss of\n",
      "     momentum.”\n",
      "126  Gome Retail (00493) Circular-[Others]. Today&#39;s Hong Kong  neutral       neutral           O\n",
      "     Economic Journal | Today&#39;s Hong Kong Economic Journal |\n",
      "     Today&#39;s Economic Journal | Today&#39;s Economic Journal\n",
      "     | Dr. Yuen Kangjuu | XinHealth Ye Yongkeng | Wealth\n",
      "     Management International Finance | Today 04:44 Current\n",
      "     Events | Today 00:36 International Finance | Today 04:10\n",
      "     International Finance | Today 04:09 Hong Kong Stocks Direct\n",
      "     Attack | Today 10:31 Important Notices | Today 08:22\n",
      "     International Finance | Today 07:59 International Finance |\n",
      "     Today 01:03 Hong Kong Stocks Direct Attack | Today 09:28\n",
      "     Hong Kong Finance | Today 03:26 Real-time News Analysis and\n",
      "     Comment Today&#39;s Hong Kong Economic Journal Series Stock\n",
      "     Wealth Management Real Estate Life/Innovation and Technology\n",
      "     Newsletter Introduction | Terms of Service | Privacy Policy\n",
      "     | Disclaimer | Advertising Enquiry | Join the Hong Kong\n",
      "     Economic Journal | Contact Economic Journal Stock and index\n",
      "     information is provided by Financial Intelligence Network\n",
      "     Limited. F\n",
      "127  Tang Jinzhao: Analysis of the latest market trend of gold     neutral         n/a\n",
      "     and crude oil and the latest operational recommendations.\n",
      "     ===Thursday, April 30 (April 29), international gold from a\n",
      "     technical point of view, from the daily line analysis, the\n",
      "     daily line recorded a long shadow line and a small positive\n",
      "     line, the current gold price daily close is still stable at\n",
      "     the overall The daily moving average technical indicator\n",
      "     support level is above the price, and the daily moving\n",
      "     average is in a glued upward cross arrangement, maintaining\n",
      "     a longer period of more shock absorption. The short-term\n",
      "     decline will still face the impact of relatively dense\n",
      "     technical indicators support prices. The hourly line shows\n",
      "     that the short-term bottoming and reversal bring the hourly\n",
      "     line technical indicators short-term upward and longer-term\n",
      "     technical indicators form, forming a short-term gold cross-\n",
      "     to-many guidance pattern. On the whole, today&#39;s short-\n",
      "     term operation ideas suggest that the callback is mainly low\n",
      "     and\n",
      "128  Gold Price Analysis: XAU/USD to extend the downside on a      negative     negative           O\n",
      "     close below 21-DMA at $1756 https://t.co/ulcdr5mDJL #Gold\n",
      "     #XAUUSD\n",
      "129  Gold Futures Discussions.   Is not about that it is about     neutral       neutral           O\n",
      "     trend. Markets do listen Powell, we traders follow trends\n",
      "     to make money. No matter Bull or Bear.\n"
     ]
    }
   ],
   "source": [
    "# !pip3 install -U tabulate\n",
    "import tabulate\n",
    "\n",
    "headers = ['Sample', 'Spacy', 'FinancialBERT', 'Comparison']\n",
    "table = tabulate.tabulate(\n",
    "    data,\n",
    "    headers=headers,\n",
    "    tablefmt='simple',\n",
    "    colalign=('center', 'left','center', 'center', 'center'),\n",
    "    maxcolwidths=[4, 60, 8, 8, 8], showindex=\"always\"\n",
    "    )\n",
    "print(table)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b9822326-9545-45a0-8cc8-d1bf62784552",
   "metadata": {},
   "source": [
    "## Fine tune FinancialBERT using the same training data that is used for training Spacy TextCategorizer\n",
    "In model overview we see that FinancialBERT include bert layer, classifier layer. The fine tuning experiment is trying to train classifier layer only, bert layer is kept unchanged during training in order to preserve pretrained performance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "43ddac7e-2754-4f3a-95dd-bcef5e87a724",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model summarization: \n",
      "\n",
      " BertForSequenceClassification(\n",
      "  (bert): BertModel(\n",
      "    (embeddings): BertEmbeddings(\n",
      "      (word_embeddings): Embedding(30873, 768, padding_idx=0)\n",
      "      (position_embeddings): Embedding(512, 768)\n",
      "      (token_type_embeddings): Embedding(2, 768)\n",
      "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
      "      (dropout): Dropout(p=0.1, inplace=False)\n",
      "    )\n",
      "    (encoder): BertEncoder(\n",
      "      (layer): ModuleList(\n",
      "        (0-11): 12 x BertLayer(\n",
      "          (attention): BertAttention(\n",
      "            (self): BertSelfAttention(\n",
      "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
      "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
      "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "            (output): BertSelfOutput(\n",
      "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
      "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (intermediate): BertIntermediate(\n",
      "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
      "            (intermediate_act_fn): GELUActivation()\n",
      "          )\n",
      "          (output): BertOutput(\n",
      "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
      "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
      "            (dropout): Dropout(p=0.1, inplace=False)\n",
      "          )\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (pooler): BertPooler(\n",
      "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
      "      (activation): Tanh()\n",
      "    )\n",
      "  )\n",
      "  (dropout): Dropout(p=0.1, inplace=False)\n",
      "  (classifier): Linear(in_features=768, out_features=3, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "from transformers import BertTokenizer, BertForSequenceClassification\n",
    "from transformers import pipeline\n",
    "from datasets import Dataset, DatasetDict\n",
    "\n",
    "model = BertForSequenceClassification.from_pretrained(\"ahmedrachid/FinancialBERT-Sentiment-Analysis\",num_labels=3)\n",
    "tokenizer = BertTokenizer.from_pretrained(\"ahmedrachid/FinancialBERT-Sentiment-Analysis\")\n",
    "print('Model summarization: \\n\\n', model)\n",
    "\n",
    "for param in model.bert.embeddings.parameters():\n",
    "  param.requires_grad = False\n",
    "\n",
    "for param in model.bert.encoder.parameters():\n",
    "  param.requires_grad = False\n",
    "\n",
    "for param in model.bert.pooler.parameters():\n",
    "  param.requires_grad = False\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ac251f66-d166-48ff-b677-f91b276e230b",
   "metadata": {},
   "source": [
    "#### Prepare training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "593b5d87-b050-4f53-b47d-823e74d65bf6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of sample:  45\n",
      "{'text': \"Gold Futures Discussions. So, any more short paper expire?\\xa0Moy said, but it's only a matter of time before the short contracts keeping the price down expire.\", 'label': 1}\n"
     ]
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "\n",
    "id2label = model.config.id2label\n",
    "label2id = model.config.label2id\n",
    "\n",
    "dataset = []\n",
    "for item in sent_train_data:\n",
    "    text = item['data']['text']\n",
    "    annotated = item['annotations'][0]\n",
    "\n",
    "    if len(text) < 20:\n",
    "        continue\n",
    "\n",
    "    result = annotated['result']\n",
    "    if len(result) == 0:\n",
    "        continue;\n",
    "    label = result[0]['value']['choices'][0]\n",
    "    label = label.lower()\n",
    "\n",
    "    dataset.append({\n",
    "        'text': text,\n",
    "        'label': label2id[label]\n",
    "        })\n",
    "\n",
    "print('number of sample: ', len(dataset))\n",
    "print(dataset[3])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "0889012b-ea1b-495f-b200-512ac2f578ee",
   "metadata": {
    "lines_to_next_cell": 0
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Map: 100%|██████████████████████████████████████████████████████████████████████████████████████████████| 45/45 [00:00<00:00, 708.93 examples/s]\n"
     ]
    }
   ],
   "source": [
    "def tokenize_function(examples):\n",
    "    return tokenizer(examples[\"text\"], padding=\"max_length\", truncation=True)\n",
    "\n",
    "dataset = Dataset.from_list(dataset)\n",
    "tokenized_datasets = dataset.map(tokenize_function, batched=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a8e68c65-5e7b-4271-b627-e51881e237f8",
   "metadata": {},
   "source": [
    "#### Training (fine tuning) \n",
    "Note: traing_dataset and eval_datasset are the same."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "aad3e6cd-d601-47e8-9dab-d23c3950a31f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import TrainingArguments, Trainer\n",
    "import numpy as np\n",
    "import evaluate\n",
    "\n",
    "metric = evaluate.load(\"accuracy\")\n",
    "\n",
    "def compute_metrics(eval_pred):\n",
    "    logits, labels = eval_pred\n",
    "    predictions = np.argmax(logits, axis=-1)\n",
    "\n",
    "    return metric.compute(predictions=predictions, references=labels)\n",
    "\n",
    "batch_size = 10\n",
    "training_args = TrainingArguments(\n",
    "  output_dir=\"output_trainer\",\n",
    "  num_train_epochs=120,\n",
    "  per_device_train_batch_size=batch_size,\n",
    "  per_device_eval_batch_size=batch_size,\n",
    "  evaluation_strategy=\"epoch\",\n",
    "  save_total_limit=2,\n",
    "  gradient_checkpointing=True,  #  If True, use gradient checkpointing to save memory at the expense of slower backward pass\n",
    ")\n",
    "\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=tokenized_datasets,\n",
    "    eval_dataset=tokenized_datasets,\n",
    "    compute_metrics=compute_metrics,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "08241411-724d-4520-bf2e-e48aed44e604",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='600' max='600' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [600/600 03:45, Epoch 120/120]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.659011</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.636894</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.614349</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.592872</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.570359</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.548936</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.530016</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.510552</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.491386</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.471930</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.453737</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.436734</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.419728</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.404612</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.387642</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.370121</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>17</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.352483</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>18</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.335292</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>19</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.317805</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>20</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.302061</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>21</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.286263</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>22</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.271982</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>23</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.257138</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>24</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.242036</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.226245</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>26</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.211173</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>27</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.196762</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>28</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.182153</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>29</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.167774</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>30</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.153065</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>31</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.138574</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>32</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.124205</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>33</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.109988</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>34</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.096584</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>35</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.084985</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>36</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.074065</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>37</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.063676</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>38</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.055366</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>39</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.046920</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>40</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.037485</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>41</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.029217</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>42</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.020984</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>43</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.012737</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>44</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.004863</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>45</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.996544</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>46</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.988180</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>47</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.980725</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>48</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.972967</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>49</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.965492</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.958243</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>51</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.950906</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>52</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.944432</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>53</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.937451</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>54</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.930818</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>55</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.924694</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>56</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.918522</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>57</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.912762</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>58</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.907031</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>59</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.901526</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>60</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.896611</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>61</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.892676</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>62</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.888940</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>63</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.885721</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>64</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.882036</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>65</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.878720</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>66</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.875829</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>67</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.872582</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>68</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.869474</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>69</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.866404</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>70</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.863878</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>71</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.861570</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>72</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.859035</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>73</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.856158</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>74</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.853376</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>75</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.850935</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>76</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.848995</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>77</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.846752</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>78</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.844928</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>79</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.843057</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>80</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.841083</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>81</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.839609</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>82</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.838216</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>83</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.836715</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>84</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.835469</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>85</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.834316</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>86</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.833015</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>87</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.831876</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>88</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.830652</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>89</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.829543</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>90</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.828527</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>91</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.827579</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>92</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.826628</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>93</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.825767</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>94</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.824931</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>95</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.824267</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>96</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.823343</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>97</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.822545</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>98</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.821902</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>99</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.821092</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>100</td>\n",
       "      <td>1.073800</td>\n",
       "      <td>0.820491</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>101</td>\n",
       "      <td>1.073800</td>\n",
       "      <td>0.819906</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>102</td>\n",
       "      <td>1.073800</td>\n",
       "      <td>0.819231</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>103</td>\n",
       "      <td>1.073800</td>\n",
       "      <td>0.818575</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>104</td>\n",
       "      <td>1.073800</td>\n",
       "      <td>0.818040</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>105</td>\n",
       "      <td>1.073800</td>\n",
       "      <td>0.817484</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>106</td>\n",
       "      <td>1.073800</td>\n",
       "      <td>0.816973</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>107</td>\n",
       "      <td>1.073800</td>\n",
       "      <td>0.816472</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>108</td>\n",
       "      <td>1.073800</td>\n",
       "      <td>0.816083</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>109</td>\n",
       "      <td>1.073800</td>\n",
       "      <td>0.815795</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>110</td>\n",
       "      <td>1.073800</td>\n",
       "      <td>0.815538</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>111</td>\n",
       "      <td>1.073800</td>\n",
       "      <td>0.815263</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>112</td>\n",
       "      <td>1.073800</td>\n",
       "      <td>0.814992</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>113</td>\n",
       "      <td>1.073800</td>\n",
       "      <td>0.814723</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>114</td>\n",
       "      <td>1.073800</td>\n",
       "      <td>0.814507</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>115</td>\n",
       "      <td>1.073800</td>\n",
       "      <td>0.814323</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>116</td>\n",
       "      <td>1.073800</td>\n",
       "      <td>0.814156</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>117</td>\n",
       "      <td>1.073800</td>\n",
       "      <td>0.814030</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>118</td>\n",
       "      <td>1.073800</td>\n",
       "      <td>0.813956</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>119</td>\n",
       "      <td>1.073800</td>\n",
       "      <td>0.813910</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>120</td>\n",
       "      <td>1.073800</td>\n",
       "      <td>0.813894</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "Checkpoint destination directory output_trainer/checkpoint-500 already exists and is non-empty. Saving will proceed but saved results may be invalid.\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/sysang/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=600, training_loss=1.0319591395060221, metrics={'train_runtime': 225.6178, 'train_samples_per_second': 23.934, 'train_steps_per_second': 2.659, 'total_flos': 1420812455731200.0, 'train_loss': 1.0319591395060221, 'epoch': 120.0})"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "93b2fa93-de6a-4249-9bbf-1f98ec471f2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save_pretrained('models/FinancialBERT')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "21db61db-36b5-4aea-86c6-da52ed6bb038",
   "metadata": {},
   "source": [
    "## Perform Spacy model, FinancialBERT model, fine tuned FinancialBERT model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "55e2f1e7-54c7-44c9-ba60-a94307f08c71",
   "metadata": {},
   "source": [
    "#### Evaluate and colllect data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "8170bfb8-e560-460f-bf64-bc8b8e11b6c8",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The BetterTransformer implementation does not support padding during training, as the fused kernels do not support attention masks. Beware that passing padded batched data during training may result in unexpected outputs. Please refer to https://huggingface.co/docs/optimum/bettertransformer/overview for more details.\n",
      "The BetterTransformer implementation does not support padding during training, as the fused kernels do not support attention masks. Beware that passing padded batched data during training may result in unexpected outputs. Please refer to https://huggingface.co/docs/optimum/bettertransformer/overview for more details.\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "from transformers import BertTokenizer, BertForSequenceClassification\n",
    "from transformers import pipeline\n",
    "from datasets import Dataset, DatasetDict\n",
    "\n",
    "# Load Spacy model\n",
    "sent = spacy.load(\"sentiment_analysis_spacy/output/model-last\")\n",
    "\n",
    "# Load FinancialBERT\n",
    "tokenizer = BertTokenizer.from_pretrained(\"ahmedrachid/FinancialBERT-Sentiment-Analysis\")\n",
    "\n",
    "# Load fine tuned model\n",
    "pretrainedFinancialBERT = BertForSequenceClassification.from_pretrained(\"ahmedrachid/FinancialBERT-Sentiment-Analysis\",num_labels=3)\n",
    "pretrainedFinancialBERT.to_bettertransformer()\n",
    "bert_sent1 = pipeline(\"sentiment-analysis\", model=pretrainedFinancialBERT, tokenizer=tokenizer)\n",
    "\n",
    "tunedFinancialBERT = BertForSequenceClassification.from_pretrained('models/FinancialBERT')\n",
    "tunedFinancialBERT.to_bettertransformer()\n",
    "bert_sent2 = pipeline(\"sentiment-analysis\", model=tunedFinancialBERT, tokenizer=tokenizer)\n",
    "\n",
    "text_length = 1000\n",
    "data2 = []\n",
    "for sample in gold_silver_com_sample:\n",
    "\n",
    "    doc = sent(sample)\n",
    "    spacy_label = max(doc.cats.items(), key=lambda x: x[1])[0]\n",
    "    spacy_label = spacy_label.lower()\n",
    "\n",
    "    # model's max_seq_length is 512, FinancialBERT is not able to process too long documents\n",
    "    if len(sample.split()) < 320:\n",
    "        bert_predicted1 = bert_sent1(sample)\n",
    "        bert_label1 = bert_predicted1[0]['label'].lower()\n",
    "        unmatched1 = 'X' if spacy_label != bert_label1 else 'O'\n",
    "\n",
    "        bert_predicted2 = bert_sent2(sample)\n",
    "        bert_label2 = bert_predicted2[0]['label'].lower()\n",
    "        unmatched2 = 'X' if spacy_label != bert_label2 else 'O'\n",
    "    else:\n",
    "        bert_label = 'n/a'\n",
    "        unmatched = ''\n",
    "\n",
    "\n",
    "    data2.append([sample[:text_length], spacy_label, bert_label1, unmatched1, bert_label2, unmatched2])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9658eb18-25b5-43c1-8965-9a542bab9e56",
   "metadata": {},
   "source": [
    "#### Display data\n",
    "Manually checking some samples that fine tuned model predicts differently from origin one, found that funed one is slightly better."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "ecdda462-0373-4a74-82de-addce7a58e60",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     Sample                                               Spacy     FinancialBERT    vs Spacy    tuned FinancialBERT    vs Spacy\n",
      "---  --------------------------------------------------  --------  ---------------  ----------  ---------------------  ----------\n",
      " 0   Silver Futures Discussions. silver 25.90            negative      neutral          X              neutral             X\n",
      " 1   Gold 💫 bagged, learned so much here, and nice      neutral       neutral          O              neutral             O\n",
      "     people too https://t.co/g9qQ8sbDue\n",
      " 2   Gold Futures Discussions. Bears game...             negative      neutral          X              neutral             X\n",
      " 3   Gold Rate Today, 30 April 2021: Gold, Silver fall,  negative      neutral          X              neutral             X\n",
      "     know – what are the 10 grams gold rate today\n",
      "     https://t.co/kSVOTVuXw5\n",
      " 4   @BTC_Archive @PeterSchiff Gold price is suppressed  negative      neutral          X              neutral             X\n",
      "     by the government. Buy Bitcoin\n",
      " 5   Silver Futures Discussions. usd/inr at support so   negative      neutral          X              neutral             X\n",
      "     the advantage to run by double engine ( plunging\n",
      "     zi price and plunging usd/inr ) is gone so now the\n",
      "     fall will depend mostly on zi  so the rate of fall\n",
      "     will not be much\n",
      " 6   Gold Futures Discussions. gold loses either way.    neutral       neutral          O             negative             X\n",
      "     if the dollar rises, gold falls. if the dollar\n",
      "     falls, the banks add more shorts so gold doesn't\n",
      "     look good\n",
      " 7   Gold Futures Discussions. gold loses either way.    neutral       neutral          O             negative             X\n",
      "     if the dollar rises, gold falls. if the dollar\n",
      "     falls, the banks add more shorts so gold doesn't\n",
      "     look good\n",
      " 8   The Edge Singapore | Rotate back to technology      negative      neutral          X              neutral             X\n",
      "     stocks, gold,..... The Edge Singapore | Rotate\n",
      "     back to technology stocks, gold, says DBS https://\n",
      "     www.theedgesingapore.com/capital/investing-\n",
      "     ideas/rotate-back-technology-stocks-gold-says-dbs\n",
      " 9   Gold Ranges for GBX - #GC1! chart                   negative      neutral          X              neutral             X\n",
      "     https://t.co/bPruM2bunq\n",
      "10   #goldprice  Upward pressure in gold prices, price   negative      neutral          X              neutral             X\n",
      "     charts show Gold is trying to break resistance\n",
      "     level of past 6 mo… https://t.co/dTklA1O9P1\n",
      "11   Gold Futures Discussions. Dis wik is crazy, wiped   neutral       neutral          O             negative             X\n",
      "     me out\n",
      "12   RT @JJK_Times: 🇯🇵 Jungkook has now two projects     neutral       neutral          O             positive             X\n",
      "     officially certified by Recording Industry\n",
      "     Association of Japan:  Gold 📀 \"Your eyes tell\" (…\n",
      "13   Can Utah Republicans survive an emerging sea of     neutral       neutral          O             positive             X\n",
      "     blue in the West?. That’s what’s at stake and why\n",
      "     the race for Utah’s next Republican leader\n",
      "     matters, outgoing Utah GOP chairman says Share\n",
      "     this story Share All sharing options for: As\n",
      "     surrounding states in the West continue to trend\n",
      "     Democratic, will Utah remain a red stronghold? Or\n",
      "     will it eventually become overtaken in an emerging\n",
      "     sea of blue? That’s what’s at stake for Utah\n",
      "     Republicans in the race for the Utah GOP’s next\n",
      "     chairperson, a position largely off the radar of a\n",
      "     public consumed by COVID-19 updates and the\n",
      "     promise of better weather. Yet when delegates make\n",
      "     their selection from five candidates at this\n",
      "     weekend’s state convention, it will help determine\n",
      "     the future of their party and its influence on\n",
      "     Utah politics, said Derek Brown, the state GOP’s\n",
      "     outgoing chairman, who announced last month he\n",
      "     wouldn’t be seeking another term to spend more\n",
      "     time with his family. “Anyone who is paying\n",
      "     attention to the trends and to the numbers who i\n",
      "14   XAU/USD - Gold Spot US Dollar. Common bull, close   negative      neutral          X              neutral             X\n",
      "     this month for 1800\n",
      "15   Gold Futures Discussions. Enjoy the 0.000001        negative      neutral          X              neutral             X\n",
      "     profit\n",
      "16   RT @dazyle_: ⎆ You're gonna love this ⎆  10K White  neutral       neutral          O             positive             X\n",
      "     Gold 3 mm Diamond Cut Hoop Earrings 0.6\" Diameter\n",
      "     Get your now ➣ https://t.co/MyrpRVXqe…\n",
      "17   Gold Futures Discussions. Fake spike, and many      negative      neutral          X              neutral             X\n",
      "     bulls trapped like $1781 past 24 hours... then\n",
      "     rapid dive -$25 and many RIP\n",
      "18   Gold Futures Discussions. kelattu pa y ur           negative     negative          O             negative             O\n",
      "     confusing\n",
      "19   Gold Futures Discussions. I heard that fridays are  neutral       neutral          O              neutral             O\n",
      "     always green, is that true? :)\n",
      "20   Recovery hopes drive shares, US Treasury yields.    neutral       neutral          O              neutral             O\n",
      "     World share indices extended gains and Treasury\n",
      "     yields rose on Thursday, after strong U.S.\n",
      "     economic data and the Federal Reserve's commitment\n",
      "     to continue supporting the economy fueled\n",
      "     confidence in a recovery.U.S. economic growth\n",
      "     accelerated in the first quarter, fueled by\n",
      "     massive government aid to households and\n",
      "     businesses, charting the course for what is\n",
      "     expected will be the strongest annual performance\n",
      "     in nearly four decades.\"Assuming that COVID\n",
      "     variants remain contained, the second quarter is\n",
      "     set for a further acceleration in growth as the\n",
      "     re-openings continue,\" said Katherine Judge,\n",
      "     senior economist at CIBC Capital Markets.New York\n",
      "     City aims to \"fully reopen\" on July 1 after more\n",
      "     than a year of closures and capacity restrictions,\n",
      "     Mayor Bill de Blasio said.MSCI's gauge of stocks\n",
      "     across the globe gained 0.33%, closing at an all-\n",
      "     time high and remaining on course for its best\n",
      "     month since November.The S&P 500 also closed at a\n",
      "     record high\n",
      "21   Check #gold on #ebay &gt;&gt;                       negative      neutral          X              neutral             X\n",
      "     https://t.co/a8cDMHn2q1 || Is Kinross Gold Stock A\n",
      "     Good Bet After Dropping 16% In Six Month…\n",
      "     https://t.co/T9XipeyJdH\n",
      "22   Gold Futures Discussions. 1776 maximum              negative      neutral          X              neutral             X\n",
      "23   Silver Futures Discussions. Bullion banks are       neutral       neutral          O             negative             X\n",
      "     about to feel the squeeze... Retail investors are\n",
      "     coming for you... And we will not stop until true\n",
      "     price Discovery is resolved... Fundamentals say\n",
      "     that it should not be this way.. Keep stacking.\n",
      "24   Gold Price News and Forecast: XAU/USD retreats on   negative     positive          X             positive             X\n",
      "     US data. Posted by: EUR Editor  in EUR  1 min ago\n",
      "     After witnessing good two-way businesses on\n",
      "     Thursday, Gold (XAU/USD) is holding on to recent\n",
      "     recovery gains amid a broadly subdued US dollar\n",
      "     and higher Treasury yields. The returns on the US\n",
      "     debt … Read Full Story at source (may require\n",
      "     registration)     Latest posts by EUR Editor ( see\n",
      "     all )\n",
      "25   Silver Futures Discussions.    If I able to         neutral       neutral          O              neutral             O\n",
      "     acquire supreme power, my first job will be hand\n",
      "     over this country to British Emperor again for\n",
      "     next 200 years or handover to a international\n",
      "     criminal .\n",
      "26   Sunshine Oil Sands (02012) Financial                neutral       neutral          O              neutral             O\n",
      "     Statement/Environmental, Social and Governance\n",
      "     Information-[Annual Report]. Today&#39;s Hong Kong\n",
      "     Economic Journal | Today&#39;s Hong Kong Economic\n",
      "     Journal | Today&#39;s Economic Journal |\n",
      "     Today&#39;s Economic Journal | Today&#39;s\n",
      "     Economic Journal | Wealth Management International\n",
      "     Finance | Today&#39;s 04:44 Important Notice |\n",
      "     Today&#39;s 08:22 Hong Kong Stocks Direct Attack |\n",
      "     Today&#39;s 10:31 International Finance | Today\n",
      "     07:59 real estate news | Today 09:50 Hong Kong\n",
      "     stocks direct attack | Today 13:38 current affairs\n",
      "     pulse | Today 11:15 current affairs pulse | Today\n",
      "     09:45 Hong Kong stocks direct attack | Today 09:28\n",
      "     immediate market review | Today 13:44 real-time\n",
      "     news analysis Comment on Today’s Hong Kong\n",
      "     Economic Journal Series Stock Financial Management\n",
      "     Real Estate Life/Innovation and Technology\n",
      "     Subscribe to Popular Economic News Brief\n",
      "     Introduction | Terms of Service | Privacy Policy |\n",
      "     Disclaimer | Advertising Enquiry | Join the Hong\n",
      "     Kong Econ\n",
      "27   Gold Futures Discussions.   I don't                 neutral       neutral          O              neutral             O\n",
      "     daytrade...time you guys start zooming out a\n",
      "     little\n",
      "28   Silver Futures Discussions. convey my wishes to     negative      neutral          X              neutral             X\n",
      "     short employee for yesterday work\n",
      "29   silver long. XAGUSD                                 negative      neutral          X              neutral             X\n",
      "30   XAU/USD - Gold Spot US Dollar. crazy crazy          neutral       neutral          O              neutral             O\n",
      "     moment....\n",
      "31   Gold futures ease below Rs 46,900, silver firm:     negative      neutral          X              neutral             X\n",
      "     How to trade at current juncture #CommodityNews\n",
      "     #Outlook https://t.co/lvSeMxmqki\n",
      "32   Shandong Gold Mining : OVERSEAS REGULATORY          positive      neutral          X              neutral             X\n",
      "     ANNOUNCEMENT- SD GOLD 2020 SOCIAL RESPONSIBILITY\n",
      "     REPORT. (a joint stock company incorporated in the\n",
      "     People's Republic of China with limited liability)\n",
      "     (Stock Code: 1787) OVERSEAS REGULATORY\n",
      "     ANNOUNCEMENT This announcement is made by Shandong\n",
      "     Gold Mining Co., Ltd. (the \"Company\") pursuant to\n",
      "     Rule 13.10B of the Rules Governing the Listing of\n",
      "     Securities on The Stock Exchange of Hong Kong\n",
      "     Limited. Reference is made to the overseas\n",
      "     regulatory announcement of the Company dated 30\n",
      "     March 2021 in relation to the contents headlined\n",
      "     \"2020 Social Responsibility Report (2020年度社會\n",
      "     責任報告)\" (the \"Report\") published on the website\n",
      "     of the Shanghai Stock Exchange. The English\n",
      "     translation of the Report is attached to this\n",
      "     announcement for the easy reference of\n",
      "     international investors only. Shandong Gold Mining\n",
      "     Co., Ltd. Li Guohong Chairman As at the date of\n",
      "     this announcement, the executive directors of the\n",
      "     Company are Mr. Liu Qin, Mr. Wang Shuhai and Mr.\n",
      "     Tang Qi; th\n",
      "33   Shandong Gold Mining : OVERSEAS REGULATORY          positive      neutral          X              neutral             X\n",
      "     ANNOUNCEMENT- SD GOLD 2020 SOCIAL RESPONSIBILITY\n",
      "     REPORT. (a joint stock company incorporated in the\n",
      "     People's Republic of China with limited liability)\n",
      "     (Stock Code: 1787) OVERSEAS REGULATORY\n",
      "     ANNOUNCEMENT This announcement is made by Shandong\n",
      "     Gold Mining Co., Ltd. (the \"Company\") pursuant to\n",
      "     Rule 13.10B of the Rules Governing the Listing of\n",
      "     Securities on The Stock Exchange of Hong Kong\n",
      "     Limited. Reference is made to the overseas\n",
      "     regulatory announcement of the Company dated 30\n",
      "     March 2021 in relation to the contents headlined\n",
      "     \"2020 Social Responsibility Report (2020年度社會\n",
      "     責任報告)\" (the \"Report\") published on the website\n",
      "     of the Shanghai Stock Exchange. The English\n",
      "     translation of the Report is attached to this\n",
      "     announcement for the easy reference of\n",
      "     international investors only. Shandong Gold Mining\n",
      "     Co., Ltd. Li Guohong Chairman As at the date of\n",
      "     this announcement, the executive directors of the\n",
      "     Company are Mr. Liu Qin, Mr. Wang Shuhai and Mr.\n",
      "     Tang Qi; th\n",
      "34   XAU/USD - Gold Spot US Dollar. time to go to 1758.  neutral       neutral          O              neutral             O\n",
      "35   XAU/USD - Gold Spot US Dollar. Mahadhir Atan  bad   neutral      negative          X             negative             X\n",
      "     for gold, usualy.\n",
      "36   Gold Futures. TP TRACKER  Around 1720-40ish is my   neutral      positive          X             positive             X\n",
      "     buy order.\n",
      "37   Zhejiang United Investment (08366) Change of        neutral      positive          X             positive             X\n",
      "     registered address or office, registered place of\n",
      "     Hong Kong business, or Hong Kong representative\n",
      "     for receiving legal process documents. Zhejiang\n",
      "     United Investment (08366) Change of Hong Kong head\n",
      "     office and principal place of business (223KB,\n",
      "     pdf) Previous post: Today&#39;s Hong Kong Economic\n",
      "     Journal | Today&#39;s Hong Kong Economic Journal |\n",
      "     Today&#39;s Economic Journal | Today&#39;s\n",
      "     Economic Journal | Today&#39;s Economic Journal\n",
      "     Important Notice | Today 08:22 International\n",
      "     Finance | Today 07:59 Hong Kong Stocks Direct\n",
      "     Attack | Today 10:31 Current Affairs Pulse | Today\n",
      "     09:45 Hong Kong Stocks Direct Attack | Today 13:38\n",
      "     Current Affairs Pulse | Today 11:15 Real Estate\n",
      "     News | Today 09:50 That Is Market Review｜ Today’s\n",
      "     13:44 Current Events｜ Today’s 12:42 Hong Kong\n",
      "     Stocks Direct Attack｜ Today’s 09:28 Real-time\n",
      "     News Analysis and Commentary Today’s Economic\n",
      "     Journal Series Stock Financial Management Real\n",
      "     Estate Life/Innovation and Technology Subs\n",
      "38   @JagexTyran hello, my account got hit with RWT      neutral       neutral          O             negative             X\n",
      "     goldfarming ban while i've never sold or bought\n",
      "     any gold. i tried al… https://t.co/8reI1kqhXq\n",
      "39   Silver Futures Discussions. At last, rates of       negative      neutral          X              neutral             X\n",
      "     silver is live....\n",
      "40   Most currencies have an asset to back them up.      neutral       neutral          O              neutral             O\n",
      "     Most countries use gold. USD used to be backed in\n",
      "     gold. Somewhere around the line (fellow ape told\n",
      "     me was Nixon in 1970-71)\n",
      "41   Most currencies have an asset to back them up.      neutral       neutral          O              neutral             O\n",
      "     Most countries use gold. USD used to be backed in\n",
      "     gold. Somewhere around the line (fellow ape told\n",
      "     me was Nixon in 1970-71)\n",
      "42   I sold the last of mine at 2.1, gold stocks have    negative      neutral          X              neutral             X\n",
      "     not been kind to me lately, still hold BRB , NXM\n",
      "     and WML but sick of going nowhere while the market\n",
      "     rallies..\n",
      "43   Gold Futures Discussions. Double top formed. Sell   negative      neutral          X              neutral             X\n",
      "44   Zhongliang Holdings (02772) Notice of Annual        neutral       neutral          X              neutral             X\n",
      "     General Meeting / Suspension of Registration\n",
      "     Procedures or Change of Date of Suspension of\n",
      "     Registration. Zhongliang Holdings (02772) Notice\n",
      "     of Annual General Meeting (339KB, pdf) Previous\n",
      "     post: Today&#39;s Hong Kong Economic Journal |\n",
      "     Today&#39;s Hong Kong Economic Journal |\n",
      "     Today&#39;s Economic Journal | Today&#39;s\n",
      "     Economic Journal | Today&#39;s Economic Journal |\n",
      "     Dr. Yuen Kangjue | Sino Health | Wealth Management\n",
      "     International Finance | Today 04 :44 Important\n",
      "     Notice | Today 08:22 Hong Kong Stocks Direct\n",
      "     Attack | Today 10:31 International Finance | Today\n",
      "     07:59 Real Estate News | Today 09:50 Hong Kong\n",
      "     Stocks Direct Attack | Today 13:38 Current Affairs\n",
      "     Pulse | Today 11:15 Current Affairs Pulse | Today\n",
      "     09: 45 Hong Kong Stocks Direct Attack |\n",
      "     Today&#39;s 09:28 Instant Market Comment |\n",
      "     Today&#39;s 13:44 Real-time News Analysis and\n",
      "     Commentary Today&#39;s Economic Journal Series\n",
      "     Stock Wealth Management Real Estate\n",
      "     Life/Innovation and Technology S\n",
      "45   Gold Futures Discussions. how can you be so sure?   positive      neutral          X              neutral             X\n",
      "     here gold moved from 1765 to 1772 and still in mcx\n",
      "     gold is at 46750 strange! absolutely strange!\n",
      "46   $MINE  today we buy gold plated lambos...           neutral       neutral          O              neutral             O\n",
      "47   Silver too if it isn’t tarnished. Copper is just    neutral       neutral          O              neutral             O\n",
      "     cheaper.\n",
      "48   Silver Futures.   if USDINR  goes up when ll see a  neutral      positive          X             positive             X\n",
      "     good  bounce,,\n",
      "49   Gold Bangles Design Making  #goldsmithjack          neutral       neutral          O              neutral             O\n",
      "     #goldbanglesdesign #goldbanglesmaking\n",
      "     #jewelrymaking https://t.co/9BlxbiUpWj\n",
      "50   China Internet Investment (00810) Others-the        neutral       neutral          O              neutral             O\n",
      "     latest situation of business development. China\n",
      "     Internet Investment (00810) Voluntary\n",
      "     Announcement-Memorandum of Understanding (204KB,\n",
      "     pdf) on a potential investment in a live\n",
      "     e-commerce marketing platform company Today&#39;s\n",
      "     Hong Kong Economic Journal, Dr. Yuen Kangjiu |\n",
      "     Shine Health Ye Yongkeng | Important Wealth\n",
      "     Management Notice | Today 08:22 Hong Kong Stocks\n",
      "     Direct Attack | Today 10:31 International Finance\n",
      "     | Today 07:59 Current Events | Today 09:45 Hong\n",
      "     Kong Stocks Direct Attack | Today 13:38 Current\n",
      "     Affairs Pulse | Today 11:15 Real Estate News |\n",
      "     Today 09:50 Current Events Pulse | Today 12:42\n",
      "     Instant Market Review | Today 13:44 Hong Kong\n",
      "     Stocks Direct Attack | Today 09:28 Real-time News\n",
      "     Analysis and Commentary Today&#39;s Economic\n",
      "     Journal Series Stock Wealth Management Real Estate\n",
      "     Life/Innovation Technology Subscription to the\n",
      "     popular Hong Kong Economic Journal | Terms of\n",
      "     Service | Privacy Policy | Disclaimer |\n",
      "     Advertising Inquiry | Jo\n",
      "51   RT @SilverChartist: 4 #Silver Stocks (+an ETF)      neutral       neutral          O              neutral             O\n",
      "     Testing 200MA Support:  1⃣ Mag Silver, $MAG 2⃣\n",
      "     Alexco Resources, $AXU 3⃣ Southern Silver, $S…\n",
      "52   Gold Prices May Fall as US PCE Data Sends Yields,   negative      neutral          X              neutral             X\n",
      "     Dollar Higher https://t.co/wfeag21tSe\n",
      "     https://t.co/ftYevFoSm1\n",
      "53   RT @eBargainsToday: Gold, Stocks, and Bitcoin:      negative      neutral          X              neutral             X\n",
      "     Weekly Overview — April 29: This week’s price\n",
      "     movements for Bitcoin (BTC), gold, and our sto…\n",
      "54   RT @eBargainsToday: Gold, Stocks, and Bitcoin:      negative      neutral          X              neutral             X\n",
      "     Weekly Overview — April 29: This week’s price\n",
      "     movements for Bitcoin (BTC), gold, and our sto…\n",
      "55   Silver Futures Discussions. According to google     neutral       neutral          O              neutral             O\n",
      "     india and china are by far the biggest consumers\n",
      "     of silver. Do you see the situation impacting\n",
      "     this? Are factories etc shutdown?\n",
      "56   Gold Futures Discussions. With US high spending     neutral       neutral          O              neutral             O\n",
      "     bills (trillions), this will fly as soon the\n",
      "     inflation will show up. Just wait...\n",
      "57   XAU/USD - Gold Spot US Dollar. gold is acting like  neutral       neutral          O              neutral             O\n",
      "     my gf 🤣🤣\n",
      "58   Gold Futures Discussions. The vaccine I still       neutral       neutral          O              neutral             O\n",
      "     catch covid and pass it on to granny like you and\n",
      "     we both die...so not sure what people are\n",
      "     thinking....sounding like fake stuff...lolz\n",
      "59   Originally posted by tobiasroy: ↑ Gold stocks       neutral       neutral          O             negative             X\n",
      "     getting hammered today as well,I reckon you will\n",
      "     be able to get back in at 3.3c.AP must be busy\n",
      "     putting his spiel together and records show that\n",
      "     when things are delivered on time means they’re\n",
      "     are good but anything late is always bad. Expand\n",
      "     Where did you get the 3.3??... pull it out of your\n",
      "     bottom??????\n",
      "60   Gold Futures Discussions. Look at silver reaction   negative      neutral          X              neutral             X\n",
      "61   Gold Futures Discussions. Zyielda loves cup and     negative      neutral          X              neutral             X\n",
      "     handle . Zyielda put in sweet and low,\n",
      "62   FUTURES: Crude Oil        $ 64.64         -0.57%    negative      neutral          X              neutral             X\n",
      "     🥵 Natural Gas    $ 2.9010       -0.34% 🥵 Gold\n",
      "     $ 176… https://t.co/c0vmEs243Q\n",
      "63   Gold Futures Discussions. Yess waterfall comming    negative      neutral          X              neutral             X\n",
      "64   Gold Futures Discussions. NO CLOSE beneath 1763.8   positive      neutral          X              neutral             X\n",
      "     is important they had an LTS bar on yesterday the\n",
      "     target 1811.2 from the 10th is still there also\n",
      "     they made another at 1786.8\n",
      "65   Simor UBS Yijia Niu E(53255) Bonds and Structured   neutral       neutral          O              neutral             O\n",
      "     Products-[Maturity Announcement-CBBC]. Smarter UBS\n",
      "     Yijia Niu E(53255) Mandatory Redemption Event and\n",
      "     Early Expiry Notice for Callable Bull/Bear\n",
      "     Contracts issued by UBS (94KB, pdf) Hong Kong\n",
      "     Economic Journal Today, Hong Kong Journalist, Dr.\n",
      "     Yuen Kangjou | Xin Health Ye Yongkeng | Wealth\n",
      "     Management International Finance | Today 04:44\n",
      "     Important Notice | Yesterday 23:26 Important\n",
      "     Notice | Yesterday 23:29 Important Notice |\n",
      "     Yesterday 23:10 Important Notice | Yesterday 23:\n",
      "     11 International Finance | Today 04:10\n",
      "     International Finance | Today 04:09 Current\n",
      "     Affairs Pulse | Today 00:36 Hong Kong Finance |\n",
      "     Yesterday 22:19 Important Notices | Today 08:22\n",
      "     Real-time News Analysis and Comments Introduction\n",
      "     to the popular Hong Kong Economic Journal | Terms\n",
      "     of Service | Privacy Policy | Disclaimer |\n",
      "     Advertising Inquiry | Join the Hong Kong Economic\n",
      "     Journal | Contact the Hong Kong Economic Journal\n",
      "     Stock and index information is provided by\n",
      "     Financial Inte\n",
      "66   Old pic but at this time it is like Gold            neutral       neutral          O              neutral             O\n",
      "     @sidharth_shukla #SidharthShukla\n",
      "     https://t.co/F9OPVwFcVp https://t.co/LHcTn8gsJO\n",
      "67   “ Tch. A fish that can talk and wears jewelries?    neutral       neutral          O              neutral             O\n",
      "     I'd eat you then use your claws to pick my teeth.\n",
      "     Gold fishhhh~” https://t.co/7z905FkmTp\n",
      "68   Silver Futures Discussions. smoking marijuana is    negative      neutral          X             positive             X\n",
      "     great experience when i was young i used to smoke\n",
      "     and sit idle for hours and imagination flew high\n",
      "     in bright colors think today silver smoke\n",
      "     marijuana  and dreaming\n",
      "69   Shipping Containers Are Falling Overboard at a      negative     negative          O             negative             O\n",
      "     Rapid Rate. This ship [0] went down offshore of\n",
      "     Cornwall in 1641, with 60 men, 100,000 pounds of\n",
      "     gold and 500,000 pieces-of-eight aboard. Cornwall\n",
      "     is not known to be friendly to wrecks. And yet,\n",
      "     nearly 400 years later, she's not been found. They\n",
      "     say.[0]\n",
      "     https://en.wikipedia.org/wiki/Merchant_Royal\n",
      "70   $AGC $AGC.AX Australia Copper and Gold Ltd  -       negative      neutral          X              neutral             X\n",
      "     Aussie #Copper and #gold  - Assays if three (out\n",
      "     of 7) drilling campa… https://t.co/vnDnBLrfaa\n",
      "71   Gold Futures Discussions. Stochs is only used in a  neutral       neutral          O              neutral             O\n",
      "     bearish trend...it can run way hot in a bullish\n",
      "     trend...101 dude\n",
      "72   Silver Futures Discussions. JAY BHOLENATH= 68130<<  negative      neutral          X              neutral             X\n",
      "73   Gold Price Today: Yellow metal trades higher post   negative     positive          X             positive             X\n",
      "     Fed meet; Silver May futures rally over 1%\n",
      "     https://t.co/7e8mVN64Of\n",
      "74   Gold Price Today: Yellow metal trades higher post   negative     positive          X             positive             X\n",
      "     Fed meet; Silver May futures rally over 1%\n",
      "     https://t.co/7e8mVN64Of\n",
      "75   Gold Futures Discussions. Let me guess this isnt    neutral       neutral          O              neutral             O\n",
      "     accumulation it's a flying saicer\n",
      "76   Gold https://t.co/VnL69bGYH0                        negative      neutral          X              neutral             X\n",
      "77   Gold Futures Discussions. Hmm, still one way        negative      neutral          X              neutral             X\n",
      "     dive... ready to hedge\n",
      "78   Gold price at $2,300 if not for bitcoin -           negative     positive          X             positive             X\n",
      "     Bloomberg Intelligence. (Kitco News) -\n",
      "     Technological innovation is benefiting many metals\n",
      "     but not gold, said Bloomberg Intelligence,\n",
      "     advising to pair gold with bitcoin in a portfolio.\n",
      "     Read more on \"Kitco\"\n",
      "79   U.S .Stocks May Give Back Ground In Early Trading.  positive     positive          X             positive             X\n",
      "     (RTTNews) - After ending the previous session\n",
      "     mostly higher, stocks may move back to the\n",
      "     downside in early trading on Friday. The major\n",
      "     index futures are currently pointing to a lower\n",
      "     open for the markets, with the Dow futures down by\n",
      "     172 points. Profit taking may contribute to\n",
      "     initial weakness on Wall Street after the upward\n",
      "     move seen on Thursday lifted the S&P 500 to a new\n",
      "     record closing high. The tech-heavy Nasdaq also\n",
      "     reached a record intraday high during yesterday's\n",
      "     trading but pulled back and ended the session only\n",
      "     modestly higher. A steep drop by shares of Twitter\n",
      "     (TWTR) may weigh on the markets, with the social\n",
      "     media giant plunging by 13.4 percent in pre-market\n",
      "     trading. The nosedive by Twitter comes after the\n",
      "     company reported better than expected first\n",
      "     quarter results but provided disappointing\n",
      "     guidance. On the other hand, shares of Amazon\n",
      "     (AMZN) are likely to see initial strength after\n",
      "     the online retail giant reported first qu\n",
      "80   $MRZ   Low EV Low mcap Low SOI and interesting      neutral       neutral          O              neutral             O\n",
      "     individuals on the top 20 register...   Bags\n",
      "     potential?  #gold… https://t.co/v04YQJ1Eh0\n",
      "81   Silver Futures Discussions. are you playing p.p.    neutral       neutral          O              neutral             O\n",
      "     rub game?\n",
      "82   @f1bridget White and Gold                           negative      neutral          X              neutral             X\n",
      "83   Gold Futures Discussions. Weak bulls...             negative      neutral          X              neutral             X\n",
      "84   Silver? I heard the world had run outta silver,,,,  neutral       neutral          O              neutral             O\n",
      "85   Silver Futures Discussions. 25 to 35 is very easy   negative      neutral          X              neutral             X\n",
      "86   Gold Futures Discussions.     yes that's followed   neutral       neutral          O              neutral             O\n",
      "     but  NPA  : )   after all .... may be it's a trick\n",
      "87   Gold prices flat as international prices fall on    negative      neutral          X              neutral             X\n",
      "     strong US data; where is MCX gold headed?  Gold\n",
      "     was trading flat t… https://t.co/Kk0M8o3RN4\n",
      "88   Gold prices flat as international prices fall on    negative      neutral          X              neutral             X\n",
      "     strong US data; where is MCX gold headed?  Gold\n",
      "     was trading flat t… https://t.co/Kk0M8o3RN4\n",
      "89   Silver Futures Discussions. buy physical. drain     negative      neutral          X              neutral             X\n",
      "     the vault!\n",
      "90   Gold Futures Discussions. Nice $3 from $1766, by    neutral       neutral          O              neutral             O\n",
      "     dips again\n",
      "91   4 #Silver Stocks (+an ETF) Testing 200MA Support:   neutral       neutral          O              neutral             O\n",
      "     1⃣ Mag Silver, $MAG 2⃣ Alexco Resources, $AXU 3⃣\n",
      "     Southern Silver… https://t.co/8sq4OkuPyl\n",
      "92   Gold Futures Discussions. Have u sold ur past       negative      neutral          X              neutral             X\n",
      "     dips?\n",
      "93   RT @healfromwithin: Check out this listing I just   neutral       neutral          O              neutral             O\n",
      "     found on #Poshmark: Christian Dior Vintage Gold\n",
      "     Compact Mirror. https://t.co/WkrN8XA12g #…\n",
      "94   Gold Futures Discussions. US Market will start      neutral       neutral          O              neutral             O\n",
      "     with the opening dump as usual this week. However\n",
      "     I don't think the support will be able to hold the\n",
      "     line today on rising dollar\n",
      "95   XAU/USD - Gold Spot US Dollar. let s break 73,      neutral       neutral          O              neutral             O\n",
      "     step by step\n",
      "96   RT @Shivfreespirit: @PeterSchiff One would expect   neutral       neutral          O              neutral             O\n",
      "     ? Yes,That one person is You.  No body else is\n",
      "     expecting is anything from Gold or Silver…\n",
      "97   RT @TigerWitAfrica: Amid Rising Inflation           neutral       neutral          O              neutral             O\n",
      "     Expectation, what is next for Gold and Dollar?\n",
      "     https://t.co/o89lTuKpKI https://t.co/olpfc0V1ZC\n",
      "98   Re: Bitcoin or Gold?. Quote from: BIN-BIN on Today  neutral       neutral          O             positive             X\n",
      "     at 07:45:23 AMSince gold is more stable for long-\n",
      "     term holding, investors will prefer to go for gold\n",
      "     against bitcoin.Bitcoin is new, just a decade-old\n",
      "     currency, but gold has been around for so many\n",
      "     years over 100 years. So the two can not be\n",
      "     compared, but again bitcoin has grown\n",
      "     magnificently, so it offers quick returns.In my\n",
      "     opinion, not all investors choose Gold for long-\n",
      "     term investments, because if we really want quick\n",
      "     returns, we should prefer Bitcoin.As we all know\n",
      "     the volatile Bitcoin price can generate a much\n",
      "     greater profit than Gold, but indeed the risk is\n",
      "     also much higher. So for investors who are looking\n",
      "     for it to be safe, they definitely prefer Gold\n",
      "     over Bitcoin. So in the end back to each of us,\n",
      "     because I prefer quick returns I prefer investing\n",
      "     in Bitcoin. And it is proven that I managed to get\n",
      "     a large profit from Bitcoin this year, because the\n",
      "     increase in Bitcoin this year is very significant\n",
      "     when compared to Gold.\n",
      "99   JD UBS Yijia Niu O (52617) Bonds and Structured     neutral       neutral          O              neutral             O\n",
      "     Products-[Maturity Announcement-CBBC]. JD UBS\n",
      "     Yijia Niu O (52617) Mandatory Redemption Event and\n",
      "     Early Expiry Notice of Callable Bull/Bear\n",
      "     Contracts issued by UBS Bank (91KB, pdf) Report to\n",
      "     Today&#39;s Hong Kong Economic Journal, Dr. Yuen\n",
      "     Kangjue | XinHealth Ye Yongkeng | Wealth\n",
      "     Management International Finance | Today 04:44\n",
      "     Important Notice | Yesterday 23:26 Important\n",
      "     Notice | Yesterday 23:29 Important Notice |\n",
      "     Yesterday 23:10 International Finance | Today\n",
      "     04:10 Important Notice | Current Events at 23:11\n",
      "     Yesterday | Today 00:36 International Finance |\n",
      "     Today 04:09 Important Notices | Today 08:22\n",
      "     International Finance | Today 07:59 Real-time News\n",
      "     Analysis and Commentary Today&#39;s Newsletter\n",
      "     Series Stock Financial Management Real Estate\n",
      "     Life/Innovation Technology Subscription to the\n",
      "     popular Hong Kong Economic Journal | Terms of\n",
      "     Service | Privacy Policy | Disclaimer |\n",
      "     Advertising Inquiry | Join the Hong Kong Economic\n",
      "     Journal | Contact The\n",
      "100  Gold rush I vote #Louies for #BestFanArmy at the    neutral       neutral          O              neutral             O\n",
      "     #iHeartAwards https://t.co/XmyXZUgo9T\n",
      "101  Gold Futures Discussions. There accumulation...     negative      neutral          X              neutral             X\n",
      "102  Silver Futures Discussions. don't short this. It    neutral      negative          X             negative             X\n",
      "     will be scary up in may.\n",
      "103  Silver Futures Discussions. feels like watching     negative      neutral          X              neutral             X\n",
      "     paint dry today...\n",
      "104  Silver Futures Discussions. Do not exept shares     neutral       neutral          O              neutral             O\n",
      "     and properties Will Come down,,, they Will go\n",
      "     higher as long we have fiatmoney....\n",
      "105  Ghost of cap’n kidd. Related Posts November 14,     neutral       neutral          O              neutral             O\n",
      "     2001   silver moon highlightsred fur on a soft\n",
      "     shouldertogether in dreams September 8, 2013   I’m\n",
      "     watching Portlandia“Season 3!”        34 others\n",
      "     are […] April 27, 2017  April 27, 2017 at 09:07AM\n",
      "     No more boogers, Sasquatch!  via Instagram\n",
      "     http://bit.ly/2qbuSmt December 16, 2001  Bu, lbh\n",
      "     rabezbhf trrx. Abg bayl qvq lbh erpbtavmr ebg13,\n",
      "     ohg lbh obgurerq gb qrpelcg vg. Lbh’er fb phgr\n",
      "     jura lbh’ir fbyirq n chmmyr.  The Three Laws of\n",
      "     Robotics are:  A robot may not injure a human\n",
      "     being, or, through […] September 9, 2010  From\n",
      "     Twitter 09-08-2010  From Twitter\n",
      "     09-08-201003:20:35: From Twitter 09-07-2010\n",
      "     http://j.mp/9COuqG04:12:32: […]\n",
      "106  $MAG.ax $MAG @MagmaticRes   Gold and Copper.        negative     positive          X             positive             X\n",
      "     17.5c eaten for afternoon tea👌  +9.4%\n",
      "107  Silver Futures Discussions. Sold 68502              positive      neutral          X              neutral             X\n",
      "108  Gold Futures Discussions.  buy time you say buy     positive      neutral          X              neutral             X\n",
      "     dips and sell time you say dive ...\n",
      "109  Check out this listing I just added to my           neutral       neutral          O              neutral             O\n",
      "     #Poshmark closet: Gold Coach Crossbody Bag.\n",
      "     https://t.co/NNAQ2Wlpwi #shopmycloset @poshmarkapp\n",
      "110  Shout out to @BlaqBonez !!! That album is forever   neutral       neutral          O              neutral             O\n",
      "     Gold bruh               SEX OVER LOVE\n",
      "111  PRECIOUS-Under-supplied palladium races past        negative      neutral          O              neutral             O\n",
      "     $3,000/oz for first time. * Graphic: 2021 asset\n",
      "     performance http://tmsnrt.rs/2yaDPgn* Spot gold\n",
      "     falls after strong U.S. economic data on Thursday\n",
      "     (Adds comments, details and updates prices)By\n",
      "     Sethuraman N R and Peter HobsonApril 30 (Reuters)\n",
      "     - Palladium pierced the $3,000 an ounce high for\n",
      "     the first time on Friday, powered by persistent\n",
      "     concerns about short supply of the metal used\n",
      "     primarily for emissions-controlling devices in\n",
      "     automobiles.Palladium XPD= was last up 1% to\n",
      "     $2,984.68 per ounce, after hitting an all-time\n",
      "     high of $3,007.73. It was on track to post a\n",
      "     monthly gain as well.\"Palladium is in a structural\n",
      "     deficit since 10 years. We have seen above ground\n",
      "     inventories falling to very low levels, and now\n",
      "     the price gives a signal to investors still\n",
      "     holding it,\" UBS analyst Giovanni Staunovo\n",
      "     said.\"The commodity ... comes with very low\n",
      "     trading volume so it can be very illiquid and this\n",
      "     is making the moves even more crazy. This rally\n",
      "     can con\n",
      "112  PRECIOUS-Under-supplied palladium races past        negative      neutral          O              neutral             O\n",
      "     $3,000/oz for first time. * Graphic: 2021 asset\n",
      "     performance http://tmsnrt.rs/2yaDPgn* Spot gold\n",
      "     falls after strong U.S. economic data on Thursday\n",
      "     (Adds comments, details and updates prices)By\n",
      "     Sethuraman N R and Peter HobsonApril 30 (Reuters)\n",
      "     - Palladium pierced the $3,000 an ounce high for\n",
      "     the first time on Friday, powered by persistent\n",
      "     concerns about short supply of the metal used\n",
      "     primarily for emissions-controlling devices in\n",
      "     automobiles.Palladium XPD= was last up 1% to\n",
      "     $2,984.68 per ounce, after hitting an all-time\n",
      "     high of $3,007.73. It was on track to post a\n",
      "     monthly gain as well.\"Palladium is in a structural\n",
      "     deficit since 10 years. We have seen above ground\n",
      "     inventories falling to very low levels, and now\n",
      "     the price gives a signal to investors still\n",
      "     holding it,\" UBS analyst Giovanni Staunovo\n",
      "     said.\"The commodity ... comes with very low\n",
      "     trading volume so it can be very illiquid and this\n",
      "     is making the moves even more crazy. This rally\n",
      "     can con\n",
      "113  PRECIOUS-Under-supplied palladium races past        negative      neutral          O              neutral             O\n",
      "     $3,000/oz for first time. * Graphic: 2021 asset\n",
      "     performance http://tmsnrt.rs/2yaDPgn* Spot gold\n",
      "     falls after strong U.S. economic data on Thursday\n",
      "     (Adds comments, details and updates prices)By\n",
      "     Sethuraman N R and Peter HobsonApril 30 (Reuters)\n",
      "     - Palladium pierced the $3,000 an ounce high for\n",
      "     the first time on Friday, powered by persistent\n",
      "     concerns about short supply of the metal used\n",
      "     primarily for emissions-controlling devices in\n",
      "     automobiles.Palladium XPD= was last up 1% to\n",
      "     $2,984.68 per ounce, after hitting an all-time\n",
      "     high of $3,007.73. It was on track to post a\n",
      "     monthly gain as well.\"Palladium is in a structural\n",
      "     deficit since 10 years. We have seen above ground\n",
      "     inventories falling to very low levels, and now\n",
      "     the price gives a signal to investors still\n",
      "     holding it,\" UBS analyst Giovanni Staunovo\n",
      "     said.\"The commodity ... comes with very low\n",
      "     trading volume so it can be very illiquid and this\n",
      "     is making the moves even more crazy. This rally\n",
      "     can con\n",
      "114  Gold Futures Discussions. Caught between still      negative      neutral          X             negative             O\n",
      "115  Silver Futures Discussions.  90% scary up in may    positive      neutral          X             negative             X\n",
      "     and the next few month. 10% go down. For me, I\n",
      "     will not short this until 13 may\n",
      "116  RT @aguila_gold: Great graphics as always from      negative      neutral          X              neutral             X\n",
      "     @VisualCap on the #gold industry.   Gold price now\n",
      "     gradually testing 2 month high  https://t…\n",
      "117  European Equities: It’s A Busy Economic Calendar    neutral       neutral          X              neutral             X\n",
      "     With GDP Figures in Focus. French GDP (QoQ)\n",
      "     (Q1)German GDP (QoQ) (Q1)German GDP (YoY)\n",
      "     (Q1)Spanish GDP (QoQ) (Q1)Italian CPI (MoM) (Apr)\n",
      "     PrelimEurozone CPI (YoY) (Apr) PrelimEurozone\n",
      "     Unemployment Rate (Mar)Eurozone GDP (QoQ)\n",
      "     (Q1)Eurozone GDP (YoY) (Q1)The Majors It was a\n",
      "     relatively bearish day for the European majors on\n",
      "     Thursday. The DAX30 fell by 0.90%, with the CAC40\n",
      "     and the EuroStoxx600 seeing losses of 0.07% and\n",
      "     0.26% respectively.Economic data from the U.S\n",
      "     failed to prevent a pullback from early gains,\n",
      "     with stats from Germany disappointing.The downside\n",
      "     came amidst positive earnings results from both\n",
      "     side of the Atlantic. From the Eurozone, Airbus\n",
      "     returned to profit, with Total SE beating\n",
      "     estimates.The Stats It was a busier day on the\n",
      "     economic calendar on Thursday. The German economy\n",
      "     was back in focus, with unemployment and inflation\n",
      "     figures the key stats of the day.Prelim April\n",
      "     inflation figures from Spain and finalized\n",
      "     consumer conf\n",
      "118  Gold Futures Discussions. ameros funding should     negative      neutral          X              neutral             X\n",
      "     get bent and impaled by a horse\n",
      "119  Gold Futures Discussions. Ouchhh... pain in the     negative      neutral          X              neutral             X\n",
      "     A/s/ss\n",
      "120  Gold is about to break the long time descending     neutral       neutral          O              neutral             O\n",
      "     channel. Gold has been on a decline path within 6\n",
      "     months or so, now by getting support of Ichimuko\n",
      "     and gathering strength from the previous bearish\n",
      "     leg and also getting provided by enough support\n",
      "     from lower boundary of the channel, IS READY to\n",
      "     make a move and find its way up. So keep an eye on\n",
      "     it and hope we'll see its drought come to an end.\n",
      "     Trade Safe ...\n",
      "121  Silver Futures Discussions.  in one hour            negative      neutral          X              neutral             X\n",
      "122  I've got no Reddit gold, but gold's decadent.\"      neutral       neutral          O              neutral             O\n",
      "123  Gold Futures Discussions. too many buyers r there   neutral       neutral          O              neutral             O\n",
      "     in mcx,, bcoz of covid situation n lockdown.. like\n",
      "     last year,, so I think it's testing the patience\n",
      "     of mcx buyers\n",
      "124  $SPY they pumped every commodity x3 x4 but short    negative      neutral          X              neutral             X\n",
      "     PMs. Thats the life blood of printers. Printer wud\n",
      "     jam if gold takes off.\n",
      "125  Silver Price Analysis: XAG/USD to resume the core   negative     positive          X             positive             X\n",
      "     uptrend above $30.72 – Credit Suisse. Strategists\n",
      "     at Credit Suisse continue to view consolidation in\n",
      "     XAG/USD as temporary with a long-term base still\n",
      "     seen in place here. What’s more, silver is\n",
      "     expected to outperform gold. See – Silver Price\n",
      "     Analysis: XAG/USD to nosedive towards $21.50 by\n",
      "     year-end – CE  Silver maintains a long-term base\n",
      "     “Silver stays trapped in a turgid range below the\n",
      "     50% retracement of the 2011/2020 bear market at\n",
      "     $30.72. With a long-term base still in place\n",
      "     though our base remains to view this a correction\n",
      "     within a broader uptrend.”  “Above $30.72 remains\n",
      "     needed to suggest the core uptrend has indeed\n",
      "     resumed for a move to our objective still at\n",
      "     $35.23/35.365 – the 61.8% retracement and key high\n",
      "     from October 2012.” “Silver is still expected to\n",
      "     on balance outperform Gold with the trend for the\n",
      "     Gold/Silver ratio still seen lower, despite the\n",
      "     recent loss of momentum.”\n",
      "126  Gome Retail (00493) Circular-[Others]. Today&#39;s  neutral       neutral          O              neutral             O\n",
      "     Hong Kong Economic Journal | Today&#39;s Hong Kong\n",
      "     Economic Journal | Today&#39;s Economic Journal |\n",
      "     Today&#39;s Economic Journal | Dr. Yuen Kangjuu |\n",
      "     XinHealth Ye Yongkeng | Wealth Management\n",
      "     International Finance | Today 04:44 Current Events\n",
      "     | Today 00:36 International Finance | Today 04:10\n",
      "     International Finance | Today 04:09 Hong Kong\n",
      "     Stocks Direct Attack | Today 10:31 Important\n",
      "     Notices | Today 08:22 International Finance |\n",
      "     Today 07:59 International Finance | Today 01:03\n",
      "     Hong Kong Stocks Direct Attack | Today 09:28 Hong\n",
      "     Kong Finance | Today 03:26 Real-time News Analysis\n",
      "     and Comment Today&#39;s Hong Kong Economic Journal\n",
      "     Series Stock Wealth Management Real Estate\n",
      "     Life/Innovation and Technology Newsletter\n",
      "     Introduction | Terms of Service | Privacy Policy |\n",
      "     Disclaimer | Advertising Enquiry | Join the Hong\n",
      "     Kong Economic Journal | Contact Economic Journal\n",
      "     Stock and index information is provided by\n",
      "     Financial Intelligence Network Limited. F\n",
      "127  Tang Jinzhao: Analysis of the latest market trend   neutral       neutral          O              neutral             O\n",
      "     of gold and crude oil and the latest operational\n",
      "     recommendations. ===Thursday, April 30 (April 29),\n",
      "     international gold from a technical point of view,\n",
      "     from the daily line analysis, the daily line\n",
      "     recorded a long shadow line and a small positive\n",
      "     line, the current gold price daily close is still\n",
      "     stable at the overall The daily moving average\n",
      "     technical indicator support level is above the\n",
      "     price, and the daily moving average is in a glued\n",
      "     upward cross arrangement, maintaining a longer\n",
      "     period of more shock absorption. The short-term\n",
      "     decline will still face the impact of relatively\n",
      "     dense technical indicators support prices. The\n",
      "     hourly line shows that the short-term bottoming\n",
      "     and reversal bring the hourly line technical\n",
      "     indicators short-term upward and longer-term\n",
      "     technical indicators form, forming a short-term\n",
      "     gold cross-to-many guidance pattern. On the whole,\n",
      "     today&#39;s short-term operation ideas suggest\n",
      "     that the callback is mainly low and\n",
      "128  Gold Price Analysis: XAU/USD to extend the          negative     negative          O             negative             O\n",
      "     downside on a close below 21-DMA at $1756\n",
      "     https://t.co/ulcdr5mDJL #Gold #XAUUSD\n",
      "129  Gold Futures Discussions.   Is not about that it    neutral       neutral          O              neutral             O\n",
      "     is about trend. Markets do listen Powell, we\n",
      "     traders follow trends  to make money. No matter\n",
      "     Bull or Bear.\n"
     ]
    }
   ],
   "source": [
    "# !pip3 install -U tabulate\n",
    "import tabulate\n",
    "\n",
    "headers = ['Sample', 'Spacy', 'FinancialBERT', 'vs Spacy', 'tuned FinancialBERT', 'vs Spacy']\n",
    "table = tabulate.tabulate(\n",
    "    data2,\n",
    "    headers=headers,\n",
    "    tablefmt='simple',\n",
    "    colalign=('center', 'left','center', 'center', 'center', 'center', 'center'),\n",
    "    maxcolwidths=[4, 50, 8, 8, 8, 8, 8], showindex=\"always\"\n",
    "    )\n",
    "print(table)"
   ]
  }
 ],
 "metadata": {
  "jupytext": {
   "cell_metadata_filter": "-all",
   "formats": "md,ipynb"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}