convert_new_weight.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import struct\n",
    "import numpy as np\n",
    "from collections import OrderedDict\n",
    "\n",
    "\n",
    "def load_dtype(fp):\n",
    "    v = struct.unpack(\"B\", fp.read(1))[0]\n",
    "    if v == 0:\n",
    "        return np.int8\n",
    "    elif v == 1:\n",
    "        return np.float16\n",
    "    elif v == 2:\n",
    "        return np.float32\n",
    "    else:\n",
    "        raise TypeError(\"Unknown dtype %d\" % v)\n",
    "\n",
    "def load_string(fp):\n",
    "    size = struct.unpack(\"I\", fp.read(4))[0]\n",
    "    v = fp.read(size)\n",
    "    return v.decode(\"utf-8\")\n",
    "\n",
    "def load_tuple(fp):\n",
    "    dim_tuple = struct.unpack(\"B\", fp.read(1))[0]\n",
    "    ret = []\n",
    "    for _ in range(dim_tuple):\n",
    "        ret.append(struct.unpack(\"I\", fp.read(4))[0]) \n",
    "    return tuple(ret)\n",
    "\n",
    "def load_parameter(fp):    \n",
    "    shape = load_tuple(fp)\n",
    "    value_size = struct.unpack(\"I\", fp.read(4))[0]\n",
    "    dtype = load_dtype(fp)\n",
    "    value = fp.read(value_size)\n",
    "    return shape, value, dtype\n",
    "\n",
    "def load(fp, parent_name=''):\n",
    "    num_parameters, num_sub_layers = struct.unpack(\"II\", fp.read(8))\n",
    "    parameters = []\n",
    "\n",
    "    for _ in range(num_parameters):\n",
    "        name = load_string(fp)\n",
    "        shape, value, dtype = load_parameter(fp)\n",
    "        parameters.append((parent_name + '.' + name, np.frombuffer(value, dtype).reshape(shape)))\n",
    "    for _ in range(num_sub_layers):\n",
    "        name = load_string(fp)\n",
    "        parameters += load(fp, parent_name + '.' + name)\n",
    "    return parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../zhiyuan/cpm/checkpoint.pt', 'rb') as fp:\n",
    "    parameters = load(fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "pindex = {x[0]: x[1] for x in parameters}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 516/516 [00:42<00:00, 12.03it/s]\n"
     ]
    }
   ],
   "source": [
    "npara = {}\n",
    "for name, value in tqdm(parameters):\n",
    "    if '_scale' not in name:\n",
    "        has_scale = name + '_scale'\n",
    "        if has_scale in pindex:\n",
    "            scale = pindex[has_scale]\n",
    "            value = value.astype(np.float16) * scale\n",
    "        npara[name] = value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ".input_embedding.weight (30000, 2560)\n",
      ".position_embedding.weight (1024, 2560)\n",
      ".layers.0.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.0.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.0.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.0.self_attention.w_project_bias (3, 2560)\n",
      ".layers.0.self_attention.w_out (2560, 2560)\n",
      ".layers.0.self_attention.w_out_bias (2560,)\n",
      ".layers.0.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.0.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.0.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.0.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.0.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.0.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.1.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.1.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.1.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.1.self_attention.w_project_bias (3, 2560)\n",
      ".layers.1.self_attention.w_out (2560, 2560)\n",
      ".layers.1.self_attention.w_out_bias (2560,)\n",
      ".layers.1.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.1.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.1.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.1.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.1.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.1.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.2.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.2.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.2.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.2.self_attention.w_project_bias (3, 2560)\n",
      ".layers.2.self_attention.w_out (2560, 2560)\n",
      ".layers.2.self_attention.w_out_bias (2560,)\n",
      ".layers.2.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.2.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.2.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.2.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.2.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.2.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.3.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.3.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.3.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.3.self_attention.w_project_bias (3, 2560)\n",
      ".layers.3.self_attention.w_out (2560, 2560)\n",
      ".layers.3.self_attention.w_out_bias (2560,)\n",
      ".layers.3.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.3.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.3.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.3.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.3.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.3.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.4.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.4.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.4.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.4.self_attention.w_project_bias (3, 2560)\n",
      ".layers.4.self_attention.w_out (2560, 2560)\n",
      ".layers.4.self_attention.w_out_bias (2560,)\n",
      ".layers.4.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.4.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.4.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.4.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.4.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.4.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.5.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.5.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.5.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.5.self_attention.w_project_bias (3, 2560)\n",
      ".layers.5.self_attention.w_out (2560, 2560)\n",
      ".layers.5.self_attention.w_out_bias (2560,)\n",
      ".layers.5.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.5.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.5.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.5.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.5.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.5.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.6.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.6.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.6.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.6.self_attention.w_project_bias (3, 2560)\n",
      ".layers.6.self_attention.w_out (2560, 2560)\n",
      ".layers.6.self_attention.w_out_bias (2560,)\n",
      ".layers.6.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.6.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.6.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.6.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.6.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.6.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.7.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.7.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.7.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.7.self_attention.w_project_bias (3, 2560)\n",
      ".layers.7.self_attention.w_out (2560, 2560)\n",
      ".layers.7.self_attention.w_out_bias (2560,)\n",
      ".layers.7.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.7.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.7.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.7.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.7.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.7.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.8.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.8.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.8.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.8.self_attention.w_project_bias (3, 2560)\n",
      ".layers.8.self_attention.w_out (2560, 2560)\n",
      ".layers.8.self_attention.w_out_bias (2560,)\n",
      ".layers.8.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.8.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.8.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.8.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.8.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.8.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.9.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.9.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.9.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.9.self_attention.w_project_bias (3, 2560)\n",
      ".layers.9.self_attention.w_out (2560, 2560)\n",
      ".layers.9.self_attention.w_out_bias (2560,)\n",
      ".layers.9.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.9.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.9.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.9.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.9.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.9.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.10.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.10.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.10.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.10.self_attention.w_project_bias (3, 2560)\n",
      ".layers.10.self_attention.w_out (2560, 2560)\n",
      ".layers.10.self_attention.w_out_bias (2560,)\n",
      ".layers.10.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.10.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.10.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.10.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.10.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.10.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.11.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.11.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.11.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.11.self_attention.w_project_bias (3, 2560)\n",
      ".layers.11.self_attention.w_out (2560, 2560)\n",
      ".layers.11.self_attention.w_out_bias (2560,)\n",
      ".layers.11.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.11.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.11.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.11.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.11.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.11.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.12.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.12.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.12.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.12.self_attention.w_project_bias (3, 2560)\n",
      ".layers.12.self_attention.w_out (2560, 2560)\n",
      ".layers.12.self_attention.w_out_bias (2560,)\n",
      ".layers.12.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.12.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.12.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.12.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.12.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.12.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.13.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.13.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.13.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.13.self_attention.w_project_bias (3, 2560)\n",
      ".layers.13.self_attention.w_out (2560, 2560)\n",
      ".layers.13.self_attention.w_out_bias (2560,)\n",
      ".layers.13.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.13.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.13.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.13.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.13.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.13.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.14.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.14.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.14.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.14.self_attention.w_project_bias (3, 2560)\n",
      ".layers.14.self_attention.w_out (2560, 2560)\n",
      ".layers.14.self_attention.w_out_bias (2560,)\n",
      ".layers.14.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.14.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.14.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.14.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.14.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.14.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.15.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.15.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.15.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.15.self_attention.w_project_bias (3, 2560)\n",
      ".layers.15.self_attention.w_out (2560, 2560)\n",
      ".layers.15.self_attention.w_out_bias (2560,)\n",
      ".layers.15.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.15.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.15.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.15.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.15.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.15.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.16.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.16.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.16.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.16.self_attention.w_project_bias (3, 2560)\n",
      ".layers.16.self_attention.w_out (2560, 2560)\n",
      ".layers.16.self_attention.w_out_bias (2560,)\n",
      ".layers.16.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.16.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.16.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.16.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.16.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.16.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.17.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.17.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.17.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.17.self_attention.w_project_bias (3, 2560)\n",
      ".layers.17.self_attention.w_out (2560, 2560)\n",
      ".layers.17.self_attention.w_out_bias (2560,)\n",
      ".layers.17.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.17.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.17.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.17.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.17.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.17.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.18.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.18.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.18.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.18.self_attention.w_project_bias (3, 2560)\n",
      ".layers.18.self_attention.w_out (2560, 2560)\n",
      ".layers.18.self_attention.w_out_bias (2560,)\n",
      ".layers.18.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.18.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.18.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.18.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.18.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.18.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.19.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.19.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.19.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.19.self_attention.w_project_bias (3, 2560)\n",
      ".layers.19.self_attention.w_out (2560, 2560)\n",
      ".layers.19.self_attention.w_out_bias (2560,)\n",
      ".layers.19.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.19.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.19.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.19.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.19.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.19.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.20.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.20.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.20.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.20.self_attention.w_project_bias (3, 2560)\n",
      ".layers.20.self_attention.w_out (2560, 2560)\n",
      ".layers.20.self_attention.w_out_bias (2560,)\n",
      ".layers.20.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.20.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.20.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.20.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.20.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.20.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.21.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.21.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.21.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.21.self_attention.w_project_bias (3, 2560)\n",
      ".layers.21.self_attention.w_out (2560, 2560)\n",
      ".layers.21.self_attention.w_out_bias (2560,)\n",
      ".layers.21.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.21.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.21.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.21.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.21.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.21.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.22.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.22.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.22.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.22.self_attention.w_project_bias (3, 2560)\n",
      ".layers.22.self_attention.w_out (2560, 2560)\n",
      ".layers.22.self_attention.w_out_bias (2560,)\n",
      ".layers.22.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.22.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.22.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.22.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.22.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.22.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.23.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.23.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.23.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.23.self_attention.w_project_bias (3, 2560)\n",
      ".layers.23.self_attention.w_out (2560, 2560)\n",
      ".layers.23.self_attention.w_out_bias (2560,)\n",
      ".layers.23.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.23.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.23.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.23.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.23.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.23.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.24.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.24.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.24.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.24.self_attention.w_project_bias (3, 2560)\n",
      ".layers.24.self_attention.w_out (2560, 2560)\n",
      ".layers.24.self_attention.w_out_bias (2560,)\n",
      ".layers.24.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.24.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.24.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.24.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.24.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.24.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.25.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.25.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.25.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.25.self_attention.w_project_bias (3, 2560)\n",
      ".layers.25.self_attention.w_out (2560, 2560)\n",
      ".layers.25.self_attention.w_out_bias (2560,)\n",
      ".layers.25.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.25.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.25.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.25.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.25.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.25.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.26.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.26.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.26.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.26.self_attention.w_project_bias (3, 2560)\n",
      ".layers.26.self_attention.w_out (2560, 2560)\n",
      ".layers.26.self_attention.w_out_bias (2560,)\n",
      ".layers.26.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.26.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.26.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.26.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.26.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.26.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.27.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.27.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.27.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.27.self_attention.w_project_bias (3, 2560)\n",
      ".layers.27.self_attention.w_out (2560, 2560)\n",
      ".layers.27.self_attention.w_out_bias (2560,)\n",
      ".layers.27.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.27.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.27.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.27.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.27.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.27.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.28.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.28.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.28.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.28.self_attention.w_project_bias (3, 2560)\n",
      ".layers.28.self_attention.w_out (2560, 2560)\n",
      ".layers.28.self_attention.w_out_bias (2560,)\n",
      ".layers.28.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.28.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.28.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.28.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.28.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.28.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.29.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.29.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.29.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.29.self_attention.w_project_bias (3, 2560)\n",
      ".layers.29.self_attention.w_out (2560, 2560)\n",
      ".layers.29.self_attention.w_out_bias (2560,)\n",
      ".layers.29.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.29.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.29.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.29.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.29.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.29.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.30.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.30.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.30.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.30.self_attention.w_project_bias (3, 2560)\n",
      ".layers.30.self_attention.w_out (2560, 2560)\n",
      ".layers.30.self_attention.w_out_bias (2560,)\n",
      ".layers.30.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.30.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.30.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.30.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.30.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.30.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.31.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.31.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.31.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.31.self_attention.w_project_bias (3, 2560)\n",
      ".layers.31.self_attention.w_out (2560, 2560)\n",
      ".layers.31.self_attention.w_out_bias (2560,)\n",
      ".layers.31.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.31.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.31.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.31.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.31.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.31.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".encoder_final_layer_nrom.weight (2560,)\n",
      ".encoder_final_layer_nrom.bias (2560,)\n"
     ]
    }
   ],
   "source": [
    "for k, v in npara.items():\n",
    "    print(k, v.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_layer(n):\n",
    "    rets = {}\n",
    "    for k, v in npara.items():\n",
    "        if f'.layers.{n}.' in k:\n",
    "            rets[k] = v\n",
    "    weights = [\n",
    "        np.transpose(rets[f'.layers.{n}.self_attention.w_project_qkv'][0]),\n",
    "        rets[f'.layers.{n}.self_attention.w_project_bias'][0],\n",
    "        np.transpose(rets[f'.layers.{n}.self_attention.w_project_qkv'][1]),\n",
    "        rets[f'.layers.{n}.self_attention.w_project_bias'][1],\n",
    "        np.transpose(rets[f'.layers.{n}.self_attention.w_project_qkv'][2]),\n",
    "        rets[f'.layers.{n}.self_attention.w_project_bias'][2],\n",
    "        np.transpose(rets[f'.layers.{n}.self_attention.w_out']),\n",
    "        rets[f'.layers.{n}.self_attention.w_out_bias'],\n",
    "        rets[f'.layers.{n}.layer_nrom_before_self_attn.weight'],\n",
    "        rets[f'.layers.{n}.layer_nrom_before_self_attn.bias'],\n",
    "        rets[f'.layers.{n}.layer_nrom_before_ff.weight'],\n",
    "        rets[f'.layers.{n}.layer_nrom_before_ff.bias'],\n",
    "        np.transpose(rets[f'.layers.{n}.dense_gelu_dense.wi.weight']),\n",
    "        rets[f'.layers.{n}.dense_gelu_dense.wi.weight_bias'],\n",
    "        np.transpose(rets[f'.layers.{n}.dense_gelu_dense.wo.weight']),\n",
    "        rets[f'.layers.{n}.dense_gelu_dense.wo.weight_bias'],\n",
    "    ]\n",
    "    \n",
    "    return weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights = [\n",
    "    npara['.input_embedding.weight'],\n",
    "    npara['.position_embedding.weight'],\n",
    "]\n",
    "for i in range(32):\n",
    "    weights += get_layer(i)\n",
    "weights += [\n",
    "    npara['.encoder_final_layer_nrom.weight'],\n",
    "    npara['.encoder_final_layer_nrom.bias'],\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "516"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "import torch\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "\n",
    "from tf2gpt.model import GPT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "gpt = GPT(\n",
    "    vocab_size=30_000,\n",
    "    layer_size=32,\n",
    "    block_size=1024,\n",
    "    embedding_dropout=0.0,\n",
    "    embedding_size=2560,\n",
    "    num_attention_heads=32,\n",
    "    attention_dropout=0.0,\n",
    "    residual_dropout=0.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "gpt._set_inputs(\n",
    "    tf.TensorSpec(shape=[None, None],\n",
    "        dtype=tf.int32,\n",
    "        name=\"input_ids\"\n",
    "    ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 1, 30000)\n"
     ]
    }
   ],
   "source": [
    "print(gpt(tf.constant([[1]])).shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gpt/embedding/embeddings:0 (30000, 2560)\n",
      "position_embeddings:0 (1024, 2560)\n",
      "gpt/layer00/attention/query_layer/kernel:0 (2560, 2560)\n",
      "gpt/layer00/attention/query_layer/bias:0 (2560,)\n",
      "gpt/layer00/attention/key_layer/kernel:0 (2560, 2560)\n",
      "gpt/layer00/attention/key_layer/bias:0 (2560,)\n",
      "gpt/layer00/attention/value_layer/kernel:0 (2560, 2560)\n",
      "gpt/layer00/attention/value_layer/bias:0 (2560,)\n",
      "gpt/layer00/attention/context_projection_layer/kernel:0 (2560, 2560)\n",
      "gpt/layer00/attention/context_projection_layer/bias:0 (2560,)\n",
      "gpt/layer00/LayerNorm_mlp_ln0/gamma:0 (2560,)\n",
      "gpt/layer00/LayerNorm_mlp_ln0/beta:0 (2560,)\n",
      "gpt/layer00/LayerNorm_mlp_ln1/gamma:0 (2560,)\n",
      "gpt/layer00/LayerNorm_mlp_ln1/beta:0 (2560,)\n",
      "gpt/layer00/intermediate/kernel:0 (2560, 10240)\n",
      "gpt/layer00/intermediate/bias:0 (10240,)\n",
      "gpt/layer00/output/kernel:0 (10240, 2560)\n",
      "gpt/layer00/output/bias:0 (2560,)\n",
      "gpt/LayerNorm_final_norm/gamma:0 (2560,)\n",
      "gpt/LayerNorm_final_norm/beta:0 (2560,)\n"
     ]
    }
   ],
   "source": [
    "for x in :\n",
    "    if 'gpt/layer' in x.name:\n",
    "        if 'gpt/layer00' in x.name:\n",
    "            print(x.name, x.shape)\n",
    "    else:\n",
    "        print(x.name, x.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "516"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(gpt.weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "gpt.set_weights(weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gpt2_tokenizer import GPT2Tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "cbpe = GPT2Tokenizer(\n",
    "    'CPM-Generate/bpe_3w_new/vocab.json',\n",
    "    'CPM-Generate/bpe_3w_new/merges.txt',\n",
    "    model_file='CPM-Generate/bpe_3w_new/chinese_vocab.model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 今天天气 不错 \n",
      "1 今天天气 不错 ,\n",
      "2 今天天气 不错 , 我\n",
      "3 今天天气 不错 , 我 带\n",
      "4 今天天气 不错 , 我 带你\n",
      "5 今天天气 不错 , 我 带你去\n",
      "6 今天天气 不错 , 我 带你去 \n",
      "7 今天天气 不错 , 我 带你去 个\n",
      "8 今天天气 不错 , 我 带你去 个 地方\n",
      "9 今天天气 不错 , 我 带你去 个 地方 \n"
     ]
    }
   ],
   "source": [
    "ids = cbpe.encode('今天天气不错')\n",
    "\n",
    "for i in range(10):\n",
    "    output = gpt(tf.constant([ids]))\n",
    "    nid = np.argmax(output[0, -1])\n",
    "    ids += [nid]\n",
    "    print(i, cbpe.decode(ids))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:Found untraced functions such as attention_layer_call_fn, attention_layer_call_and_return_conditional_losses, LayerNorm_mlp_ln0_layer_call_fn, LayerNorm_mlp_ln0_layer_call_and_return_conditional_losses, LayerNorm_mlp_ln1_layer_call_fn while saving (showing 5 of 2080). These functions will not be directly callable after loading.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: cpm_saved_model/assets\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: cpm_saved_model/assets\n"
     ]
    }
   ],
   "source": [
    "gpt.save('cpm_saved_model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-10-12 16:31:26.635547: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory\n",
      "2021-10-12 16:31:26.635574: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n",
      "/opt/conda/lib/python3.8/runpy.py:127: RuntimeWarning: 'tf2onnx.convert' found in sys.modules after import of package 'tf2onnx', but prior to execution of 'tf2onnx.convert'; this may result in unpredictable behaviour\n",
      "  warn(RuntimeWarning(msg))\n",
      "2021-10-12 16:31:27.831891: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory\n",
      "2021-10-12 16:31:27.831915: W tensorflow/stream_executor/cuda/cuda_driver.cc:269] failed call to cuInit: UNKNOWN ERROR (303)\n",
      "2021-10-12 16:31:27.831942: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (iZuf6fokcl2k1pwfopz0n4Z): /proc/driver/nvidia/version does not exist\n",
      "2021-10-12 16:31:27.832198: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 AVX512F FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2021-10-12 16:31:27,842 - WARNING - '--tag' not specified for saved_model. Using --tag serve\n",
      "2021-10-12 16:31:45,968 - INFO - Signatures found in model: [serving_default].\n",
      "2021-10-12 16:31:45,969 - WARNING - '--signature_def' not specified, using first signature: serving_default\n",
      "2021-10-12 16:31:45,971 - INFO - Output names: ['output_1']\n",
      "2021-10-12 16:31:46.122281: I tensorflow/core/grappler/devices.cc:66] Number of eligible GPUs (core count >= 8, compute capability >= 0.0): 0\n",
      "2021-10-12 16:31:46.122457: I tensorflow/core/grappler/clusters/single_machine.cc:357] Starting new session\n",
      "2021-10-12 16:31:46.578145: I tensorflow/core/grappler/optimizers/meta_optimizer.cc:1137] Optimization results for grappler item: graph_to_optimize\n",
      "  function_optimizer: Graph size after: 9077 nodes (8557), 11780 edges (11260), time = 272.039ms.\n",
      "  function_optimizer: function_optimizer did nothing. time = 9.448ms.\n",
      "\n",
      "2021-10-12 16:32:35,050 - INFO - Using tensorflow=2.6.0, onnx=1.9.0, tf2onnx=1.9.2/0f28b7\n",
      "2021-10-12 16:32:35,050 - INFO - Using opset <onnx, 13>\n",
      "2021-10-12 16:33:51,010 - INFO - Computed 741 values for constant folding\n",
      "2021-10-12 16:34:16,253 - INFO - folding node using tf type=Identity, name=Func/StatefulPartitionedCall/input/_1\n",
      "2021-10-12 16:34:16,804 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer00/attention/key_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,804 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer00/attention/value_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,805 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer00/attention/query_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,805 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer00/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,805 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer00/intermediate/Tensordot/concat\n",
      "2021-10-12 16:34:16,805 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer00/output/Tensordot/concat\n",
      "2021-10-12 16:34:16,806 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer01/attention/key_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,807 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer01/attention/value_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,807 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer01/attention/query_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,807 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer01/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,807 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer01/intermediate/Tensordot/concat\n",
      "2021-10-12 16:34:16,807 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer01/output/Tensordot/concat\n",
      "2021-10-12 16:34:16,807 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer02/attention/key_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,807 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer02/attention/value_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,807 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer02/attention/query_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,808 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer02/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,808 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer02/intermediate/Tensordot/concat\n",
      "2021-10-12 16:34:16,808 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer02/output/Tensordot/concat\n",
      "2021-10-12 16:34:16,808 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer03/attention/key_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,808 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer03/attention/value_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,808 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer03/attention/query_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,808 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer03/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,808 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer03/intermediate/Tensordot/concat\n",
      "2021-10-12 16:34:16,809 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer03/output/Tensordot/concat\n",
      "2021-10-12 16:34:16,809 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer04/attention/key_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,809 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer04/attention/value_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,809 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer04/attention/query_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,809 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer04/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,809 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer04/intermediate/Tensordot/concat\n",
      "2021-10-12 16:34:16,809 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer04/output/Tensordot/concat\n",
      "2021-10-12 16:34:16,809 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer05/attention/key_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,810 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer05/attention/value_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,810 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer05/attention/query_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,810 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer05/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,810 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer05/intermediate/Tensordot/concat\n",
      "2021-10-12 16:34:16,810 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer05/output/Tensordot/concat\n",
      "2021-10-12 16:34:16,810 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer06/attention/key_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,810 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer06/attention/value_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,810 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer06/attention/query_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,811 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer06/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,811 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer06/intermediate/Tensordot/concat\n",
      "2021-10-12 16:34:16,811 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer06/output/Tensordot/concat\n",
      "2021-10-12 16:34:16,811 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer07/attention/key_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,811 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer07/attention/value_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,811 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer07/attention/query_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,811 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer07/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,811 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer07/intermediate/Tensordot/concat\n",
      "2021-10-12 16:34:16,811 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer07/output/Tensordot/concat\n",
      "2021-10-12 16:34:16,812 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer08/attention/key_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,812 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer08/attention/value_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,812 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer08/attention/query_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,812 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer08/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,812 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer08/intermediate/Tensordot/concat\n",
      "2021-10-12 16:34:16,812 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer08/output/Tensordot/concat\n",
      "2021-10-12 16:34:16,812 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer09/attention/key_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,812 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer09/attention/value_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,813 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer09/attention/query_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,813 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer09/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,813 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer09/intermediate/Tensordot/concat\n",
      "2021-10-12 16:34:16,813 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer09/output/Tensordot/concat\n",
      "2021-10-12 16:34:16,813 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer10/attention/key_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,813 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer10/attention/value_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,813 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer10/attention/query_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,813 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer10/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,814 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer10/intermediate/Tensordot/concat\n",
      "2021-10-12 16:34:16,814 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer10/output/Tensordot/concat\n",
      "2021-10-12 16:34:16,814 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer11/attention/key_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,814 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer11/attention/value_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,814 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer11/attention/query_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,814 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer11/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,814 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer11/intermediate/Tensordot/concat\n",
      "2021-10-12 16:34:16,814 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer11/output/Tensordot/concat\n",
      "2021-10-12 16:34:16,814 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer12/attention/key_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,815 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer12/attention/value_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,815 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer12/attention/query_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,815 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer12/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,815 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer12/intermediate/Tensordot/concat\n",
      "2021-10-12 16:34:16,815 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer12/output/Tensordot/concat\n",
      "2021-10-12 16:34:16,815 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer13/attention/key_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,815 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer13/attention/value_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,815 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer13/attention/query_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,816 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer13/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,816 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer13/intermediate/Tensordot/concat\n",
      "2021-10-12 16:34:16,816 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer13/output/Tensordot/concat\n",
      "2021-10-12 16:34:16,816 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer14/attention/key_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,816 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer14/attention/value_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,816 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer14/attention/query_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,816 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer14/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,816 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer14/intermediate/Tensordot/concat\n",
      "2021-10-12 16:34:16,816 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer14/output/Tensordot/concat\n",
      "2021-10-12 16:34:16,817 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer15/attention/key_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,817 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer15/attention/value_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,817 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer15/attention/query_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,817 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer15/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,817 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer15/intermediate/Tensordot/concat\n",
      "2021-10-12 16:34:16,817 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer15/output/Tensordot/concat\n",
      "2021-10-12 16:34:16,817 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer16/attention/key_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,817 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer16/attention/value_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,818 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer16/attention/query_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,818 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer16/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,818 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer16/intermediate/Tensordot/concat\n",
      "2021-10-12 16:34:16,818 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer16/output/Tensordot/concat\n",
      "2021-10-12 16:34:16,818 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer17/attention/key_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,818 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer17/attention/value_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,818 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer17/attention/query_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,818 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer17/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,819 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer17/intermediate/Tensordot/concat\n",
      "2021-10-12 16:34:16,819 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer17/output/Tensordot/concat\n",
      "2021-10-12 16:34:16,819 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer18/attention/key_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,819 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer18/attention/value_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,819 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer18/attention/query_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,819 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer18/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,819 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer18/intermediate/Tensordot/concat\n",
      "2021-10-12 16:34:16,819 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer18/output/Tensordot/concat\n",
      "2021-10-12 16:34:16,819 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer19/attention/key_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,820 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer19/attention/value_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,820 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer19/attention/query_layer/Tensordot/concat\n",
      "2021-10-12 16:34:16,820 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer19/attention/context_projection_layer/Tensordot/concat\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-10-12 16:34:16,820 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer19/intermediate/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,820 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer19/output/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,820 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer20/attention/key_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,820 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer20/attention/value_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,820 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer20/attention/query_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,821 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer20/attention/context_projection_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,821 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer20/intermediate/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,821 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer20/output/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,821 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer21/attention/key_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,821 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer21/attention/value_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,821 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer21/attention/query_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,821 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer21/attention/context_projection_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,821 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer21/intermediate/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,821 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer21/output/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,822 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer22/attention/key_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,822 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer22/attention/value_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,822 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer22/attention/query_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,822 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer22/attention/context_projection_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,822 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer22/intermediate/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,822 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer22/output/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,822 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer23/attention/key_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,822 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer23/attention/value_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,822 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer23/attention/query_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,823 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer23/attention/context_projection_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,823 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer23/intermediate/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,823 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer23/output/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,823 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer24/attention/key_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,823 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer24/attention/value_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,823 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer24/attention/query_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,823 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer24/attention/context_projection_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,823 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer24/intermediate/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,823 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer24/output/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,824 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer25/attention/key_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,824 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer25/attention/value_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,824 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer25/attention/query_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,824 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer25/attention/context_projection_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,824 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer25/intermediate/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,824 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer25/output/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,824 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer26/attention/key_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,824 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer26/attention/value_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,824 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer26/attention/query_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,825 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer26/attention/context_projection_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,825 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer26/intermediate/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,825 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer26/output/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,825 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer27/attention/key_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,825 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer27/attention/value_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,825 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer27/attention/query_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,825 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer27/attention/context_projection_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,825 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer27/intermediate/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,826 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer27/output/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,826 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer28/attention/key_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,826 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer28/attention/value_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,826 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer28/attention/query_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,826 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer28/attention/context_projection_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,826 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer28/intermediate/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,826 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer28/output/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,826 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer29/attention/key_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,826 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer29/attention/value_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,827 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer29/attention/query_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,827 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer29/attention/context_projection_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,827 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer29/intermediate/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,827 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer29/output/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,827 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer30/attention/key_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,827 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer30/attention/value_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,827 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer30/attention/query_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,827 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer30/attention/context_projection_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,827 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer30/intermediate/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,828 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer30/output/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,828 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer31/attention/key_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,828 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer31/attention/value_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,828 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer31/attention/query_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,828 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer31/attention/context_projection_layer/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,828 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer31/intermediate/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,828 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt/layer31/output/Tensordot/concat\r\n",
      "2021-10-12 16:34:16,828 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt/layer00/activation/PartitionedCall/Sqrt\r\n",
      "2021-10-12 16:34:16,829 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt/layer01/activation_1/PartitionedCall/Sqrt\r\n",
      "2021-10-12 16:34:16,829 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt/layer02/activation_2/PartitionedCall/Sqrt\r\n",
      "2021-10-12 16:34:16,829 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt/layer03/activation_3/PartitionedCall/Sqrt\r\n",
      "2021-10-12 16:34:16,829 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt/layer04/activation_4/PartitionedCall/Sqrt\r\n",
      "2021-10-12 16:34:16,829 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt/layer05/activation_5/PartitionedCall/Sqrt\r\n",
      "2021-10-12 16:34:16,829 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt/layer06/activation_6/PartitionedCall/Sqrt\r\n",
      "2021-10-12 16:34:16,829 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt/layer07/activation_7/PartitionedCall/Sqrt\r\n",
      "2021-10-12 16:34:16,829 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt/layer08/activation_8/PartitionedCall/Sqrt\r\n",
      "2021-10-12 16:34:16,829 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt/layer09/activation_9/PartitionedCall/Sqrt\r\n",
      "2021-10-12 16:34:16,830 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt/layer10/activation_10/PartitionedCall/Sqrt\r\n",
      "2021-10-12 16:34:16,830 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt/layer11/activation_11/PartitionedCall/Sqrt\r\n",
      "2021-10-12 16:34:16,830 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt/layer12/activation_12/PartitionedCall/Sqrt\r\n",
      "2021-10-12 16:34:16,830 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt/layer13/activation_13/PartitionedCall/Sqrt\r\n",
      "2021-10-12 16:34:16,830 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt/layer14/activation_14/PartitionedCall/Sqrt\r\n",
      "2021-10-12 16:34:16,830 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt/layer15/activation_15/PartitionedCall/Sqrt\r\n",
      "2021-10-12 16:34:16,830 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt/layer16/activation_16/PartitionedCall/Sqrt\r\n",
      "2021-10-12 16:34:16,830 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt/layer17/activation_17/PartitionedCall/Sqrt\r\n",
      "2021-10-12 16:34:16,831 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt/layer18/activation_18/PartitionedCall/Sqrt\r\n",
      "2021-10-12 16:34:16,831 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt/layer19/activation_19/PartitionedCall/Sqrt\r\n",
      "2021-10-12 16:34:16,831 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt/layer20/activation_20/PartitionedCall/Sqrt\r\n",
      "2021-10-12 16:34:16,831 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt/layer21/activation_21/PartitionedCall/Sqrt\r\n",
      "2021-10-12 16:34:16,831 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt/layer22/activation_22/PartitionedCall/Sqrt\r\n",
      "2021-10-12 16:34:16,831 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt/layer23/activation_23/PartitionedCall/Sqrt\r\n",
      "2021-10-12 16:34:16,831 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt/layer24/activation_24/PartitionedCall/Sqrt\r\n",
      "2021-10-12 16:34:16,831 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt/layer25/activation_25/PartitionedCall/Sqrt\r\n",
      "2021-10-12 16:34:16,832 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt/layer26/activation_26/PartitionedCall/Sqrt\r\n",
      "2021-10-12 16:34:16,832 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt/layer27/activation_27/PartitionedCall/Sqrt\r\n",
      "2021-10-12 16:34:16,832 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt/layer28/activation_28/PartitionedCall/Sqrt\r\n",
      "2021-10-12 16:34:16,832 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt/layer29/activation_29/PartitionedCall/Sqrt\r\n",
      "2021-10-12 16:34:16,832 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt/layer30/activation_30/PartitionedCall/Sqrt\r\n",
      "2021-10-12 16:34:16,832 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt/layer31/activation_31/PartitionedCall/Sqrt\r\n",
      "2021-10-12 16:34:16,832 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/position_embedding/ReadVariableOp\r\n",
      "2021-10-12 16:34:16,839 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer00/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\r\n",
      "2021-10-12 16:34:16,839 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer00/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\r\n",
      "2021-10-12 16:34:16,839 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer00/attention/key_layer/Tensordot/ReadVariableOp\r\n",
      "2021-10-12 16:34:16,855 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer00/attention/key_layer/BiasAdd/ReadVariableOp\r\n",
      "2021-10-12 16:34:16,855 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer00/attention/value_layer/Tensordot/ReadVariableOp\r\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-10-12 16:34:16,881 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer00/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:16,881 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer00/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:16,897 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer00/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:16,897 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer00/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:16,913 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer00/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:16,913 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer00/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:16,914 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer00/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:16,914 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer00/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:17,108 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer00/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:17,109 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer00/output/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:17,298 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer00/output/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:17,298 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer01/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:17,298 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer01/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:17,298 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer01/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:17,325 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer01/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:17,325 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer01/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:17,351 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer01/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:17,351 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer01/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:17,377 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer01/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:17,377 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer01/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:17,402 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer01/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:17,402 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer01/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:17,403 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer01/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:17,403 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer01/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:17,595 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer01/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:17,595 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer01/output/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:17,782 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer01/output/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:17,783 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer02/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:17,783 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer02/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:17,783 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer02/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:17,809 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer02/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:17,809 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer02/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:17,835 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer02/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:17,835 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer02/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:17,862 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer02/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:17,862 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer02/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:17,889 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer02/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:17,889 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer02/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:17,890 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer02/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:17,890 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer02/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:18,079 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer02/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:18,079 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer02/output/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:18,268 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer02/output/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:18,268 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer03/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:18,268 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer03/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:18,268 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer03/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:18,295 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer03/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:18,296 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer03/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:18,322 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer03/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:18,323 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer03/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:18,349 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer03/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:18,350 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer03/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:18,376 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer03/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:18,376 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer03/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:18,377 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer03/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:18,377 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer03/intermediate/Tensordot/ReadVariableOp\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-10-12 16:34:18,568 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer03/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:18,568 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer03/output/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:18,756 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer03/output/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:18,756 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer04/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:18,756 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer04/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:18,756 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer04/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:18,783 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer04/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:18,783 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer04/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:18,810 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer04/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:18,810 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer04/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:18,836 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer04/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:18,836 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer04/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:18,863 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer04/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:18,863 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer04/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:18,863 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer04/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:18,863 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer04/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:19,054 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer04/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:19,055 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer04/output/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:19,243 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer04/output/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:19,243 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer05/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:19,244 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer05/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:19,244 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer05/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:19,271 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer05/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:19,271 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer05/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:19,298 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer05/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:19,298 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer05/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:19,324 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer05/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:19,324 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer05/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:19,351 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer05/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:19,351 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer05/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:19,351 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer05/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:19,351 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer05/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:19,542 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer05/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:19,543 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer05/output/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:19,732 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer05/output/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:19,733 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer06/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:19,733 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer06/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:19,733 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer06/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:19,760 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer06/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:19,760 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer06/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:19,787 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer06/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:19,787 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer06/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:19,814 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer06/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:19,815 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer06/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:19,841 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer06/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:19,841 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer06/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:19,841 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer06/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:19,841 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer06/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:20,032 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer06/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:20,033 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer06/output/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:20,222 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer06/output/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:20,222 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer07/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:20,222 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer07/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:20,222 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer07/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:20,249 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer07/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:20,250 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer07/attention/value_layer/Tensordot/ReadVariableOp\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-10-12 16:34:20,276 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer07/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:20,277 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer07/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:20,303 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer07/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:20,303 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer07/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:20,330 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer07/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:20,330 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer07/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:20,330 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer07/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:20,330 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer07/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:20,521 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer07/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:20,521 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer07/output/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:20,711 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer07/output/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:20,711 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer08/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:20,712 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer08/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:20,712 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer08/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:20,739 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer08/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:20,739 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer08/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:20,766 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer08/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:20,766 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer08/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:20,792 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer08/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:20,792 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer08/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:20,819 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer08/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:20,819 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer08/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:20,819 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer08/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:20,820 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer08/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:21,010 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer08/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:21,011 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer08/output/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:21,200 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer08/output/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:21,200 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer09/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:21,201 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer09/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:21,201 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer09/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:21,227 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer09/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:21,227 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer09/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:21,254 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer09/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:21,254 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer09/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:21,281 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer09/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:21,281 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer09/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:21,307 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer09/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:21,308 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer09/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:21,308 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer09/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:21,308 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer09/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:21,498 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer09/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:21,499 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer09/output/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:21,687 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer09/output/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:21,688 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer10/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:21,688 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer10/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:21,688 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer10/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:21,715 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer10/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:21,715 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer10/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:21,742 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer10/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:21,742 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer10/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:21,768 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer10/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:21,768 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer10/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:21,794 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer10/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:21,795 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer10/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:21,795 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer10/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:21,795 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer10/intermediate/Tensordot/ReadVariableOp\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-10-12 16:34:21,986 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer10/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:21,986 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer10/output/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:22,175 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer10/output/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:22,176 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer11/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:22,176 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer11/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:22,176 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer11/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:22,203 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer11/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:22,203 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer11/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:22,229 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer11/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:22,230 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer11/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:22,255 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer11/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:22,255 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer11/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:22,283 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer11/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:22,283 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer11/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:22,283 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer11/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:22,283 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer11/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:22,474 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer11/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:22,475 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer11/output/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:22,664 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer11/output/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:22,665 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer12/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:22,665 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer12/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:22,665 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer12/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:22,692 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer12/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:22,692 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer12/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:22,718 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer12/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:22,719 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer12/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:22,745 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer12/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:22,745 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer12/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:22,772 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer12/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:22,772 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer12/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:22,772 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer12/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:22,772 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer12/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:22,964 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer12/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:22,964 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer12/output/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:23,153 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer12/output/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:23,154 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer13/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:23,154 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer13/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:23,154 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer13/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:23,180 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer13/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:23,180 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer13/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:23,207 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer13/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:23,207 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer13/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:23,234 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer13/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:23,234 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer13/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:23,260 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer13/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:23,261 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer13/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:23,261 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer13/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:23,261 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer13/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:23,452 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer13/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:23,452 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer13/output/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:23,642 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer13/output/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:23,642 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer14/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:23,642 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer14/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:23,642 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer14/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:23,669 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer14/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:23,669 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer14/attention/value_layer/Tensordot/ReadVariableOp\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-10-12 16:34:23,695 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer14/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:23,695 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer14/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:23,723 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer14/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:23,723 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer14/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:23,751 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer14/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:23,751 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer14/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:23,752 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer14/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:23,752 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer14/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:23,941 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer14/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:23,941 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer14/output/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:24,131 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer14/output/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:24,131 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer15/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:24,131 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer15/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:24,132 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer15/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:24,159 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer15/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:24,159 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer15/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:24,185 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer15/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:24,186 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer15/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:24,212 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer15/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:24,212 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer15/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:24,241 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer15/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:24,241 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer15/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:24,241 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer15/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:24,241 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer15/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:24,430 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer15/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:24,431 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer15/output/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:24,620 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer15/output/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:24,621 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer16/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:24,621 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer16/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:24,621 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer16/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:24,656 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer16/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:24,656 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer16/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:24,683 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer16/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:24,683 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer16/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:24,710 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer16/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:24,710 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer16/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:24,737 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer16/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:24,737 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer16/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:24,737 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer16/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:24,737 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer16/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:24,927 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer16/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:24,927 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer16/output/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:25,117 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer16/output/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:25,117 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer17/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:25,117 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer17/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:25,117 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer17/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:25,144 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer17/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:25,144 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer17/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:25,172 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer17/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:25,172 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer17/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:25,199 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer17/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:25,199 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer17/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:25,226 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer17/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:25,226 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer17/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:25,227 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer17/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:25,227 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer17/intermediate/Tensordot/ReadVariableOp\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-10-12 16:34:25,416 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer17/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:25,417 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer17/output/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:25,606 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer17/output/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:25,607 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer18/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:25,607 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer18/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:25,607 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer18/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:25,634 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer18/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:25,634 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer18/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:25,661 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer18/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:25,661 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer18/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:25,688 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer18/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:25,688 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer18/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:25,715 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer18/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:25,715 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer18/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:25,716 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer18/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:25,716 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer18/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:25,905 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer18/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:25,905 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer18/output/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:26,095 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer18/output/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:26,095 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer19/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:26,096 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer19/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:26,096 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer19/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:26,122 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer19/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:26,122 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer19/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:26,149 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer19/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:26,149 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer19/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:26,176 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer19/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:26,176 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer19/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:26,203 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer19/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:26,203 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer19/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:26,203 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer19/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:26,204 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer19/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:26,393 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer19/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:26,394 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer19/output/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:26,583 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer19/output/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:26,583 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer20/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:26,583 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer20/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:26,584 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer20/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:26,611 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer20/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:26,611 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer20/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:26,637 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer20/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:26,637 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer20/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:26,664 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer20/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:26,664 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer20/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:26,691 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer20/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:26,691 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer20/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:26,691 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer20/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:26,691 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer20/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:26,881 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer20/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:26,881 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer20/output/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:27,070 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer20/output/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:27,071 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer21/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:27,071 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer21/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:27,071 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer21/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:27,098 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer21/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:27,098 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer21/attention/value_layer/Tensordot/ReadVariableOp\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-10-12 16:34:27,125 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer21/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:27,125 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer21/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:27,152 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer21/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:27,152 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer21/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:27,179 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer21/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:27,179 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer21/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:27,179 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer21/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:27,179 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer21/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:27,369 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer21/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:27,369 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer21/output/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:27,559 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer21/output/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:27,559 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer22/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:27,559 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer22/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:27,559 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer22/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:27,586 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer22/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:27,586 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer22/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:27,612 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer22/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:27,612 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer22/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:27,639 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer22/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:27,639 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer22/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:27,666 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer22/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:27,666 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer22/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:27,666 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer22/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:27,666 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer22/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:27,856 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer22/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:27,856 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer22/output/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:28,045 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer22/output/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:28,046 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer23/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:28,046 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer23/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:28,046 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer23/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:28,073 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer23/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:28,073 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer23/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:28,100 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer23/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:28,100 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer23/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:28,127 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer23/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:28,127 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer23/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:28,154 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer23/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:28,154 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer23/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:28,154 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer23/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:28,154 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer23/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:28,343 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer23/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:28,344 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer23/output/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:28,533 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer23/output/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:28,533 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer24/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:28,534 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer24/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:28,534 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer24/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:28,559 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer24/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:28,560 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer24/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:28,586 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer24/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:28,587 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer24/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:28,613 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer24/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:28,614 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer24/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:28,640 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer24/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:28,641 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer24/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:28,641 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer24/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:28,641 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer24/intermediate/Tensordot/ReadVariableOp\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-10-12 16:34:28,830 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer24/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:28,831 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer24/output/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:29,020 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer24/output/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:29,020 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer25/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:29,020 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer25/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:29,021 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer25/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:29,048 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer25/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:29,048 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer25/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:29,074 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer25/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:29,074 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer25/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:29,101 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer25/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:29,102 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer25/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:29,128 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer25/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:29,128 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer25/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:29,128 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer25/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:29,128 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer25/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:29,319 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer25/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:29,320 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer25/output/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:29,510 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer25/output/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:29,510 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer26/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:29,510 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer26/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:29,511 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer26/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:29,538 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer26/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:29,538 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer26/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:29,564 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer26/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:29,564 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer26/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:29,591 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer26/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:29,591 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer26/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:29,618 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer26/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:29,618 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer26/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:29,618 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer26/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:29,618 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer26/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:29,808 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer26/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:29,809 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer26/output/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:29,998 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer26/output/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:29,998 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer27/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:29,999 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer27/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:29,999 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer27/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:30,026 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer27/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:30,026 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer27/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:30,052 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer27/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:30,052 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer27/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:30,078 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer27/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:30,079 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer27/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:30,106 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer27/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:30,106 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer27/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:30,106 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer27/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:30,106 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer27/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:30,296 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer27/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:30,296 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer27/output/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:30,486 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer27/output/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:30,486 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer28/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:30,486 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer28/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:30,486 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer28/attention/key_layer/Tensordot/ReadVariableOp\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-10-12 16:34:30,513 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer28/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:30,514 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer28/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:30,542 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer28/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:30,542 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer28/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:30,568 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer28/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:30,569 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer28/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:30,595 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer28/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:30,595 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer28/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:30,596 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer28/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:30,596 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer28/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:30,785 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer28/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:30,786 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer28/output/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:30,975 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer28/output/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:30,975 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer29/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:30,975 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer29/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:30,975 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer29/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:31,002 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer29/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:31,003 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer29/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:31,029 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer29/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:31,029 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer29/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:31,057 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer29/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:31,057 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer29/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:31,083 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer29/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:31,084 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer29/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:31,084 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer29/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:31,084 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer29/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:31,273 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer29/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:31,274 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer29/output/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:31,463 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer29/output/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:31,464 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer30/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:31,464 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer30/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:31,464 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer30/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:31,491 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer30/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:31,492 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer30/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:31,520 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer30/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:31,520 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer30/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:31,546 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer30/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:31,547 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer30/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:31,572 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer30/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:31,573 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer30/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:31,573 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer30/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:31,573 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer30/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:31,762 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer30/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:31,763 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer30/output/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:31,952 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer30/output/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:31,952 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer31/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:31,953 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer31/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:31,953 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer31/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:31,981 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer31/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:31,981 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer31/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:32,008 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer31/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:32,008 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer31/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:32,035 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer31/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:32,035 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer31/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:32,062 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer31/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:32,062 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer31/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:32,062 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer31/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:32,063 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer31/intermediate/Tensordot/ReadVariableOp\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-10-12 16:34:32,252 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer31/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:32,252 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer31/output/Tensordot/ReadVariableOp\n",
      "2021-10-12 16:34:32,442 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/layer31/output/BiasAdd/ReadVariableOp\n",
      "2021-10-12 16:34:32,442 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/LayerNorm_final_norm/batchnorm/mul/ReadVariableOp\n",
      "2021-10-12 16:34:32,442 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/LayerNorm_final_norm/batchnorm/ReadVariableOp\n",
      "2021-10-12 16:34:32,442 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt/Identity\n",
      "2021-10-12 16:34:37,117 - INFO - Optimizing ONNX model\n",
      "2021-10-12 16:36:30,567 - INFO - After optimization: Cast -514 (901->387), Concat -320 (545->225), Const -4223 (4765->542), Gather -256 (385->129), GlobalAveragePool +130 (0->130), Identity -266 (266->0), ReduceMean -130 (130->0), ReduceProd -384 (384->0), Reshape -129 (609->480), Shape -96 (257->161), Squeeze -33 (33->0), Transpose -225 (353->128), Unsqueeze -930 (995->65)\n",
      "2021-10-12 16:36:51,459 - INFO - \n",
      "2021-10-12 16:36:51,459 - INFO - Successfully converted TensorFlow model cpm_saved_model to ONNX\n",
      "2021-10-12 16:36:51,460 - INFO - Model inputs: ['input_ids']\n",
      "2021-10-12 16:36:51,460 - INFO - Model outputs: ['output_1']\n",
      "2021-10-12 16:36:51,461 - INFO - Zipped ONNX model is saved at onnx/cpm.onnx. Unzip before opening in onnxruntime.\n"
     ]
    }
   ],
   "source": [
    "!mkdir -p onnx\n",
    "!python -m tf2onnx.convert --saved-model cpm_saved_model --output onnx/cpm.zip --large_model --opset=13"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# to unzip it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "import onnx\n",
    "from onnxruntime.quantization import quantize_dynamic, QuantType"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "!mkdir -p onnx_q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "quantized_model = quantize_dynamic(\n",
    "    './onnx/__MODEL_PROTO.onnx',\n",
    "    './onnx_q/cpm.onnx',\n",
    "    weight_type=QuantType.QUInt8,\n",
    "    use_external_data_format=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}