Upload remove-donut-tokens.ipynb

remove-donut-tokens.ipynb

@@ -106,26 +106,26 @@
     {
      "data": {
       "text/plain": [
-       "['덟',\n",
-       " '引起',\n",
-       " 'ärer',\n",
-       " '▁vaziyat',\n",
-       " '☻',\n",
-       " '▁līderi',\n",
-       " '▁సంతోష',\n",
-       " '▁פלילי',\n",
-       " '▁obvi',\n",
-       " '▁автомобиля',\n",
-       " '赂',\n",
-       " '400',\n",
-       " 'ॠ',\n",
-       " '摩擦',\n",
-       " '▁Rund',\n",
-       " 'ضخم',\n",
-       " '▁ruin',\n",
-       " '▁Center',\n",
-       " 'DOWNLOAD',\n",
-       " '😇']"
+       "['▁1',\n",
+       " '▁kilala',\n",
+       " 'ସୀ',\n",
+       " '错误',\n",
+       " '▁https',\n",
+       " '▁жасаған',\n",
+       " '病院',\n",
+       " '▁effort',\n",
+       " '▁기사',\n",
+       " 'ptu',\n",
+       " '윽',\n",
+       " 'jit',\n",
+       " '▁college',\n",
+       " '383',\n",
+       " '▁dveře',\n",
+       " 'masse',\n",
+       " '\\x9c',\n",
+       " 'rait',\n",
+       " '闰',\n",
+       " '▁gewoonlik']"
       ]
      },
      "execution_count": 3,
@@ -185,7 +185,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -203,7 +203,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [
     {
@@ -212,7 +212,7 @@
        "27510"
       ]
      },
-     "execution_count": 10,
+     "execution_count": 7,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -234,7 +234,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [
     {
@@ -250,7 +250,7 @@
        "[0, 56881, 3, 2]"
       ]
      },
-     "execution_count": 11,
+     "execution_count": 8,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -264,7 +264,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [
     {
@@ -283,7 +283,7 @@
        " (2, '</s>')]"
       ]
      },
-     "execution_count": 12,
+     "execution_count": 9,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -298,7 +298,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [
     {
@@ -309,7 +309,7 @@
        " type: NORMAL]"
       ]
      },
-     "execution_count": 13,
+     "execution_count": 10,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -320,7 +320,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 11,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -339,7 +339,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 19,
+   "execution_count": 12,
    "metadata": {},
    "outputs": [
     {
@@ -348,7 +348,7 @@
        "27511"
       ]
      },
-     "execution_count": 19,
+     "execution_count": 12,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -366,7 +366,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": 13,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -383,7 +383,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 17,
+   "execution_count": 14,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -394,7 +394,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 18,
+   "execution_count": 15,
    "metadata": {},
    "outputs": [
     {
@@ -403,7 +403,7 @@
        "(27513, 57525)"
       ]
      },
-     "execution_count": 18,
+     "execution_count": 15,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -421,7 +421,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 20,
+   "execution_count": 16,
    "metadata": {},
    "outputs": [
     {
@@ -430,7 +430,7 @@
        "['<s_iitcdip>', '<s_synthdog>']"
       ]
      },
-     "execution_count": 20,
+     "execution_count": 16,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -441,7 +441,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 21,
+   "execution_count": 17,
    "metadata": {},
    "outputs": [
     {
@@ -450,7 +450,7 @@
        "2"
       ]
      },
-     "execution_count": 21,
+     "execution_count": 17,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -461,7 +461,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 22,
+   "execution_count": 18,
    "metadata": {},
    "outputs": [
     {
@@ -470,7 +470,7 @@
        "['▁<', 's', '>']"
       ]
      },
-     "execution_count": 22,
+     "execution_count": 18,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -481,7 +481,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 23,
+   "execution_count": 19,
    "metadata": {},
    "outputs": [
     {
@@ -497,7 +497,7 @@
        " 'additional_special_tokens': ['<s_iitcdip>', '<s_synthdog>']}"
       ]
      },
-     "execution_count": 23,
+     "execution_count": 19,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -508,7 +508,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 24,
+   "execution_count": 20,
    "metadata": {},
    "outputs": [
     {
@@ -520,7 +520,7 @@
        " 'donut-base-ascii/added_tokens.json')"
       ]
      },
-     "execution_count": 24,
+     "execution_count": 20,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -535,7 +535,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 25,
+   "execution_count": 21,
    "metadata": {},
    "outputs": [
     {
@@ -544,7 +544,7 @@
        "['<s>']"
       ]
      },
-     "execution_count": 25,
+     "execution_count": 21,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -555,7 +555,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 26,
+   "execution_count": 22,
    "metadata": {},
    "outputs": [
     {
@@ -564,7 +564,7 @@
        "(27515, 57525)"
       ]
      },
-     "execution_count": 26,
+     "execution_count": 22,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -575,7 +575,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 28,
+   "execution_count": 23,
    "metadata": {},
    "outputs": [
     {
@@ -584,7 +584,7 @@
        "['<s>']"
       ]
      },
-     "execution_count": 28,
+     "execution_count": 23,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -614,7 +614,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 29,
+   "execution_count": 24,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -632,7 +632,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 30,
+   "execution_count": 25,
    "metadata": {},
    "outputs": [
     {
@@ -644,7 +644,7 @@
        " ('<s_synthdog>', 27514)]"
       ]
      },
-     "execution_count": 30,
+     "execution_count": 25,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -666,9 +666,59 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 32,
+   "execution_count": 26,
    "metadata": {},
    "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "===================================BUG REPORT===================================\n",
+      "Welcome to bitsandbytes. For bug reports, please run\n",
+      "\n",
+      "python -m bitsandbytes\n",
+      "\n",
+      " and submit this information together with your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n",
+      "================================================================================\n",
+      "bin /home/nicholas/miniconda3/lib/python3.10/site-packages/bitsandbytes/libbitsandbytes_cpu.so\n",
+      "/home/nicholas/miniconda3/lib/python3.10/site-packages/bitsandbytes/libbitsandbytes_cpu.so: undefined symbol: cadam32bit_grad_fp32\n",
+      "CUDA_SETUP: WARNING! libcudart.so not found in any environmental path. Searching in backup paths...\n",
+      "CUDA SETUP: Highest compute capability among GPUs detected: 8.9\n",
+      "CUDA SETUP: Detected CUDA version 117\n",
+      "CUDA SETUP: Loading binary /home/nicholas/miniconda3/lib/python3.10/site-packages/bitsandbytes/libbitsandbytes_cpu.so...\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/nicholas/miniconda3/lib/python3.10/site-packages/bitsandbytes/cextension.py:34: UserWarning: The installed version of bitsandbytes was compiled without GPU support. 8-bit optimizers, 8-bit multiplication, and GPU quantization are unavailable.\n",
+      "  warn(\"The installed version of bitsandbytes was compiled without GPU support. \"\n",
+      "/home/nicholas/miniconda3/lib/python3.10/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: /home/nicholas/miniconda3 did not contain ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] as expected! Searching further paths...\n",
+      "  warn(msg)\n",
+      "/home/nicholas/miniconda3/lib/python3.10/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/usr/share/gconf/ubuntu.default.path')}\n",
+      "  warn(msg)\n",
+      "/home/nicholas/miniconda3/lib/python3.10/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('0'), PosixPath('1')}\n",
+      "  warn(msg)\n",
+      "/home/nicholas/miniconda3/lib/python3.10/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/usr/share/gconf/ubuntu.mandatory.path')}\n",
+      "  warn(msg)\n",
+      "/home/nicholas/miniconda3/lib/python3.10/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('@/tmp/.ICE-unix/2404,unix/nicholas-MS-7D25'), PosixPath('local/nicholas-MS-7D25')}\n",
+      "  warn(msg)\n",
+      "/home/nicholas/miniconda3/lib/python3.10/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('vs/workbench/api/node/extensionHostProcess')}\n",
+      "  warn(msg)\n",
+      "/home/nicholas/miniconda3/lib/python3.10/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/etc/xdg/xdg-ubuntu')}\n",
+      "  warn(msg)\n",
+      "/home/nicholas/miniconda3/lib/python3.10/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/home/nicholas/.local/share/flatpak/exports/share')}\n",
+      "  warn(msg)\n",
+      "/home/nicholas/miniconda3/lib/python3.10/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('module'), PosixPath('//matplotlib_inline.backend_inline')}\n",
+      "  warn(msg)\n",
+      "/home/nicholas/miniconda3/lib/python3.10/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/usr/local/cuda/lib64')}\n",
+      "  warn(msg)\n",
+      "/home/nicholas/miniconda3/lib/python3.10/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: No libcudart.so found! Install CUDA or the cudatoolkit package (anaconda)!\n",
+      "  warn(msg)\n"
+     ]
+    },
     {
      "name": "stdout",
      "output_type": "stream",
@@ -693,1022 +743,22 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 33,
+   "execution_count": 27,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "[0,\n",
-       " 1,\n",
-       " 2,\n",
-       " 3,\n",
-       " 4,\n",
-       " 5,\n",
-       " 6,\n",
-       " 7,\n",
-       " 8,\n",
-       " 10,\n",
-       " 12,\n",
-       " 13,\n",
-       " 17,\n",
-       " 18,\n",
-       " 26,\n",
-       " 28,\n",
-       " 30,\n",
-       " 32,\n",
-       " 33,\n",
-       " 35,\n",
-       " 36,\n",
-       " 38,\n",
-       " 40,\n",
-       " 43,\n",
-       " 44,\n",
-       " 45,\n",
-       " 47,\n",
-       " 48,\n",
-       " 51,\n",
-       " 52,\n",
-       " 54,\n",
-       " 56,\n",
-       " 57,\n",
-       " 58,\n",
-       " 60,\n",
-       " 61,\n",
-       " 63,\n",
-       " 64,\n",
-       " 69,\n",
-       " 70,\n",
-       " 72,\n",
-       " 74,\n",
-       " 75,\n",
-       " 76,\n",
-       " 77,\n",
-       " 79,\n",
-       " 82,\n",
-       " 84,\n",
-       " 85,\n",
-       " 87,\n",
-       " 88,\n",
-       " 90,\n",
-       " 93,\n",
-       " 94,\n",
-       " 95,\n",
-       " 103,\n",
-       " 104,\n",
-       " 105,\n",
-       " 107,\n",
-       " 110,\n",
-       " 111,\n",
-       " 112,\n",
-       " 113,\n",
-       " 118,\n",
-       " 119,\n",
-       " 120,\n",
-       " 123,\n",
-       " 124,\n",
-       " 130,\n",
-       " 133,\n",
-       " 136,\n",
-       " 139,\n",
-       " 146,\n",
-       " 148,\n",
-       " 149,\n",
-       " 150,\n",
-       " 152,\n",
-       " 153,\n",
-       " 155,\n",
-       " 157,\n",
-       " 160,\n",
-       " 162,\n",
-       " 168,\n",
-       " 172,\n",
-       " 179,\n",
-       " 182,\n",
-       " 187,\n",
-       " 191,\n",
-       " 195,\n",
-       " 197,\n",
-       " 199,\n",
-       " 201,\n",
-       " 202,\n",
-       " 205,\n",
-       " 207,\n",
-       " 208,\n",
-       " 209,\n",
-       " 210,\n",
-       " 211,\n",
-       " 212,\n",
-       " 214,\n",
-       " 215,\n",
-       " 216,\n",
-       " 220,\n",
-       " 226,\n",
-       " 228,\n",
-       " 230,\n",
-       " 233,\n",
-       " 234,\n",
-       " 238,\n",
-       " 240,\n",
-       " 241,\n",
-       " 243,\n",
-       " 244,\n",
-       " 245,\n",
-       " 248,\n",
-       " 249,\n",
-       " 250,\n",
-       " 251,\n",
-       " 252,\n",
-       " 256,\n",
-       " 258,\n",
-       " 261,\n",
-       " 262,\n",
-       " 264,\n",
-       " 265,\n",
-       " 266,\n",
-       " 267,\n",
-       " 271,\n",
-       " 272,\n",
-       " 273,\n",
-       " 274,\n",
-       " 275,\n",
-       " 281,\n",
-       " 285,\n",
-       " 288,\n",
-       " 290,\n",
-       " 294,\n",
-       " 301,\n",
-       " 302,\n",
-       " 303,\n",
-       " 304,\n",
-       " 306,\n",
-       " 308,\n",
-       " 311,\n",
-       " 313,\n",
-       " 315,\n",
-       " 317,\n",
-       " 322,\n",
-       " 333,\n",
-       " 336,\n",
-       " 337,\n",
-       " 341,\n",
-       " 342,\n",
-       " 348,\n",
-       " 353,\n",
-       " 354,\n",
-       " 355,\n",
-       " 356,\n",
-       " 359,\n",
-       " 360,\n",
-       " 361,\n",
-       " 363,\n",
-       " 373,\n",
-       " 375,\n",
-       " 378,\n",
-       " 379,\n",
-       " 380,\n",
-       " 381,\n",
-       " 383,\n",
-       " 385,\n",
-       " 389,\n",
-       " 390,\n",
-       " 391,\n",
-       " 394,\n",
-       " 398,\n",
-       " 400,\n",
-       " 401,\n",
-       " 404,\n",
-       " 409,\n",
-       " 410,\n",
-       " 416,\n",
-       " 417,\n",
-       " 419,\n",
-       " 422,\n",
-       " 426,\n",
-       " 428,\n",
-       " 430,\n",
-       " 436,\n",
-       " 437,\n",
-       " 438,\n",
-       " 443,\n",
-       " 446,\n",
-       " 448,\n",
-       " 452,\n",
-       " 454,\n",
-       " 456,\n",
-       " 458,\n",
-       " 460,\n",
-       " 463,\n",
-       " 465,\n",
-       " 468,\n",
-       " 469,\n",
-       " 470,\n",
-       " 472,\n",
-       " 477,\n",
-       " 478,\n",
-       " 479,\n",
-       " 480,\n",
-       " 482,\n",
-       " 483,\n",
-       " 485,\n",
-       " 486,\n",
-       " 487,\n",
-       " 489,\n",
-       " 491,\n",
-       " 497,\n",
-       " 498,\n",
-       " 500,\n",
-       " 502,\n",
-       " 505,\n",
-       " 506,\n",
-       " 507,\n",
-       " 510,\n",
-       " 511,\n",
-       " 513,\n",
-       " 516,\n",
-       " 517,\n",
-       " 520,\n",
-       " 526,\n",
-       " 527,\n",
-       " 530,\n",
-       " 531,\n",
-       " 534,\n",
-       " 535,\n",
-       " 543,\n",
-       " 546,\n",
-       " 550,\n",
-       " 552,\n",
-       " 553,\n",
-       " 554,\n",
-       " 556,\n",
-       " 559,\n",
-       " 560,\n",
-       " 566,\n",
-       " 568,\n",
-       " 571,\n",
-       " 575,\n",
-       " 580,\n",
-       " 583,\n",
-       " 584,\n",
-       " 586,\n",
-       " 589,\n",
-       " 593,\n",
-       " 595,\n",
-       " 596,\n",
-       " 597,\n",
-       " 600,\n",
-       " 602,\n",
-       " 605,\n",
-       " 609,\n",
-       " 610,\n",
-       " 612,\n",
-       " 613,\n",
-       " 614,\n",
-       " 617,\n",
-       " 620,\n",
-       " 621,\n",
-       " 622,\n",
-       " 626,\n",
-       " 628,\n",
-       " 631,\n",
-       " 632,\n",
-       " 633,\n",
-       " 641,\n",
-       " 644,\n",
-       " 645,\n",
-       " 646,\n",
-       " 647,\n",
-       " 648,\n",
-       " 651,\n",
-       " 655,\n",
-       " 656,\n",
-       " 657,\n",
-       " 658,\n",
-       " 659,\n",
-       " 660,\n",
-       " 661,\n",
-       " 662,\n",
-       " 663,\n",
-       " 666,\n",
-       " 669,\n",
-       " 670,\n",
-       " 673,\n",
-       " 674,\n",
-       " 675,\n",
-       " 676,\n",
-       " 678,\n",
-       " 679,\n",
-       " 680,\n",
-       " 683,\n",
-       " 687,\n",
-       " 696,\n",
-       " 699,\n",
-       " 700,\n",
-       " 701,\n",
-       " 702,\n",
-       " 705,\n",
-       " 707,\n",
-       " 710,\n",
-       " 711,\n",
-       " 715,\n",
-       " 716,\n",
-       " 718,\n",
-       " 720,\n",
-       " 721,\n",
-       " 724,\n",
-       " 735,\n",
-       " 736,\n",
-       " 737,\n",
-       " 739,\n",
-       " 741,\n",
-       " 742,\n",
-       " 744,\n",
-       " 746,\n",
-       " 747,\n",
-       " 750,\n",
-       " 751,\n",
-       " 753,\n",
-       " 755,\n",
-       " 758,\n",
-       " 761,\n",
-       " 762,\n",
-       " 765,\n",
-       " 766,\n",
-       " 776,\n",
-       " 778,\n",
-       " 782,\n",
-       " 783,\n",
-       " 785,\n",
-       " 786,\n",
-       " 787,\n",
-       " 788,\n",
-       " 789,\n",
-       " 790,\n",
-       " 791,\n",
-       " 794,\n",
-       " 796,\n",
-       " 797,\n",
-       " 800,\n",
-       " 802,\n",
-       " 803,\n",
-       " 805,\n",
-       " 809,\n",
-       " 812,\n",
-       " 815,\n",
-       " 816,\n",
-       " 827,\n",
-       " 829,\n",
-       " 830,\n",
-       " 832,\n",
-       " 833,\n",
-       " 844,\n",
-       " 846,\n",
-       " 847,\n",
-       " 851,\n",
-       " 853,\n",
-       " 854,\n",
-       " 857,\n",
-       " 861,\n",
-       " 864,\n",
-       " 865,\n",
-       " 867,\n",
-       " 868,\n",
-       " 870,\n",
-       " 871,\n",
-       " 878,\n",
-       " 879,\n",
-       " 883,\n",
-       " 885,\n",
-       " 888,\n",
-       " 889,\n",
-       " 892,\n",
-       " 897,\n",
-       " 898,\n",
-       " 899,\n",
-       " 901,\n",
-       " 903,\n",
-       " 906,\n",
-       " 908,\n",
-       " 911,\n",
-       " 917,\n",
-       " 918,\n",
-       " 921,\n",
-       " 924,\n",
-       " 929,\n",
-       " 937,\n",
-       " 944,\n",
-       " 948,\n",
-       " 949,\n",
-       " 950,\n",
-       " 954,\n",
-       " 955,\n",
-       " 957,\n",
-       " 958,\n",
-       " 959,\n",
-       " 960,\n",
-       " 962,\n",
-       " 965,\n",
-       " 969,\n",
-       " 970,\n",
-       " 972,\n",
-       " 973,\n",
-       " 975,\n",
-       " 977,\n",
-       " 979,\n",
-       " 982,\n",
-       " 983,\n",
-       " 984,\n",
-       " 990,\n",
-       " 992,\n",
-       " 993,\n",
-       " 998,\n",
-       " 999,\n",
-       " 1001,\n",
-       " 1002,\n",
-       " 1003,\n",
-       " 1004,\n",
-       " 1007,\n",
-       " 1009,\n",
-       " 1010,\n",
-       " 1012,\n",
-       " 1016,\n",
-       " 1017,\n",
-       " 1018,\n",
-       " 1023,\n",
-       " 1024,\n",
-       " 1027,\n",
-       " 1028,\n",
-       " 1030,\n",
-       " 1031,\n",
-       " 1034,\n",
-       " 1036,\n",
-       " 1039,\n",
-       " 1051,\n",
-       " 1054,\n",
-       " 1056,\n",
-       " 1059,\n",
-       " 1062,\n",
-       " 1069,\n",
-       " 1074,\n",
-       " 1076,\n",
-       " 1077,\n",
-       " 1079,\n",
-       " 1080,\n",
-       " 1082,\n",
-       " 1085,\n",
-       " 1088,\n",
-       " 1089,\n",
-       " 1095,\n",
-       " 1096,\n",
-       " 1097,\n",
-       " 1098,\n",
-       " 1103,\n",
-       " 1105,\n",
-       " 1107,\n",
-       " 1108,\n",
-       " 1110,\n",
-       " 1112,\n",
-       " 1114,\n",
-       " 1116,\n",
-       " 1119,\n",
-       " 1121,\n",
-       " 1122,\n",
-       " 1123,\n",
-       " 1124,\n",
-       " 1126,\n",
-       " 1128,\n",
-       " 1129,\n",
-       " 1131,\n",
-       " 1132,\n",
-       " 1133,\n",
-       " 1138,\n",
-       " 1139,\n",
-       " 1140,\n",
-       " 1144,\n",
-       " 1145,\n",
-       " 1148,\n",
-       " 1152,\n",
-       " 1153,\n",
-       " 1156,\n",
-       " 1157,\n",
-       " 1160,\n",
-       " 1161,\n",
-       " 1162,\n",
-       " 1164,\n",
-       " 1165,\n",
-       " 1166,\n",
-       " 1169,\n",
-       " 1172,\n",
-       " 1178,\n",
-       " 1181,\n",
-       " 1182,\n",
-       " 1185,\n",
-       " 1186,\n",
-       " 1190,\n",
-       " 1191,\n",
-       " 1194,\n",
-       " 1197,\n",
-       " 1198,\n",
-       " 1201,\n",
-       " 1202,\n",
-       " 1205,\n",
-       " 1207,\n",
-       " 1208,\n",
-       " 1210,\n",
-       " 1211,\n",
-       " 1214,\n",
-       " 1215,\n",
-       " 1217,\n",
-       " 1218,\n",
-       " 1220,\n",
-       " 1221,\n",
-       " 1224,\n",
-       " 1227,\n",
-       " 1228,\n",
-       " 1229,\n",
-       " 1230,\n",
-       " 1235,\n",
-       " 1239,\n",
-       " 1241,\n",
-       " 1244,\n",
-       " 1246,\n",
-       " 1247,\n",
-       " 1248,\n",
-       " 1250,\n",
-       " 1253,\n",
-       " 1257,\n",
-       " 1260,\n",
-       " 1261,\n",
-       " 1263,\n",
-       " 1264,\n",
-       " 1265,\n",
-       " 1267,\n",
-       " 1271,\n",
-       " 1277,\n",
-       " 1279,\n",
-       " 1280,\n",
-       " 1290,\n",
-       " 1293,\n",
-       " 1296,\n",
-       " 1297,\n",
-       " 1302,\n",
-       " 1303,\n",
-       " 1304,\n",
-       " 1310,\n",
-       " 1313,\n",
-       " 1314,\n",
-       " 1321,\n",
-       " 1322,\n",
-       " 1323,\n",
-       " 1324,\n",
-       " 1325,\n",
-       " 1326,\n",
-       " 1330,\n",
-       " 1333,\n",
-       " 1334,\n",
-       " 1338,\n",
-       " 1340,\n",
-       " 1342,\n",
-       " 1347,\n",
-       " 1348,\n",
-       " 1350,\n",
-       " 1353,\n",
-       " 1354,\n",
-       " 1356,\n",
-       " 1358,\n",
-       " 1359,\n",
-       " 1360,\n",
-       " 1363,\n",
-       " 1364,\n",
-       " 1365,\n",
-       " 1366,\n",
-       " 1367,\n",
-       " 1370,\n",
-       " 1371,\n",
-       " 1376,\n",
-       " 1378,\n",
-       " 1379,\n",
-       " 1380,\n",
-       " 1382,\n",
-       " 1383,\n",
-       " 1385,\n",
-       " 1389,\n",
-       " 1390,\n",
-       " 1392,\n",
-       " 1393,\n",
-       " 1399,\n",
-       " 1403,\n",
-       " 1405,\n",
-       " 1410,\n",
-       " 1411,\n",
-       " 1416,\n",
-       " 1418,\n",
-       " 1419,\n",
-       " 1421,\n",
-       " 1424,\n",
-       " 1426,\n",
-       " 1427,\n",
-       " 1428,\n",
-       " 1430,\n",
-       " 1432,\n",
-       " 1433,\n",
-       " 1434,\n",
-       " 1435,\n",
-       " 1440,\n",
-       " 1448,\n",
-       " 1451,\n",
-       " 1452,\n",
-       " 1454,\n",
-       " 1456,\n",
-       " 1457,\n",
-       " 1464,\n",
-       " 1465,\n",
-       " 1466,\n",
-       " 1468,\n",
-       " 1470,\n",
-       " 1475,\n",
-       " 1480,\n",
-       " 1481,\n",
-       " 1482,\n",
-       " 1483,\n",
-       " 1489,\n",
-       " 1490,\n",
-       " 1493,\n",
-       " 1501,\n",
-       " 1504,\n",
-       " 1506,\n",
-       " 1507,\n",
-       " 1509,\n",
-       " 1510,\n",
-       " 1512,\n",
-       " 1514,\n",
-       " 1515,\n",
-       " 1516,\n",
-       " 1518,\n",
-       " 1521,\n",
-       " 1522,\n",
-       " 1524,\n",
-       " 1525,\n",
-       " 1531,\n",
-       " 1537,\n",
-       " 1538,\n",
-       " 1539,\n",
-       " 1540,\n",
-       " 1541,\n",
-       " 1542,\n",
-       " 1544,\n",
-       " 1547,\n",
-       " 1550,\n",
-       " 1551,\n",
-       " 1552,\n",
-       " 1554,\n",
-       " 1555,\n",
-       " 1556,\n",
-       " 1557,\n",
-       " 1558,\n",
-       " 1560,\n",
-       " 1561,\n",
-       " 1565,\n",
-       " 1566,\n",
-       " 1568,\n",
-       " 1570,\n",
-       " 1572,\n",
-       " 1573,\n",
-       " 1575,\n",
-       " 1576,\n",
-       " 1577,\n",
-       " 1579,\n",
-       " 1581,\n",
-       " 1582,\n",
-       " 1587,\n",
-       " 1592,\n",
-       " 1595,\n",
-       " 1598,\n",
-       " 1601,\n",
-       " 1602,\n",
-       " 1603,\n",
-       " 1604,\n",
-       " 1606,\n",
-       " 1609,\n",
-       " 1612,\n",
-       " 1615,\n",
-       " 1618,\n",
-       " 1619,\n",
-       " 1623,\n",
-       " 1628,\n",
-       " 1629,\n",
-       " 1634,\n",
-       " 1635,\n",
-       " 1638,\n",
-       " 1640,\n",
-       " 1641,\n",
-       " 1642,\n",
-       " 1643,\n",
-       " 1646,\n",
-       " 1649,\n",
-       " 1652,\n",
-       " 1653,\n",
-       " 1656,\n",
-       " 1663,\n",
-       " 1665,\n",
-       " 1667,\n",
-       " 1672,\n",
-       " 1673,\n",
-       " 1674,\n",
-       " 1675,\n",
-       " 1676,\n",
-       " 1677,\n",
-       " 1681,\n",
-       " 1682,\n",
-       " 1683,\n",
-       " 1686,\n",
-       " 1689,\n",
-       " 1690,\n",
-       " 1694,\n",
-       " 1695,\n",
-       " 1699,\n",
-       " 1707,\n",
-       " 1709,\n",
-       " 1713,\n",
-       " 1716,\n",
-       " 1722,\n",
-       " 1723,\n",
-       " 1725,\n",
-       " 1727,\n",
-       " 1729,\n",
-       " 1730,\n",
-       " 1732,\n",
-       " 1734,\n",
-       " 1737,\n",
-       " 1741,\n",
-       " 1745,\n",
-       " 1747,\n",
-       " 1748,\n",
-       " 1749,\n",
-       " 1750,\n",
-       " 1752,\n",
-       " 1755,\n",
-       " 1757,\n",
-       " 1758,\n",
-       " 1761,\n",
-       " 1765,\n",
-       " 1766,\n",
-       " 1771,\n",
-       " 1772,\n",
-       " 1774,\n",
-       " 1777,\n",
-       " 1780,\n",
-       " 1782,\n",
-       " 1784,\n",
-       " 1785,\n",
-       " 1786,\n",
-       " 1788,\n",
-       " 1789,\n",
-       " 1792,\n",
-       " 1794,\n",
-       " 1798,\n",
-       " 1799,\n",
-       " 1802,\n",
-       " 1803,\n",
-       " 1809,\n",
-       " 1810,\n",
-       " 1817,\n",
-       " 1818,\n",
-       " 1821,\n",
-       " 1825,\n",
-       " 1826,\n",
-       " 1827,\n",
-       " 1829,\n",
-       " 1831,\n",
-       " 1832,\n",
-       " 1834,\n",
-       " 1836,\n",
-       " 1837,\n",
-       " 1838,\n",
-       " 1839,\n",
-       " 1840,\n",
-       " 1841,\n",
-       " 1843,\n",
-       " 1845,\n",
-       " 1846,\n",
-       " 1847,\n",
-       " 1853,\n",
-       " 1855,\n",
-       " 1858,\n",
-       " 1861,\n",
-       " 1863,\n",
-       " 1866,\n",
-       " 1868,\n",
-       " 1871,\n",
-       " 1872,\n",
-       " 1873,\n",
-       " 1874,\n",
-       " 1878,\n",
-       " 1881,\n",
-       " 1882,\n",
-       " 1884,\n",
-       " 1886,\n",
-       " 1887,\n",
-       " 1888,\n",
-       " 1889,\n",
-       " 1890,\n",
-       " 1893,\n",
-       " 1895,\n",
-       " 1898,\n",
-       " 1900,\n",
-       " 1901,\n",
-       " 1902,\n",
-       " 1908,\n",
-       " 1913,\n",
-       " 1916,\n",
-       " 1917,\n",
-       " 1918,\n",
-       " 1919,\n",
-       " 1921,\n",
-       " 1922,\n",
-       " 1924,\n",
-       " 1927,\n",
-       " 1928,\n",
-       " 1932,\n",
-       " 1934,\n",
-       " 1935,\n",
-       " 1938,\n",
-       " 1939,\n",
-       " 1941,\n",
-       " 1944,\n",
-       " 1947,\n",
-       " 1949,\n",
-       " 1951,\n",
-       " 1953,\n",
-       " 1957,\n",
-       " 1958,\n",
-       " 1960,\n",
-       " 1961,\n",
-       " 1963,\n",
-       " 1967,\n",
-       " 1968,\n",
-       " 1969,\n",
-       " 1970,\n",
-       " 1971,\n",
-       " 1973,\n",
-       " 1974,\n",
-       " 1978,\n",
-       " 1979,\n",
-       " 1980,\n",
-       " 1986,\n",
-       " 1987,\n",
-       " 1988,\n",
-       " 1989,\n",
-       " 1991,\n",
-       " 1992,\n",
-       " 1995,\n",
-       " 1996,\n",
-       " 1997,\n",
-       " 1999,\n",
-       " 2001,\n",
-       " 2004,\n",
-       " 2005,\n",
-       " 2006,\n",
-       " 2007,\n",
-       " 2008,\n",
-       " 2011,\n",
-       " 2012,\n",
-       " 2015,\n",
-       " 2016,\n",
-       " 2017,\n",
-       " 2019,\n",
-       " 2021,\n",
-       " 2023,\n",
-       " 2025,\n",
-       " 2028,\n",
-       " 2033,\n",
-       " 2036,\n",
-       " 2037,\n",
-       " 2039,\n",
-       " 2045,\n",
-       " 2051,\n",
-       " 2053,\n",
-       " 2054,\n",
-       " 2055,\n",
-       " 2057,\n",
-       " 2059,\n",
-       " 2061,\n",
-       " 2064,\n",
-       " 2065,\n",
-       " 2067,\n",
-       " 2068,\n",
-       " 2069,\n",
-       " 2070,\n",
-       " 2074,\n",
-       " 2076,\n",
-       " 2080,\n",
-       " 2081,\n",
-       " 2082,\n",
-       " 2084,\n",
-       " 2085,\n",
-       " 2087,\n",
-       " 2088,\n",
-       " 2090,\n",
-       " 2092,\n",
-       " 2096,\n",
-       " 2106,\n",
-       " 2109,\n",
-       " 2110,\n",
-       " 2111,\n",
-       " 2115,\n",
-       " 2116,\n",
-       " 2118,\n",
-       " 2122,\n",
-       " 2123,\n",
-       " 2126,\n",
-       " 2128,\n",
-       " 2135,\n",
-       " 2137,\n",
-       " 2139,\n",
-       " 2142,\n",
-       " 2144,\n",
-       " 2145,\n",
-       " 2146,\n",
-       " 2150,\n",
-       " 2153,\n",
-       " 2157,\n",
-       " 2159,\n",
-       " 2161,\n",
-       " 2162,\n",
-       " 2163,\n",
-       " 2165,\n",
-       " 2166,\n",
-       " 2167,\n",
-       " 2169,\n",
-       " 2170,\n",
-       " 2171,\n",
-       " 2173,\n",
-       " 2174,\n",
-       " 2177,\n",
-       " 2182,\n",
-       " 2183,\n",
-       " 2187,\n",
-       " 2189,\n",
-       " 2192,\n",
-       " 2194,\n",
-       " 2198,\n",
-       " 2199,\n",
-       " 2200,\n",
-       " 2205,\n",
-       " 2206,\n",
-       " 2209,\n",
-       " 2210,\n",
-       " 2211,\n",
-       " 2213,\n",
-       " 2216,\n",
-       " 2218,\n",
-       " 2222,\n",
-       " 2223,\n",
-       " 2225,\n",
-       " 2228,\n",
-       " 2230,\n",
-       " 2231,\n",
-       " 2235,\n",
-       " 2236,\n",
-       " 2238,\n",
-       " 2239,\n",
-       " 2242,\n",
-       " 2243,\n",
-       " 2250,\n",
-       " 2251,\n",
-       " 2252,\n",
-       " 2253,\n",
-       " 2255,\n",
-       " 2256,\n",
-       " 2257,\n",
-       " 2258,\n",
-       " 2263,\n",
-       " 2264,\n",
-       " 2265,\n",
-       " 2267,\n",
-       " 2273,\n",
-       " 2276,\n",
-       " 2277,\n",
-       " 2278,\n",
-       " 2279,\n",
-       " 2280,\n",
-       " 2284,\n",
-       " 2286,\n",
-       " 2288,\n",
-       " 2289,\n",
-       " 2290,\n",
-       " 2291,\n",
-       " 2293,\n",
-       " ...]"
+       "[0, 1, 2, 3, 4, 5, 6, 7, 8, 10]"
       ]
      },
-     "execution_count": 33,
+     "execution_count": 27,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "embed_indexes"
+    "embed_indexes[:10]"
    ]
   },
   {
@@ -1723,7 +773,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 34,
+   "execution_count": 28,
    "metadata": {},
    "outputs": [
     {
@@ -1739,7 +789,7 @@
        "torch.Size([27515, 1024])"
       ]
      },
-     "execution_count": 34,
+     "execution_count": 28,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -1773,7 +823,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 35,
+   "execution_count": 29,
    "metadata": {},
    "outputs": [
     {
@@ -1782,7 +832,7 @@
        "torch.Size([27520, 1024])"
       ]
      },
-     "execution_count": 35,
+     "execution_count": 29,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -1797,7 +847,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 36,
+   "execution_count": 30,
    "metadata": {},
    "outputs": [
     {
@@ -1806,7 +856,7 @@
        "430.0"
       ]
      },
-     "execution_count": 36,
+     "execution_count": 30,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -1824,7 +874,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 37,
+   "execution_count": 31,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -1846,7 +896,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 38,
+   "execution_count": 32,
    "metadata": {},
    "outputs": [
     {
@@ -1868,7 +918,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 39,
+   "execution_count": 33,
    "metadata": {},
    "outputs": [
     {
@@ -1877,7 +927,7 @@
        "tensor(True)"
       ]
      },
-     "execution_count": 39,
+     "execution_count": 33,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -1900,7 +950,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 40,
+   "execution_count": 34,
    "metadata": {},
    "outputs": [
     {
@@ -1916,7 +966,7 @@
        "['donut-base-ascii/preprocessor_config.json']"
       ]
      },
-     "execution_count": 40,
+     "execution_count": 34,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -1940,7 +990,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 41,
+   "execution_count": 35,
    "metadata": {},
    "outputs": [
     {
@@ -1959,7 +1009,7 @@
        " (2, '</s>')]"
       ]
      },
-     "execution_count": 41,
+     "execution_count": 35,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -1980,7 +1030,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 42,
+   "execution_count": 36,
    "metadata": {},
    "outputs": [
     {
@@ -2006,7 +1056,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 43,
+   "execution_count": 37,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -2022,7 +1072,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 44,
+   "execution_count": 38,
    "metadata": {},
    "outputs": [
     {
@@ -2032,7 +1082,7 @@
        "<PIL.PngImagePlugin.PngImageFile image mode=RGB size=1701x2386>"
       ]
      },
-     "execution_count": 44,
+     "execution_count": 38,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -2043,44 +1093,74 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 48,
+   "execution_count": 39,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "CPU times: user 2.66 s, sys: 208 ms, total: 2.87 s\n",
-      "Wall time: 579 ms\n"
+      "467 ms ± 42.1 ms per loop (mean ± std. dev. of 2 runs, 5 loops each)\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%timeit -n 5 -r 2\n",
+    "\n",
+    "pixel_values = processor(image, return_tensors=\"pt\").pixel_values\n",
+    "\n",
+    "outputs = model.generate(\n",
+    "    pixel_values.to(device),\n",
+    "    decoder_input_ids=decoder_input_ids.to(device),\n",
+    "    early_stopping=True,\n",
+    "    pad_token_id=processor.tokenizer.pad_token_id,\n",
+    "    eos_token_id=processor.tokenizer.eos_token_id,\n",
+    "    use_cache=True,\n",
+    "    num_beams=1,\n",
+    "    bad_words_ids=[[processor.tokenizer.unk_token_id]],\n",
+    "    return_dict_in_generate=True,   \n",
+    "    min_length=10,\n",
+    "    max_length=10\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 40,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/nicholas/miniconda3/lib/python3.10/site-packages/transformers/generation/utils.py:1369: UserWarning: Using `max_length`'s default (20) to control the generation length. This behaviour is deprecated and will be removed from the config in v5 of Transformers -- we recommend using `max_new_tokens` to control the maximum length of the generation.\n",
+      "  warnings.warn(\n"
      ]
     },
     {
      "data": {
       "text/plain": [
-       "\"<s_synthdog> LOVE DELIGHTFULLY SOFT SKIN? GET INDIAS FIRST GET. BAR! WITH SKIN CONDITIONERS The Modern's take by traditions and the idea.</s>\""
+       "'<s_synthdog> LOVE DELIGHTFULLY SOFT SKIN? GET INDIAS FIR</s>'"
       ]
      },
-     "execution_count": 48,
+     "execution_count": 40,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "%%time\n",
-    "\n",
     "pixel_values = processor(image, return_tensors=\"pt\").pixel_values\n",
     "\n",
     "outputs = model.generate(\n",
     "    pixel_values.to(device),\n",
     "    decoder_input_ids=decoder_input_ids.to(device),\n",
-    "    max_length=model.decoder.config.max_position_embeddings,\n",
     "    early_stopping=True,\n",
     "    pad_token_id=processor.tokenizer.pad_token_id,\n",
     "    eos_token_id=processor.tokenizer.eos_token_id,\n",
     "    use_cache=True,\n",
     "    num_beams=1,\n",
     "    bad_words_ids=[[processor.tokenizer.unk_token_id]],\n",
-    "    return_dict_in_generate=True,\n",
+    "    return_dict_in_generate=True,   \n",
     ")\n",
     "\n",
     "sequence = processor.batch_decode(outputs.sequences)[0]\n",
@@ -2089,7 +1169,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 49,
+   "execution_count": 41,
    "metadata": {},
    "outputs": [
     {
@@ -2098,7 +1178,7 @@
        "tensor([[27514]])"
       ]
      },
-     "execution_count": 49,
+     "execution_count": 41,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -2106,69 +1186,84 @@
    "source": [
     "model_name = \"./donut-base-ascii\"\n",
     "\n",
-    "processor = AutoProcessor.from_pretrained(model_name)\n",
-    "model = VisionEncoderDecoderModel.from_pretrained(model_name)\n",
+    "new_processor = AutoProcessor.from_pretrained(model_name)\n",
+    "new_model = VisionEncoderDecoderModel.from_pretrained(model_name)\n",
     "\n",
     "device = 0\n",
     "\n",
-    "model.to(device);\n",
+    "new_model.to(device);\n",
     "\n",
-    "decoder_input_ids = processor.tokenizer(task_prompt, add_special_tokens=False, return_tensors=\"pt\").input_ids\n",
-    "decoder_input_ids"
+    "new_decoder_input_ids = new_processor.tokenizer(task_prompt, add_special_tokens=False, return_tensors=\"pt\").input_ids\n",
+    "new_decoder_input_ids"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 56,
+   "execution_count": 42,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "CPU times: user 2.53 s, sys: 39.2 ms, total: 2.57 s\n",
-      "Wall time: 530 ms\n"
+      "382 ms ± 2.19 ms per loop (mean ± std. dev. of 2 runs, 5 loops each)\n"
      ]
-    },
+    }
+   ],
+   "source": [
+    "%%timeit -n 5 -r 2\n",
+    "\n",
+    "pixel_values = new_processor(image, return_tensors=\"pt\").pixel_values\n",
+    "\n",
+    "outputs = new_model.generate(\n",
+    "    pixel_values.to(device),\n",
+    "    decoder_input_ids=new_decoder_input_ids.to(device),\n",
+    "    early_stopping=True,\n",
+    "    pad_token_id=new_processor.tokenizer.pad_token_id,\n",
+    "    eos_token_id=new_processor.tokenizer.eos_token_id,\n",
+    "    use_cache=True,\n",
+    "    num_beams=1,\n",
+    "    bad_words_ids=[[new_processor.tokenizer.unk_token_id]],\n",
+    "    return_dict_in_generate=True,   \n",
+    "    min_length=10,\n",
+    "    max_length=10\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 43,
+   "metadata": {},
+   "outputs": [
     {
      "data": {
       "text/plain": [
-       "\"<s_synthdog> LOVE DELIGHTFULLY SOFT SKIN? GET INDIAS FIRST GET. BAR! WITH SKIN CONDITIONERS The Modern's take by traditions and the idea.</s>\""
+       "'<s_synthdog> LOVE DELIGHTFULLY SOFT SKIN? GET INDIAS FIR</s>'"
       ]
      },
-     "execution_count": 56,
+     "execution_count": 43,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "%%time\n",
-    "\n",
-    "pixel_values = processor(image, return_tensors=\"pt\").pixel_values\n",
+    "pixel_values = new_processor(image, return_tensors=\"pt\").pixel_values\n",
     "\n",
-    "outputs = model.generate(\n",
+    "outputs = new_model.generate(\n",
     "    pixel_values.to(device),\n",
-    "    decoder_input_ids=decoder_input_ids.to(device),\n",
-    "    max_length=model.decoder.config.max_position_embeddings,\n",
+    "    decoder_input_ids=new_decoder_input_ids.to(device),\n",
     "    early_stopping=True,\n",
-    "    pad_token_id=processor.tokenizer.pad_token_id,\n",
-    "    eos_token_id=processor.tokenizer.eos_token_id,\n",
+    "    pad_token_id=new_processor.tokenizer.pad_token_id,\n",
+    "    eos_token_id=new_processor.tokenizer.eos_token_id,\n",
     "    use_cache=True,\n",
     "    num_beams=1,\n",
-    "    bad_words_ids=[[processor.tokenizer.unk_token_id]],\n",
-    "    return_dict_in_generate=True,\n",
+    "    bad_words_ids=[[new_processor.tokenizer.unk_token_id]],\n",
+    "    return_dict_in_generate=True,   \n",
     ")\n",
     "\n",
-    "sequence = processor.batch_decode(outputs.sequences)[0]\n",
+    "sequence = new_processor.batch_decode(outputs.sequences)[0]\n",
     "sequence"
    ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "\"<s_synthdog> LOVE DELIGHTFULLY SOFT SKIN? GET INDIAS FIRST GET. BAR! WITH SKIN CONDITIONERS The Modern's take by traditions and the idea.</s>\""
-   ]
   }
  ],
  "metadata": {