vocab_and_tokenization_comparison.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "793f0b05",
   "metadata": {},
   "source": [
    "### General Corpus vs. Story Point (SP) Corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b07b529b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "general_bpe_vocab = \"./all_tokenizers/general_bpe/vocab.json\"\n",
    "sp_bpe_vocab = \"./all_tokenizers/sp_bpe/sp_bpe_vocab.json\"\n",
    "\n",
    "with open(general_bpe_vocab, encoding=\"utf8\") as f:\n",
    "    general_vocab = json.load(f)\n",
    "\n",
    "with open(sp_bpe_vocab, encoding=\"utf8\") as f:\n",
    "    sp_vocab = json.load(f)\n",
    "    \n",
    "general_words = []\n",
    "sp_words = []\n",
    "for key, val in general_vocab.items():\n",
    "    general_words.append(key)\n",
    "for key, val in sp_vocab.items():\n",
    "    sp_words.append(key)\n",
    "    \n",
    "overlap = 0\n",
    "\n",
    "total = len(general_words)\n",
    "for word in general_words:\n",
    "    if word in sp_words:\n",
    "        overlap += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "f068e3ba",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Length of BPE vocab:  50257\n",
      "Length of Word-level vocab:  50257\n",
      "Number of overlapping words:  17818\n"
     ]
    }
   ],
   "source": [
    "print(\"Length of BPE vocab: \", len(general_words))\n",
    "print(\"Length of Word-level vocab: \", len(sp_words))\n",
    "print(\"Number of overlapping words: \", overlap)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "123b47ca",
   "metadata": {},
   "source": [
    "### BPE Tokenization vs Word-level Tokenization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "c8ff9151",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "sp_bpe_vocab = \"./all_tokenizers/sp_bpe/sp_bpe_vocab.json\"\n",
    "sp_wordlevel_vocab = \"./all_tokenizers/sp_word_level/wordlevel.json\"\n",
    "\n",
    "with open(sp_bpe_vocab, encoding=\"utf8\") as f:\n",
    "    bpe_vocab = json.load(f)\n",
    "\n",
    "with open(sp_wordlevel_vocab, encoding=\"utf8\") as f:\n",
    "    wordlevel_vocab = json.load(f)\n",
    "    wordlevel_vocab = wordlevel_vocab[\"model\"][\"vocab\"]\n",
    "\n",
    "bpe_words = []\n",
    "wordlevel_words = []\n",
    "for key, val in bpe_vocab.items():\n",
    "    bpe_words.append(key)\n",
    "for key, val in wordlevel_vocab.items():\n",
    "    wordlevel_words.append(key)\n",
    "\n",
    "bpe_wordlevel_overlap = 0\n",
    "\n",
    "total = len(wordlevel_words)\n",
    "for word in bpe_words:\n",
    "    if word in wordlevel_words:\n",
    "        bpe_wordlevel_overlap += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "18ffeed9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Length of BPE vocab:  50257\n",
      "Length of Word-level vocab:  34321\n",
      "Number of overlapping words:  16621\n"
     ]
    }
   ],
   "source": [
    "print(\"Length of BPE vocab: \", len(bpe_words))\n",
    "print(\"Length of Word-level vocab: \", len(wordlevel_words))\n",
    "print(\"Number of overlapping words: \", bpe_wordlevel_overlap)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c461aba8",
   "metadata": {},
   "source": [
    "### Plot the result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "efc0a540",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAo8AAAFgCAYAAADaapOxAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAkQUlEQVR4nO3de3RU1aHH8d+eySQkQAhIiCQgIAESXhEIQSLP1irlFpRiKy+1cilXW0Gl92rtVURbqdpbtFpFAYvWB6igyEvFgoAWJCbQCAihpPIyEAKEYB6QZGbfPzJJAwbY8krE72etrk7OObNnn4lr5Zt95gRjrRUAAADgwlPbEwAAAMC3B/EIAAAAZ8QjAAAAnBGPAAAAcEY8AgAAwFlIbU/guyQjI6NZSEjILEmdRbjju8Xv9/t/1b179wW1PREAwNkhHi+gkJCQWZdeemlidHR0vsfj4W8k4TujqKioXnZ29q8lLajtuQAAzg6rXxdW5+jo6COEI75rIiIijlprY2p7HgCAs0c8XlgewhHfRcYYSTK1PQ8AwNkjHgEAAOCMzzzWkpzrUnqc7Rix76RlnGr/4sWLGz744IOxHo9HMTExZbNnz94ZHR3tP5PX6ty5c+KmTZu2nNlM/23UqFGtXnvttZ1nO843NeDNUWf1fq/8yWsnfa+zsrJCExISuixfvnzr9773vaJ58+ZFrlmzpsG0adNyzuS1iouLzbhx41pu27Yt3O/3mwkTJuSOHTs2/0znPmnSpNiePXsWjRw5suBMx5Ckp5566pKOHTsevfrqq4vOZhwAwLcbK48XqdzcXO+kSZNaLly4MPvTTz/NGjx48OFx48ZddiZj+f1n1Js1qo1wvBDatm179LHHHrv0XIx17733xrZu3bo0PT09a+XKldumTZt2aWZmZtiZjHUuv3cTJ048SDgCAIjHi9Sbb74ZNXjw4MNxcXHlkvSLX/ziUHp6eoMrrrgiofKYIUOGtNm8eXPY6tWrI3r16tW+R48eHSZPnhwjVaxW/fjHP27dr1+/dhkZGeGVz3nmmWea9O7du33Hjh0Tn3nmmSaVxw4bNqz1VVdd1W7gwIHxxcXFZvHixQ0HDBgQP2DAgPguXbokbty4MUyqWMGUpOHDh7ceMWJEq9TU1Pbf//732wYCAZWVlWnQoEGXp6amtr/llltaDh8+vPUFfMvOSnx8fEl5ebk2bNhQr/r25557rknXrl0TunbtmjBv3rxISUpJSekwduzYlsnJyR3GjBnztaBfvHhx1AMPPJArSY0aNQqMHTs275VXXmnywAMPxMyaNauxJGVmZoYNGzasdSAQ0C233NKyV69e7VNTU9tlZ2f7JKlt27adhg0b1vq2225rUTmu3+/XVVdd1S4lJaVDampqu0OHDnkqj73++uvbJCYmdnz55ZejKuc4bty4Ft27d0+444474qSK7/OcOXMaZWVlhXbr1i1h0KBBl7dv377jggULGkrSW2+9FZmYmNjxhz/84eW9evVqn5WVFXoe3moAQC0jHi9Se/fu9cXFxZVW3xYdHV0WHx9/NC0tLbywsNDs37/f16lTp2P33ntvi8WLF2dnZGRkrV27tsHu3btDJKlly5alq1ev/mdKSkpJ5Rg333zz4bVr127LyMjYOn369Kq7Z+Pi4sr+/ve//7NXr16Fs2bNaiJJX331lXfFihXbp02btmvKlCnNT5xjnz59CtesWbOtfv36gU8//TT85ZdfbhwfH39szZo125KSkkpOPL6uu+eee3J///vfV70n5eXlmjZt2qXr1q3LWrFixT8nT54cV7nvJz/5SX56enrWli1bwg8ePOitPk5ZWZknPDy86saqVq1alebk5Phuvvnm/Hnz5jWWpJdffrnJjTfemP/66683aty4sX/dunXbHnvssS8feuih5pKUm5sbOmPGjN0zZ87cUzmO1+vV+++/vz0tLS1r8ODBBbNnz24iSXv37g19/vnnd61Zs2bro48+WvV9Gjx48JH169dvzczMjNi6detxIZifnx+yaNGif82fPz/72WefbSZJU6ZMiV25cmXWW2+99UVOTg7hCAAXKeLxIhUbG1t64g/wQ4cOhYwcOfLQK6+80viNN96IGjRoUIEkZWVlhf/Hf/xHfEpKSoddu3aFffHFF6GSlJKS8rVLlO+8805kz549O/Tr16/9zp07qy6lJicnF0lSz549i7OzsytXGYs9Ho/69OlTvGPHjq9ddk1JSSmWpBYtWpQeOHDAu3379rDKcXr37v2tuzx67bXXFu7atSts9+7doZKUk5MT0qJFi9Lw8HDbtGlTf2hoqC0rK5MkXXnllcWS1Lx587IT49Hn8wVKSkqq7kzetWtXaGxsbFm7du1KCwoKQgoKCjwrV66MHDZs2JHNmzfXW7p0aVRKSkqHSZMmtSwoKPBK0mWXXXb0xM+3HjlyxDN69OjWycnJHf761782zcnJ8UlSixYtjsXExPgbN24cqD7H1NTUIknq2rVrcVZW1nHfvw4dOpT4fD61bdu2tPI1A4GAiYmJ8YeHh9uEhIRvXfwDANwQjxepG264oWDJkiVROTk5IZL0/PPPN+nevXvR0KFDj3z88ceR8+bNa3zTTTcdkqSEhITid999d3taWlrW5s2bP+/Tp0+xJHk8X//PY+rUqc3fe++97atWrdoWHh4eqNy+fv36CEnKyMiIaNu27TFJ2rRpU0QgENCaNWsiWrdufezEsYwxVatr1loTHx9/rHKcdevWRZzTN+QCmTBhQu706dObSVJsbGz5nj17QktKSsyBAwe8ZWVlxufzSTr+vbX2+L/eNGTIkPxHHnmkmVQRfH/5y1+ix4wZc0iSBg8efPj+++9vHh8ffzQsLMwmJiYeve666/LT0tKyPv3006w33nhjx4njV3rrrbcimzdvXpaenp518803H7DWGkn68ssvww4cOOA9cuSIp7S0tGqOn3zySYQkbdy4MaJDhw7Hff9O/N5JktfrtXl5ed6jR4+arKyscAEALkrcbV1LTnen9NmKiYnxT5s2bfeQIUPaVr/b2ufzKSEhoTg7O7tefHx8mSQ99thjX/7oRz9qGwgE5PP57Hvvvbf9ZOMOGTIkv2/fvu07d+5c3KhRo6qVrZ07d4alpqa2CwsLs0uWLMlesWJFgwYNGvgHDhwYf+jQId9rr732r9PNefTo0Yfnzp3bpHfv3u3btGlzzOfznbO/iXmqu6XPpZEjRxZMnjy5hSSFhIRo0qRJ+3r16tVBkh566KEvXcZ49NFH944bN+6y5OTkDn6/39x55537kpKSjknSTTfdlN+uXbsu8+bN+2fl6y1fvjyyV69e7SVpxIgRh+6+++4DNY3bv3//oscff7z5gAED4mNiYsri4uLKJOnSSy8tvfXWWy/Lzs6ud9999+2tPH7hwoVRDz/8cFxqaupXCQkJpTWNWd0DDzyQ069fvw6XXXbZsejo6LLQ0FD+pikAXITMiaseOH8yMzN3JCUl1fiD/duspj8Fs3jx4oYLFy5sNGPGjD2neu6Jjh07ZsLCwuy0adOaHjx4MOSRRx7Zd+5njOpq+jNMKSkpHT744IN/NmrUKHCy552o8ntXUlJiunXrlrhp06bPQ0L+/ftpZmZm06SkpNbnbuYAgNrAyiPqlGuuuaZtUVGRNzQ0NPD222+fdrUSdcerr74aNX369GbFxcWe2267Lbd6OAIALh6sPF5AmZmZ/+rSpUs+/0Qhvmustfrss88uSUpKalPbcwEAnB1umLmwNuXl5TUKBAL8G7/4TikuLq5njMmt7XkAAM4e15UuoPLy8nH79u2btW/fvs4i3PHd4vf7/ffX9iQAAGePy9YAAABwxuoXAAAAnBGPAAAAcMZnHs+jpk2b2tatW9f2NAAAAL6RjIyMA9ba6Jr2EY/nUevWrZWenl7b0wAAAPhGjDE7T7aPy9YAAABwRjwCAADAGfEIAAAAZ8QjAAAAnBGPAAAAcEY8AgAAwBnxCAAAAGfEIwAAAJwRjwAAAHBGPAIAAMAZ8QgAAABnxCMAAACcEY8AAABwFlLbE7iYlW3fopzrUmp7GkCdM2pMfG1PAQBwhlh5BAAAgDPiEQAAAM6IRwAAADgjHgEAAOCMeAQAAIAz4hEAAADOiEcAAAA4Ix4BAADgjHgEAACAM+IRAAAAzohHAAAAOCMeAQAA4Ix4BAAAgDPiEQAAAM6IRwAAADgjHgEAAOCMeAQAAIAz4hEAAADOiEcAAAA4Ix4BAADgjHgEAACAM+IRAAAAzohHAAAAOCMeAQAA4Ix4BAAAgDPiEQAAAM6IRwAAADgjHgEAAOCMeAQAAIAz4hEAAADOiEcAAAA4Ix4BAADgjHgEAACAM+IRAAAAzohHAAAAOCMeAQAA4Ix4BAAAgDPiEQAAAM6IRwAAADgjHgEAAOCMeAQAAIAz4hEAAADOQmp7AoCrqAn3Kyy5jwIF+cqbOFKS1Ph/HlFIbCtJkqnfQLaoUHl3j5G8XkXdcb98l3eQvF6VfLhUhfNfkiQ1HHO7IgYOlqnfUPtGDKga39s0RlF3PShP/YaSx6Mjf31GxzLWXPDzBFA7Qj0+/WngZPk8IfIar1btWacXP5+v27qOUmrz7ioLlCunKFePffq8CsuKq57XLPwSvTToD3px83y9vm2JJOnxvveqSb0oeY1XGw9s1ZPrZysgq5iIpronebyiwiL1VWmhHkl7Vnklh2rrlIEzctp4NMbESHpC0pWS8iWVSnrcWvv2eZ7byebzM0nJ1to7qm1rLeljSZdZawPVtv9D0nhrbdo5eN2Vkv7bWpt+tmPhzBQvX6KiJW8q6q4pVdvy//C/VY8jb71TgeJCSVL4VVfL+HzKu3OUTGiYov/8uko+Wib//r06mvaRipa8oWbT5x83foOfjlXJx8tV/N58hbRsoyYPPKH946+/EKcGoA4oDZRp0srfqcR/TF7j1dMDH1Tavkyl527UzI1z5bcBje8yQqMShmrGxrlVz/vlFTdp3d7M48aasvYpFZeXSJIe6n2XBrS8Uit2r9XtXUdr2c6P9P7Oj9QtuqN+3uVGTU2bfkHPEzhbp7xsbYwxkhZIWm2tvdxa20PSCEktzuekjDHfaEXUWrtD0m5JfauNkSCp4bkIR9QNpZ9vUKDwyEn3h/e5WiWrl1V8Ya1MWLjk8Uph9aTycgWKiyRJZds2KZB/8OsDWCtPRH1JkolooED+gXN+DgDqthL/MUlSiMerEI9XVlbpuRvlD65LfH5wu6LDL6k6vk9ssvYW7deOI3uOG6cyHL3GK58nRNZaSVKryDit379ZkrQh73NdFdvjvJ8TcK6d7jOP35NUaq19rnKDtXantfZpSTLGeI0xfzDGfGqM+cwY81/B7QOMMSuNMfOMMVuNMa8GQ1TGmB7GmFXGmAxjzPvGmObB7SuNMVONMask3WmMGWKMWWeM2WCM+VtwBfRU5qgibCuNkDTHGFPPGDPbGLMxONbAanP/v+D2z4wxE4LbJwfPZ5MxZkblvIPGGGPWBPelnO7NxYUT2rGb/IcPyb93tySpZM1y2WMlinlxqWJmLVThgldkTxGekvTV3JkK7z9IMS8s0iWTn1DBjP+7EFMHUId4ZDTrB1O1YOhzSs/dqC2Hso/bP7jNAKXt+4ckqZ43TCMThuilzfO/PpCkx/v+WguGPqfi8hKt2rNOkpRdsFP94ip+fPSN66n6vghFhjY4b+cDnA+ni8dOktafYv9/Siqw1vaU1FPSz40xbYL7ukm6S1JHSZdLusoY45P0tKQbgquYf5H0SLXxoqy1/a21f1TFZegrrbXdJM2VdM9p5vqGpOurrVreGHzeLyXJWttF0khJLxlj6kkaL6mNpG7W2q6SXg0+78/W2p7W2s6SwiX9qNpr1LfWpkr6RXDuX2OMGW+MSTfGpB8sLT/NlHGuhPe7RiWr36/6OrRdJ9lAQLm3Dtb+8derwfWj5Y2JPfUYfa9V8YrFyv3PITr48N2KunuKdNzvDgAudgFZjfvgN/rJ4juU2KSt2kT++0LbmITr5Ld+fbDr75KkWzsN15vbllatVp7ono8e1fBFv5DP41O3Zp0kSdMzX1VSdIJmXj1VSdGJyis+KH/Af/5PDDiHvtHlYWPMM5L6qGI1sqekayR1NcbcEDykkaR2qvhcZJq1dk/wef+Q1FrSYUmdJX0QXNDzStpb7SVer/a4haTXgyuToZK+ONXcrLX7jDGbJX3fGJMrqcxau8kY81tVBKustVuNMTsltZd0taTnrLXlwX2Vn1geaIy5R1KEpCaSNktaFNw3J3jsamNMpDEmylp7+IR5zJA0Q5KSourbU80Z54jHq3q9Byhv0i1Vm8L7X6tj69dKfr8CBfkq3ZIpX3xH+XNzTjpMxA+G6uBDEyVJZVkbZXxh8kRGKVCQf95PAUDdUlhWrH/kbVHKpUn64sgeXduqr3rHdtekVf9e70hsEq/+LXrptq6j1MAXoYCsSv1lejt7WdUxpYEyrcnJUJ+4ZGXs36SDRw9r8tonJUnh3jD1j+upouAlbuDb4nTxuFnS8MovrLW/NMY0lVR504iRNMFa+371JxljBkiq/quYP/haRtJma23vk7xeUbXHT0uaZq1dGBxvymnmKv370nVu8HHlHGtiJB0Xd8EVyWdVcUPObmPMFEn1qh1yYgwSh3VAWFJPle/ZqcDB/VXb/Hm5CuuarJKV78qE1ZOvQ2cVLpx7ilEkf94+hXXtqZIVSxTSorVMaCjhCHyHNAptKL/1q7CsWKEen3o066w5WYuUEtNVIxOG6M4Pf6tj/tKq4yeufLjq8c86DldJ+VG9nb1M4d4whfvCdejoYXmNR72aX6GNeVlVr3GktFBWVqMSr9PSHasu+HkCZ+t08bhC0lRjzO3W2srbwSKq7X9f0u3GmBXW2jJjTHtJX55ivCxJ0caY3tbatcHL2O2ttZtrOLZRtbFuqWF/TeZLmiqpWBWf15Sk1ZJGS1oRnN9lwXksk3SbMWaltbbcGNNEUuWd2geMMQ0k3SBpXrXxb5T0oTGmjyou1xc4zgvnQNSvfquwzj3kiYxSzAuL9NWcmSr+20KF971GJR8tO+7YoqVvKmriZEU/PVcyUvHyxSrfuV2SFHnLBIX3u0YmrJ5iXlik4g8W6qu5M3Vk9p8U9cvfqMHQUZK1Ovynh2uaBoCL1CXhUbqv5+3yGI88xujD3Z9o7d4NevWH0+Tz+PTH/vdJqrhpZtr6Gj+5JEmqFxKmqVf9Sj6PTx7j0Yb9m7XwX3+TJF0RnaifdxkhK6vP8rbqyQ2zL8i5AeeSqbwD7KQHVFw2fkJSL0l5qlgdfM5a+7oxxiPpd5KGqGIlL0/S9ar4vON/W2t/FBzjz5LSrbUvGmOukPSUKuIwRNKT1tqZJ/4pHGPMdcHX/VLSJ5J6WmsH1PSnek6Y7zuSYqy1Vwa/rifpOUk9JJVLmmSt/TD42cjHJQ2SVCZpprX2z8aY36li9XKHKu7g3mmtnRKc31pJ/SVFShp7uju5k6Lq23f7dzrl+wt8F40aE1/bUwAAnMKqn87JsNYm17TvtPGIM0c8AjUjHgGgbjtVPPLPEwIAAMAZ8QgAAABnxCMAAACcEY8AAABwRjwCAADAGfEIAAAAZ8QjAAAAnBGPAAAAcEY8AgAAwBnxCAAAAGfEIwAAAJwRjwAAAHBGPAIAAMAZ8QgAAABnxCMAAACcEY8AAABwRjwCAADAGfEIAAAAZ8QjAAAAnBGPAAAAcEY8AgAAwBnxCAAAAGfEIwAAAJwRjwAAAHBGPAIAAMAZ8QgAAABnxCMAAACcEY8AAABwRjwCAADAGfEIAAAAZ8QjAAAAnBGPAAAAcEY8AgAAwBnxCAAAAGfEIwAAAJwRjwAAAHBGPAIAAMAZ8QgAAABnxCMAAACcEY8AAABwRjwCAADAGfEIAAAAZyG1PYGLmS8+UbHvpNX2NIA6Z2VtTwAAcEpGc066j5VHAAAAOCMeAQAA4Ix4BAAAgDPiEQAAAM6IRwAAADgjHgEAAOCMeAQAAIAz4hEAAADOiEcAAAA4Ix4BAADgjHgEAACAM+IRAAAAzohHAAAAOCMeAQAA4Ix4BAAAgDPiEQAAAM6IRwAAADgjHgEAAOCMeAQAAIAz4hEAAADOiEcAAAA4Ix4BAADgjHgEAACAM+IRAAAAzohHAAAAOCMeAQAA4Ix4BAAAgDPiEQAAAM6IRwAAADgjHgEAAOCMeAQAAIAz4hEAAADOiEcAAAA4Ix4BAADgjHgEAACAM+IRAAAAzohHAAAAOCMeAQAA4Ix4BAAAgDPiEQAAAM6IRwAAADgjHgEAAOCMeAQAAIAz4hEAAADOiEcAAAA4Ix4BAADgjHgEAACAM+IRAAAAzohHAAAAOCMeAQAA4Ix4BAAAgDPiEQAAAM6IRwAAADgjHgEAAOCMeAQAAIAz4hEAAADOiEcAAAA4Ix4BAADgjHgEAACAM+IRAAAAzohHAAAAOCMeAQAA4Ix4BAAAgDPiEQAAAM6IRwAAADgjHgEAAOCMeAQAAIAz4hEAAADOiEcAAAA4Ix4BAADgjHgEAACAM+IRAAAAzohHAAAAOCMeAQAA4Ix4BAAAgDPiEQAAAM6IRwAAADgjHgEAAOCMeAQAAIAz4hEAAADOiEcAAAA4Ix4BAADgjHgEAACAM+IRAAAAzohHAAAAOCMeAQAA4Ix4BAAAgDPiEQAAAM6IRwAAADgjHgEAAOCMeAQAAIAz4hEAAADOiEcAAAA4Ix4BAADgjHgEAACAs5DansDFrGz7FuVcl1Lb0wDqnFFj4mt7CgCAM8TKIwAAAJwRjwAAAHBGPAIAAMAZ8QgAAABnxCMAAACcEY8AAABwRjwCAADAGfEIAAAAZ8QjAAAAnBGPAAAAcEY8AgAAwBnxCAAAAGfEIwAAAJwRjwAAAHBGPAIAAMAZ8QgAAABnxCMAAACcEY8AAABwRjwCAADAGfEIAAAAZ8QjAAAAnBGPAAAAcEY8AgAAwBnxCAAAAGfEIwAAAJwRjwAAAHBGPAIAAMAZ8QgAAABnxCMAAACcEY8AAABwRjwCAADAGfEIAAAAZ8QjAAAAnBGPAAAAcEY8AgAAwBnxCAAAAGfEIwAAAJwRjwAAAHBGPAIAAMAZ8QgAAABnxCMAAACcEY8AAABwFlLbEwBcRU24X2HJfRQoyFfexJGSpMb/84hCYltJkkz9BrJFhcq7e4zk9Srqjvvlu7yD5PWq5MOlKpz/kiSp4ZjbFTFwsEz9hto3YkDV+N6mMYq660F56jeUPB4d+eszOpax5oKfJ4DaEerx6U8DJ8vnCZHXeLVqzzq9+Pl83dZ1lFKbd1dZoFw5Rbl67NPnVVhWXPW8ZuGX6KVBf9CLm+fr9W1LJEmP971XTepFyWu82nhgq55cP1sBWcVENNU9yeMVFRapr0oL9Ujas8orOVRbpwyckToVj8aY/5U0SpJfUkDSf1lr1xljVkpqLumopEJJY621WdWe9zNJ11prR1bb1lTSFkktrLXHzsHcCq21Dc52HJy54uVLVLTkTUXdNaVqW/4f/rfqceStdypQXChJCr/qahmfT3l3jpIJDVP0n19XyUfL5N+/V0fTPlLRkjfUbPr848Zv8NOxKvl4uYrfm6+Qlm3U5IEntH/89Rfi1ADUAaWBMk1a+TuV+I/Ja7x6euCDStuXqfTcjZq5ca78NqDxXUZoVMJQzdg4t+p5v7ziJq3bm3ncWFPWPqXi8hJJ0kO979KAlldqxe61ur3raC3b+ZHe3/mRukV31M+73KipadMv6HkCZ6vOXLY2xvSW9CNJ3a21XSVdLWl3tUNGW2uTJL0k6Q8nPP0tST8wxkRU23aDpIXnIhxRN5R+vkGBwiMn3R/e52qVrF5W8YW1MmHhkscrhdWTyssVKC6SJJVt26RA/sGvD2CtPBH1JUkmooEC+QfO+TkAqNtK/BU/MkI8XoV4vLKySs/dKL8NSJI+P7hd0eGXVB3fJzZZe4v2a8eRPceNUxmOXuOVzxMia60kqVVknNbv3yxJ2pD3ua6K7XHezwk41+pMPKpiZfFAZexZaw9Ya3NqOG61pPjqG6y1R4Lbh1TbPELSHGNMK2PMcmPMZ8H/v0ySjDExxpi3jTGZwf+lBrcvMMZkGGM2G2PGV38dY8wfjTHrg+NEn7Mzx1kL7dhN/sOH5N9b8ftGyZrlssdKFPPiUsXMWqjCBa/IniI8JemruTMV3n+QYl5YpEsmP6GCGf93IaYOoA7xyGjWD6ZqwdDnlJ67UVsOZR+3f3CbAUrb9w9JUj1vmEYmDNFLm+d/fSBJj/f9tRYMfU7F5SVatWedJCm7YKf6xaVIkvrG9VR9X4QiQ7mohW+XuhSPyyS1NMZsM8Y8a4zpf5LjhkjaWMP2OaoIRhljYiW1l/ShpD9L+mtwNfNVSU8Fj39K0qrgamZ3SZuD28daa3tISpY00RhT+StmfUnrrbXdJa2S9GBNkzPGjDfGpBtj0g+WlrueO85SeL9rVLL6/aqvQ9t1kg0ElHvrYO0ff70aXD9a3pjYU4/R91oVr1is3P8cooMP362ou6dIxpznmQOoSwKyGvfBb/STxXcosUlbtYlsUbVvTMJ18lu/Ptj1d0nSrZ2G681tS6tWK090z0ePaviiX8jn8albs06SpOmZryopOkEzr56qpOhE5RUflD/gP/8nBpxDdeYzj9baQmNMD0l9JQ2U9Lox5tfW2heDh7xqjCmRtEPShBqGWCzpWWNMpKSfSppnrfUHL4f/OHjMy5IeDz7+nqSbg6/tl1QQ3D7RGDMs+LilpHaSDqriM5ivB7e/oopL5TWdxwxJMyQpKaq+dX4DcOY8XtXrPUB5k26p2hTe/1odW79W8vsVKMhX6ZZM+eI7yp9b02J2hYgfDNXBhyZKksqyNsr4wuSJjFKgIP+8nwKAuqWwrFj/yNuilEuT9MWRPbq2VV/1ju2uSaseqTomsUm8+rfopdu6jlIDX4QCsir1l+nt7GVVx5QGyrQmJ0N94pKVsX+TDh49rMlrn5QkhXvD1D+up4qCl7iBb4s6E49SVcStlLTSGLNR0i2SXgzuHm2tTT/Fc0uMMe9JGqaKFci7T3boycYwxgxQxWcte1tri4M36tT7puPgwgpL6qnyPTsVOLi/aps/L1dhXZNVsvJdmbB68nXorMKFc08xiuTP26ewrj1VsmKJQlq0lgkNJRyB75BGoQ3lt34VlhUr1ONTj2adNSdrkVJiumpkwhDd+eFvdcxfWnX8xJUPVz3+WcfhKik/qrezlyncG6ZwX7gOHT0sr/GoV/MrtDEvq+o1jpQWyspqVOJ1Wrpj1QU/T+Bs1Zl4NMZ0kBSw1v4zuOkKSTu/4TBzJP1eUqSkT4Lb1qgiJl+WNFrSx8HtyyXdLulJY4xXFZelG0nKD4ZjgqQrq43tUcVNOHNVcUf4x8IFFfWr3yqscw95IqMU88IifTVnpor/tlDhfa9RyUfLjju2aOmbipo4WdFPz5WMVLx8scp3bpckRd4yQeH9rpEJq6eYFxap+IOF+mruTB2Z/SdF/fI3ajB0lGStDv/p4ZqmAeAidUl4lO7rebs8xiOPMfpw9ydau3eDXv3hNPk8Pv2x/32SKm6ambb+Lycdp15ImKZe9Sv5PD55jEcb9m/Wwn/9TZJ0RXSift5lhKysPsvbqic3zL4g5wacS6byDrDaFrxk/bSkKEnlkrZLGm+tPRBcAfzvU608BscIkbRX0gvW2l8Ht7WW9BdJTSXlSbrVWrvLGBOjisvLl6viTwPdLmm9pAWS4iRlSYqWNMVau9IYUyjpCUmDVXGJ+0Zrbd6p5pMUVd++27/TN3sjgO+AUWPiT38QAKDWrPrpnAxrbXJN++rMyqO1NkNS6kn2DXAco1wVwVd92w5VfL7xxGNzJV1XwzA/PMnYlbfDPeAyFwAAgItRXbrbGgAAAHUc8QgAAABnxCMAAACcEY8AAABwRjwCAADAGfEIAAAAZ8QjAAAAnBGPAAAAcEY8AgAAwBnxCAAAAGfEIwAAAJwRjwAAAHBGPAIAAMAZ8QgAAABnxCMAAACcEY8AAABwRjwCAADAGfEIAAAAZ8QjAAAAnBGPAAAAcEY8AgAAwBnxCAAAAGfEIwAAAJwRjwAAAHBGPAIAAMAZ8QgAAABnxCMAAACcEY8AAABwRjwCAADAGfEIAAAAZ8QjAAAAnBGPAAAAcEY8AgAAwBnxCAAAAGfEIwAAAJwRjwAAAHBGPAIAAMAZ8QgAAABnxCMAAACcEY8AAABwRjwCAADAGfEIAAAAZyG1PYGLmS8+UbHvpNX2NIA6Z2VtTwAAcEpGc066j5VHAAAAOCMeAQAA4Ix4BAAAgDPiEQAAAM6IRwAAADgjHgEAAOCMeAQAAIAz4hEAAADOiEcAAAA4Ix4BAADgjHgEAACAM+IRAAAAzohHAAAAODPW2tqew0XLGJMnaWdtzwMAAOAbamWtja5pB/EIAAAAZ1y2BgAAgDPiEQAAAM6IRwAAADgjHgEAAOCMeAQAAIAz4hEAAADOiEcAAAA4Ix4BAADgjHgEAACAs/8Hx71csEPShfoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 662.4x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "category_names = ['Overlapping', 'Non Overlapping']\n",
    "results = {'General Vocab': [overlap, len(general_words)-overlap],\n",
    "           'SP Vocab': [overlap, len(sp_words)-overlap]}\n",
    "\n",
    "\n",
    "def stack(results, category_names):\n",
    "    \"\"\"\n",
    "    Parameters\n",
    "    ----------\n",
    "    results : dict\n",
    "        A mapping from question labels to a list of answers per category.\n",
    "        It is assumed all lists contain the same number of entries and that\n",
    "        it matches the length of *category_names*.\n",
    "    category_names : list of str\n",
    "        The category labels.\n",
    "    \"\"\"\n",
    "    labels = list(results.keys())\n",
    "    data = np.array(list(results.values()))\n",
    "    data_cum = data.cumsum(axis=1)\n",
    "    category_colors = plt.get_cmap('RdYlGn')(\n",
    "        np.linspace(0.15, 0.85, data.shape[1]))\n",
    "\n",
    "    fig, ax = plt.subplots(figsize=(9.2, 5))\n",
    "    ax.invert_yaxis()\n",
    "    ax.xaxis.set_visible(False)\n",
    "    ax.set_xlim(0, np.sum(data, axis=1).max())\n",
    "\n",
    "    for i, (colname, color) in enumerate(zip(category_names, category_colors)):\n",
    "        widths = data[:, i]\n",
    "        starts = data_cum[:, i] - widths\n",
    "        ax.barh(labels, widths, left=starts, height=0.5,\n",
    "                label=colname, color=color)\n",
    "        xcenters = starts + widths / 2\n",
    "\n",
    "        r, g, b, _ = color\n",
    "        text_color = 'white' if r * g * b < 0.5 else 'darkgrey'\n",
    "        for y, (x, c) in enumerate(zip(xcenters, widths)):\n",
    "            ax.text(x, y, str(int(c)), ha='center', va='center',\n",
    "                    color=text_color)\n",
    "    ax.legend(ncol=len(category_names), bbox_to_anchor=(0, 1),\n",
    "              loc='lower left', fontsize='small')\n",
    "\n",
    "    return fig, ax\n",
    "\n",
    "\n",
    "fig, ax = stack(results, category_names)\n",
    "plt.tight_layout()\n",
    "fig.savefig('./corpus_tokenization_comparison/corpus_comparison.png', dpi=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "f9e496d3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAo8AAAFgCAYAAADaapOxAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAkB0lEQVR4nO3deXiU1cH38d+ZyUJYQgKEAIEYSiAB2QmJhEXQPpWiVKlFQVBbXuqrVnxabLX0EQo+Wpe3jS0WV9wVrOICgtqqFBfCGgQBJUIUgoadkEBCksnMef+YSQwQ4IjAQPl+rotLci9nzp3Y+uXccw/GWisAAADAhSfcEwAAAMDZg3gEAACAM+IRAAAAzohHAAAAOCMeAQAA4Cwi3BPAkfLy8lpGRETMlNRVBD7OLX6/339b79693wj3RAAA9SMez0AREREzW7Vq1TkhIaHY4/HwWUo4Z5SVlTUoKCj4vaQ3wj0XAED9WNU6M3VNSEgoJRxxrmnYsGGFtTYx3PMAABwd8Xhm8hCOOBcZYyTJhHseAICjIx4BAADgjPc8nuGKLs/s833HaDN3ed6x9s+fP7/JH//4xzYej0eJiYm+p59+ektCQoL/RF6ra9eundetW/f5ic30W9dcc815s2bN2vJ9x/muBr9yzff6fi8aOeuo3+v8/Pyo9PT0bu+///6Giy66qGzOnDmxubm5jXNycopO5LXKy8vN+PHj233xxRcxfr/fTJgwYce4ceOKT3TuEydObNO3b9+y0aNHl5zoGJI0ffr05l26dKn44Q9/WPZ9xgEAnJlYeTzH7dixwztx4sR28+bNK1ixYkX+sGHD9o0fPz75RMby+0+oN+sVjnA8HTp06FBx//33tzoZY91xxx1tUlJSqlauXJm/aNGiL3JyclqtWbMm+kTGOpk/u1tvvXUP4QgA/7mIx3PcK6+8Ejds2LB9SUlJ1ZJ088037125cmXjnj17ptccM3z48Pbr16+P/vDDDxtmZWV16tOnT9qUKVMSpeBq1U9/+tOUQYMGdczLy4upOWfGjBnN+vXr16lLly6dZ8yY0azm2BEjRqT079+/45AhQ1LLy8vN/PnzmwwePDh18ODBqd26deu8du3aaCm4gilJV155ZcqoUaPOy87O7nTxxRd3CAQC8vl8Gjp06A+ys7M7XX/99e2uvPLKlNP4LfteUlNTD1ZXV+uTTz5pUHf7o48+2qx79+7p3bt3T58zZ06sJGVmZqaNGzeuXUZGRtrYsWOPCPr58+fHTZ48eYckNW3aNDBu3LhdL7zwQrPJkycnzpw5M16S1qxZEz1ixIiUQCCg66+/vl1WVlan7OzsjgUFBZGS1KFDh/NHjBiRcuONN7atGdfv96t///4dMzMz07Kzszvu3bvXU3PsFVdc0b5z585dnn/++biaOY4fP75t796902+55ZYkKfhznj17dtP8/PyoXr16pQ8dOvQHnTp16vLGG280kaTXXnsttnPnzl1+/OMf/yArK6tTfn5+1Cn4VgMAThHi8Ry3bdu2yKSkpKq62xISEnypqakVy5cvjzlw4IDZuXNn5Pnnn195xx13tJ0/f35BXl5e/pIlSxpv3bo1QpLatWtX9eGHH27MzMw8WDPGddddt2/JkiVf5OXlbXjkkUdqn55NSkryLV68eGNWVtaBmTNnNpOk/fv3excuXLgpJyencOrUqa0Pn+OAAQMO5ObmftGoUaPAihUrYp5//vn41NTUytzc3C969Ohx8PDjz3S33377jnvvvbf2e1JdXa2cnJxWy5Yty1+4cOHGKVOmJNXsGzlyZPHKlSvzP//885g9e/Z4647j8/k8MTExtQ9WnXfeeVVFRUWR1113XfGcOXPiJen5559vdvXVVxf/4x//aBofH+9ftmzZF/fff/8306ZNay1JO3bsiHr88ce3PvHEE1/XjOP1evXPf/5z0/Lly/OHDRtW8vTTTzeTpG3btkU99thjhbm5uRvuu+++2p/TsGHDSletWrVhzZo1DTds2HBICBYXF0e8+eabX7766qsFDz/8cEtJmjp1aptFixblv/baa18VFRURjgBwliEez3Ft2rSpOvw/4Hv37o0YPXr03hdeeCH+5Zdfjhs6dGiJJOXn58dceumlqZmZmWmFhYXRX331VZQkZWZmHnGLcu7cubF9+/ZNGzRoUKctW7bU3krNyMgok6S+ffuWFxQU1Kwylns8Hg0YMKB88+bNR9x2zczMLJektm3bVu3evdu7adOm6Jpx+vXrd9bdHr3kkksOFBYWRm/dujVKkoqKiiLatm1bFRMTY1u0aOGPioqyPp9PknTBBReUS1Lr1q19h8djZGRk4ODBg7VPJhcWFka1adPG17Fjx6qSkpKIkpISz6JFi2JHjBhRun79+gZvvfVWXGZmZtrEiRPblZSUeCUpOTm54vD3t5aWlnrGjBmTkpGRkfbcc8+1KCoqipSktm3bViYmJvrj4+MDdeeYnZ1dJkndu3cvz8/PP+Tnl5aWdjAyMlIdOnSoqnnNQCBgEhMT/TExMTY9Pf2si38AONcRj+e4n/3sZyULFiyIKyoqipCkxx57rFnv3r3LfvKTn5R+/PHHsXPmzIm/9tpr90pSenp6+dtvv71p+fLl+evXr/9swIAB5ZLk8Rz5r9Gf/vSn1u+8886mDz744IuYmJhAzfZVq1Y1lKS8vLyGHTp0qJSkdevWNQwEAsrNzW2YkpJSefhYxpja1TVrrUlNTa2sGWfZsmUNT+o35DSZMGHCjkceeaSlJLVp06b666+/jjp48KDZvXu31+fzmcjISEmHfm+tPfTTm4YPH158zz33tJSCwffUU08ljB07dq8kDRs2bN+dd97ZOjU1tSI6Otp27ty54vLLLy9evnx5/ooVK/JffvnlzYePX+O1116Lbd26tW/lypX511133W5rrZGkb775Jnr37t3e0tJST1VVVe0cly5d2lCS1q5d2zAtLe2Qn9/hPztJ8nq9dteuXd6KigqTn58fIwDAWYWnrc9wx3tS+vtKTEz05+TkbB0+fHiHuk9bR0ZGKj09vbygoKBBamqqT5Luv//+by677LIOgUBAkZGR9p133tl0tHGHDx9ePHDgwE5du3Ytb9q0ae3K1pYtW6Kzs7M7RkdH2wULFhQsXLiwcePGjf1DhgxJ3bt3b+SsWbO+PN6cx4wZs++ll15q1q9fv07t27evjIyMPGmfiXmsp6VPptGjR5dMmTKlrSRFRERo4sSJ27OystIkadq0ad+4jHHfffdtGz9+fHJGRkaa3+83//3f/729R48elZJ07bXXFnfs2LHbnDlzNta83vvvvx+blZXVSZJGjRq19ze/+c3u+sa98MILyx544IHWgwcPTk1MTPQlJSX5JKlVq1ZVv/jFL5ILCgoaTJo0aVvN8fPmzYu76667krKzs/enp6dX1TdmXZMnTy4aNGhQWnJycmVCQoIvKiqKzzQFgLOIOXw1A+G3Zs2azT169Kj3P+xns/o+Cmb+/PlN5s2b1/Txxx//+ljnHq6ystJER0fbnJycFnv27Im45557tp/8GaOu+j6GKTMzM+3dd9/d2LRp08DRzjtczc/u4MGDplevXp3XrVv3WUTEt3+OXbNmTYsePXqknLyZAwBOJlYecVb60Y9+1KGsrMwbFRUVeP3114+7Wokzx4svvhj3yCOPtCwvL/fceOONO+qGIwDgzMfK4xlozZo1X3br1q2Yv6IQ5xprrT799NPmPXr0aB/uuQAA6scDM2emdbt27WoaCAT4O35xTikvL29gjNkR7nkAAI6O+0VnoOrq6vHbt2+fuX379q4i8HFu8fv9/jvDPQkAwNFx2xoAAADOWNUCAACAM+IRAAAAznjP4xmgRYsWNiUlJdzTAAAAkCTl5eXtttYm1LePeDwDpKSkaOXKleGeBgAAgCTJGLPlaPu4bQ0AAABnxCMAAACcEY8AAABwRjwCAADAGfEIAAAAZ8QjAAAAnBGPAAAAcEY8AgAAwBnxCAAAAGfEIwAAAJwRjwAAAHBGPAIAAMAZ8QgAAABnEeGeACTfps9VdHlmuKcBnFTXjE0N9xQAAKcAK48AAABwRjwCAADAGfEIAAAAZ8QjAAAAnBGPAAAAcEY8AgAAwBnxCAAAAGfEIwAAAJwRjwAAAHBGPAIAAMAZ8QgAAABnxCMAAACcEY8AAABwRjwCAADAGfEIAAAAZ8QjAAAAnBGPAAAAcEY8AgAAwBnxCAAAAGfEIwAAAJwRjwAAAHBGPAIAAMAZ8QgAAABnxCMAAACcEY8AAABwRjwCAADAGfEIAAAAZ8QjAAAAnBGPAAAAcEY8AgAAwBnxCAAAAGfEIwAAAJwRjwAAAHBGPAIAAMAZ8QgAAABnxCMAAACcEY8AAABwRjwCAADAGfEIAAAAZ8QjAAAAnBGPAAAAcEY8AgAAwFlEuCcAfFdxE+5UdMYABUqKtevW0bXbG116lRpdOlLW71flysUqffYhSVLEeamKu3mSTMNGUiCgXb/9uYwxir/jXnlbtZUCAVWs+Ej7n5shSYrq0kux43+jyJRUFf/5TlXkLgzLdQI4taI8kfrbkCmK9ETIa7z64OtleuazVzXu/JHq36aPrAIqrijVfSse1Z6KfZKkHzRtp9v6jFfDiBhZBXTje5NVFfDpgYF3qFmDOHmNV2t3b9BfVz2tgKwkaXDbLP38/CtlrVRQskV3L5sRxqsGvr8zOh6NMX5JayUZSX5Jt1hrc40xKZI+l5QvKUrSh5JulpRcZ3uNHGvtc3XGnCop2lo7qc62npJmW2s7n4Q5p0iab63t+n3HQv3K31+gsgWvKO7XU2u3RXXrowZZg7Tz1mukap88TeODOzxexU+cpuIHp6p680aZJk0lf7UUEakDb7yoqrV5UkSEmt/1sKJ791PlqiXy796ufX+7S41HjA3PBQI4LaoCPk1cdLcO+ivlNV49NOSPWr59jV7Kn6+n1r8iSfpp6iW6vstPlbPqKXmNR/+T+Sv9afnDKigpVGxUY1UHqiVJU5dMV3n1QUnStH6/1uB2F2jh1iVKatxKY9Iv1y0Lp+mAr0xx0bFhu17gZDmj41HSQWttT0kyxlwi6V5JF4b2FVhrexpjIiQtlHSFpFU1248x5mxJb0uaVGfbKEmzTurMccpUffaJvC1bH7Kt0dArtf/VZ6VqnyQpUFIsSYrulSXf5k2q3rxRkmT3lwT/WVUZDEdJqq6W78sN8jZvKUny79wW3B4InOpLARBmB/2VkqQIj1cRHq+sbG0ESlKDiOjQ+qGUkdhdX5YUqqCkUJJUWnWg9riac7zGq0hPhKwNnnVZ+yF6o+BfOuArkyTtqyw91ZcEnHJnejzWFSup+PCN1tpqY0yupFQF4/GYrLX5xph9xpgsa+2y0OarJF0SWoF8VFJDSQWSxllri40xqaHtCQqugI6UtEPSXEnxkiIl3WmtnRsaL8IY86ykXpK+kHSdtbb8BK8bDiLaJCu6S0/Fjr1JtqpKpU//Tb5NnyuiTbJkrZpNnS5PbJwqPnpXB15//pBzTaPGatB3oPa8+VKYZg8gXDwyevy/7lFS41Z6fdO/9PneAknS/+l6lS45b6DKfOX69aK7JUntmrSSldUDA3+vuOgmWrh1iV7Kn1871gMDf6/OzTpo2fbV+uDrZaFzgn/QfWjIH+U1Hj2z/lUt3/Hpab5K4OQ60x+YiTHGrDbGbJA0U9L/Hn6AMaahpIsVvL0tSR1C59T8GljPuLMVXG2UMeYCSXustRslPSfpDmtt99B4fwwd/6KkGdbaHpKyJW2TVCFphLW2t6Qhkv5ijDGh49MkPR4ap1TBW+qHz/sGY8xKY8zKPVXV3/X7gsN5vTKNY7X7d+NU+sx0xd9+b+32qC49te8vk7Xn979UgwsGK6p732/P83gVf9vdKpv/D/l3FIVn7gDCJiCr8e/+QSPn36LOzTqofWxbSdKT617WVQsm6N3CxRqR+iNJwVXFbi3SdM+yGZrw72kamNRXvVueXzvW7R/dpyvfvFmRnkj1Cm33Go/aNmmlXy+6W3ct/bt+l/FLNY5sePovFDiJzvR4PGit7WmtTZc0VNJzdQKtgzFmtaTFkhZYa98ObS8InVPz66N6xn1J0s+MMR4FI3K2MaappDhr7QehY56VNMgY00RSkrX2dUmy1laEVhGNpD8ZYz6V9J6kJEmJoXO3WmsXh37/gqQBh0/AWvu4tTbDWpvRPOpsWgA+M/n37FTFkn9LknwbP5MCAXli4+Tfs1NV61YpsL9EtqpSFXmLFdkhrfa8uF9NUvW2rSpj1RE4px3wlWv1rs+V2arHIdvfL8zVhW0zJUm7Du7Vml2fq6Rqvyr9VVq6bbU6xrU/5PiqgE+5RXkakJRRe87ib/Lkt35tL9+lwv3blNS41em5KOAUOdPjsZa1domkFgreOpa+jcRe1tqp33GsrZI2K/j+ySslvXyMw81Rto8JzaVP6D2WOyQ1qHmJw1/yu8wP313Fsg8U3T34f9beNskykZEKlO5T5aqlikhJlYmKljxeRXftrerCryRJTcbcKNOwsUpn5oRz6gDCpGlUk9pVwChPpPq07KrC/UWHxF12m94q3B+8K7F8+6f6QdNkRXuj5DUe9UzorC2lXyvGG61mDeIkBVcas1r3VGFp8JyPv1mpni271L5euyatta1s52m8SuDkO2uWvIwx6ZK8kvYo+J7E72u2pAcVjNCvQ69RbIwZGFqtvFbSB9baUmPM18aYK6y1bxhjokPzaCppp7XWZ4wZIum8OmMnG2P6hYJ3tKSPT8J8ERJ32/8qumsfeWLjlPjkm9o/+wmVvzdPcRMmK2H6bNlqn4r/Ok2SZMv2q2zuLLX4y7OStarMy1Vl3mJ5mrdUk6vGybf1KyXkBN8DWfbWKyp/d64iUzur2aQHZBrHqkHfgfKPvkG7JowK5yUDOAWax8RpUt+b5DEeeYzRv7cu1ZJtn2hav18ruUlrBazVjvLdysl7UpJ0wFemV754S49efLckq6XbVmvp9tWKj47Vn/rfpkhPpDzGo092rte8L9+TJC3f8akyWnXXM5c8oIAN6NFPZx3yoA1wNjI1T4Sdiep8VI8UXAH8g7V2wdE+Duewj/Cp8ZS1dno9YydIKpI0wVr7aGhbT337wMyXkn4RemCmo6THFFz59Cn4wEyppDcVfFhmtaT+kn4cGv4tBT8+KFvSRknXHuuBmR5xjezbF55/tN3AWemasanhngIA4AR9cNXsPGttRn37zuiVR2ut9yjbN0s64nMUQ9tjHMfepWD41d22WtIF9Ry7UdJF9QzT7yjDd3GZAwAAwNnmrHnPIwAAAMKPeAQAAIAz4hEAAADOiEcAAAA4Ix4BAADgjHgEAACAM+IRAAAAzohHAAAAOCMeAQAA4Ix4BAAAgDPiEQAAAM6IRwAAADgjHgEAAOCMeAQAAIAz4hEAAADOiEcAAAA4Ix4BAADgjHgEAACAM+IRAAAAzohHAAAAOCMeAQAA4Ix4BAAAgDPiEQAAAM6IRwAAADgjHgEAAOCMeAQAAIAz4hEAAADOiEcAAAA4Ix4BAADgjHgEAACAM+IRAAAAzohHAAAAOCMeAQAA4Ix4BAAAgDPiEQAAAM6IRwAAADgjHgEAAOCMeAQAAIAz4hEAAADOiEcAAAA4Ix4BAADgjHgEAACAs4hwTwBSZGpntZm7PNzTAE6qReGeAADghBnNPuo+Vh4BAADgjHgEAACAM+IRAAAAzohHAAAAOCMeAQAA4Ix4BAAAgDPiEQAAAM6IRwAAADgjHgEAAOCMeAQAAIAz4hEAAADOiEcAAAA4Ix4BAADgjHgEAACAM+IRAAAAzohHAAAAOCMeAQAA4Ix4BAAAgDPiEQAAAM6IRwAAADgjHgEAAOCMeAQAAIAz4hEAAADOiEcAAAA4Ix4BAADgjHgEAACAM+IRAAAAzohHAAAAOCMeAQAA4Ix4BAAAgDPiEQAAAM6IRwAAADgjHgEAAOCMeAQAAIAz4hEAAADOiEcAAAA4Ix4BAADgjHgEAACAM+IRAAAAzohHAAAAOCMeAQAA4Ix4BAAAgDPiEQAAAM6IRwAAADgjHgEAAOCMeAQAAIAz4hEAAADOiEcAAAA4Ix4BAADgjHgEAACAM+IRAAAAzohHAAAAOCMeAQAA4Ix4BAAAgDPiEQAAAM6IRwAAADgjHgEAAOCMeAQAAIAz4hEAAADOiEcAAAA4Ix4BAADgjHgEAACAM+IRAAAAzohHAAAAOCMeAQAA4Ix4BAAAgDPiEQAAAM6IRwAAADgjHgEAAOCMeAQAAIAz4hEAAADOiEcAAAA4Ix4BAADgjHgEAACAM+IRAAAAzohHAAAAOCMeAQAA4Ix4BAAAgDPiEQAAAM6IRwAAADgjHgEAAOCMeAQAAIAz4hEAAADOiEcAAAA4Ix4BAADgjHgEAACAM+IRAAAAzohHAAAAOCMeAQAA4Ix4BAAAgDPiEQAAAM6IRwAAADgjHgEAAOCMeAQAAIAz4hEAAADOIsI9AUi+TZ+r6PLMcE8DOKmuGZsa7ikAYbdo5KxwTwE46Vh5BAAAgDPiEQAAAM6IRwAAADgjHgEAAOCMeAQAAIAz4hEAAADOiEcAAAA4Ix4BAADgjHgEAACAM+IRAAAAzohHAAAAOCMeAQAA4Ix4BAAAgDPiEQAAAM6IRwAAADgjHgEAAOCMeAQAAIAz4hEAAADOiEcAAAA4Ix4BAADgjHgEAACAM+IRAAAAzohHAAAAOCMeAQAA4Ix4BAAAgDPiEQAAAM6IRwAAADgjHgEAAOCMeAQAAIAz4hEAAADOiEcAAAA4Ix4BAADgjHgEAACAM+IRAAAAzohHAAAAOCMeAQAA4Ix4BAAAgDPiEQAAAM6IRwAAADgjHgEAAOCMeAQAAIAz4hEAAADOIsI9AeC7iptwp6IzBihQUqxdt46u3d7o0qvU6NKRsn6/KlcuVumzD0mSIs5LVdzNk2QaNpICAe367c9ljFH8HffK26qtFAioYsVH2v/cDElSVJdeih3/G0WmpKr4z3eqIndhWK4TkKTbM25Qv9a9tK+yVL/41x2SpCkXTFByk9aSpMaRjXTAV6bx7/5BP0zur1Fpl9ae+4Omybrh3f/RppIt6hTXXr/P/L+K9kZp6bbVemj1c5KkSE+EJmXepLT49iqpPKC7lk7X9vLdp/9CAZw1jhmPxpgHJW2x1v419PU/JW211o4Pff0XSd9Ya3O+6wsbYwZL+q219jJjzM8lZVhrb/mu4xxj/CPGNMakSPpYUrK1NlBn+2pJN1hrl5+E112k4HWt/L5joX7l7y9Q2YJXFPfrqbXborr1UYOsQdp56zVStU+epvHBHR6v4idOU/GDU1W9eaNMk6aSv1qKiNSBN15U1do8KSJCze96WNG9+6ly1RL5d2/Xvr/dpcYjxobnAoE63tn8oV7f9C/9IfOm2m13LX2o9vc3dR+jMl+5JOm9wsV6r3CxJKl9bDvd0/82bSrZIkn6TZ9x+vPKJ/XZ3o26f8DtymzVQ8u3r9Gw9oN1oKpMY96eqIva9dMN3UcfMj4AHO54t61zJWVLkjHGI6mFpPPr7M+WtNjlhYwx3hOZ4Mlkrd0saaukgTXbjDHpkpqcjHDE6VH12ScKHCg9ZFujoVdq/6vPStU+SVKgpFiSFN0rS77Nm1S9eaMkye4vkQIB2arKYDhKUnW1fF9ukLd5S0mSf+c2VW/ZJAUCAsLt090btL/qwFH3D2l3gd7fuuSI7RcnZ+v9rbmSpGYN4tQoIkaf7Q3+7+CfWz7SgDYZkqT+bTL0zuaPJEkffL1MfVp2PdmXAOA/zPHicbFC8ahgNK6TtN8YE2+MiZbUWdInxpiLjTGfGGPWGmOeCu2TMWazMWaKMeZjSSONMUONMRtCX/+0vhc0xiQYY141xqwI/epvjPGExoqrc9wmY0xifccf55pmSxpV5+tRkmYbYxoYY54OXcMnxpghodfxGmP+HNr+qTFmQmj7lNDrrTPGPG6MMXXGHGuMyQ3tyzzOfHASRLRJVnSXnmrx/55S83seVWRq59rtslbNpk5Xi5zn1HjEtUecaxo1VoO+A1X56YrTPW3ge+neIl3FFSX65sD2I/YNaXeBFhYG4zEhJl67Du6t3bfr4F4lxMTX2bdHkuS3AR3wlatpVJPTMHsAZ6tjxqO1tkhStTEmWcGIXCJpmaR+kjIkfRoa4xlJV1truyl4K/ymOsNUWGsHSHpD0hOShiu48tfqKC/7N0kPWmv7SrpS0szQLea5kkZIkjEmS9Jma+2O+o4/zjW/LOkKY0zNLfurJb0k6Veha+4mabSkZ40xDSTdIKm9pF7W2u6SXgyd93drbV9rbVdJMZIuq/Majay12ZJulvRUfZMwxtxgjFlpjFm5p6r6OFPGcXm9Mo1jtft341T6zHTF335v7faoLj217y+Ttef3v1SDCwYrqnvfb8/zeBV/290qm/8P+XcUhWfuwAmqu7pYV+dmHVTpr9RXpV+HtpgjjrG1v6tvnz1iGwDUcHnaumb1sSYel9T5OldSmqSvrLVfhI5/VtKgOuf/I/TP9NBxG621VtILR3m9H0r6e+h9iPMkxRpjmoTGuTp0zKg64x7t+HpZa7dLWi/pYmNMT0k+a+06SQMkPR86ZoOkLZI6hcZ/1FpbHdpX88f3IcaYZcaYtZIu0qG382eHjv0wNJ+4eubxuLU2w1qb0TyK55a+L/+enapY8m9Jkm/jZ1IgIE9snPx7dqpq3SoF9pfIVlWqIm+xIjuk1Z4X96tJqt62VWVvvhSuqQMnxGs8GpjUV//euvSIfRe166f3C7+9lR1caWxW+3VCTDPtPlhcZ1/z2jEbRzZU6TFukwOASzzWvO+xm4K3rZcquPJY837HI//YeqiyOr93+eOsR1I/a23P0K8ka+1+BaM11RiTIOkKSa8d5/hjqbl1PSr0ex3jOszh8w6tSD4s6WehlconJDU4xnXyx/hTrGLZB4ruHnwPl7dNskxkpAKl+1S5aqkiUlJloqIlj1fRXXuruvArSVKTMTfKNGys0pnf+XkvIOz6tOyqwv1Fh9yOliQjo8Fts7Swzvsg91bsU3n1QXVplipJuuS8gVpcFHzPb25RnoamBN8GfmHbLK3auf40XQGAs5XryuNlkvZaa/2hlbc4BQNyiaQNklKMMamh46+V9EE942yQ1N4Y0yH09eh6jpGkf0mq+4R0T0kKrVa+LilH0ufW2j3HOv44XpU0TN/espakDyWNCY3RSVKypPzQ+DfW3OY2xjTTt6G42xjTWNLPDhv/6tCxAySVWGtLHOYER3G3/a9a3P+kIpLOU+KTb6rhD3+i8vfmyZuYpITpsxX/27tV/NdpkiRbtl9lc2epxV+eVcJfX5CvIF+VeYvlad5STa4ap4h27ZWQ87wSHnxBDf/rcklSZGpnJT75phr0v1hxN01SwkOsSiJ8JmfdohkXTVO7Jq31yqUPaVjKYEnSRcn9at/TWFePhHTtOrhX28p2HrL9wVVP6XcZv9SLP35QRWU7tWz7aknSW18tUmxUE7344xyN7DRMj6/l33cAx2aCTXaMA4JPSRdLmm6tvTO07RkFV/vSQl9fLOnPCr7fcYWkm6y1lcaYzQp+XM7u0HFDJf1V0m4FPzKn6+Ef1WOMaSFphoIP40RI+tBae2Po/IzQ+D+31j4b2lbv8cf7+B9jzFxJidbaC0JfN5D0qKQ+kqolTbTW/jsUjQ9IGirJJ+kJa+3fjTF3K7hyuVnBJ7i3WGunhj6qZ4mkCyXFShp3vCe5e8Q1sm9feP6xDgHOOteMTT3+QcB/uEUjZ4V7CsAJMcbkWWsz6t13vHjEqUc84j8R8QgQjzh7HSse+esJAQAA4Ix4BAAAgDPiEQAAAM6IRwAAADgjHgEAAOCMeAQAAIAz4hEAAADOiEcAAAA4Ix4BAADgjHgEAACAM+IRAAAAzohHAAAAOCMeAQAA4Ix4BAAAgDPiEQAAAM6IRwAAADgjHgEAAOCMeAQAAIAz4hEAAADOiEcAAAA4Ix4BAADgjHgEAACAM+IRAAAAzohHAAAAOCMeAQAA4Ix4BAAAgDPiEQAAAM6IRwAAADgjHgEAAOCMeAQAAIAz4hEAAADOiEcAAAA4Ix4BAADgjHgEAACAM+IRAAAAzohHAAAAOCMeAQAA4Ix4BAAAgDPiEQAAAM6IRwAAADgjHgEAAOCMeAQAAICziHBPAFJkame1mbs83NMATqpF4Z4AAOCUYOURAAAAzohHAAAAOCMeAQAA4Ix4BAAAgDPiEQAAAM6IRwAAADgjHgEAAOCMeAQAAIAz4hEAAADOiEcAAAA4Ix4BAADgjHgEAACAM+IRAAAAzoy1NtxzOOcZY3ZJ2hLueQAAAIScZ61NqG8H8QgAAABn3LYGAACAM+IRAAAAzohHAAAAOCMeAQAA4Ix4BAAAgDPiEQAAAM6IRwAAADgjHgEAAOCMeAQAAICz/w/dfZW4qF0cogAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 662.4x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "tok_results = {'BPE Vocab': [bpe_wordlevel_overlap, len(bpe_words)-bpe_wordlevel_overlap],\n",
    "               'Wordlevel Vocab': [bpe_wordlevel_overlap, len(wordlevel_words)-bpe_wordlevel_overlap]}\n",
    "fig, ax = stack(tok_results, category_names)\n",
    "plt.tight_layout()\n",
    "fig.savefig('./corpus_tokenization_comparison/tokenization_comparison.png', dpi=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bfb29691",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}