texts_similarity.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Find two sentences that are closest in meaning to the first one."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['felis', 'it', 'redirection', 'undergone', 'arguments', 'offered', 'according', 'concatenate', 'may', 'single', 'later', 'recent', 'error', 'closest', 'installs', 'safari', 'predators', 'starting', 'legibility', 'process', 'simply', 'can', 'faint', 'are', 'installation', 'place', 'command', 'marks', 'roughly', 'features', 'purchase', 'some', 'used', 'left', 'safer', 'those', 'sounds', 'human', 'learned', 'download', 'drive', 'from', 'any', 'october', 'for', 'versions', 'won', 'genes', 'concern', 'genus', 'moved', 'store', 'cat', 'to', 'world', 'where', 'however', 'connected', 'streams', 'permanently', 'clear', 'pipes', 'receives', 'has', 'os', 'releasing', 'off', 'tiger', 'linux', 'output', 'two', 'entirely', 'in', 'stdout', 'available', 'file', 'mice', 'bytes', 'you', 'high', 'instead', 'is', 'contains', 'also', 'predecessor', 'members', 'second', 'release', 'separate', 'named', 'so', 'stdin', 'mac', 'yosemite', 'now', 'or', 'count', 'changes', 'selection', 'firmware', 'will', 'their', 'upgrade', 'need', 'wrong', 'unix', 'type', 'mid', 'using', 'symbols', 'typically', 'piped', 'factory', 'started', 'app', 'files', 'fifth', 'adjacent', 'if', 'ancestor', 'update', 'such', 'when', 'organisms', 'standard', 'with', 'intel', 't', 'have', 'tamed', 'through', 'content', 'leopard', 'cats', 'part', 'its', 'symbol', 'community', 'artificial', 'x', 'an', 'mavericks', 'other', 'similar', 'default', 'new', 'interactive', 'on', 'lines', 'every', 'most', 'the', 'keyboards', 'were', 'read', 'since', 'editions', 'hear', 'redirected', 'domestication', 'year', 'possess', 'right', 'both', 'chromosomes', 'small', 'version', 'online', 'a', 'switch', 'external', 'run', 'weighing', 'received', 'diploid', 'which', 'apple', 'catenates', 'was', 'of', 'as', 'just', 'sequence', 'disk', 'released', 'binary', 'no', 'non', 's', 'basic', 'rather', 'displays', 'useful', 'too', 'information', 'lion', 'terms', 'made', 'installed', 'longer', 'than', 'during', 'more', 'vermin', 'that', 'frequency', 'common', 'they', 'developed', 'computers', 'based', 'use', 'needing', 'patch', 'comparison', 'major', 'lb', 'often', 'deliberately', 'be', 'wild', 'domestic', 'commands', 'running', 'domesticated', 'unnecessary', 'processors', 'releases', 'not', 'enhancements', 'animals', 'ears', 'create', 'july', 'delete', 'and', 'flow', 'without', 'mountain', 'three', 'between', 'dogs', 'incremental', 'over', 'osx', 'time', 'size', 'allows', 'allow', 'successor', 'kg', 'people', 'by', 'one']\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import re\n",
    "\n",
    "a = []\n",
    "\n",
    "f = open('sentences.txt', 'r') # extracting data\n",
    "for line in f:\n",
    "    line = line.lower()\n",
    "    line = re.split('[^a-z]', line) # using regexp for tokenization\n",
    "    \n",
    "    # removing empty strings\n",
    "    for word in line:\n",
    "        if (word != ''):\n",
    "            a.append(word)\n",
    "            \n",
    "a = list(set(a)) # removing duplicates\n",
    "\n",
    "print(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{0: 'felis', 1: 'it', 2: 'redirection', 3: 'undergone', 4: 'arguments', 5: 'offered', 6: 'according', 7: 'concatenate', 8: 'may', 9: 'single', 10: 'later', 11: 'recent', 12: 'error', 13: 'closest', 14: 'installs', 15: 'safari', 16: 'predators', 17: 'starting', 18: 'legibility', 19: 'process', 20: 'simply', 21: 'can', 22: 'faint', 23: 'are', 24: 'installation', 25: 'place', 26: 'command', 27: 'marks', 28: 'roughly', 29: 'features', 30: 'purchase', 31: 'some', 32: 'used', 33: 'left', 34: 'safer', 35: 'those', 36: 'sounds', 37: 'human', 38: 'learned', 39: 'download', 40: 'drive', 41: 'from', 42: 'any', 43: 'october', 44: 'for', 45: 'versions', 46: 'won', 47: 'genes', 48: 'concern', 49: 'genus', 50: 'moved', 51: 'store', 52: 'cat', 53: 'to', 54: 'world', 55: 'where', 56: 'however', 57: 'connected', 58: 'streams', 59: 'permanently', 60: 'clear', 61: 'pipes', 62: 'receives', 63: 'has', 64: 'os', 65: 'releasing', 66: 'off', 67: 'tiger', 68: 'linux', 69: 'output', 70: 'two', 71: 'entirely', 72: 'in', 73: 'stdout', 74: 'available', 75: 'file', 76: 'mice', 77: 'bytes', 78: 'you', 79: 'high', 80: 'instead', 81: 'is', 82: 'contains', 83: 'also', 84: 'predecessor', 85: 'members', 86: 'second', 87: 'release', 88: 'separate', 89: 'named', 90: 'so', 91: 'stdin', 92: 'mac', 93: 'yosemite', 94: 'now', 95: 'or', 96: 'count', 97: 'changes', 98: 'selection', 99: 'firmware', 100: 'will', 101: 'their', 102: 'upgrade', 103: 'need', 104: 'wrong', 105: 'unix', 106: 'type', 107: 'mid', 108: 'using', 109: 'symbols', 110: 'typically', 111: 'piped', 112: 'factory', 113: 'started', 114: 'app', 115: 'files', 116: 'fifth', 117: 'adjacent', 118: 'if', 119: 'ancestor', 120: 'update', 121: 'such', 122: 'when', 123: 'organisms', 124: 'standard', 125: 'with', 126: 'intel', 127: 't', 128: 'have', 129: 'tamed', 130: 'through', 131: 'content', 132: 'leopard', 133: 'cats', 134: 'part', 135: 'its', 136: 'symbol', 137: 'community', 138: 'artificial', 139: 'x', 140: 'an', 141: 'mavericks', 142: 'other', 143: 'similar', 144: 'default', 145: 'new', 146: 'interactive', 147: 'on', 148: 'lines', 149: 'every', 150: 'most', 151: 'the', 152: 'keyboards', 153: 'were', 154: 'read', 155: 'since', 156: 'editions', 157: 'hear', 158: 'redirected', 159: 'domestication', 160: 'year', 161: 'possess', 162: 'right', 163: 'both', 164: 'chromosomes', 165: 'small', 166: 'version', 167: 'online', 168: 'a', 169: 'switch', 170: 'external', 171: 'run', 172: 'weighing', 173: 'received', 174: 'diploid', 175: 'which', 176: 'apple', 177: 'catenates', 178: 'was', 179: 'of', 180: 'as', 181: 'just', 182: 'sequence', 183: 'disk', 184: 'released', 185: 'binary', 186: 'no', 187: 'non', 188: 's', 189: 'basic', 190: 'rather', 191: 'displays', 192: 'useful', 193: 'too', 194: 'information', 195: 'lion', 196: 'terms', 197: 'made', 198: 'installed', 199: 'longer', 200: 'than', 201: 'during', 202: 'more', 203: 'vermin', 204: 'that', 205: 'frequency', 206: 'common', 207: 'they', 208: 'developed', 209: 'computers', 210: 'based', 211: 'use', 212: 'needing', 213: 'patch', 214: 'comparison', 215: 'major', 216: 'lb', 217: 'often', 218: 'deliberately', 219: 'be', 220: 'wild', 221: 'domestic', 222: 'commands', 223: 'running', 224: 'domesticated', 225: 'unnecessary', 226: 'processors', 227: 'releases', 228: 'not', 229: 'enhancements', 230: 'animals', 231: 'ears', 232: 'create', 233: 'july', 234: 'delete', 235: 'and', 236: 'flow', 237: 'without', 238: 'mountain', 239: 'three', 240: 'between', 241: 'dogs', 242: 'incremental', 243: 'over', 244: 'osx', 245: 'time', 246: 'size', 247: 'allows', 248: 'allow', 249: 'successor', 250: 'kg', 251: 'people', 252: 'by', 253: 'one'}\n"
     ]
    }
   ],
   "source": [
    "dict = {}\n",
    "d = len(a)\n",
    "\n",
    "for i in range(d): # dict indexing\n",
    "    dict[i] = a[i]\n",
    "print(dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(22, 254)"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# creating the matrix of occurrences\n",
    "\n",
    "matrix = np.zeros((22, d), np.int) \n",
    "matrix.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "f = open('sentences.txt', 'r')\n",
    "row = 0\n",
    "\n",
    "for line in f:\n",
    "    line = line.lower()\n",
    "    line = re.split('[^a-z]', line)\n",
    "    for key,value in dict.items():\n",
    "        if value in line:\n",
    "            matrix[[row],[key]] += 1\n",
    "            \n",
    "#            print(value)\n",
    "#    print(line)\n",
    "#    print('(line ', row + 1, ')', sep='')\n",
    "#    print('----------------')\n",
    "\n",
    "    row += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0 0 0 ... 0 0 0]\n",
      " [0 1 0 ... 0 0 0]\n",
      " [0 0 0 ... 0 0 0]\n",
      " ...\n",
      " [0 0 0 ... 0 0 0]\n",
      " [0 0 0 ... 0 0 0]\n",
      " [0 0 0 ... 0 1 0]]\n",
      "[0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0\n",
      " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      " 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 1 1 0 0 0 0 0 0\n",
      " 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0]\n",
      "[0]\n",
      "[1]\n"
     ]
    }
   ],
   "source": [
    "print(matrix)\n",
    "print(matrix[0])\n",
    "\n",
    "print(matrix[[0],[52]])  # line 1 doesn't contain the 'cat'\n",
    "print(matrix[[0],[133]]) # but it does contain the 'cats'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Distance to 2: 0.9402385695332803\n",
      "Distance to 3: 0.8619868881315291\n",
      "Distance to 4: 0.8885443574849294\n",
      "Distance to 5: 0.8055388829343507\n",
      "Distance to 6: 0.9351796276447836\n",
      "Distance to 7: 0.7547442642060137\n",
      "Distance to 8: 0.9154845745271484\n",
      "Distance to 9: 0.9069515789601529\n",
      "Distance to 10: 0.8833576312960392\n",
      "Distance to 11: 0.8703592552895671\n",
      "Distance to 12: 0.8703592552895671\n",
      "Distance to 13: 0.8773721321030068\n",
      "Distance to 14: 0.8703592552895671\n",
      "Distance to 15: 0.8619868881315291\n",
      "Distance to 16: 0.9402385695332803\n",
      "Distance to 17: 0.85361498905772\n",
      "Distance to 18: 0.9503708333014534\n",
      "Distance to 19: 0.9442721787424647\n",
      "Distance to 20: 0.9402385695332803\n",
      "Distance to 21: 0.8427572744917122\n",
      "Distance to 22: 0.8250364469440588\n"
     ]
    }
   ],
   "source": [
    "from scipy.spatial import distance\n",
    "\n",
    "for k in range(1, 22):\n",
    "    print('Distance to ', k + 1, ': ', distance.cosine(matrix[0],matrix[k]), sep='')\n",
    "    # cosine distance from sentence 1 to sentence k"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 7 and 5 is the answer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}