NLP/Notebooks/1_Tokenization_examples.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "orig_nbformat": 4,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.7.6 64-bit ('ml': conda)"
  },
  "interpreter": {
   "hash": "774e39f5f21f7b60062f3abb90b8afe0139135a3792f728ee4bd9a31085203c9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "source": [
    "## Tokenization"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Manual\n",
    "def tokenizer(txt):\n",
    "    tokens = re.split('\\W+', txt)\n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "['Muestra', 'de', 'un', 'Tokenizer', 'sencillo', 'podría', 'mejorarse', '']"
      ]
     },
     "metadata": {},
     "execution_count": 31
    }
   ],
   "source": [
    "tokenizer('Muestra de un Tokenizer sencillo :),@podría mejorarse.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Usando librerias   \n",
    "import nltk\n",
    "from nltk.corpus import stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "['Tokenizer', ',', 'de', 'libreria', 'NLTK', ',', '@', 'con', 'mejores', 'funcionalidades', ':', ')', '.']\n"
     ]
    }
   ],
   "source": [
    "Text = 'Tokenizer, de libreria NLTK, @con mejores funcionalidades :) .'\n",
    "Text_tokenized = nltk.word_tokenize(Text)\n",
    "print(Text_tokenized)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "['Mary,', 'don’t', 'slap', 'the', 'green', 'witch']"
      ]
     },
     "metadata": {},
     "execution_count": 39
    }
   ],
   "source": [
    "import spacy\n",
    "nlp = spacy.load('en_core_web_sm')\n",
    "texta = \"Mary, don’t slap the green witch\"\n",
    "#print([str(token) for token in nlp(text.lower())])\n",
    "#print([token.text for token in nlp(text.lower())])\n",
    "nlp(texta).text.split(\" \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "['snow', 'white', 'and', 'the', 'seven', 'degrees', ':d', '#makeamoviecold', '@midnight']\n"
     ]
    }
   ],
   "source": [
    "from nltk.tokenize import TweetTokenizer\n",
    "tweet=u\"Snow White and the Seven Degrees :D #MakeAMovieCold@midnight\"\n",
    "tokenizern = TweetTokenizer()\n",
    "print(tokenizern.tokenize(tweet.lower()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "['OMG', '@twitterguy', 'that', 'was', 'sooooooooo', '#happy', 'cool', ':D', ':D', ':D', '!', '!', '!']\n"
     ]
    }
   ],
   "source": [
    "from nltk.tokenize.casual import casual_tokenize\n",
    "\n",
    "tweet = \"OMG @twitterguy that was sooooooooo #happy cool :D :D :D!!!!\"\n",
    "print(casual_tokenize(tweet))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "['hola', 'como', 'estas', '?']\n<generator object <genexpr> at 0x7f98abe37c50>\n"
     ]
    }
   ],
   "source": [
    "from nltk.tokenize import word_tokenize\n",
    "from gensim.corpora.dictionary import Dictionary\n",
    "\n",
    "sometext = \"hola como estas?\"\n",
    "\n",
    "tokens = word_tokenize(sometext)\n",
    "my_vocab = Dictionary([tokens])\n",
    "print(tokens)\n",
    "losit_id = (my_vocab.token2id[x] for x in tokens)"
   ]
  },
  {
   "source": [
    "### Using Transformers Library"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "import transformers\n",
    "tokenizer = transformers.BertTokenizer.from_pretrained('bert-base-uncased')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "['mari', ',', 'don', '’', 't', 'slap', 'the', 'green', 'witch']\n9\n"
     ]
    }
   ],
   "source": [
    "sample_txt = \"Mari, don’t slap the green witch\"\n",
    "tokens = tokenizer.tokenize(sample_txt)\n",
    "print(tokens)\n",
    "print(len(tokens))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "[16266, 1010, 2123, 1521, 1056, 14308, 1996, 2665, 6965]\n9\n"
     ]
    }
   ],
   "source": [
    "token_ids = tokenizer.convert_tokens_to_ids(tokens)\n",
    "print(token_ids)\n",
    "print(len(token_ids))"
   ]
  }
 ]
}