{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/dbamman/anlp25/blob/main/4.topics/TopicModel.ipynb)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "r9IMbBbn2vK1"
   },
   "source": [
    "# Topic modeling movie summaries\n",
    "\n",
    "In this notebook we'll use topic modeling to discover broad themes in a collection of movie summaries."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install gensim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "LiJYciPG2vK4",
    "outputId": "f6b7fb84-2c15-4a0e-e687-c034c54ba8dd"
   },
   "outputs": [],
   "source": [
    "import operator\n",
    "import re\n",
    "\n",
    "import gensim\n",
    "import nltk\n",
    "from gensim import corpora\n",
    "from tqdm import tqdm  # for progress bars\n",
    "\n",
    "nltk.download(\"stopwords\")\n",
    "nltk.download(\"punkt\")\n",
    "nltk.download(\"punkt_tab\")\n",
    "import random\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "random.seed(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "7ZF3Si83290-",
    "outputId": "10237cff-9a10-45af-8ad1-3ef8e7b925f5"
   },
   "outputs": [],
   "source": [
    "!wget https://raw.githubusercontent.com/dbamman/anlp25/main/data/jockers.stopwords\n",
    "!wget https://raw.githubusercontent.com/dbamman/anlp25/main/data/movie.metadata.tsv\n",
    "!wget https://raw.githubusercontent.com/dbamman/anlp25/main/data/plot_summaries.txt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "X1phZTyi2vK6"
   },
   "source": [
    "## Loading stopwords\n",
    "\n",
    "Since we're running topic modeling on texts with lots of names, we'll add the Jockers list of stopwords (which includes character names) to our stoplist. We'll also filter out any words that don't contain at least one letter."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "BWKhujgi2vK6"
   },
   "outputs": [],
   "source": [
    "def read_stopwords(filename):\n",
    "    \"\"\"Reads a file of stopwords into a set.\"\"\"\n",
    "    stopwords = set([\n",
    "        line.rstrip() for line in open(filename)\n",
    "    ])\n",
    "    return stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "H8-0-X_-2vK7"
   },
   "outputs": [],
   "source": [
    "stop_words = set(stopwords.words('english'))\n",
    "stop_words = stop_words | read_stopwords(\"jockers.stopwords\")\n",
    "stop_words.add(\"'s\")\n",
    "stop_words=list(stop_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "iAMInj_R2vK7"
   },
   "outputs": [],
   "source": [
    "pattern = re.compile(r\"[A-Za-z]\")\n",
    "def stopword_filter(word, stopwords):\n",
    "    \"\"\" Function to exclude words from a text.\"\"\"\n",
    "\n",
    "    # no stopwords\n",
    "    if word in stopwords:\n",
    "        return False\n",
    "\n",
    "    # has to contain at least one letter\n",
    "    if pattern.search(word) is not None:\n",
    "        return True\n",
    "\n",
    "    return False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "7AF6QANy2vK8"
   },
   "source": [
    "## Loading summaries\n",
    "\n",
    "We'll read in summaries of the 5,000 movies with the highest box office revenues. This may take 3-4 minutes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "wIxxxBWu2vK7"
   },
   "outputs": [],
   "source": [
    "def read_docs(plot_filename, metadata_filename, stopwords):\n",
    "    n=5000\n",
    "\n",
    "    # only get box office top N\n",
    "    metadata = pd.read_csv(metadata_filename, sep=\"\\t\", names=[\"movie_id\", \"_\", \"title\", \"year\", \"box_office\", \"_1\", \"_2\", \"_3\", \"_4\"])\n",
    "    metadata = metadata.dropna(subset=[\"box_office\"]).sort_values(by=\"box_office\", ascending=False)\n",
    "    metadata = metadata.iloc[:n].set_index(\"movie_id\")\n",
    "\n",
    "    plots = pd.read_csv(plot_filename, sep=\"\\t\", names=[\"movie_id\", \"summary\"])\n",
    "    plots = plots.set_index(\"movie_id\")\n",
    "    plots = metadata.join(plots)\n",
    "\n",
    "    def tokenize_and_process(text):\n",
    "        return [\n",
    "            x for x in nltk.word_tokenize(text.lower()) if stopword_filter(x, stopwords)\n",
    "        ]\n",
    "\n",
    "    docs = []\n",
    "    for summary in tqdm(plots.summary.fillna(\"\")):\n",
    "        docs.append(tokenize_and_process(summary))\n",
    "\n",
    "    names = plots.title.to_list()\n",
    "    return docs, list(names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "XaYqSs8F2vK8"
   },
   "outputs": [],
   "source": [
    "docs, names = read_docs(\"plot_summaries.txt\", \"movie.metadata.tsv\", stop_words)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "aQUWExT22vK9"
   },
   "source": [
    "We will convert the movie summaries into a bag-of-words representation using gensim's [corpora.dictionary](https://radimrehurek.com/gensim/corpora/dictionary.html) methods."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "R0zxKTmJ2vK9"
   },
   "outputs": [],
   "source": [
    "# Create vocab from data; restrict vocab to only the top 10K terms that show up in at least 5 documents\n",
    "# and no more than 50% of all documents\n",
    "\n",
    "dictionary = corpora.Dictionary(docs)\n",
    "dictionary.filter_extremes(no_below=5, no_above=.5, keep_n=10000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "tAAsrxaI2vK9"
   },
   "outputs": [],
   "source": [
    "# Replace dataset with numeric ids words in vocab (and exclude all other words)\n",
    "corpus = [dictionary.doc2bow(text) for text in docs]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "qJqjhoFL2vK-"
   },
   "source": [
    "## Running topic model\n",
    "\n",
    "Now let's run a topic model on this data using gensim's built-in LDA."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "4hE64x_22vK9"
   },
   "outputs": [],
   "source": [
    "num_topics = 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "373pP7-02vK-"
   },
   "outputs": [],
   "source": [
    "lda_model = gensim.models.ldamodel.LdaModel(\n",
    "    corpus=corpus,\n",
    "    id2word=dictionary,\n",
    "    num_topics=num_topics,\n",
    "    passes=10,\n",
    "    alpha='auto'\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "FUduxAS92vK-"
   },
   "source": [
    "## Interpreting topic model\n",
    "\n",
    "**Topic word distributions**\n",
    "\n",
    "We can get a sense of what the topics are by printing the top 10 words with highest $P(word \\mid topic)$ for each topic."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "pG3HiUOC2vK-",
    "outputId": "cfb76fd4-756e-4cbf-e775-8b3cfe1556ef"
   },
   "outputs": [],
   "source": [
    "for i in range(num_topics):\n",
    "    print(\"topic %s:\\t%s\" % (i, ' '.join([term for term, freq in lda_model.show_topic(i, topn=10)])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Yirq9huh2vK-"
   },
   "source": [
    "**Topic document distributions**\n",
    "\n",
    "Another way of understanding topics is to print out the documents that have the highest topic representation -- i.e., for a given topic $k$, the documents with highest $P(\\text{topic}=k \\mid \\text{document})$.  How much do the documents listed here align with your understanding of the topics?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "rDrsC_zz2vK-",
    "outputId": "f61bcab5-8826-4779-8c97-58f8e0bf4c8c",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "topic_model = lda_model\n",
    "topic_docs = []\n",
    "\n",
    "for i in range(num_topics):\n",
    "    topic_docs.append({})\n",
    "\n",
    "for doc_id in range(len(corpus)):\n",
    "    doc_topics = topic_model.get_document_topics(corpus[doc_id])\n",
    "    for topic_num, topic_prob in doc_topics:\n",
    "        topic_docs[topic_num][doc_id] = topic_prob\n",
    "\n",
    "for i in range(num_topics):\n",
    "    top_topic_terms = [term for term, _ in topic_model.show_topic(i, topn=10)]\n",
    "    sorted_docs = sorted(topic_docs[i].items(), key=lambda x: x[1], reverse=True)\n",
    "\n",
    "    print(\" \".join(top_topic_terms))\n",
    "    print()\n",
    "\n",
    "    for doc_id, prob in sorted_docs[:5]:\n",
    "        print(f\"{i}\\t{prob}\\t{names[doc_id]}\")\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ps1k2yMu2vK-"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}