{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/dbamman/anlp25/blob/main/6.tests/PermutationTest.ipynb)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "sWi802SCcvCC"
   },
   "source": [
    "This notebook explores the use of the permutation test to assess the significance of coefficents learned in logistic regression (testing against the null that each $\\beta$ = 0)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "KO0UOYCLcvCG"
   },
   "outputs": [],
   "source": [
    "import copy\n",
    "import sys\n",
    "from random import choices, shuffle\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn import linear_model, preprocessing\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from tqdm import tqdm\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "L-GgtiqJc5xo"
   },
   "outputs": [],
   "source": [
    "# get LMRD data\n",
    "!wget --no-check-certificate https://raw.githubusercontent.com/dbamman/anlp25/refs/heads/main/data/lmrd/train.tsv -O lmrd_train.tsv\n",
    "!wget --no-check-certificate https://raw.githubusercontent.com/dbamman/anlp25/refs/heads/main/data/lmrd/dev.tsv -O lmrd_dev.tsv\n",
    "!wget --no-check-certificate https://raw.githubusercontent.com/dbamman/anlp25/refs/heads/main/data/lmrd/test.tsv -O lmrd_test.tsv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "VwhM7f38c6aj"
   },
   "outputs": [],
   "source": [
    "# get Convote data\n",
    "!wget --no-check-certificate https://raw.githubusercontent.com/dbamman/anlp25/refs/heads/main/data/convote/train.tsv -O convote_train.tsv\n",
    "!wget --no-check-certificate https://raw.githubusercontent.com/dbamman/anlp25/refs/heads/main/data/convote/dev.tsv -O convote_dev.tsv\n",
    "!wget --no-check-certificate https://raw.githubusercontent.com/dbamman/anlp25/refs/heads/main/data/convote/test.tsv -O convote_test.tsv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "dALCXYcTc6qn"
   },
   "outputs": [],
   "source": [
    "# get LoC data\n",
    "!wget --no-check-certificate https://raw.githubusercontent.com/dbamman/anlp25/refs/heads/main/data/loc/train.tsv -O loc_train.tsv\n",
    "!wget --no-check-certificate https://raw.githubusercontent.com/dbamman/anlp25/refs/heads/main/data/loc/dev.tsv -O loc_dev.tsv\n",
    "!wget --no-check-certificate https://raw.githubusercontent.com/dbamman/anlp25/refs/heads/main/data/loc/test.tsv -O loc_test.tsv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "_zqzfRM2cvCI"
   },
   "outputs": [],
   "source": [
    "def read_data(filename):\n",
    "    df = pd.read_csv(filename, names=[\"label\", \"text\"], sep=\"\\t\")\n",
    "    return df.text.to_list(), df.label.to_list()\n",
    "\n",
    "# Change this to the directory with the data you will be using.\n",
    "# The directory should contain train.tsv, dev.tsv and test.tsv\n",
    "data = \"convote\"\n",
    "\n",
    "x_train, y_train = read_data(\"%s_train.tsv\" % data)\n",
    "x_dev, y_dev = read_data(\"%s_dev.tsv\" % data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "agiLpsKacvCJ"
   },
   "outputs": [],
   "source": [
    "def featurize(x_train, x_dev):\n",
    "    vectorizer = CountVectorizer(max_features=10000, analyzer=str.split, lowercase=False, strip_accents=None, binary=True)\n",
    "\n",
    "    x_train = vectorizer.fit_transform(x_train)\n",
    "    x_dev = vectorizer.transform(x_dev)\n",
    "\n",
    "    return x_train, x_dev, vectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "aTsL3wxNcvCK"
   },
   "outputs": [],
   "source": [
    "def train(x_train, y_train, le):\n",
    "    y_train = le.transform(y_train)\n",
    "    logreg = linear_model.LogisticRegression(C=100, solver='lbfgs', penalty='l2', max_iter=10000)\n",
    "    logreg.fit(x_train, y_train)\n",
    "    return logreg\n",
    "    return logreg.coef_[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "H68lmfITcvCK"
   },
   "outputs": [],
   "source": [
    "def test(logreg, x_dev_feats, y_dev, le):\n",
    "    y_dev = le.transform(y_dev)\n",
    "    print(\"Accuracy: %.3f\" % logreg.score(x_dev_feats, y_dev))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "fnJBf-mncvCL"
   },
   "outputs": [],
   "source": [
    "def analyze_weights(coefs, label_encoder, vocab, p_values):\n",
    "    reverse_vocab = {v: k for k, v in vocab.items()}\n",
    "\n",
    "    sort_index = np.argsort(coefs)\n",
    "\n",
    "    print(label_encoder.inverse_transform([0])[0])\n",
    "    for k in sort_index[:25]:\n",
    "        print (\"%.5f\\t%s\\t%.4f\" % (coefs[k], reverse_vocab[k], p_values[k] ))\n",
    "\n",
    "    print(label_encoder.inverse_transform([1])[0])\n",
    "\n",
    "    for k in reversed(sort_index[-25:]):\n",
    "        print (\"%.5f\\t%s\\t%.4f\" % (coefs[k], reverse_vocab[k], p_values[k] ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "BRdNJnrBcvCL"
   },
   "outputs": [],
   "source": [
    "x_train_feats, x_dev_feats, vectorizer=featurize(x_train, x_dev)\n",
    "le = preprocessing.LabelEncoder()\n",
    "le.fit(y_train)\n",
    "\n",
    "logreg=train(x_train_feats, y_train, le)\n",
    "test(logreg, x_dev_feats, y_dev, le)\n",
    "\n",
    "true_coefficients=logreg.coef_[0]\n",
    "\n",
    "# We'll set P=100 here to finish running in class, but set higher (e.g., 10000) for real applications\n",
    "P=100\n",
    "\n",
    "p_values = np.zeros(len(true_coefficients))\n",
    "y_permuted = copy.deepcopy(y_train)\n",
    "\n",
    "for i in tqdm(range(P)):\n",
    "    # permute the values of Y so that they're now attached to random data points in X\n",
    "    shuffle(y_permuted)\n",
    "\n",
    "    # train logistic regression on that permuted dataset\n",
    "    permuted_logreg = train(x_train_feats, y_permuted, le)\n",
    "    coefficients = permuted_logreg.coef_[0]\n",
    "\n",
    "    # test how often the coefficients learned from the permuted data are as extreme as\n",
    "    # the coefficients from the true data\n",
    "    for idx, coef in enumerate(coefficients):\n",
    "        if abs(true_coefficients[idx]) < abs(coef):\n",
    "            p_values[idx] += 1. / P"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "TPN8pvtUcvCM"
   },
   "outputs": [],
   "source": [
    "inverse_vocab = {v: k for k, v in vectorizer.vocabulary_.items()}\n",
    "out = open(\"weights.txt\", \"w\")\n",
    "for idx, coef in enumerate(true_coefficients):\n",
    "    out.write(\"%.3f\\t%s\\t%.5f\\n\" % (coef, inverse_vocab[idx], p_values[idx]))\n",
    "out.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "7Y1mnqKIcvCN"
   },
   "outputs": [],
   "source": [
    "analyze_weights(true_coefficients, le, vectorizer.vocabulary_, p_values)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}