{"cells":[{"cell_type":"markdown","id":"b2d0ab15-c334-4f13-8952-12572f5123fc","metadata":{"id":"b2d0ab15-c334-4f13-8952-12572f5123fc"},"source":["### Imports and Plotting Functions"]},{"cell_type":"code","execution_count":null,"id":"bf82d617-d639-46b8-a638-c8b9660a3eb0","metadata":{"id":"bf82d617-d639-46b8-a638-c8b9660a3eb0"},"outputs":[],"source":["import itertools\n","import random\n","import numpy as np\n","import matplotlib.pyplot as plt\n","from sklearn import datasets\n","from sklearn.inspection import DecisionBoundaryDisplay\n","\n","# Fix seed for reproducibility\n","np.random.seed(231)\n","np.set_printoptions(precision=4)"]},{"cell_type":"code","execution_count":null,"id":"58d020f1-e651-4f21-a81f-488670eb72cc","metadata":{"jupyter":{"source_hidden":true},"id":"58d020f1-e651-4f21-a81f-488670eb72cc"},"outputs":[],"source":["def plot_decision_boundary(\n","    estimator,\n","    X,\n","    *,\n","    grid_resolution=100,\n","    eps=1.0,\n","    **kwargs,\n","):\n","    \"\"\"Generates a contour plot of the estimator's prediction thresholds.\"\"\"\n","    x0, x1 = X[:,0],X[:,1]\n","\n","    x0_min, x0_max = x0.min() - eps, x0.max() + eps\n","    x1_min, x1_max = x1.min() - eps, x1.max() + eps\n","\n","    xx0, xx1 = np.meshgrid(\n","        np.linspace(x0_min, x0_max, grid_resolution),\n","        np.linspace(x1_min, x1_max, grid_resolution),\n","    )\n","\n","    X_grid = np.c_[xx0.ravel(), xx1.ravel()]\n","\n","    response = estimator.forward(X_grid)\n","\n","    display = DecisionBoundaryDisplay(\n","        xx0=xx0,\n","        xx1=xx1,\n","        response=response.reshape(xx0.shape),\n","    )\n","    display.plot(plot_method=\"contourf\", **kwargs)\n","    return display\n","\n","def visualize_dataset(x, y, title=\"Visualization of dataset\"):\n","    \"\"\"Generates a scatter plot of the data.\"\"\"\n","    scatter = plt.scatter(*x.T, c=y)\n","    plt.title(title)\n","    plt.legend(handles=scatter.legend_elements()[0], labels=[0,1])\n","    plt.grid(visible=True, which='both')\n","    plt.figure(figsize=(6, 6))\n","    plt.show()\n","\n","def visualize_weights(mlp, gradients=True):\n","    \"\"\"Visualize the first layer weights and gradients of an MLPClassifier.\"\"\"\n","    z = np.zeros(mlp.w1.shape)\n","    plt.quiver(z[0, :], z[1, :], mlp.w1[0, :], mlp.w1[1, :],\n","               scale=1, width=0.005, angles='xy', scale_units='xy')\n","    if gradients:\n","        plt.quiver(mlp.w1[0, :], mlp.w1[1, :], -mlp.lr * mlp.grad_w1[0, :], -mlp.lr * mlp.grad_w1[1, :],\n","                   scale=1, width=0.01, angles='xy', scale_units='xy', color='r')\n","    plt.grid(visible=True, which='both')\n","    plt.xlim(-3, 3)\n","    plt.ylim(-3, 3)\n","    plt.show()"]},{"cell_type":"markdown","id":"08fdb5e0-774d-4fb4-ae02-df39879eb6d4","metadata":{"id":"08fdb5e0-774d-4fb4-ae02-df39879eb6d4"},"source":["### Generate Example Dataset"]},{"cell_type":"code","execution_count":null,"id":"7a94479f-83a7-4261-813e-88c1e6977b57","metadata":{"id":"7a94479f-83a7-4261-813e-88c1e6977b57"},"outputs":[],"source":["X, y = datasets.make_circles(n_samples=1000, noise=0.05, factor=0.5)\n","y = y[:, np.newaxis]\n","print(f\"The data has dimensions {X.shape} and the labels have dimensions {y.shape}.\")"]},{"cell_type":"code","execution_count":null,"id":"5916ac86-724f-414b-8aa7-282923592a63","metadata":{"id":"5916ac86-724f-414b-8aa7-282923592a63"},"outputs":[],"source":["visualize_dataset(X, y)"]},{"cell_type":"markdown","id":"3f4283d5-9e07-43ed-8f19-a8693b2dfeb8","metadata":{"id":"3f4283d5-9e07-43ed-8f19-a8693b2dfeb8"},"source":["### Define Activation functions and Classifier"]},{"cell_type":"code","execution_count":null,"id":"4f131369-7cde-4c5c-be9b-e40422cfbda6","metadata":{"id":"4f131369-7cde-4c5c-be9b-e40422cfbda6"},"outputs":[],"source":["def relu(x):\n","    \"\"\"Rectified linear activation function.\"\"\"\n","    return np.maximum(x, 0)\n","\n","def sigmoid(x):\n","    \"\"\"Sigmoid activation function.\"\"\"\n","    return 1 / (1 + np.exp(-x))\n","\n","def cross_entropy(preds, labels):\n","    \"\"\"\n","    Binary cross entropy loss.\n","\n","    Args:\n","        preds:  predicted probabiltiies\n","        labels: ground truth labels\n","    \"\"\"\n","    return -np.mean(labels * np.log(preds) + (1 - labels) * np.log(1 - preds))"]},{"cell_type":"code","execution_count":null,"id":"6d4320fc-cfa3-49ba-8317-bf24d1b141ad","metadata":{"id":"6d4320fc-cfa3-49ba-8317-bf24d1b141ad"},"outputs":[],"source":["class MLPClassifier():\n","    def __init__(self, lr=1e-2, in_dim=2, out_dim=1, h_dim=3):\n","        \"\"\"\n","        Defines the learning rate and initializes the weight matrices\n","        for a simple multilayer perceptron.\n","\n","        Args:\n","            lr:      learning rate\n","            in_dim:  input dimension\n","            out_dim: output dimension\n","            h_dim:   hidden dimension\n","        \"\"\"\n","        self.lr = lr\n","\n","        # Randomly initialize weights from a Gaussian distribution\n","        self.w1 = np.random.randn(in_dim, h_dim) * np.sqrt(2 / in_dim)\n","        self.w2 = np.random.randn(h_dim, out_dim) * np.sqrt(2 / h_dim)\n","        self.b = np.zeros(out_dim) + 0.1\n","\n","    def forward(self, x):\n","        \"\"\"\n","        Forward pass of the MLP.\n","\n","        Args:\n","            x: input data with shape (dataset_size, in_dim)\n","\n","        Returns:\n","            y_pred: predicted probabilities of being in class 1\n","        \"\"\"\n","        self.x = x                            # Cache inputs for gradient computation\n","        self.h = relu(self.x @ self.w1)       # Cache relu output for gradient computation\n","        self.z = self.h @ self.w2 + self.b    # Cache for viewing purposes (not used in backwards)\n","        y_pred = sigmoid(self.z)\n","        return y_pred\n","\n","    def backward(self, y_pred, grad_y_pred):\n","        \"\"\"\n","        Backwards pass of the MLP.\n","\n","        Args:\n","            y_pred:      predicted probabilities of being in class 1\n","            grad_y_pred: gradient of the loss w.r.t y_pred\n","        \"\"\"\n","        self.grad_w2 = self.h.T @ (y_pred * (1-y_pred) * grad_y_pred)     # Gradient w.r.t. w2\n","        self.grad_b  = np.sum(y_pred * (1-y_pred) * grad_y_pred)          # Gradient w.r.t. b\n","        self.grad_h  = self.w2.T * (y_pred * (1-y_pred) * grad_y_pred)    # Gradient w.r.t. relu\n","        self.grad_w1 = self.x.T @ ((self.h > 0) * self.grad_h)            # Gradient w.r.t. w1\n","\n","    def update(self):\n","        \"\"\"\n","        Update weights with gradients.\n","        \"\"\"\n","        self.w1 -= self.lr * self.grad_w1\n","        self.w2 -= self.lr * self.grad_w2\n","        self.b  -= self.lr * self.grad_b"]},{"cell_type":"markdown","id":"b17eb82c-9dcc-4c64-9e3c-11d2afee3ee9","metadata":{"id":"b17eb82c-9dcc-4c64-9e3c-11d2afee3ee9"},"source":["### Train MLPClassifier"]},{"cell_type":"code","execution_count":null,"id":"b1282d43-167f-4561-a99f-197e47c33519","metadata":{"id":"b1282d43-167f-4561-a99f-197e47c33519"},"outputs":[],"source":["mlp = MLPClassifier(lr=5e-3, h_dim=3)\n","loss_history, loss = [np.inf], np.inf\n","visualize_weights(mlp, gradients=False)"]},{"cell_type":"code","execution_count":null,"id":"5ce26b36-0edb-443e-8928-9f965d20d16b","metadata":{"scrolled":true,"id":"5ce26b36-0edb-443e-8928-9f965d20d16b"},"outputs":[],"source":["epochs = 10+1\n","for t in range(epochs):\n","\n","    # Predict on data (forward pass)\n","    y_pred = mlp.forward(X)\n","\n","    # Calculate and store loss\n","    loss = cross_entropy(y_pred, y)\n","    loss_history.append(loss)\n","\n","    # Calculate loss gradient and backpropogate errors (backwards pass)\n","    grad_y_pred = -(np.divide(y, y_pred) - np.divide(1-y, 1-y_pred))\n","    mlp.backward(y_pred, grad_y_pred)\n","\n","    # Visualize classifier\n","    if t % (epochs // 10) == 0:\n","        display = plot_decision_boundary(mlp, X)\n","        plt.colorbar(display.ax_.collections[0])\n","        visualize_dataset(X, y, title=f\"t = {t}, Loss = {round(loss, 4)}\")\n","        visualize_weights(mlp)\n","\n","    mlp.update()"]},{"cell_type":"code","execution_count":null,"id":"e2ba685c-751b-4c3d-b41f-83217a1fe68f","metadata":{"id":"e2ba685c-751b-4c3d-b41f-83217a1fe68f"},"outputs":[],"source":["plt.plot(loss_history)\n","plt.title(\"Training Loss\")\n","plt.xlabel(\"iterations\")\n","plt.show()"]},{"cell_type":"code","execution_count":null,"id":"01e8cc66-66f6-49c8-bac1-280e2abeb689","metadata":{"id":"01e8cc66-66f6-49c8-bac1-280e2abeb689"},"outputs":[],"source":["threshold = 0.5\n","acc = np.sum((y_pred > threshold) == y) / X.shape[0]\n","print(f\"Accuracy = {acc}\")"]},{"cell_type":"code","execution_count":null,"id":"8696ed7b-03ef-4ae9-8b36-92a4a8f544b6","metadata":{"id":"8696ed7b-03ef-4ae9-8b36-92a4a8f544b6"},"outputs":[],"source":["idx = [0, 1, 2]\n","for i, j in itertools.combinations(idx, 2):\n","    wi, wj = mlp.w1[:, i], mlp.w1[:, j]\n","    print(f\"Angle between weight {i} and {j} is {np.degrees(np.arccos((wi @ wj) / (np.linalg.norm(wi) * np.linalg.norm(wj))))} degrees.\")"]},{"cell_type":"code","execution_count":null,"id":"776381d6-ca4f-4ffd-9394-7466406bc5fd","metadata":{"id":"776381d6-ca4f-4ffd-9394-7466406bc5fd"},"outputs":[],"source":[]}],"metadata":{"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.13.2"},"colab":{"provenance":[]}},"nbformat":4,"nbformat_minor":5}