{
"cells": [
{
"cell_type": "markdown",
"id": "9c8253d5",
"metadata": {},
"source": [
"\n",
"\n",
"Lecture 12: Neural Networks (1) – CS 189, Fall 2026
\n",
"\n",
"Logistic regression fits a linear boundary on whatever features $\\phi(x)$ we give it. Today the model learns the features. We will:\n",
"- Watch logistic regression fail on XOR, then solve XOR with a two-layer network whose weights we set by hand.\n",
"- Train a deep network with no activation function and check that it is still a linear model.\n",
"- Add a non-linearity and watch the boundary bend.\n",
"- Compare sigmoid, tanh, ReLU, and GELU by where their derivatives vanish, and watch a ReLU unit die.\n",
"- Match the output layer and loss to the target, compute softmax cross-entropy from logits, and see that the loss is no longer convex.\n",
"\n",
"*Reference: Bishop & Bishop,* Deep Learning: Foundations and Concepts, *Ch. 6, §6.1, §6.2.1-6.2.3, §6.3.1, §6.4.*"
]
},
{
"cell_type": "markdown",
"id": "9647f69a",
"metadata": {},
"source": [
"## Imports"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "976cb924",
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"from matplotlib.colors import LinearSegmentedColormap\n",
"from sklearn.datasets import make_moons\n",
"from sklearn.linear_model import LogisticRegression\n",
"from sklearn.metrics import log_loss\n",
"\n",
"import torch\n",
"import torch.nn as nn\n",
"import torch.nn.functional as F\n",
"\n",
"import warnings\n",
"warnings.filterwarnings(\"ignore\")\n",
"\n",
"import ipywidgets as widgets\n",
"\n",
"%config InlineBackend.figure_format = \"retina\"\n",
"\n",
"C_BLUE, C_GOLD, C_RED, C_GREEN = \"#003262\", \"#FDB515\", \"#D55E00\", \"#00553A\"\n",
"CLASS_COLORS = [C_BLUE, C_GOLD, C_RED]\n",
"plt.rcParams.update({\"figure.figsize\": (8, 5), \"axes.grid\": True, \"grid.alpha\": 0.3,\n",
" \"font.size\": 13, \"axes.titlesize\": 15, \"axes.labelsize\": 14})"
]
},
{
"cell_type": "markdown",
"id": "70b966fd",
"metadata": {},
"source": [
"## Helper Code\n",
"\n",
"Run these two cells before the rest. You do not need to read them today: Step 8 opens up `MLP` and `train`, and Lecture 13 explains `loss.backward()`."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c63b2bff",
"metadata": {},
"outputs": [],
"source": [
"def make_xor(n=400, noise=0.15, seed=189):\n",
" \"\"\"n points split evenly over the corners of the unit square, plus Gaussian noise.\n",
" The label is x1 XOR x2 of the corner.\"\"\"\n",
" rng = np.random.default_rng(seed)\n",
" corners = np.repeat(np.array([[0, 0], [0, 1], [1, 0], [1, 1]]), n // 4, axis=0)\n",
" X = corners + noise * rng.standard_normal(corners.shape)\n",
" return X, corners[:, 0] ^ corners[:, 1]\n",
"\n",
"\n",
"def make_spiral(n_per_class=150, K=3, noise=0.2, seed=189):\n",
" \"\"\"K interleaved spiral arms, one class per arm.\"\"\"\n",
" rng = np.random.default_rng(seed)\n",
" r = np.linspace(0.2, 1, n_per_class)\n",
" X, t = [], []\n",
" for k in range(K):\n",
" theta = np.linspace(0, 4, n_per_class) + 2 * np.pi * k / K + noise * rng.standard_normal(n_per_class)\n",
" X.append(np.c_[r * np.cos(theta), r * np.sin(theta)])\n",
" t.append(np.full(n_per_class, k))\n",
" return np.vstack(X), np.concatenate(t)\n",
"\n",
"\n",
"PROB_CMAP = LinearSegmentedColormap.from_list(\"blue_white_gold\", [\"#B9CCE2\", \"#FFFFFF\", \"#FEE3A0\"])\n",
"\n",
"\n",
"def plot_data(X, t, ax=None, title=None):\n",
" ax = ax or plt.gca()\n",
" for k in np.unique(t):\n",
" ax.scatter(*X[t == k].T, s=16, color=CLASS_COLORS[k], edgecolor=\"k\", linewidth=0.3, label=f\"class {k}\")\n",
" ax.set(xlabel=\"$x_1$\", ylabel=\"$x_2$\", title=title, aspect=\"equal\")\n",
" return ax\n",
"\n",
"\n",
"def plot_boundary(predict, X, t, ax=None, title=None, pad=0.5, n=300):\n",
" \"\"\"Shade the plane by the model's prediction, then overlay the data.\n",
" predict maps an (m, 2) array to P(class 1) with shape (m,), or to class probabilities (m, K).\"\"\"\n",
" ax = ax or plt.gca()\n",
" lo, hi = X.min(0) - pad, X.max(0) + pad\n",
" g1, g2 = np.meshgrid(np.linspace(lo[0], hi[0], n), np.linspace(lo[1], hi[1], n))\n",
" P = predict(np.c_[g1.ravel(), g2.ravel()])\n",
" if P.ndim == 1:\n",
" ax.contourf(g1, g2, P.reshape(g1.shape), levels=np.linspace(0, 1, 21), cmap=PROB_CMAP)\n",
" ax.contour(g1, g2, P.reshape(g1.shape), levels=[0.5], colors=\"k\", linewidths=1.5)\n",
" else:\n",
" K = P.shape[1]\n",
" ax.contourf(g1, g2, P.argmax(1).reshape(g1.shape), levels=np.arange(K + 1) - 0.5,\n",
" colors=[c + \"40\" for c in CLASS_COLORS[:K]])\n",
" plot_data(X, t, ax, title)\n",
" ax.grid(False)\n",
" return ax"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "603cd793",
"metadata": {},
"outputs": [],
"source": [
"ACTIVATIONS = {\"identity\": lambda a: a, \"relu\": torch.relu, \"tanh\": torch.tanh,\n",
" \"sigmoid\": torch.sigmoid, \"gelu\": F.gelu}\n",
"\n",
"\n",
"class MLP(nn.Module):\n",
" \"\"\"Fully connected network. dims = [inputs, hidden_1, ..., hidden_L, outputs].\n",
" Every hidden layer applies the activation. The output layer returns logits (no activation).\"\"\"\n",
"\n",
" def __init__(self, dims, activation=\"relu\"):\n",
" super().__init__()\n",
" self.layers = nn.ModuleList(nn.Linear(d_in, d_out) for d_in, d_out in zip(dims[:-1], dims[1:]))\n",
" self.act = ACTIVATIONS[activation]\n",
"\n",
" def forward(self, x):\n",
" for layer in self.layers[:-1]:\n",
" x = self.act(layer(x))\n",
" return self.layers[-1](x)\n",
"\n",
"\n",
"def train(model, X, t, epochs=500, lr=0.01, loss_fn=None):\n",
" \"\"\"Full-batch Adam (Lecture 11). Unless a loss is given, it follows the output layer:\n",
" 1 output -> binary cross-entropy on logits; K outputs -> softmax cross-entropy on logits.\"\"\"\n",
" X = torch.as_tensor(X, dtype=torch.float32)\n",
" K = model.layers[-1].out_features\n",
" if loss_fn is None:\n",
" loss_fn = nn.BCEWithLogitsLoss() if K == 1 else nn.CrossEntropyLoss()\n",
" if isinstance(loss_fn, nn.CrossEntropyLoss):\n",
" T = torch.as_tensor(t, dtype=torch.long)\n",
" else:\n",
" T = torch.as_tensor(t, dtype=torch.float32).reshape(-1, K)\n",
" optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n",
" history = []\n",
" for epoch in range(epochs):\n",
" optimizer.zero_grad()\n",
" loss = loss_fn(model(X), T)\n",
" loss.backward() # gradients: Lecture 13\n",
" optimizer.step()\n",
" history.append(loss.item())\n",
" return history\n",
"\n",
"\n",
"def predict_proba(model):\n",
" \"\"\"Wrap a trained classifier as a NumPy function: P(class 1), or class probabilities.\"\"\"\n",
" def f(X):\n",
" with torch.no_grad():\n",
" z = model(torch.as_tensor(X, dtype=torch.float32))\n",
" return torch.sigmoid(z)[:, 0].numpy() if z.shape[1] == 1 else torch.softmax(z, dim=1).numpy()\n",
" return f\n",
"\n",
"\n",
"def accuracy(model, X, t):\n",
" P = predict_proba(model)(X)\n",
" pred = (P > 0.5).astype(int) if P.ndim == 1 else P.argmax(1)\n",
" return (pred == t).mean()"
]
},
{
"cell_type": "markdown",
"id": "04aa72ee",
"metadata": {},
"source": [
"---\n",
"## The Data: XOR\n",
"\n",
"Four clusters at the corners of the unit square. The label is $x_1 \\text{ XOR } x_2$ of the corner: class 1 when exactly one coordinate is 1."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "530449cb",
"metadata": {},
"outputs": [],
"source": [
"X, t = make_xor()\n",
"plot_data(X, t, title=\"XOR: 400 points, 100 per corner\")\n",
"plt.legend(loc=\"center\")\n",
"plt.show()"
]
},
{
"cell_type": "markdown",
"id": "0a712cc1",
"metadata": {},
"source": [
"---\n",
"## Step 1: Logistic Regression on XOR **(Demo 1)**\n",
"\n",
"Logistic regression from Lectures 8-9, with the raw coordinates as features, $\\phi(x) = (x_1, x_2)$, and no regularization."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7ced5a4b",
"metadata": {},
"outputs": [],
"source": [
"logreg = LogisticRegression(C=np.inf).fit(X, t)\n",
"p_lr = logreg.predict_proba(X)[:, 1]\n",
"print(f\"weights {logreg.coef_.ravel().round(3)}, bias {logreg.intercept_.round(3)}\")\n",
"print(f\"accuracy {logreg.score(X, t):.3f}, cross-entropy {log_loss(t, p_lr):.4f} (ln 2 = {np.log(2):.4f})\")\n",
"print(f\"predicted P(class 1) ranges from {p_lr.min():.3f} to {p_lr.max():.3f}\")"
]
},
{
"cell_type": "markdown",
"id": "d26a6ff0",
"metadata": {},
"source": [
"### The failure: the best line is no line\n",
"\n",
"Any line leaves at least one cluster on the wrong side. Cross-entropy punishes confident mistakes, so the best fit shrinks the weights toward zero and predicts about 0.5 everywhere. Its loss is $\\ln 2$, the loss of a coin flip."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "62fed5be",
"metadata": {},
"outputs": [],
"source": [
"fig, axes = plt.subplots(1, 2, figsize=(11, 5))\n",
"plot_data(X, t, axes[0], \"The data\")\n",
"plot_boundary(lambda Z: logreg.predict_proba(Z)[:, 1], X, t, axes[1],\n",
" f\"Logistic regression: accuracy {logreg.score(X, t):.0%}\")\n",
"plt.tight_layout()\n",
"plt.show()"
]
},
{
"cell_type": "markdown",
"id": "f385295e",
"metadata": {},
"source": [
"---\n",
"## Step 2: One Neuron, Wired by Hand\n",
"\n",
"A neuron takes a weighted sum of its inputs plus a bias, then applies an activation: $z = h(w_1 x_1 + w_2 x_2 + w_0)$. With the step activation, $h(a) = 1$ if $a > 0$ and $0$ otherwise, a neuron draws one line and reports which side of it you are on."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "4791ff31",
"metadata": {},
"outputs": [],
"source": [
"def step(a):\n",
" return (a > 0).astype(int)\n",
"\n",
"\n",
"def neuron(X, w, w0, h=step):\n",
" return h(X @ np.asarray(w) + w0)\n",
"\n",
"\n",
"corners = np.array([[0, 0], [0, 1], [1, 0], [1, 1]])\n",
"print(\"x1 x2 | AND OR\")\n",
"for x, a, o in zip(corners, neuron(corners, [1, 1], -1.5), neuron(corners, [1, 1], -0.5)):\n",
" print(f\" {x[0]} {x[1]} | {a} {o}\")"
]
},
{
"cell_type": "markdown",
"id": "b60b46dd",
"metadata": {},
"source": [
"AND and OR each need one line. XOR needs class 1 at $(0, 1)$ and $(1, 0)$ and class 0 at $(0, 0)$ and $(1, 1)$. No single line does that, so no single neuron does."
]
},
{
"cell_type": "markdown",
"id": "ae5d4451",
"metadata": {},
"source": [
"---\n",
"## Step 3: Two Layers Solve XOR\n",
"\n",
"Write XOR as $(x_1 \\text{ AND NOT } x_2) \\text{ OR } (\\text{NOT } x_1 \\text{ AND } x_2)$. Each piece is one neuron:\n",
"\n",
"- hidden unit 1: $z_1 = h(x_1 - x_2 - 0.5)$, on only at $(1, 0)$\n",
"- hidden unit 2: $z_2 = h(-x_1 + x_2 - 0.5)$, on only at $(0, 1)$\n",
"- output unit: $y = h(z_1 + z_2 - 0.5)$, the OR of the two\n",
"\n",
"Three neurons and nine weights, all set by hand."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "6dc65585",
"metadata": {},
"outputs": [],
"source": [
"W1 = np.array([[1, -1], [-1, 1]]); b1 = np.array([-0.5, -0.5]) # hidden layer: 2 units\n",
"w2 = np.array([1, 1]); b2 = -0.5 # output unit: OR\n",
"\n",
"\n",
"def xor_net(X, h=step):\n",
" Z = h(X @ W1.T + b1) # hidden layer\n",
" return step(Z @ w2 + b2), Z # output, hidden activations\n",
"\n",
"\n",
"y, Z = xor_net(corners)\n",
"print(\"x1 x2 | z1 z2 | y\")\n",
"for x, z, v in zip(corners, Z, y):\n",
" print(f\" {x[0]} {x[1]} | {z[0]} {z[1]} | {v}\")\n",
"print(f\"\\naccuracy on the 400 noisy points: {(xor_net(X)[0] == t).mean():.4f}\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f73a8e55",
"metadata": {},
"outputs": [],
"source": [
"plot_boundary(lambda Z: xor_net(Z)[0].astype(float), X, t,\n",
" title=f\"Hand-wired network: accuracy {(xor_net(X)[0] == t).mean():.1%}\")\n",
"plt.show()"
]
},
{
"cell_type": "markdown",
"id": "ddf4ad87",
"metadata": {},
"source": [
"### The hidden layer is a new $\\phi(x)$\n",
"\n",
"Plot each point by its hidden activations $(z_1, z_2)$ instead of its inputs. Here the hidden step is replaced by a steep sigmoid, $\\sigma(5a)$, so the points spread out instead of piling onto three corners. In the new coordinates one line separates the classes: the output unit is logistic regression on features that the hidden layer computed. (The points fall on a single curve because both hidden units read the same quantity, $x_1 - x_2$.)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e2615d6e",
"metadata": {},
"outputs": [],
"source": [
"steep = lambda a: 1 / (1 + np.exp(-5 * a))\n",
"_, Z_smooth = xor_net(X, h=steep)\n",
"\n",
"fig, axes = plt.subplots(1, 2, figsize=(11, 5))\n",
"plot_data(X, t, axes[0], \"Input space $(x_1, x_2)$\")\n",
"plot_data(Z_smooth, t, axes[1], \"Hidden space $(z_1, z_2)$\")\n",
"axes[1].plot([-0.1, 0.6], [0.6, -0.1], color=\"k\", lw=1.5) # the output unit's line: z1 + z2 = 0.5\n",
"axes[1].set(xlabel=\"$z_1$\", ylabel=\"$z_2$\")\n",
"plt.tight_layout()\n",
"plt.show()"
]
},
{
"cell_type": "markdown",
"id": "fb99a719",
"metadata": {},
"source": [
"---\n",
"## Step 4: Depth Without a Non-Linearity **(Demo 1, continued)**\n",
"\n",
"**Poll 1 comes before this cell.** Now we *train* a network instead of wiring one: three hidden layers of 100 units each, with the identity as the activation, $h(a) = a$."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "81277714",
"metadata": {},
"outputs": [],
"source": [
"torch.manual_seed(189)\n",
"deep_linear = MLP([2, 100, 100, 100, 1], activation=\"identity\")\n",
"history = train(deep_linear, X, t, epochs=500, lr=0.01)\n",
"n_params = sum(p.numel() for p in deep_linear.parameters())\n",
"print(f\"deep linear network ({n_params:,} parameters): accuracy {accuracy(deep_linear, X, t):.3f}, loss {history[-1]:.4f}\")\n",
"print(f\"logistic regression (3 parameters): accuracy {logreg.score(X, t):.3f}, loss {log_loss(t, p_lr):.4f}\")"
]
},
{
"cell_type": "markdown",
"id": "59b66b34",
"metadata": {},
"source": [
"### The failure: three hidden layers, one straight line\n",
"\n",
"The same loss as logistic regression, to four decimals. Multiply the layers out and the whole network is a single linear map $Wx + b$:"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "950007c1",
"metadata": {},
"outputs": [],
"source": [
"with torch.no_grad():\n",
" W, b = torch.eye(2), torch.zeros(2)\n",
" for layer in deep_linear.layers: # x -> W_l (W x + b) + b_l\n",
" W, b = layer.weight @ W, layer.weight @ b + layer.bias\n",
" Xt = torch.as_tensor(X, dtype=torch.float32)\n",
" gap = (deep_linear(Xt) - (Xt @ W.T + b)).abs().max().item()\n",
"print(f\"collapsed weights {W.numpy().round(3)}, bias {b.numpy().round(3)}\")\n",
"print(f\"largest difference from the full network's output: {gap:.1e}\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c6c9ef2a",
"metadata": {},
"outputs": [],
"source": [
"fig, axes = plt.subplots(1, 2, figsize=(11, 5))\n",
"plot_boundary(predict_proba(deep_linear), X, t, axes[0],\n",
" f\"3 hidden layers of 100, identity: {accuracy(deep_linear, X, t):.0%}\")\n",
"plot_boundary(lambda Z: xor_net(Z)[0].astype(float), X, t, axes[1],\n",
" f\"2 step units, set by hand: {(xor_net(X)[0] == t).mean():.0%}\")\n",
"plt.tight_layout()\n",
"plt.show()"
]
},
{
"cell_type": "markdown",
"id": "f0988c66",
"metadata": {},
"source": [
"---\n",
"## Step 5: Train a ReLU Network\n",
"\n",
"Same training code as Step 4, with ReLU, $h(a) = \\max(0, a)$, in the hidden layer. One hidden layer of 8 units is enough."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7d0af45e",
"metadata": {},
"outputs": [],
"source": [
"torch.manual_seed(189)\n",
"relu_net = MLP([2, 8, 1], activation=\"relu\")\n",
"history = train(relu_net, X, t, epochs=500, lr=0.05)\n",
"n_params = sum(p.numel() for p in relu_net.parameters())\n",
"print(f\"ReLU network ({n_params} parameters): accuracy {accuracy(relu_net, X, t):.3f}, loss {history[-1]:.4f}\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "9f037e33",
"metadata": {},
"outputs": [],
"source": [
"fig, axes = plt.subplots(1, 2, figsize=(11, 5))\n",
"plot_boundary(lambda Z: xor_net(Z)[0].astype(float), X, t, axes[0],\n",
" f\"2 step units, set by hand: {(xor_net(X)[0] == t).mean():.0%}\")\n",
"plot_boundary(predict_proba(relu_net), X, t, axes[1],\n",
" f\"8 ReLU units, trained: {accuracy(relu_net, X, t):.0%}\")\n",
"plt.tight_layout()\n",
"plt.show()"
]
},
{
"cell_type": "markdown",
"id": "71c3ec5f",
"metadata": {},
"source": [
"---\n",
"## Step 6: Activation Functions and Their Derivatives\n",
"\n",
"Any non-linear $h$ prevents the collapse. Which one we pick matters for training. Gradient descent (Lectures 10-11) moves each weight in proportion to its gradient, and the gradient that reaches a unit's incoming weights is multiplied by $h'(a)$ at that unit (Lecture 13 makes this precise). Where $h'(a) \\approx 0$, those weights barely move."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "5fa67209",
"metadata": {},
"outputs": [],
"source": [
"names, colors = [\"sigmoid\", \"tanh\", \"relu\", \"gelu\"], [C_BLUE, C_GOLD, C_RED, C_GREEN]\n",
"a = torch.linspace(-6, 6, 1201)\n",
"eps = 1e-3\n",
"\n",
"fig, axes = plt.subplots(1, 2, figsize=(12, 4.5))\n",
"for name, color in zip(names, colors):\n",
" h = ACTIVATIONS[name]\n",
" axes[0].plot(a, h(a), color=color, lw=2.5, label=name)\n",
" axes[1].plot(a, (h(a + eps) - h(a - eps)) / (2 * eps), color=color, lw=2.5, label=name) # numerical derivative\n",
"axes[0].set(title=\"$h(a)$\", xlabel=\"$a$\", ylim=(-1.5, 3))\n",
"axes[1].set(title=\"$h'(a)$\", xlabel=\"$a$\", ylim=(-0.2, 1.2))\n",
"axes[0].legend(loc=\"upper left\")\n",
"plt.tight_layout()\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "68694aaf",
"metadata": {},
"outputs": [],
"source": [
"pts = torch.tensor([-5.0, -1.0, 1.0, 5.0])\n",
"print(\" h'(-5) h'(-1) h'(1) h'(5)\")\n",
"for name in names:\n",
" h = ACTIVATIONS[name]\n",
" d = (h(pts + eps) - h(pts - eps)) / (2 * eps)\n",
" print(f\"{name:8s}\" + \"\".join(f\"{v:9.4f}\" for v in d.tolist()))"
]
},
{
"cell_type": "markdown",
"id": "e43ac8a1",
"metadata": {},
"source": [
"Sigmoid's derivative is at most 0.25, and both sigmoid and tanh flatten out for large $|a|$: they *saturate*. ReLU's derivative is exactly 1 for $a > 0$, one reason it became the default hidden activation. GELU, used in transformers later in the course, is a smooth version of ReLU. For $a < 0$ ReLU's derivative is exactly 0, which has its own failure.\n",
"\n",
"### The failure: a dead ReLU unit\n",
"\n",
"**Poll 2 comes before this cell.** Train an 8-unit ReLU network on the moons data, but first set unit 0's bias to $-10$ so its pre-activation is negative for every training point. Then measure how far each unit's incoming weights move."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c05cce51",
"metadata": {},
"outputs": [],
"source": [
"Xm, tm = make_moons(400, noise=0.2, random_state=189)\n",
"torch.manual_seed(189)\n",
"net = MLP([2, 8, 1], activation=\"relu\")\n",
"with torch.no_grad():\n",
" net.layers[0].bias[0] = -10.0\n",
" pre = torch.as_tensor(Xm, dtype=torch.float32) @ net.layers[0].weight.T + net.layers[0].bias\n",
"print(f\"unit 0 pre-activation, largest value over the data: {pre[:, 0].max():.2f}\")\n",
"\n",
"W_before = net.layers[0].weight.detach().clone()\n",
"_ = train(net, Xm, tm, epochs=500, lr=0.01)\n",
"moved = (net.layers[0].weight.detach() - W_before).norm(dim=1)\n",
"print(\"distance each unit's incoming weights moved:\", moved.numpy().round(3))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "9a1939a3",
"metadata": {},
"outputs": [],
"source": [
"fig, ax = plt.subplots(figsize=(9, 4.5))\n",
"ax.bar(range(8), moved.numpy(), color=C_BLUE)\n",
"ax.text(0, 0.05, \"0.000\", ha=\"center\", va=\"bottom\", color=C_RED, fontsize=14, fontweight=\"bold\")\n",
"ax.set(xlabel=\"hidden unit\", ylabel=\"distance moved\", xticks=range(8),\n",
" title=\"How far each unit's incoming weights moved in training\")\n",
"ax.get_xticklabels()[0].set_color(C_RED)\n",
"plt.show()"
]
},
{
"cell_type": "markdown",
"id": "bd12c9f1",
"metadata": {},
"source": [
"Unit 0 outputs 0 for every input, its derivative is 0 at every input, so its incoming weights get zero gradient and never move. It stays dead for the rest of training. Lecture 14 covers initialization, which makes this less likely."
]
},
{
"cell_type": "markdown",
"id": "75d46812",
"metadata": {},
"source": [
"---\n",
"## Step 7: Output Layers and Loss\n",
"\n",
"The hidden layers compute features $z = \\phi(x;\\, W^{(1)}, \\dots, W^{(L-1)})$. The output layer is a model from Lectures 8-9 applied to those features, so the output activation and the loss come from the same maximum-likelihood argument:\n",
"\n",
"| Target | Output activation | Likelihood | Loss | PyTorch loss |\n",
"|---|---|---|---|---|\n",
"| a real number | identity | Gaussian | squared error | `nn.MSELoss` |\n",
"| one of 2 classes | sigmoid | Bernoulli | binary cross-entropy | `nn.BCEWithLogitsLoss` |\n",
"| one of $K$ classes | softmax | categorical | cross-entropy | `nn.CrossEntropyLoss` |\n",
"| any subset of $K$ labels | $K$ sigmoids | $K$ Bernoullis | sum of $K$ binary cross-entropies | `nn.BCEWithLogitsLoss` |\n",
"\n",
"The two cross-entropy losses take **logits** and apply the sigmoid or softmax themselves, so our `MLP` returns logits.\n",
"\n",
"### Same hidden layers, different output layers\n",
"\n",
"Regression: a noisy sine curve, identity output, squared error."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "50c01ea7",
"metadata": {},
"outputs": [],
"source": [
"rng = np.random.default_rng(189)\n",
"x_reg = np.sort(rng.uniform(-3, 3, 200))[:, None]\n",
"t_reg = np.sin(2 * x_reg[:, 0]) + 0.2 * rng.standard_normal(200)\n",
"\n",
"torch.manual_seed(189)\n",
"reg_net = MLP([1, 32, 32, 1], activation=\"tanh\")\n",
"history = train(reg_net, x_reg, t_reg, epochs=1000, lr=0.01, loss_fn=nn.MSELoss())\n",
"print(f\"final mean squared error {history[-1]:.4f} (noise variance 0.04)\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c76cceed",
"metadata": {},
"outputs": [],
"source": [
"xg = np.linspace(-3.5, 3.5, 300)[:, None]\n",
"with torch.no_grad():\n",
" yg = reg_net(torch.as_tensor(xg, dtype=torch.float32)).numpy()\n",
"plt.scatter(x_reg, t_reg, s=14, color=C_BLUE, alpha=0.6, label=\"data\")\n",
"plt.plot(xg, yg, color=C_GOLD, lw=3, label=\"network\")\n",
"plt.plot(xg, np.sin(2 * xg), \"k--\", lw=1, label=r\"$\\sin(2x)$\")\n",
"plt.xlabel(\"$x$\")\n",
"plt.legend()\n",
"plt.show()"
]
},
{
"cell_type": "markdown",
"id": "f5f7b6a9",
"metadata": {},
"source": [
"Multi-class: three interleaved spirals, softmax output, cross-entropy. Compare with softmax regression on the raw inputs (no hidden layer)."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a0cb0826",
"metadata": {},
"outputs": [],
"source": [
"Xs, ts = make_spiral()\n",
"torch.manual_seed(189)\n",
"softmax_reg = MLP([2, 3]) # no hidden layer\n",
"_ = train(softmax_reg, Xs, ts, epochs=500, lr=0.01)\n",
"torch.manual_seed(189)\n",
"spiral_net = MLP([2, 32, 32, 3], activation=\"relu\")\n",
"_ = train(spiral_net, Xs, ts, epochs=500, lr=0.01)\n",
"\n",
"fig, axes = plt.subplots(1, 2, figsize=(11, 5))\n",
"plot_boundary(predict_proba(softmax_reg), Xs, ts, axes[0], f\"Softmax regression: {accuracy(softmax_reg, Xs, ts):.0%}\")\n",
"plot_boundary(predict_proba(spiral_net), Xs, ts, axes[1], f\"2 hidden layers + softmax: {accuracy(spiral_net, Xs, ts):.0%}\")\n",
"plt.tight_layout()\n",
"plt.show()"
]
},
{
"cell_type": "markdown",
"id": "41f665d9",
"metadata": {},
"source": [
"### The failure: softmax overflows\n",
"\n",
"Softmax exponentiates the logits, and large logits overflow:"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0d1d898d",
"metadata": {},
"outputs": [],
"source": [
"logits = np.array([1000.0, 1001.0, 1002.0])\n",
"naive = np.exp(logits) / np.exp(logits).sum()\n",
"print(\"naive softmax: \", naive)\n",
"shifted = np.exp(logits - logits.max()) # softmax(a) = softmax(a - c) for any constant c\n",
"print(\"shifted softmax:\", (shifted / shifted.sum()).round(4))"
]
},
{
"cell_type": "markdown",
"id": "ffe299d8",
"metadata": {},
"source": [
"PyTorch's cross-entropy takes logits and does this shift internally (the log-sum-exp trick), so it never forms the overflowing exponentials:"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c9c0a9e8",
"metadata": {},
"outputs": [],
"source": [
"z = torch.tensor([[1000.0, 1001.0, 1002.0]])\n",
"print(\"F.cross_entropy: \", round(F.cross_entropy(z, torch.tensor([2])).item(), 4))\n",
"print(\"-log softmax, by hand:\", round(-F.log_softmax(z, dim=1)[0, 2].item(), 4))"
]
},
{
"cell_type": "markdown",
"id": "a4a02516",
"metadata": {},
"source": [
"### Pass logits, not probabilities\n",
"\n",
"A common bug: apply softmax inside the model, then use `nn.CrossEntropyLoss`, which applies softmax again."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a2a31b74",
"metadata": {},
"outputs": [],
"source": [
"z = torch.tensor([[10.0, 0.0, 0.0]]) # confident and correct: the target is class 0\n",
"target = torch.tensor([0])\n",
"print(\"logits in: \", round(F.cross_entropy(z, target).item(), 5))\n",
"print(\"probabilities in:\", round(F.cross_entropy(F.softmax(z, dim=1), target).item(), 5))"
]
},
{
"cell_type": "markdown",
"id": "ceb53f2f",
"metadata": {},
"source": [
"---\n",
"## Step 8: The PyTorch Pieces\n",
"\n",
"Everything above used two helpers, `MLP` and `train`. This section opens them up. Nothing after it depends on it, so it can be skipped.\n",
"\n",
"Four ideas:\n",
"1. **`torch.tensor`**: an array like a NumPy array, which can live on a GPU and records how it was computed.\n",
"2. **`nn.Module`**: a model. `__init__` creates the layers (and their parameters); `forward` computes the output.\n",
"3. **Autograd**: `loss.backward()` computes the gradient of the loss with respect to every parameter. How it does that is Lecture 13.\n",
"4. **Optimizers**: `torch.optim.SGD` and `torch.optim.Adam` apply the update rules from Lecture 11."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "35edeeb1",
"metadata": {},
"outputs": [],
"source": [
"x = torch.tensor([[1.0, 2.0], [3.0, 4.0]])\n",
"print(x.shape, x.dtype)\n",
"print(x @ x.T) # the same operators as NumPy\n",
"print(torch.from_numpy(X[:2]).float()) # NumPy -> torch\n",
"print(x.numpy()) # torch -> NumPy"
]
},
{
"cell_type": "markdown",
"id": "b7890bd7",
"metadata": {},
"source": [
"`nn.Linear(d_in, d_out)` holds a weight matrix of shape `(d_out, d_in)` and a bias of length `d_out`. Counting the parameters of a network with layer sizes 784, 128, 64, and 10 (slides 18-19):"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2a5d63e1",
"metadata": {},
"outputs": [],
"source": [
"model = MLP([784, 128, 64, 10])\n",
"print(model)\n",
"print(f\"{sum(p.numel() for p in model.parameters()):,} parameters\")\n",
"print(f\"{(784*128 + 128) + (128*64 + 64) + (64*10 + 10):,} by hand: weights + biases, layer by layer\")"
]
},
{
"cell_type": "markdown",
"id": "d71deee8",
"metadata": {},
"source": [
"One training step, written out. `train` repeats this `epochs` times:"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c2278401",
"metadata": {},
"outputs": [],
"source": [
"torch.manual_seed(189)\n",
"model = MLP([2, 8, 1], activation=\"relu\")\n",
"optimizer = torch.optim.Adam(model.parameters(), lr=0.05)\n",
"loss_fn = nn.BCEWithLogitsLoss()\n",
"X_torch = torch.as_tensor(X, dtype=torch.float32)\n",
"t_torch = torch.as_tensor(t, dtype=torch.float32)[:, None]\n",
"\n",
"optimizer.zero_grad() # clear gradients left from the previous step\n",
"loss = loss_fn(model(X_torch), t_torch) # forward pass: inputs -> logits -> loss\n",
"loss.backward() # gradient of the loss w.r.t. every parameter (Lecture 13)\n",
"optimizer.step() # Adam update (Lecture 11)\n",
"print(f\"loss {loss.item():.4f}\")\n",
"print(\"gradient of the first layer's weights:\\n\", model.layers[0].weight.grad)"
]
},
{
"cell_type": "markdown",
"id": "585184e4",
"metadata": {},
"source": [
"A row of exact zeros is a unit whose pre-activation is negative on every training point at this initialization: the dead-unit case from Step 6, this time by chance."
]
},
{
"cell_type": "markdown",
"id": "215d4d7c",
"metadata": {},
"source": [
"---\n",
"## Step 9: Width, Depth, and Activation **(Demo 2)**\n",
"\n",
"Pick a dataset and an architecture, press **Run Interact**, and look at the boundary and the loss curve. Things to try:\n",
"- moons, depth 1, width 1, then 8, then 64: what happens to the boundary as units are added?\n",
"- spiral, identity vs. ReLU at the same size\n",
"- depth 4, sigmoid vs. ReLU (Step 6 says why sigmoid is slower)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "eeb088e5",
"metadata": {},
"outputs": [],
"source": [
"DATASETS = {\"xor\": make_xor(), \"moons\": make_moons(400, noise=0.2, random_state=189), \"spiral\": make_spiral()}\n",
"\n",
"\n",
"def explore(dataset=\"moons\", depth=1, width=8, activation=\"relu\", epochs=500, lr=0.01):\n",
" Xd, td = DATASETS[dataset]\n",
" K = len(np.unique(td))\n",
" torch.manual_seed(189)\n",
" model = MLP([2] + [width] * depth + [1 if K == 2 else K], activation)\n",
" history = train(model, Xd, td, epochs=epochs, lr=lr)\n",
" fig, axes = plt.subplots(1, 2, figsize=(12, 5))\n",
" plot_boundary(predict_proba(model), Xd, td, axes[0], f\"accuracy {accuracy(model, Xd, td):.1%}\")\n",
" axes[1].plot(history, color=C_BLUE, lw=2)\n",
" axes[1].set(xlabel=\"epoch\", ylabel=\"training loss\", yscale=\"log\")\n",
" plt.tight_layout()\n",
" plt.show()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f1c8f48c",
"metadata": {},
"outputs": [],
"source": [
"widgets.interact_manual(\n",
" explore,\n",
" dataset=[\"xor\", \"moons\", \"spiral\"],\n",
" depth=widgets.IntSlider(value=1, min=1, max=5),\n",
" width=widgets.IntSlider(value=8, min=1, max=64),\n",
" activation=[\"identity\", \"relu\", \"tanh\", \"sigmoid\", \"gelu\"],\n",
" epochs=widgets.IntSlider(value=500, min=50, max=2000, step=50),\n",
" lr=widgets.FloatLogSlider(value=0.01, base=10, min=-3, max=-1),\n",
");"
]
},
{
"cell_type": "markdown",
"id": "8196e065",
"metadata": {},
"source": [
"The same comparison as a fixed figure: moons with one hidden layer of 1, 8, and 64 ReLU units, then two layers of 32."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d711b406",
"metadata": {},
"outputs": [],
"source": [
"Xd, td = DATASETS[\"moons\"]\n",
"fig, axes = plt.subplots(1, 4, figsize=(18, 4.8))\n",
"for ax, hidden in zip(axes, [[1], [8], [64], [32, 32]]):\n",
" torch.manual_seed(189)\n",
" m = MLP([2] + hidden + [1], activation=\"relu\")\n",
" _ = train(m, Xd, td, epochs=500, lr=0.01)\n",
" plot_boundary(predict_proba(m), Xd, td, ax, f\"hidden {hidden}: {accuracy(m, Xd, td):.0%}\")\n",
"plt.tight_layout()\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a884a407",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.6"
}
},
"nbformat": 4,
"nbformat_minor": 5
}