============================================================================= Lecture 5 - Neural Networks & Sequence Models Practical Session: Building a Neural Text Classifier with PyTorch # Methods Seminar: Multimodal Computational Methods for Political Science # Computational Social Science Program, LMU Munich ============================================================================= In this practical we will: 1. Load the UK House of Commons corpus (same as Lectures 1-2) 2. Build a vocabulary and tokenize 3. Build a simple neural classifier (Embedding -> Average Pool -> Dense) 4. Train it and evaluate against the Lecture 2 logistic regression baseline 5. Upgrade to a BiLSTM and compare 6. Experiment: pre-trained embeddings, freezing, hyperparameters This is the course's transition from R to Python. The deep learning ecosystem (PyTorch, Hugging Face) is Python-first. Runs on a laptop CPU (slow) or Google Colab's free GPU (fast, recommended). Setup: pip install torch pandas scikit-learn numpy ============================================================================= """ # ----------------------------------------------------------------------------- # Dataset # ----------------------------------------------------------------------------- In the first lectures we were working with .rds file that can not be directly read in Python (Corp_HouseOfCommons_V2.rds). Just save the original .rds in R as a csv. Use the code below (to be run in R) to make basic filtering and to rewrite .rds into .csv format. library(dplyr) speeches_raw <- readRDS(".../Corp_HouseOfCommons_V2.rds") speeches <- speeches_raw %>% filter(date >= "2017-06-08", date <= "2019-11-06") %>% filter(terms >= 50) %>% filter(!is.na(party), party != "") %>% filter(party %in% c("Con", "Lab", "LibDem", "SNP")) %>% mutate(label = ifelse(party == "Con", "Government", "Opposition")) %>% select(text, label, party, date) write.csv(speeches, ".../uk_commons_gov_opp.csv", row.names = FALSE) # ----------------------------------------------------------------------------- # 0. SETUP # ----------------------------------------------------------------------------- import re import random import numpy as np import pandas as pd import torch import torch.nn as nn from torch.utils.data import Dataset, DataLoader from sklearn.model_selection import train_test_split from sklearn.metrics import (accuracy_score, precision_recall_fscore_support, confusion_matrix, classification_report) # Reproducibility: set ALL the seeds SEED = 42 random.seed(SEED) np.random.seed(SEED) torch.manual_seed(SEED) if torch.cuda.is_available(): torch.cuda.manual_seed_all(SEED) # Use GPU if available (Colab), else CPU device = torch.device("cuda" if torch.cuda.is_available() else "cpu") print(f"Using device: {device}") # ----------------------------------------------------------------------------- # 1. LOAD AND PREPARE THE DATA # ----------------------------------------------------------------------------- # # We use the same UK Commons Gov vs. Opposition task as Lecture 2. # Export your data from R (Lecture 2) to CSV, or load ParlSpeech here. # # Expected columns: 'text' (speech), 'label' ("Government"/"Opposition") df = pd.read_csv("uk_commons_gov_opp.csv") # Basic cleaning df = df.dropna(subset=["text", "label"]) df = df[df["text"].str.len() > 100] # drop very short speeches # Encode labels as integers: Government=0, Opposition=1 df["y"] = (df["label"] == "Opposition").astype(int) print(f"\nTotal speeches: {len(df)}") print(df["label"].value_counts()) # Optional: sample for faster in-class runs df = df.sample(n=min(5000, len(df)), random_state=SEED).reset_index(drop=True) # ----------------------------------------------------------------------------- # 2. TOKENIZATION AND VOCABULARY # ----------------------------------------------------------------------------- # # Neural nets need integer token IDs, not raw text. # We build a simple whitespace tokenizer and a vocabulary. def simple_tokenize(text): """Lowercase, keep only word characters, split on whitespace.""" text = text.lower() text = re.sub(r"[^a-z\s]", " ", text) # strip punctuation/numbers tokens = text.split() return tokens # Build vocabulary from the training data only (avoid leakage!) from collections import Counter # First split, so vocab is built only on training data train_df, test_df = train_test_split( df, test_size=0.2, stratify=df["y"], random_state=SEED) # Count word frequencies in training set counter = Counter() for text in train_df["text"]: counter.update(simple_tokenize(text)) # Keep the most frequent words; reserve 0 for padding, 1 for unknown MAX_VOCAB = 20000 vocab = {"": 0, "": 1} for word, _ in counter.most_common(MAX_VOCAB - 2): vocab[word] = len(vocab) vocab_size = len(vocab) print(f"\nVocabulary size: {vocab_size}") def encode(text, vocab, max_len=200): """Convert text to a list of token IDs, truncated/padded to max_len.""" tokens = simple_tokenize(text) ids = [vocab.get(tok, 1) for tok in tokens] # 1 = ids = ids[:max_len] # truncate return ids # ----------------------------------------------------------------------------- # 3. PYTORCH DATASET AND DATALOADER # ----------------------------------------------------------------------------- class SpeechDataset(Dataset): """Wraps texts and labels for PyTorch.""" def __init__(self, texts, labels, vocab, max_len=200): self.texts = list(texts) self.labels = list(labels) self.vocab = vocab self.max_len = max_len def __len__(self): return len(self.texts) def __getitem__(self, idx): ids = encode(self.texts[idx], self.vocab, self.max_len) return ids, self.labels[idx] def collate_batch(batch): """Pad sequences in a batch to the same length; build a mask.""" ids_list, labels = zip(*batch) max_len = max(len(ids) for ids in ids_list) padded, masks = [], [] for ids in ids_list: pad_len = max_len - len(ids) padded.append(ids + [0] * pad_len) # 0 = masks.append([1] * len(ids) + [0] * pad_len) # 1 = real token return (torch.tensor(padded, dtype=torch.long), torch.tensor(masks, dtype=torch.long), torch.tensor(labels, dtype=torch.long)) # Further split training into train + validation train_df2, val_df = train_test_split( train_df, test_size=0.15, stratify=train_df["y"], random_state=SEED) train_ds = SpeechDataset(train_df2["text"], train_df2["y"], vocab) val_ds = SpeechDataset(val_df["text"], val_df["y"], vocab) test_ds = SpeechDataset(test_df["text"], test_df["y"], vocab) BATCH_SIZE = 32 train_loader = DataLoader(train_ds, batch_size=BATCH_SIZE, shuffle=True, collate_fn=collate_batch) val_loader = DataLoader(val_ds, batch_size=64, shuffle=False, collate_fn=collate_batch) test_loader = DataLoader(test_ds, batch_size=64, shuffle=False, collate_fn=collate_batch) print(f"\nTrain: {len(train_ds)}, Val: {len(val_ds)}, Test: {len(test_ds)}") # ----------------------------------------------------------------------------- # 4. MODEL 1: SIMPLE NEURAL CLASSIFIER # ----------------------------------------------------------------------------- # # Architecture: Embedding -> Average Pool -> Dense -> Dense (output) # This is essentially logistic regression on averaged embeddings, # plus one hidden layer. class SimpleTextClassifier(nn.Module): def __init__(self, vocab_size, embed_dim=300, hidden_dim=128, n_classes=2): super().__init__() self.embedding = nn.Embedding(vocab_size, embed_dim, padding_idx=0) self.fc1 = nn.Linear(embed_dim, hidden_dim) self.fc2 = nn.Linear(hidden_dim, n_classes) self.dropout = nn.Dropout(0.3) def forward(self, x, mask): emb = self.embedding(x) # [batch, seq_len, embed_dim] # Masked mean pooling: average over real tokens only, ignore padding mask = mask.unsqueeze(-1).float() # [batch, seq_len, 1] summed = (emb * mask).sum(dim=1) # sum over sequence counts = mask.sum(dim=1).clamp(min=1) # number of real tokens pooled = summed / counts # [batch, embed_dim] h = torch.relu(self.fc1(pooled)) h = self.dropout(h) return self.fc2(h) # logits [batch, n_classes] # ----------------------------------------------------------------------------- # 5. TRAINING AND EVALUATION FUNCTIONS # ----------------------------------------------------------------------------- def evaluate(model, loader): """Return accuracy and F1 on a data loader.""" model.eval() all_preds, all_labels = [], [] with torch.no_grad(): for ids, mask, labels in loader: ids, mask = ids.to(device), mask.to(device) logits = model(ids, mask) preds = logits.argmax(dim=1).cpu().numpy() all_preds.extend(preds) all_labels.extend(labels.numpy()) acc = accuracy_score(all_labels, all_preds) _, _, f1, _ = precision_recall_fscore_support( all_labels, all_preds, average="binary", zero_division=0) return acc, f1, all_preds, all_labels def train_model(model, train_loader, val_loader, epochs=10, lr=1e-3): """Standard PyTorch training loop with early stopping on val F1.""" model = model.to(device) optimizer = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=0.01) criterion = nn.CrossEntropyLoss() best_val_f1 = 0.0 best_state = None patience, no_improve = 3, 0 for epoch in range(epochs): model.train() total_loss = 0.0 for ids, mask, labels in train_loader: ids, mask, labels = (ids.to(device), mask.to(device), labels.to(device)) optimizer.zero_grad() logits = model(ids, mask) loss = criterion(logits, labels) loss.backward() optimizer.step() total_loss += loss.item() val_acc, val_f1, _, _ = evaluate(model, val_loader) avg_loss = total_loss / len(train_loader) print(f"Epoch {epoch+1:2d} | train_loss={avg_loss:.4f} | " f"val_acc={val_acc:.4f} | val_f1={val_f1:.4f}") # Early stopping: keep the best model by validation F1 if val_f1 > best_val_f1: best_val_f1 = val_f1 best_state = {k: v.cpu().clone() for k, v in model.state_dict().items()} no_improve = 0 else: no_improve += 1 if no_improve >= patience: print(f"Early stopping at epoch {epoch+1}") break # Restore best model if best_state is not None: model.load_state_dict(best_state) return model # ----------------------------------------------------------------------------- # 6. TRAIN THE SIMPLE MODEL # ----------------------------------------------------------------------------- print("\n" + "="*60) print(" MODEL 1: SIMPLE NEURAL CLASSIFIER") print("="*60) model_simple = SimpleTextClassifier(vocab_size, embed_dim=300, hidden_dim=128, n_classes=2) model_simple = train_model(model_simple, train_loader, val_loader, epochs=15, lr=1e-3) # Final evaluation on the held-out TEST set test_acc, test_f1, preds, labels = evaluate(model_simple, test_loader) print(f"\nSimple model TEST accuracy: {test_acc:.4f}") print(f"Simple model TEST F1: {test_f1:.4f}") print("\nClassification report:") print(classification_report(labels, preds, target_names=["Government", "Opposition"])) # ----------------------------------------------------------------------------- # 7. MODEL 2: BIDIRECTIONAL LSTM # ----------------------------------------------------------------------------- # # Architecture: Embedding -> BiLSTM -> Dense (output) # The LSTM processes the sequence in order (both directions), # so it captures word order -- unlike average pooling. class LSTMClassifier(nn.Module): def __init__(self, vocab_size, embed_dim=300, hidden_dim=128, n_classes=2): super().__init__() self.embedding = nn.Embedding(vocab_size, embed_dim, padding_idx=0) self.lstm = nn.LSTM(embed_dim, hidden_dim, batch_first=True, bidirectional=True) self.fc = nn.Linear(hidden_dim * 2, n_classes) # *2 for bidirectional self.dropout = nn.Dropout(0.3) def forward(self, x, mask): emb = self.embedding(x) # [batch, seq, embed] # Pack to let the LSTM ignore padding (more correct + faster) lengths = mask.sum(dim=1).cpu() packed = nn.utils.rnn.pack_padded_sequence( emb, lengths, batch_first=True, enforce_sorted=False) _, (h_n, _) = self.lstm(packed) # h_n shape: [2, batch, hidden_dim] -- concatenate both directions h = torch.cat([h_n[0], h_n[1]], dim=1) # [batch, hidden*2] h = self.dropout(h) return self.fc(h) print("\n" + "="*60) print(" MODEL 2: BIDIRECTIONAL LSTM") print("="*60) model_lstm = LSTMClassifier(vocab_size, embed_dim=300, hidden_dim=128, n_classes=2) model_lstm = train_model(model_lstm, train_loader, val_loader, epochs=15, lr=1e-3) lstm_acc, lstm_f1, lstm_preds, lstm_labels = evaluate(model_lstm, test_loader) print(f"\nBiLSTM model TEST accuracy: {lstm_acc:.4f}") print(f"BiLSTM model TEST F1: {lstm_f1:.4f}") print("\nClassification report:") print(classification_report(lstm_labels, lstm_preds, target_names=["Government", "Opposition"])) # ----------------------------------------------------------------------------- # 8. COMPARISON WITH LECTURE 2 BASELINE # ----------------------------------------------------------------------------- # # Fill in your actual Lecture 2 logistic regression F1 here for comparison. # (From the R practical - e.g., LR with TF-IDF got ~0.83) LECTURE2_LR_F1 = 0.83 # <-- replace with YOUR Lecture 2 result print("\n" + "="*60) print(" MODEL COMPARISON") print("="*60) comparison = pd.DataFrame({ "Model": ["Logistic Regression (L2)", "Simple Neural (avg pool)", "BiLSTM"], "Test F1": [LECTURE2_LR_F1, test_f1, lstm_f1], "Captures word order?": ["No", "No", "Yes"], }) print(comparison.to_string(index=False)) print("\nKey question: does the added complexity of neural models") print("justify the improvement (if any) over logistic regression?") # ----------------------------------------------------------------------------- # 9. EXPERIMENT: PRE-TRAINED EMBEDDINGS (OPTIONAL) # ----------------------------------------------------------------------------- # # Initialize the embedding layer with GloVe vectors (from Lecture 3) # instead of random initialization. Often helps, especially with # limited training data. def load_glove_embeddings(glove_path, vocab, embed_dim=300): """Build an embedding matrix from GloVe, aligned to our vocabulary.""" # Start with small random values embeddings = np.random.normal(0, 0.1, (len(vocab), embed_dim)) embeddings[0] = 0 # = zeros found = 0 with open(glove_path, "r", encoding="utf-8") as f: for line in f: parts = line.split() word = parts[0] if word in vocab: vec = np.array(parts[1:], dtype=np.float32) embeddings[vocab[word]] = vec found += 1 print(f"Found GloVe vectors for {found}/{len(vocab)} words") return torch.tensor(embeddings, dtype=torch.float) # Uncomment to use (requires glove.6B.300d.txt from Lecture 3): # # glove_matrix = load_glove_embeddings("glove.6B.300d.txt", vocab, 300) # # model_glove = SimpleTextClassifier(vocab_size, embed_dim=300, # hidden_dim=128, n_classes=2) # # Copy in the pre-trained embeddings # model_glove.embedding.weight.data.copy_(glove_matrix) # # Optionally freeze them: # # model_glove.embedding.weight.requires_grad = False # # model_glove = train_model(model_glove, train_loader, val_loader, # epochs=15, lr=1e-3) # glove_acc, glove_f1, _, _ = evaluate(model_glove, test_loader) # print(f"\nGloVe-initialized model TEST F1: {glove_f1:.4f}") # ============================================================================= # DISCUSSION QUESTIONS # ============================================================================= # # 1. How does the simple neural classifier compare with the Lecture 2 # logistic regression? Is the gap large enough to justify the complexity? # # 2. Does the BiLSTM beat the simple averaging model? By how much? # On which kinds of speeches might word order help most? # # 3. Watch the training output: does train_loss keep falling while # val_f1 plateaus or drops? That's overfitting. Did early stopping help? # # 4. Try the GloVe experiment (section 9). Does pre-trained initialization # help? Try both fine-tuning and freezing the embeddings. # # 5. Experiment with hyperparameters: # - hidden_dim: 64, 128, 256 # - dropout: 0.1, 0.3, 0.5 # - learning rate: 1e-2, 1e-3, 1e-4 # - max_len: 100, 200, 400 # What's the cost of careless choices? # # =============================================================================