""" ============================================================================= Lecture 7 - Transfer Learning & Fine-Tuning BERT Practical Session: Fine-Tuning BERT on UK House of Commons Speeches (Google Colab version) # Methods Seminar: Multimodal Computational Methods for Political Science # Computational Social Science Program, LMU Munich ============================================================================= THE CULMINATION OF THE COURSE. In this practical we will: 1. Load the same Gov vs. Opposition task from Lectures 2 and 5 2. Fine-tune a pre-trained BERT model with Hugging Face 3. Evaluate: accuracy, precision, recall, F1, confusion matrix 4. Compare against the Lecture 2 (LR) and Lecture 5 (BiLSTM) baselines 5. Run multiple seeds to check stability 6. Inspect the documents BERT misclassifies GPU STRONGLY RECOMMENDED. In Colab: Runtime -> Change runtime type -> Hardware accelerator -> GPU (T4) ============================================================================= """ # ============================================================================= # CELL 1 -- INSTALL (run once, at the top of the notebook) # ============================================================================= # !pip install -q transformers datasets evaluate accelerate scikit-learn # ----------------------------------------------------------------------------- # 0. SETUP # ----------------------------------------------------------------------------- import datasets datasets.config.TORCHVISION_AVAILABLE = False import numpy as np import pandas as pd import torch from datasets import Dataset from transformers import (AutoTokenizer, AutoModelForSequenceClassification, TrainingArguments, Trainer, DataCollatorWithPadding, set_seed) from sklearn.model_selection import train_test_split from sklearn.metrics import (accuracy_score, precision_recall_fscore_support, confusion_matrix, classification_report) device = "cuda" if torch.cuda.is_available() else "cpu" print(f"Using device: {device}") if device == "cpu": print("WARNING: no GPU detected. Fine-tuning will be SLOW.") print("In Colab: Runtime -> Change runtime type -> GPU (T4)") MODEL_NAME = "bert-base-uncased" SEED = 42 set_seed(SEED) # ----------------------------------------------------------------------------- # 1. LOAD DATA # ----------------------------------------------------------------------------- # # Option A (recommended): a CSV exported from R (same as Lecture 5) # Columns: 'text', 'label' ("Government"/"Opposition") df = pd.read_csv("uk_commons_gov_opp.csv") # --- Option B: read the .rds directly (uncomment to use) --- # !pip install pyreadr -q # import pyreadr # result = pyreadr.read_r("Corp_HouseOfCommons_V2.rds") # raw = list(result.values())[0] # raw["date"] = pd.to_datetime(raw["date"], errors="coerce") # df = raw[(raw["date"] >= "2017-06-08") & (raw["date"] <= "2019-11-06")].copy() # if "terms" in df.columns: # df = df[df["terms"] >= 50] # df = df[df["party"].isin(["Con", "Lab", "LibDem", "SNP"])].copy() # df["label"] = np.where(df["party"] == "Con", "Government", "Opposition") # Clean and encode df = df.dropna(subset=["text", "label"]) df = df[df["text"].str.len() > 100] df["labels"] = (df["label"] == "Opposition").astype(int) # Sample for a manageable in-class run df = df.sample(n=min(5000, len(df)), random_state=SEED).reset_index(drop=True) print(f"\nTotal speeches: {len(df)}") print(df["label"].value_counts()) # ----------------------------------------------------------------------------- # 2. TRAIN / VAL / TEST SPLIT # ----------------------------------------------------------------------------- # # Hold out a TEST set at the very start. Do not touch it until the end. train_df, test_df = train_test_split( df[["text", "labels"]], test_size=0.2, stratify=df["labels"], random_state=SEED) train_df, val_df = train_test_split( train_df, test_size=0.15, stratify=train_df["labels"], random_state=SEED) print(f"\nTrain: {len(train_df)}, Val: {len(val_df)}, Test: {len(test_df)}") train_ds = Dataset.from_pandas(train_df, preserve_index=False) val_ds = Dataset.from_pandas(val_df, preserve_index=False) test_ds = Dataset.from_pandas(test_df, preserve_index=False) # ----------------------------------------------------------------------------- # 3. TOKENIZATION # ----------------------------------------------------------------------------- # # Use BERT's own tokenizer. Truncate to 512 tokens (BERT's max). # We do NOT pad here -- the data collator pads each batch dynamically # (faster than padding everything to 512). Remember: 512 TOKENS, not words. tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME) def tokenize_function(batch): return tokenizer(batch["text"], truncation=True, max_length=512) print("\nTokenizing...") train_ds = train_ds.map(tokenize_function, batched=True) val_ds = val_ds.map(tokenize_function, batched=True) test_ds = test_ds.map(tokenize_function, batched=True) # Drop the raw text column so the collator only sees model inputs. train_ds = train_ds.remove_columns(["text"]) val_ds = val_ds.remove_columns(["text"]) test_ds = test_ds.remove_columns(["text"]) # Dynamic padding: pads each batch to its own longest sequence. data_collator = DataCollatorWithPadding(tokenizer=tokenizer) # ----------------------------------------------------------------------------- # 4. METRICS # ----------------------------------------------------------------------------- def compute_metrics(eval_pred): logits, labels = eval_pred preds = np.argmax(logits, axis=1) acc = accuracy_score(labels, preds) prec, rec, f1, _ = precision_recall_fscore_support( labels, preds, average="binary", zero_division=0) return {"accuracy": acc, "precision": prec, "recall": rec, "f1": f1} # ----------------------------------------------------------------------------- # 5. FINE-TUNE BERT # ----------------------------------------------------------------------------- def fine_tune_bert(seed=SEED): """Fine-tune BERT and return the trainer + test metrics.""" set_seed(seed) # Load pre-trained BERT with a fresh (randomly initialized) head. # The "some weights were not used / newly initialized" message that # prints here is NORMAL: the MLM/NSP pre-training heads are dropped and # your classification head is created fresh. Nothing is wrong. model = AutoModelForSequenceClassification.from_pretrained( MODEL_NAME, num_labels=2) args = TrainingArguments( output_dir=f"./bert_gov_opp_seed{seed}", learning_rate=2e-5, # SMALL LR -- standard for fine-tuning num_train_epochs=3, # 2-4 epochs is typical per_device_train_batch_size=16, per_device_eval_batch_size=32, weight_decay=0.01, # regularization eval_strategy="epoch", save_strategy="epoch", load_best_model_at_end=True, metric_for_best_model="f1", logging_steps=50, seed=seed, report_to="none", # disable wandb etc. ) trainer = Trainer( model=model, args=args, train_dataset=train_ds, eval_dataset=val_ds, # validate on VAL, not TEST compute_metrics=compute_metrics, data_collator=data_collator, # dynamic padding tokenizer=tokenizer, ) trainer.train() # Final evaluation on the HELD-OUT TEST set (touched only once) test_metrics = trainer.evaluate(test_ds) return trainer, test_metrics print("\n" + "="*60) print(" FINE-TUNING BERT (seed=42)") print("="*60) trainer, test_metrics = fine_tune_bert(seed=42) print(f"\nBERT TEST results:") print(f" Accuracy: {test_metrics['eval_accuracy']:.4f}") print(f" Precision: {test_metrics['eval_precision']:.4f}") print(f" Recall: {test_metrics['eval_recall']:.4f}") print(f" F1: {test_metrics['eval_f1']:.4f}") # ----------------------------------------------------------------------------- # 6. DETAILED EVALUATION # ----------------------------------------------------------------------------- print("\n" + "="*60) print(" DETAILED EVALUATION") print("="*60) preds_output = trainer.predict(test_ds) y_pred = np.argmax(preds_output.predictions, axis=1) y_true = preds_output.label_ids print("\nClassification report:") print(classification_report(y_true, y_pred, target_names=["Government", "Opposition"])) print("Confusion matrix:") cm = confusion_matrix(y_true, y_pred) print(pd.DataFrame(cm, index=["true_Gov", "true_Opp"], columns=["pred_Gov", "pred_Opp"])) # ----------------------------------------------------------------------------- # 7. COMPARISON WITH BASELINES # ----------------------------------------------------------------------------- # # Fill in your ACTUAL results from Lectures 2 and 5. LECTURE2_LR_F1 = 0.83 # <-- your logistic regression F1 LECTURE5_BILSTM_F1 = 0.85 # <-- your BiLSTM F1 print("\n" + "="*60) print(" MODEL COMPARISON") print("="*60) comparison = pd.DataFrame({ "Model": ["Logistic Regression (L2)", "BiLSTM (L5)", "BERT fine-tuned (L7)"], "Test F1": [LECTURE2_LR_F1, LECTURE5_BILSTM_F1, test_metrics["eval_f1"]], "Compute": ["seconds", "minutes (CPU)", "minutes (GPU)"], }) print(comparison.to_string(index=False)) print("\nThe honest question: is BERT's gain worth the extra compute?") print("On Gov/Opp (a relatively EASY task with rich word signal),") print("the gap is often small. On subtler tasks, BERT pulls ahead more.") # ----------------------------------------------------------------------------- # 8. INSPECT MISCLASSIFIED DOCUMENTS # ----------------------------------------------------------------------------- # # Error analysis: what does BERT get wrong, and why? print("\n" + "="*60) print(" ERROR ANALYSIS") print("="*60) test_texts = test_df["text"].tolist() errors = np.where(y_pred != y_true)[0] print(f"\nBERT misclassified {len(errors)}/{len(y_true)} test speeches " f"({100*len(errors)/len(y_true):.1f}%)") label_names = ["Government", "Opposition"] print("\nFirst 3 misclassified speeches:") for i in errors[:3]: print("-" * 50) print(f"TRUE: {label_names[y_true[i]]} | " f"PREDICTED: {label_names[y_pred[i]]}") print(f"TEXT: {test_texts[i][:300]}...") # Are the errors long? Ambiguous? Procedural? Look for patterns. # ----------------------------------------------------------------------------- # 9. MULTI-SEED STABILITY (OPTIONAL -- takes longer) # ----------------------------------------------------------------------------- # # BERT fine-tuning is stochastic. Report mean +/- std over several seeds. RUN_MULTISEED = False # set True if you have time / GPU if RUN_MULTISEED: print("\n" + "="*60) print(" MULTI-SEED STABILITY") print("="*60) seed_f1s = [] for s in [42, 123, 456]: print(f"\n--- Seed {s} ---") _, m = fine_tune_bert(seed=s) seed_f1s.append(m["eval_f1"]) print(f"Seed {s} test F1: {m['eval_f1']:.4f}") print(f"\nBERT F1 across seeds: {np.mean(seed_f1s):.4f} " f"+/- {np.std(seed_f1s):.4f}") print("If this std is similar to the gap vs. the baseline,") print("the 'improvement' may not be meaningful!") # ----------------------------------------------------------------------------- # 10. SAVE THE FINE-TUNED MODEL (OPTIONAL) # ----------------------------------------------------------------------------- # trainer.save_model("./my_finetuned_bert") # tokenizer.save_pretrained("./my_finetuned_bert") # # Reload later: # from transformers import pipeline # clf = pipeline("text-classification", model="./my_finetuned_bert") # clf("The government has failed the British people.") # ============================================================================= # DISCUSSION QUESTIONS # ============================================================================= # # 1. How does fine-tuned BERT compare with logistic regression (Lecture 2)? # Is the gap meaningful, given the extra compute? # # 2. Run multiple seeds (section 9). How much do results vary? Does this # change how you interpret a 1-2% difference between models? # # 3. Look at the misclassified speeches (section 8). Are they hard cases? # Long? Ambiguous? Procedural? What made them difficult? # # 4. Try a HARDER task: predict the specific party (Con/Lab/LibDem/SNP), # not just Gov/Opp. Set num_labels=4 and re-encode. Does BERT's # advantage over LR grow on the harder task? # # 5. Try the LOW-DATA regime: retrain with only 200, then 500, then 1000 # labeled examples. How do BERT and LR compare as labels grow? # (Hint: BERT often wins MORE decisively when labels are scarce -- # that's the transfer-learning advantage.) # # =============================================================================