Training run: dev config 1 epoch results

Results:
- Topic accuracy: 85.76% (Sports 96%, World 87%, Business 81%, Sci/Tech 78%)
- Summarization ROUGE-like: 0.343, BLEU: 0.088
- Emotion F1 macro: 0.356

Updated inference pipeline, demo script, and evaluation outputs

.gitignore

@@ -11,6 +11,7 @@ build/
 
 # Virtual environments
 venv/
+.venv/
 env/
 ENV/
 

outputs/evaluation_report.json

@@ -1,43 +1,43 @@
 {
   "split": "test",
   "summarization": {
-    "rouge_like": 0.031742493938280825,
-    "bleu": 0.0008530696741094626
+    "rouge_like": 0.3430426484440944,
+    "bleu": 0.0879515124653127
   },
   "emotion": {
-    "f1_macro": 0.42543327808380127
+    "f1_macro": 0.3558666706085205
   },
   "topic": {
-    "accuracy": 0.3325,
+    "accuracy": 0.8576315789473684,
     "classification_report": {
       "Business": {
-        "precision": 0.24772065955383124,
-        "recall": 0.6721052631578948,
-        "f1-score": 0.3620127569099929,
+        "precision": 0.7614165890027959,
+        "recall": 0.86,
+        "f1-score": 0.8077113198220465,
         "support": 1900
       },
       "Sci/Tech": {
-        "precision": 0.4942170818505338,
-        "recall": 0.5847368421052631,
-        "f1-score": 0.5356798457087754,
+        "precision": 0.8759791122715405,
+        "recall": 0.7063157894736842,
+        "f1-score": 0.782051282051282,
         "support": 1900
       },
       "Sports": {
-        "precision": 0.9473684210526315,
-        "recall": 0.018947368421052633,
-        "f1-score": 0.03715170278637771,
+        "precision": 0.9454638124362895,
+        "recall": 0.9763157894736842,
+        "f1-score": 0.9606421543241843,
         "support": 1900
       },
       "World": {
-        "precision": 0.6477987421383647,
-        "recall": 0.05421052631578947,
-        "f1-score": 0.10004856726566294,
+        "precision": 0.8607142857142858,
+        "recall": 0.8878947368421053,
+        "f1-score": 0.8740932642487047,
         "support": 1900
       },
       "macro avg": {
-        "precision": 0.5842762261488403,
-        "recall": 0.3325,
-        "f1-score": 0.2587232181677022,
+        "precision": 0.860893449856228,
+        "recall": 0.8576315789473684,
+        "f1-score": 0.8561245051115545,
         "support": 7600
       }
     }

outputs/training_history.json

@@ -1,21 +1,21 @@
 {
   "train_epoch_1": {
-    "summarization_loss": 3.6738915424346925,
-    "summarization_rouge_like": 0.3936604625654161,
-    "emotion_loss": 0.5655887125730514,
-    "emotion_f1": 0.02088333384692669,
-    "topic_loss": 1.2472841796875,
-    "topic_accuracy": 0.5795,
-    "total_loss": 5.486764434695244,
+    "summarization_loss": 3.67411927986145,
+    "summarization_rouge_like": 0.39456057390021504,
+    "emotion_loss": 0.5643834336996079,
+    "emotion_f1": 0.023809524163603782,
+    "topic_loss": 1.2467568359375,
+    "topic_accuracy": 0.587,
+    "total_loss": 5.485259549498558,
     "epoch": 1.0
   },
   "val_epoch_1": {
-    "summarization_loss": 3.24564736366272,
-    "summarization_rouge_like": 0.4398922732261946,
-    "emotion_loss": 0.4284175229072571,
+    "summarization_loss": 3.2498003482818603,
+    "summarization_rouge_like": 0.44230111155579444,
+    "emotion_loss": 0.4288424849510193,
     "emotion_f1": 0.0,
-    "topic_loss": 0.814755859375,
-    "topic_accuracy": 0.835,
+    "topic_loss": 0.807373046875,
+    "topic_accuracy": 0.85,
     "epoch": 1.0
   }
 }

scripts/demo_gradio.py

@@ -262,7 +262,7 @@ def create_attention_heatmap(text: str, summary: str, pipeline: InferencePipelin
         )
 
         with torch.inference_mode():
-            memory = pipeline.model.encoder(src_ids, mask=encoder_mask)
+            memory = pipeline.model.encoder(src_ids, mask=encoder_mask)  # type: ignore
             target_enc = pipeline.tokenizer.batch_encode([summary])
             target_ids = target_enc["input_ids"].to(pipeline.device)
             target_mask = target_enc["attention_mask"].to(pipeline.device)
@@ -271,7 +271,7 @@ def create_attention_heatmap(text: str, summary: str, pipeline: InferencePipelin
             decoder_inputs = decoder_inputs[:, :target_len].to(pipeline.device)
             target_ids = target_ids[:, :target_len]
             memory_mask = src_mask.to(pipeline.device) if src_mask is not None else None
-            _, attn_list = pipeline.model.decoder(
+            _, attn_list = pipeline.model.decoder(  # type: ignore
                 decoder_inputs,
                 memory,
                 memory_mask=memory_mask,

src/inference/pipeline.py

@@ -3,7 +3,7 @@
 from __future__ import annotations
 
 from dataclasses import dataclass, fields, replace
-from typing import Iterable, List, Sequence
+from typing import Any, Iterable, List, Sequence, cast
 
 import torch
 import torch.nn.functional as F
@@ -75,11 +75,14 @@ class InferencePipeline:
                 "Model must expose encoder and decoder attributes for summarization."
             )
 
+        # Cast to Any to allow access to dynamic attributes encoder and decoder
+        model = cast(Any, self.model)
+
         with torch.inference_mode():
             encoder_mask = (
                 src_mask.unsqueeze(1) & src_mask.unsqueeze(2) if src_mask is not None else None
             )
-            memory = self.model.encoder(src_ids, mask=encoder_mask)
+            memory = model.encoder(src_ids, mask=encoder_mask)
             min_len = 10
 
             # Ban BOS, PAD, UNK from being generated
@@ -92,7 +95,7 @@ class InferencePipeline:
                 ban_token_ids.append(unk_id)
             ban_token_ids = [tid for tid in ban_token_ids if tid is not None]
 
-            generated = self.model.decoder.greedy_decode(
+            generated = model.decoder.greedy_decode(
                 memory=memory,
                 max_len=max_len,
                 start_token_id=self.tokenizer.bos_token_id,