NLP Evaluation Metrics — BLEU, ROUGE & Perplexity

from nltk.translate.bleu_score import corpus_bleu, sentence_bleu, SmoothingFunction
from rouge_score import rouge_scorer
import numpy as np

# ── BLEU for machine translation ──
# References: one or more human translations (list of lists)
# Hypotheses: machine translations (list of token lists)
references = [
    [['the', 'cat', 'is', 'on', 'the', 'mat']],      # 1 reference for sent 1
    [['the', 'dog', 'barked', 'at', 'the', 'mailman']],  # 1 reference for sent 2
]
hypotheses = [
    ['the', 'cat', 'is', 'on', 'mat'],           # Missing one 'the'
    ['the', 'dog', 'barked', 'at', 'a', 'man'],  # 'the mailman' → 'a man'
]

# Corpus-level BLEU (standard for MT evaluation)
bleu_score = corpus_bleu(references, hypotheses)
print(f"Corpus BLEU: {bleu_score:.4f}")

# Sentence-level BLEU with smoothing
smooth = SmoothingFunction().method1
for ref, hyp in zip(references, hypotheses):
    score = sentence_bleu(ref, hyp, smoothing_function=smooth)
    print(f"Sentence BLEU: {score:.4f} | Hyp: {' '.join(hyp)}")

# Individual n-gram precision breakdown
from nltk.translate.bleu_score import modified_precision
from fractions import Fraction
ref = [['the', 'cat', 'is', 'on', 'the', 'mat']]
hyp =  ['the', 'cat', 'is', 'on', 'mat']
for n in range(1, 5):
    prec = modified_precision(ref, hyp, n=n)
    print(f"  P_{n}: {float(prec):.3f}")
# P_1: 0.800 (4 of 5 unigrams match)
# P_2: 0.750 (3 of 4 bigrams match)
# P_3: 0.667 ...
# P_4: 0.500 ...

# ── ROUGE for summarisation ──
scorer = rouge_scorer.RougeScorer(['rouge1', 'rouge2', 'rougeL'], use_stemmer=True)

reference_summary = """The transformer architecture revolutionised NLP by replacing
recurrent networks with self-attention mechanisms for parallel processing."""

hypothesis1 = """Transformers changed NLP by using self-attention instead of
recurrent networks, enabling faster parallel training."""

hypothesis2 = """Scientists invented a new cooking technique in 2019."""

for hyp_name, hyp in [("Good", hypothesis1), ("Bad", hypothesis2)]:
    scores = scorer.score(reference_summary, hyp)
    print(f"
{hyp_name} summary ROUGE:")
    for metric, score in scores.items():
        print(f"  {metric}: P={score.precision:.3f}, R={score.recall:.3f}, F1={score.fmeasure:.3f}")
# ROUGE-1: measures unigram overlap
# ROUGE-2: measures bigram overlap
# ROUGE-L: measures longest common subsequence

# ── BERTScore: semantic similarity beyond exact match ──
try:
    from bert_score import score as bert_score
    # BERTScore uses contextual BERT embeddings to compare semantic similarity
    # Better than BLEU/ROUGE for paraphrases (same meaning, different words)
    cands = ["The cat sat on the mat"]
    refs  = ["A feline rested on the rug"]   # Paraphrase — different words, same meaning
    P, R, F1 = bert_score(cands, refs, lang='en', verbose=False)
    print(f"
BERTScore F1 for paraphrase: {F1.mean():.3f}")  # ~0.88 (high!)
    # BLEU would give 0 (no word overlap). BERTScore captures semantic equivalence.
except ImportError:
    print("pip install bert-score")

from transformers import GPT2LMHeadModel, GPT2Tokenizer
import torch, math

tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
model     = GPT2LMHeadModel.from_pretrained('gpt2')
model.eval()

def perplexity(text: str, stride: int = 512) -> float:
    """Compute perplexity with stride trick for long texts."""
    encodings = tokenizer(text, return_tensors='pt')
    max_length = model.config.n_positions   # 1024 for GPT-2
    seq_len    = encodings.input_ids.size(1)
    nlls, n_tokens = [], 0

    for begin in range(0, seq_len, stride):
        end       = min(begin + max_length, seq_len)
        target_len = end - begin
        input_ids  = encodings.input_ids[:, begin:end]
        target_ids = input_ids.clone()
        target_ids[:, :-target_len] = -100   # Ignore context tokens in loss

        with torch.no_grad():
            nll = model(input_ids, labels=target_ids).loss
        nlls.append(nll * target_len)
        n_tokens += target_len

    return math.exp(torch.stack(nlls).sum() / n_tokens)

texts = {
    'Simple English': "The cat sat on the mat. The dog ran in the park.",
    'Scientific': "Transformer architectures utilise multi-head self-attention mechanisms.",
    'Random': "Purple elephant dances quantum philosophy seventeen banana.",
    'Code': "def fibonacci(n): return n if n < 2 else fibonacci(n-1) + fibonacci(n-2)",
}
for label, text in texts.items():
    ppl = perplexity(text)
    print(f"PPL={ppl:6.1f}: {label}")
# Simple English: low PPL (well-represented in training)
# Scientific: medium PPL (technical vocabulary)
# Random: very high PPL (semantically incoherent)
# Code: medium-high PPL (GPT-2 saw little code)