Language Modelling — N-gram, Neural & Transformer LMs

from nltk.lm import MLE, Laplace, KneserNeyInterpolated
from nltk.lm.preprocessing import padded_everygram_pipeline
from nltk.tokenize import word_tokenize, sent_tokenize
import math

# Training corpus
corpus_text = """
The cat sat on the mat. The cat ate the rat.
The dog sat on the log. The dog ran in the fog.
A quick brown fox jumps over the lazy dog.
The quick fox ran fast. The brown dog barked loudly.
"""

# Tokenise into sentences and words
sentences = [word_tokenize(s.lower()) for s in sent_tokenize(corpus_text) if s.strip()]

# Prepare n-gram training data (2-gram bigram model)
n = 2
train_data, padded_vocab = padded_everygram_pipeline(n, sentences)

# MLE bigram model (maximum likelihood estimation)
lm_mle = MLE(n)
lm_mle.fit(train_data, padded_vocab)

# Laplace smoothed bigram model (handles unseen n-grams)
train_data2, padded_vocab2 = padded_everygram_pipeline(n, sentences)
lm_laplace = Laplace(n)
lm_laplace.fit(train_data2, padded_vocab2)

# Compute probabilities
print("MLE P('mat'|'the') =", lm_mle.score('mat', ['the']))
print("MLE P('elephant'|'the') =", lm_mle.score('elephant', ['the']))   # 0 — never seen
print("Laplace P('elephant'|'the') =", lm_laplace.score('elephant', ['the']))  # Small but non-zero

# Perplexity: how surprised is the model by the test text?
# Lower = better model
test_data, _ = padded_everygram_pipeline(n, [word_tokenize("the cat ran fast")])
test_list = list(test_data)
if test_list:
    ppl_laplace = lm_laplace.perplexity(test_list[0] if test_list else [])
    print(f"Laplace perplexity: {ppl_laplace:.2f}")

# Perplexity formula: PP(W) = P(w1...wN)^(-1/N) = exp(-1/N * sum(log P(wi|context)))
def compute_perplexity(model, test_sentence, n):
    tokens = word_tokenize(test_sentence.lower())
    log_prob = sum(math.log(max(model.score(tokens[i], tokens[max(0,i-n+1):i]), 1e-10))
                   for i in range(1, len(tokens)))
    return math.exp(-log_prob / (len(tokens) - 1)) if len(tokens) > 1 else float('inf')

ppl = compute_perplexity(lm_laplace, "the cat sat on the mat", n)
print(f"Manual perplexity (in-domain): {ppl:.2f}")      # Should be low
ppl2 = compute_perplexity(lm_laplace, "the elephant flew over", n)
print(f"Manual perplexity (out-domain): {ppl2:.2f}")    # Should be high

from transformers import (pipeline, GPT2LMHeadModel, GPT2Tokenizer,
                            BertForMaskedLM, BertTokenizer)
import torch

# ── CAUSAL LM: GPT-2 (predict next token, auto-regressive) ──
generator = pipeline('text-generation', model='gpt2', max_new_tokens=30)
result = generator("The capital of France is", num_return_sequences=1)
print("GPT-2 completion:", result[0]['generated_text'])
# "The capital of France is Paris, and the city has a population of..."

# ── MASKED LM: BERT (fill in [MASK] using bidirectional context) ──
unmasker = pipeline('fill-mask', model='bert-base-uncased')
results  = unmasker("The capital of France is [MASK].")
for r in results[:3]:
    print(f"  {r['token_str']:15} (score: {r['score']:.3f})")
# paris          (score: 0.871)
# lyon           (score: 0.041)
# marseille      (score: 0.023)

# Computing perplexity with a neural LM
tokenizer_gpt2 = GPT2Tokenizer.from_pretrained('gpt2')
model_gpt2     = GPT2LMHeadModel.from_pretrained('gpt2')
model_gpt2.eval()

def neural_lm_perplexity(text: str, model, tokenizer) -> float:
    """Compute perplexity of text under a causal language model."""
    inputs = tokenizer(text, return_tensors='pt')
    with torch.no_grad():
        outputs = model(**inputs, labels=inputs['input_ids'])
    return torch.exp(outputs.loss).item()  # e^(avg negative log-likelihood)

easy_text = "The dog barked at the cat"
hard_text = "The cat barked at the dog"  # Plausible but less common
random_text = "Colourless green ideas sleep furiously"  # Grammatical but nonsensical

print(f"
Perplexity comparison (GPT-2):")
for text in [easy_text, hard_text, random_text]:
    ppl = neural_lm_perplexity(text, model_gpt2, tokenizer_gpt2)
    print(f"  PPL={ppl:.1f}: {text}")