Sequence Models & RNNs

import numpy as np

# One-hot: every word pair is orthogonal
V = 50000                       # vocabulary size
cat     = np.zeros(V); cat[8432]     = 1
kitten  = np.zeros(V); kitten[8433]  = 1
carb    = np.zeros(V); carb[31007]   = 1

print(cat @ kitten)   # 0.0 -- 'cat' and 'kitten' unrelated?!
print(cat @ carb)     # 0.0 -- same similarity as 'carburettor'

# Dense embeddings (here: pre-trained GloVe-style vectors, illustrative)
emb = {
  'cat':    np.array([0.61, 0.21, -0.43, 0.18]),
  'kitten': np.array([0.58, 0.25, -0.39, 0.22]),
  'carb':   np.array([-0.31, 0.74, 0.12, -0.55]),
}

def cos(a, b): return a @ b / (np.linalg.norm(a) * np.linalg.norm(b))

print(f"cos(cat, kitten) = {cos(emb['cat'], emb['kitten']):.3f}")
print(f"cos(cat, carb)   = {cos(emb['cat'], emb['carb']):.3f}")
# cos(cat, kitten) = 0.995   <- semantically similar = geometrically close
# cos(cat, carb)   = -0.468  <- unrelated = far apart

import numpy as np
from collections import defaultdict, Counter

corpus = """the cat sat on the mat . the dog sat on the rug .
            the cat chased the dog . the kitten purred softly .""".split()

def build_cooccurrence(tokens, window=2):
    vocab  = sorted(set(tokens))
    w2i    = {w: i for i, w in enumerate(vocab)}
    V      = len(vocab)
    X      = np.zeros((V, V))
    for i, word in enumerate(tokens):
        for j in range(max(0, i-window), min(len(tokens), i+window+1)):
            if i != j:
                X[w2i[word], w2i[tokens[j]]] += 1
    return X, vocab, w2i

X, vocab, w2i = build_cooccurrence(corpus, window=2)

# 'cat' and 'dog' co-occur with similar words -> similar rows
def cos(a, b): return a@b / (np.linalg.norm(a)*np.linalg.norm(b) + 1e-9)
print(f"cos(cat, dog)    = {cos(X[w2i['cat']], X[w2i['dog']]):.3f}")
print(f"cos(cat, softly) = {cos(X[w2i['cat']], X[w2i['softly']]):.3f}")
# cos(cat, dog)    = 0.667  -- both are 'sat on / chased' animals
# cos(cat, softly) = 0.000  -- no shared context in this tiny corpus

import numpy as np

class SkipGram:
    def __init__(self, V, dim=50, lr=0.05, seed=0):
        rng    = np.random.default_rng(seed)
        self.V_in  = rng.normal(0, 0.1, (V, dim))  # centre embeddings v_w
        self.V_out = rng.normal(0, 0.1, (V, dim))  # context embeddings u_c
        self.lr = lr

    def step(self, centre, context):  # one (w, c) training pair
        v = self.V_in[centre]              # (dim,)
        scores = self.V_out @ v            # (V,) logits over vocab
        scores -= scores.max()             # stable softmax
        p = np.exp(scores); p /= p.sum()   # (V,) probabilities

        # Gradient of cross-entropy: p - onehot(context)
        grad = p.copy(); grad[context] -= 1.0

        # Backprop into both embedding matrices
        dV_in  = self.V_out.T @ grad       # (dim,)
        dV_out = np.outer(grad, v)         # (V, dim)
        self.V_in[centre] -= self.lr * dV_in
        self.V_out        -= self.lr * dV_out
        return -np.log(p[context] + 1e-12)  # loss for monitoring

# Generate training pairs from corpus
def make_pairs(tokens, w2i, window=2):
    pairs = []
    for i, w in enumerate(tokens):
        for j in range(max(0, i-window), min(len(tokens), i+window+1)):
            if i != j: pairs.append((w2i[w], w2i[tokens[j]]))
    return pairs

pairs = make_pairs(corpus, w2i)
sg    = SkipGram(V=len(vocab), dim=20)
for epoch in range(200):
    loss = np.mean([sg.step(w, c) for w, c in pairs])
    if epoch % 50 == 0: print(f"epoch {epoch}: loss {loss:.3f}")
# epoch 0:   loss 2.973
# epoch 150: loss 1.213   -- 'cat' and 'dog' now have cos ≈ 0.7

for each (centre w, context c) pair in corpus:

  # Positive update: this pair is real
  g  ←  σ(u_c · v_w) − 1        # gradient of −log σ(score)
  u_c ← u_c − lr · g · v_w
  acc ← g · u_c                # accumulate gradient for v_w

  # Negative updates: k random fake pairs
  for i = 1…k:
    n  ←  sample from Pₙ(w) ∝ count(w)^0.75
    g  ←  σ(u_n · v_w)           # want score → −∞, so σ → 0
    u_n ← u_n − lr · g · v_w
    acc ← acc + g · u_n

  v_w ← v_w − lr · acc          # single update for the centre word

import numpy as np
from collections import Counter

class SGNS:
    def __init__(self, tokens, dim=100, k=5, lr=0.025, seed=0):
        self.vocab = sorted(set(tokens))
        self.w2i   = {w:i for i,w in enumerate(self.vocab)}
        V          = len(self.vocab)
        rng        = np.random.default_rng(seed)
        self.W_in  = rng.uniform(-0.5/dim, 0.5/dim, (V, dim))
        self.W_out = np.zeros((V, dim))  # standard word2vec init
        self.k, self.lr = k, lr

        # Unigram^0.75 noise distribution
        counts = Counter(tokens)
        freqs  = np.array([counts[w] for w in self.vocab], dtype=float)
        self.noise = freqs**0.75; self.noise /= self.noise.sum()
        self.rng = rng

    def _sigmoid(self, x): return 1 / (1 + np.exp(-np.clip(x, -20, 20)))

    def step(self, centre, context):
        v   = self.W_in[centre]
        acc = np.zeros_like(v)

        # --- Positive pair ---
        u = self.W_out[context]
        g = self._sigmoid(u @ v) - 1.0       # ∂/∂score of -log σ(score)
        acc += g * u
        self.W_out[context] -= self.lr * g * v

        # --- k negative samples ---
        negs = self.rng.choice(len(self.vocab), self.k, p=self.noise)
        for n in negs:
            if n == context: continue
            u = self.W_out[n]
            g = self._sigmoid(u @ v)           # want σ → 0 for fakes
            acc += g * u
            self.W_out[n] -= self.lr * g * v

        self.W_in[centre] -= self.lr * acc

    def most_similar(self, word, topn=5):
        v = self.W_in[self.w2i[word]]
        sims = self.W_in @ v / (np.linalg.norm(self.W_in, axis=1) * np.linalg.norm(v) + 1e-9)
        best = np.argsort(-sims)[1:topn+1]  # skip self
        return [(self.vocab[i], sims[i]) for i in best]

# Cost per pair: O(k·dim) instead of O(V·dim).
# For V=1M, k=5: a 200,000x speedup. This is what made Word2Vec feasible.

import numpy as np

class GloVe:
    def __init__(self, X, dim=50, x_max=100, alpha=0.75, lr=0.05, seed=0):
        V = X.shape[0]
        rng = np.random.default_rng(seed)
        # Two embedding matrices + two bias vectors
        self.W  = rng.normal(0, 0.1, (V, dim))   # word vectors
        self.Wc = rng.normal(0, 0.1, (V, dim))   # context vectors
        self.b  = np.zeros(V); self.bc = np.zeros(V)
        self.X, self.x_max, self.alpha, self.lr = X, x_max, alpha, lr
        # Pre-compute the non-zero co-occurrence entries
        self.nz = np.argwhere(X > 0)

    def _weight(self, x):
        return np.minimum((x / self.x_max)**self.alpha, 1.0)

    def train_epoch(self):
        total = 0.0
        np.random.shuffle(self.nz)
        for i, j in self.nz:
            xij  = self.X[i, j]
            # Prediction error: w_i·w̃_j + b_i + b̃_j − log X_ij
            diff = self.W[i] @ self.Wc[j] + self.b[i] + self.bc[j] - np.log(xij)
            fw   = self._weight(xij)
            total += fw * diff**2

            # Gradients (weighted squared loss)
            g = 2 * fw * diff
            gW, gWc = g * self.Wc[j], g * self.W[i]
            self.W[i]  -= self.lr * gW
            self.Wc[j] -= self.lr * gWc
            self.b[i]  -= self.lr * g
            self.bc[j] -= self.lr * g
        return total / len(self.nz)

    @property
    def embeddings(self):
        return self.W + self.Wc   # GloVe paper: sum the two matrices

glove = GloVe(X, dim=20)
for epoch in range(100):
    loss = glove.train_epoch()
    if epoch % 25 == 0: print(f"epoch {epoch}: weighted loss {loss:.4f}")
# epoch 0:  weighted loss 1.8042
# epoch 75: weighted loss 0.0031   -- dot products now ≈ log co-occurrences

def char_ngrams(word, n_min=3, n_max=6):
    """fastText-style subword decomposition with boundary markers."""
    w = '<' + word + '>'
    grams = []
    for n in range(n_min, min(n_max, len(w)) + 1):
        for i in range(len(w) - n + 1):
            grams.append(w[i:i+n])
    grams.append(w)        # the full word is also a 'gram'
    return grams

print(char_ngrams('where', 3, 4))
# ['<wh', 'whe', 'her', 'ere', 're>',          <- 3-grams
#  '<whe', 'wher', 'here', 'ere>',             <- 4-grams
#  '<where>']                                   <- full word

# Word vector = sum of n-gram vectors:
#   v(where) = z(<wh) + z(whe) + z(her) + ... + z(<where>)
# 'her' inside 'where' shares the SAME z(her) used in 'her', 'here', 'there'

# pip install fasttext  (or gensim's FastText)
from gensim.models import FastText

# Train on a small corpus (in practice: Wikipedia, Common Crawl)
sentences = [s.split() for s in [
    'the cat sat on the mat',
    'the kitten purred softly',
    'dogs and cats are running in the park',
]]

ft = FastText(sentences, vector_size=50, window=3, min_count=1,
              min_n=3, max_n=6, epochs=100)

# OOV word: 'catlike' was NEVER in training data
v_oov = ft.wv['catlike']  # works! built from <ca, cat, atl, ... n-grams
print(ft.wv.similarity('catlike', 'cat'))
# 0.71  -- shares 'cat' n-grams, lands near 'cat' in embedding space

# Typo robustness
print(ft.wv.similarity('runing', 'running'))
# 0.89  -- overlapping n-grams make typos land near correct spellings

import numpy as np
import gensim.downloader as api

# Load 100-d GloVe trained on Wikipedia + Gigaword (~130k vocab)
glv = api.load('glove-wiki-gigaword-100')

# --- The classic ---
print(glv.most_similar(positive=['king', 'woman'], negative=['man'], topn=3))
# [('queen', 0.770), ('monarch', 0.684), ('throne', 0.683)]

# --- Syntactic analogies work too ---
print(glv.most_similar(positive=['walking', 'swam'], negative=['walked'], topn=1))
# [('swimming', 0.789)]   walked:walking :: swam:swimming

# --- Geographic ---
print(glv.most_similar(positive=['paris', 'italy'], negative=['france'], topn=1))
# [('rome', 0.842)]   france:paris :: italy:rome

# --- Nearest neighbours reveal local structure ---
print(glv.most_similar('transformer', topn=5))
# [('transformers',...), ('amplifier',...), ('voltage',...)]
# 2014 vectors: 'transformer' is electrical, not neural. Embeddings freeze
# the corpus's snapshot of meaning -- a key limitation.

import numpy as np

E = glv.vectors                       # (V, 100) all embeddings

# Average pairwise cosine of random word pairs (should be ~0 if isotropic)
rng   = np.random.default_rng(0)
idx   = rng.choice(len(E), (2000, 2))
def cos_rows(A, B):
    return np.sum(A*B, axis=1) / (np.linalg.norm(A,axis=1)*np.linalg.norm(B,axis=1) + 1e-9)
avg_cos = cos_rows(E[idx[:,0]], E[idx[:,1]]).mean()
print(f"Mean random-pair cosine (raw):       {avg_cos:.3f}")
# 0.318  -- far from 0: vectors live in a narrow cone

# All-but-the-top: centre, then remove top-k principal components
E_c   = E - E.mean(axis=0)
U, S, Vt = np.linalg.svd(E_c, full_matrices=False)
k     = 3                                  # remove top 3 PCs
E_iso = E_c - (E_c @ Vt[:k].T) @ Vt[:k]
avg_cos_iso = cos_rows(E_iso[idx[:,0]], E_iso[idx[:,1]]).mean()
print(f"Mean random-pair cosine (corrected): {avg_cos_iso:.3f}")
# 0.011  -- near-isotropic; similarity scores are now meaningful

import numpy as np
from scipy.stats import spearmanr

# SimLex-999 style: (word1, word2, human_score 0-10)
pairs = [
    ('old', 'new', 1.58),    # antonyms: related but NOT similar
    ('smart', 'intelligent', 9.20),
    ('hard', 'difficult', 8.77),
    ('happy', 'cheerful', 9.55),
    ('coast', 'shore', 9.00),
    ('movie', 'popcorn', 2.50),  # associated, not similar
]

model_sims = [glv.similarity(a, b) for a, b, _ in pairs]
human_sims = [h for _, _, h in pairs]

rho, p = spearmanr(model_sims, human_sims)
print(f"Spearman ρ = {rho:.3f}  (p={p:.3f})")
# Spearman ρ = 0.829
# Note: GloVe scores 'old'/'new' as quite similar (they share contexts!)
# but humans rate them dissimilar. Distributional similarity conflates
# similarity with relatedness -- a fundamental limitation.

# Static embeddings: one vector regardless of context
print(glv.most_similar('bank', topn=6))
# [('banks',.79), ('banking',.74), ('credit',.69),
#  ('financial',.68), ('lending',.67), ('lender',.66)]
# The financial sense dominates -- river banks are invisible.

# Contextual embeddings (BERT): different vector per occurrence
import torch
from transformers import AutoTokenizer, AutoModel

tok   = AutoTokenizer.from_pretrained('bert-base-uncased')
model = AutoModel.from_pretrained('bert-base-uncased').eval()

def word_vec_in_context(sentence, word):
    """Return BERT's contextual embedding of `word` inside `sentence`."""
    enc = tok(sentence, return_tensors='pt')
    with torch.no_grad():
        h = model(**enc).last_hidden_state[0]  # (T, 768)
    w_id = tok.convert_tokens_to_ids(word)
    pos  = (enc.input_ids[0] == w_id).nonzero()[0, 0]
    return h[pos]

v_fin   = word_vec_in_context('I deposited cash at the bank', 'bank')
v_river = word_vec_in_context('We fished from the river bank', 'bank')
v_fin2  = word_vec_in_context('The bank approved my loan', 'bank')

cos = lambda a, b: torch.cosine_similarity(a, b, dim=0).item()
print(f"finance vs finance: {cos(v_fin, v_fin2):.3f}")
print(f"finance vs river:   {cos(v_fin, v_river):.3f}")
# finance vs finance: 0.85   <- same sense, similar vectors
# finance vs river:   0.46   <- different senses, BERT separates them