1990-Elman循環神經網路#

import numpy as np
rng = np.random.default_rng(0)
words = ["boy", "girl", "chase", "see", "dog", "cat"]
V = len(words); w2i = {w: i for i, w in enumerate(words)}

def sample_sentence():
    # 簡單文法: [N-life N] [V] [N-life]
    s = [["boy", "girl"][rng.random() < .5], ["dog", "cat"][rng.random() < .5]]
    s += [["chase", "see"][rng.random() < .5], ["boy", "girl"][rng.random() < .5]]
    return s

def onehot(i):
    v = np.zeros(V); v[i] = 1.0; return v

d = 12  # Elman 網路參數
Wx = rng.normal(0, .3, (d, V)); Wh = rng.normal(0, .3, (d, d)); b = np.zeros(d)
Wy = rng.normal(0, .3, (V, d)); by = np.zeros(V)
sigmoid = lambda z: 1 / (1 + np.exp(-z))
softmax = lambda z: np.exp(z - z.max()) / np.exp(z - z.max()).sum()
lr = 0.3
for epoch in range(300):
    sent = sample_sentence()
    xs = [onehot(w2i[w]) for w in sent]
    T = len(xs)
    hs = [np.zeros(d)]
    for t in range(T):                     # 前向
        h = sigmoid(Wx @ xs[t] + Wh @ hs[-1] + b)
        hs.append(h)
    # 簡化 BPTT：只對最後數步截斷反傳（truncated BPTT）
    K = min(3, T)
    dWh = np.zeros_like(Wh); dWx = np.zeros_like(Wx)
    dWy = np.zeros_like(Wy); dby = np.zeros_like(by); db = np.zeros_like(b)
    dh_next = np.zeros(d)
    for t in range(T - 1, T - 1 - K, -1):
        y = softmax(Wy @ hs[t + 1] + by)
        dy = y.copy(); dy[w2i[sent[t]]] -= 1
        dh = Wy.T @ dy + dh_next
        dWy += np.outer(dy, hs[t + 1]); dby += dy
        dh_raw = dh * hs[t + 1] * (1 - hs[t + 1])
        dWh += np.outer(dh_raw, hs[t]); dWx += np.outer(dh_raw, xs[t])
        dh_next = Wh.T @ dh_raw
    for P, G in [(Wx, dWx), (Wh, dWh), (b, db), (Wy, dWy), (by, dby)]:
        P -= lr * G

# Elman 式觀察：收集每個詞在各上下文中出現時的隱層狀態，比較類內/跨類相似度
H = {w: [] for w in words}
for _ in range(3000):
    sent = sample_sentence(); h = np.zeros(d)
    for w in sent:
        h = sigmoid(Wx @ onehot(w2i[w]) + Wh @ h + b)
        H[w].append(h.copy())
cos = lambda a, c: float(a @ c / (np.linalg.norm(a) * np.linalg.norm(c)))
mn = lambda w: np.mean(H[w], axis=0)
nouns, verbs = ["boy", "girl", "dog", "cat"], ["chase", "see"]
intra = np.mean([cos(mn(a), mn(c)) for a in nouns for c in nouns if a < c])
cross = np.mean([cos(mn(a), mn(v)) for a in nouns for v in verbs])
print("類內(名詞-名詞)相似度:", round(intra, 3), "| 跨類(名詞-動詞):", round(cross, 3))
print("差異:", round(intra - cross, 3), "(>0 代表語法類別在隱層中自動分離)")