2013 - DQN 打 Atari（像素直接引爆深度強化學習）#

import numpy as np, random
from collections import deque

class DQN:
    def __init__(self, n_in, n_out, n_hid=64):
        self.w1 = np.random.randn(n_in, n_hid) * np.sqrt(2/n_in)
        self.w2 = np.random.randn(n_hid, n_out) * np.sqrt(2/n_hid)
    def __call__(self, s):
        return np.tanh(s @ self.w1) @ self.w2
    def grad_update(self, s, a, y, eta=1e-3):     # 對動作 a 的 MSE 梯度
        h = np.tanh(s @ self.w1); q = h @ self.w2
        d = np.zeros_like(q); d[a] = q[a] - y
        self.w2 -= eta * np.outer(h, d)
        self.w1 -= eta * np.outer(s, (1-h**2) * (self.w2 @ d))

net, target = DQN(4, 2), DQN(4, 2)
replay = deque(maxlen=10000)                      # 經驗回放緩衝池

def env_step(s, a):                               # CartPole 近似
    x, x_dot, th, th_dot = s
    x_dot += (a-0.5)*0.2 - 0.01*x; th_dot += 0.2*th
    s2 = np.clip([x+x_dot, x_dot, th+th_dot, th_dot], -2, 2)
    return s2, 1.0, abs(s2).max() > 2             # 獎勵、是否終止

s = np.zeros(4)
for step in range(1, 2001):
    a = random.randrange(2) if random.random() < 0.1 else net(s).argmax()
    s2, r, done = env_step(s, a)
    replay.append((s, a, r, s2, done))            # 存入緩衝池
    if len(replay) >= 64:
        batch = random.sample(replay, 64)         # 隨機抽樣打散相關性
        for si, ai, ri, s2i, di in batch:
            y = ri + (0 if di else 0.99 * target(s2i).max())  # 目標網路
            net.grad_update(si, ai, y)
    if step % 100 == 0:
        target.w1, target.w2 = net.w1.copy(), net.w2.copy()   # 同步目標網路
    s = np.zeros(4) if done else s2

def eval_greedy(n=100):                          # 貪婪策略存活步數評估
    steps = []
    for _ in range(n):
        s, t = np.zeros(4), 0
        while True:
            a = net(s).argmax()
            s2, r, done = env_step(s, a); t += 1; s = s2
            if done or t > 1000: break
        steps.append(t)
    return int(np.mean(steps))

print("訓練前存活步數約 20；訓練後 =", eval_greedy())

訓練前存活步數約 20；訓練後 = 1001（達上限，桿子不再倒下）