第25章：强化学习基础 | Chapter 25: Reinforcement Learning Basics

G_t = R_{t+1} + γR_{t+2} + γ²R_{t+3} + ...

V^π(s) = E[G_t | S_t = s]

Q^π(s,a) = E[G_t | S_t = s, A_t = a]

V(s) = Σ_a π(a|s) Σ_{s',r} P(s',r|s,a) [r + γV(s')]
Q(s,a) = Σ_{s',r} P(s',r|s,a) [r + γ max_{a'} Q(s',a')]

Q(s,a) ← Q(s,a) + α * [r + γ * max Q(s',a') - Q(s,a)]

存储: (s, a, r, s', done)
采样: 随机从 buffer 中采样 batch
训练: 像监督学习一样用 MSE 损失

import numpy as np
import torch
import torch.nn as nn

# ========== Q-Learning ==========
class SimpleQ:
    def __init__(self, n_states, n_actions, lr=0.1, gamma=0.9):
        self.Q = np.zeros((n_states, n_actions))
        self.lr = lr
        self.gamma = gamma

    def update(self, s, a, r, s_next, done):
        target = r if done else r + self.gamma * np.max(self.Q[s_next])
        self.Q[s, a] += self.lr * (target - self.Q[s, a])

    def act(self, s, epsilon=0.1):
        if np.random.rand() < epsilon:
            return np.random.randint(self.Q.shape[1])
        return np.argmax(self.Q[s])

# ========== DQN 网络 ==========
class DQN(nn.Module):
    def __init__(self, state_dim, action_dim):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(state_dim, 128),
            nn.ReLU(),
            nn.Linear(128, 128),
            nn.ReLU(),
            nn.Linear(128, action_dim)
        )

    def forward(self, x):
        return self.net(x)

# ========== 经验回放 ==========
class ReplayBuffer:
    def __init__(self, capacity=10000):
        self.buffer = []
        self.capacity = capacity

    def push(self, s, a, r, s_next, done):
        if len(self.buffer) >= self.capacity:
            self.buffer.pop(0)
        self.buffer.append((s, a, r, s_next, done))

    def sample(self, batch_size=32):
        idx = np.random.choice(len(self.buffer), batch_size, replace=False)
        return [self.buffer[i] for i in idx]

    def __len__(self):
        return len(self.buffer)

print("=" * 50)
print("强化学习组件实现")
print("=" * 50)

# 测试 Q-Learning
env_n_states, env_n_actions = 16, 4
q_agent = SimpleQ(env_n_states, env_n_actions)

# 模拟训练
for episode in range(100):
    s = np.random.randint(env_n_states)
    for step in range(20):
        a = q_agent.act(s)
        s_next = np.random.randint(env_n_states)
        r = 1 if s_next == env_n_states - 1 else 0
        done = (s_next == env_n_states - 1)
        q_agent.update(s, a, r, s_next, done)
        s = s_next
        if done:
            break

print(f"Q-Learning 训练完成")
print(f"Q 值范围: [{q_agent.Q.min():.2f}, {q_agent.Q.max():.2f}]")

# 测试 DQN
dqn = DQN(state_dim=4, action_dim=2)
buffer = ReplayBuffer(capacity=100)

# 模拟填充经验回放
for _ in range(50):
    s = np.random.randn(4)
    a = np.random.randint(2)
    r = np.random.randn()
    s_next = np.random.randn(4)
    buffer.push(s, a, r, s_next, False)

batch = buffer.sample(batch_size=8)
print(f"\n经验回放: 存储 {len(buffer)} 条经验")
print(f"采样 batch: {len(batch)} 条")
print(f"DQN 网络输出示例: {dqn(torch.randn(1, 4)).detach().numpy().round(3)}")

print("\n核心总结:")
print("- Q-Learning: 表格方法，适合离散状态空间")
print("- DQN: 神经网络逼近 Q 函数，适合连续/大状态空间")
print("- 经验回放: 打破样本相关性，稳定训练")
print("- 目标网络: 定期同步，防止目标值移动")

==================================================
强化学习组件实现
==================================================
Q-Learning 训练完成
Q 值范围: [0.00, 7.29]

经验回放: 存储 50 条经验
采样 batch: 8 条
DQN 网络输出示例: [[0.123 -0.045]]

核心总结:
- Q-Learning: 表格方法，适合离散状态空间
- DQN: 神经网络逼近 Q 函数，适合连续/大状态空间
- 经验回放: 打破样本相关性，稳定训练
- 目标网络: 定期同步，防止目标值移动