# Transformer的重要性
transformer_importance = {
    "架构基础": "几乎所有现代大模型都基于Transformer",
    "并行计算": "支持并行训练，大幅提升训练效率",
    "长距离依赖": "有效处理长序列依赖关系",
    "可扩展性": "易于扩展到更大规模"
}

# 自注意力机制原理
def self_attention(Q, K, V):
    """
    Q: Query矩阵 (查询)
    K: Key矩阵 (键)
    V: Value矩阵 (值)
    """
    # 计算注意力分数
    scores = Q @ K.T / sqrt(d_k)
    
    # 应用softmax
    attention_weights = softmax(scores)
    
    # 加权求和
    output = attention_weights @ V
    
    return output

# 多头注意力实现
def multi_head_attention(X, num_heads=8):
    """
    X: 输入矩阵
    num_heads: 注意力头数
    """
    d_model = X.shape[-1]
    d_k = d_model // num_heads
    
    # 为每个头创建Q、K、V
    Q = linear(X, d_model)  # 投影到Q
    K = linear(X, d_model)  # 投影到K
    V = linear(X, d_model)  # 投影到V
    
    # 分割为多个头
    Q = reshape(Q, (batch, seq_len, num_heads, d_k))
    K = reshape(K, (batch, seq_len, num_heads, d_k))
    V = reshape(V, (batch, seq_len, num_heads, d_k))
    
    # 每个头计算注意力
    heads = []
    for i in range(num_heads):
        head = attention(Q[:, :, i], K[:, :, i], V[:, :, i])
        heads.append(head)
    
    # 拼接所有头
    concat = concatenate(heads, axis=-1)
    
    # 输出投影
    output = linear(concat, d_model)
    
    return output

# 位置编码实现
import numpy as np
import math

def positional_encoding(seq_len, d_model):
    """
    生成位置编码
    seq_len: 序列长度
    d_model: 模型维度
    """
    pe = np.zeros((seq_len, d_model))
    
    for pos in range(seq_len):
        for i in range(0, d_model, 2):
            # 正弦编码
            pe[pos, i] = math.sin(pos / (10000 ** (i / d_model)))
            # 余弦编码
            if i + 1 < d_model:
                pe[pos, i + 1] = math.cos(pos / (10000 ** (i / d_model)))
    
    return pe

# 前馈网络实现
def feed_forward(x, d_model=512, d_ff=2048):
    """
    前馈网络
    """
    # 第一层：扩展到d_ff
    x = linear(x, d_ff)
    
    # 激活函数
    x = gelu(x)  # 或 relu(x)
    
    # 第二层：投影回d_model
    x = linear(x, d_model)
    
    return x

# 编码器层实现
class EncoderLayer:
    def __init__(self, d_model, num_heads, d_ff, dropout=0.1):
        self.self_attn = MultiHeadAttention(d_model, num_heads)
        self.feed_forward = FeedForward(d_model, d_ff)
        self.norm1 = LayerNorm(d_model)
        self.norm2 = LayerNorm(d_model)
        self.dropout = Dropout(dropout)
    
    def forward(self, x, mask=None):
        # 自注意力 + 残差连接
        attn_output = self.self_attn(x, x, x, mask)
        x = self.norm1(x + self.dropout(attn_output))
        
        # 前馈网络 + 残差连接
        ff_output = self.feed_forward(x)
        x = self.norm2(x + self.dropout(ff_output))
        
        return x

# 掩码机制
def create_look_ahead_mask(size):
    """
    创建前瞻掩码，防止解码器看到未来信息
    """
    mask = np.triu(np.ones((size, size)), k=1)
    return mask == 0  # True表示可以关注，False表示掩码