Stable Diffusion LoRA 微调：从 Dreambooth 到社区生态的完整指南

1
┌──────────────────────────────────────────────────────────────┐
2
│  SD 微调方法对比                                             │
3
├──────────────────────────────────────────────────────────────┤
4
│                                                              │
5
│  1. DreamBooth (2023, Google)                               │
6
│     ──────────────────────────────────────────────────────  │
7
│     方法: 给定 3-5 张图像，微调整个 SD (UNet + Text Encoder)  │
8
│     目标: 让模型学会特定主体的完整表示                        │
9
│     效果: ★★★★★ (最强，捕捉主体细节)                     │
10
│     成本: 高 (全量微调，显存需求大)                          │
11
│                                                              │
12
│  2. Textual Inversion (2022, Google)                        │
13
│     ──────────────────────────────────────────────────────  │
14
│     方法: 学习新的词嵌入向量代表新概念                        │
15
│     目标: 用新词触发新概念                                   │
16
│     效果: ★★★☆☆ (弱，概念表示有限)                      │
17
│     成本: 低 (只训练 embedding)                              │
18
│                                                              │
19
│  3. LoRA (Low-Rank Adaptation)                             │
20
│     ──────────────────────────────────────────────────────  │
21
│     方法: 在权重矩阵旁边添加低秩适配器                        │
22
│     目标: 高效学习新概念/风格                                │
23
│     效果: ★★★★☆ (好，效率和效果平衡)                    │
24
│     成本: 中 (比 DreamBooth 低，比 TI 高)                   │
25
│                                                              │
26
│  4. LyCORIS (LoRA Beyond Correction)                       │
27
│     ──────────────────────────────────────────────────────  │
28
│     方法: LoRA 的扩展集合（含 LoCon、LoHA、DiP+ 等)         │
29
│     目标: 更精细的风格/概念控制                              │
30
│     效果: ★★★★☆ (灵活，支持更多变体)                    │
31
│     成本: 中低                                              │
32
│                                                              │
33
└──────────────────────────────────────────────────────────────┘

1
def dreambooth_loss():
2
    """
3
    DreamBooth 的训练损失。
4
    """
5
    return {
6
        "扩散损失": {
7
            "公式": "L_diff = E_{x,c,ε,t}[‖ε - ε_θ(α_t x + σ_t c, t)‖²]",
8
            "说明": "标准 DDPM 噪声预测损失",
9
            "作用": "让模型学会从噪声中重建图像",
10
        },
11
        "身份保留损失": {
12
            "公式": "L_ID = E[‖x_recon - x_class‖²]",
13
            "说明": "微调后的输出应保留主体特征",
14
            "作用": "防止主体被过度改变",
15
        },
16
        "KL 散度": {
17
            "公式": "L_KL = KL(θ_new || θ_pretrained)",
18
            "说明": "防止新权重偏离预训练太远",
19
            "作用": "保持模型的通用生成能力",
20
        },
21
        "总损失": {
22
            "公式": "L_total = L_diff + λ_id * L_ID + λ_kl * L_KL",
23
            "说明": "三损失加权求和",
24
            "作用": "平衡生成质量与主体保真度",
25
        },
26
    }
27

28

29
def dreambooth_training_loop():
30
    """
31
    DreamBooth 训练循环。
32
    """
33
    def train(model, subject_images, class_images, optimizer, device):
34
        """
35
        参数:
36
            subject_images: 3-5 张目标主体的图像
37
            class_images: 同类别的通用图像 (用于身份保留)
38
            optimizer: AdamW
39
        """
40
        for batch in dataloader:
41
            # 1) 主体图像 + 特殊 prompt
42
            subject_imgs = batch["subject"].to(device)
43
            prompt = "a photo of a [V] dog"  # [V] 是特殊 token
44

45
            # 2) 加噪
46
            t = sample_timesteps(len(subject_imgs))
47
            noise = torch.randn_like(subject_imgs)
48
            alpha_bar = diffusion_schedule[t]
49
            noisy_imgs = alpha_bar.sqrt() * subject_imgs + (1 - alpha_bar).sqrt() * noise
50

51
            # 3) 文本编码
52
            context = text_encoder(prompt)
53

54
            # 4) UNet 前向
55
            noise_pred = unet(noisy_imgs, t, context)
56

57
            # 5) 重建损失
58
            loss_diff = F.mse_loss(noise_pred, noise)
59

60
            # 6) 身份保留损失 (与类别图像的重建一致)
61
            with torch.no_grad():
62
                class_imgs = batch["class"].to(device)
63
                class_noisy = add_noise(class_imgs, t)
64
                class_pred = unet(class_noisy, t, context)
65
                class_loss = F.mse_loss(class_pred, noise)
66

67
            # 7) KL 散度
68
            loss_kl = kl_divergence(model.weights(), pretrained_weights)
69

70
            # 8) 总损失
71
            loss = loss_diff + 0.1 * class_loss + 0.01 * loss_kl
72

73
            # 9) 反向传播
74
            loss.backward()
75
            optimizer.step()
76
            optimizer.zero_grad()
77

78
        return loss

1
def dreambooth_vram():
2
    """
3
    DreamBooth 微调的显存需求。
4
    """
5
    return {
6
        "SD 1.5 (FP16)": {
7
            "模型权重": "~7GB",
8
            "梯度": "~7GB (全量)",
9
            "优化器": "~14GB",
10
            "激活值": "~5GB",
11
            "总计": "~33GB",
12
            "实际需要": "24GB 显存 (如 RTX 3090)",
13
        },
14
        "SDXL (FP16)": {
15
            "模型权重": "~30GB",
16
            "梯度": "~30GB",
17
            "优化器": "~60GB",
18
            "激活值": "~20GB",
19
            "总计": "~140GB",
20
            "实际需要": "80GB 显存 (如 A100)",
21
        },
22
        "SD 1.5 + LoRA": {
23
            "模型权重": "~7GB (冻结)",
24
            "LoRA 梯度": "~0.1GB",
25
            "优化器": "~0.2GB",
26
            "激活值": "~5GB",
27
            "总计": "~12GB",
28
            "实际需要": "16GB 显存 (如 RTX 4060 Ti)",
29
        },
30
    }

1
def textual_inversion_training():
2
    """
3
    Textual Inversion 训练。
4
    """
5
    def train(text_encoder, vae, unet, images, concept_token, optimizer, device):
6
        """
7
        参数:
8
            text_encoder: CLIP 文本编码器
9
            images: 3-5 张目标图像
10
            concept_token: 新概念的 token 名 (如 "SKS")
11
        """
12
        # 1) 在 tokenizer 中添加新 token
13
        tokenizer = text_encoder.tokenizer
14
        num_added = tokenizer.add_tokens([concept_token])
15
        text_encoder.resize_token_embeddings(len(tokenizer))
16

17
        # 获取新 token 的 embedding
18
        token_id = tokenizer.convert_tokens_to_ids(concept_token)
19
        concept_embed = text_encoder.get_input_embeddings()
20

21
        # 2) 训练循环
22
        for img in tqdm(images):
23
            # 图像 → 潜空间
24
            with torch.no_grad():
25
                z0 = vae.encode(img)
26

27
            # 3) 加噪
28
            t = sample_timesteps(1)
29
            noise = torch.randn_like(z0)
30
            alpha_bar = diffusion_schedule[t]
31
            z_t = alpha_bar.sqrt() * z0 + (1 - alpha_bar).sqrt() * noise
32

33
            # 4) 文本编码 (新 token 的 embedding)
34
            # 新 token 的 embedding 可训练
35
            prompt_embeds = concept_embed.weight[token_id]  # 可训练
36
            context = prompt_embeds.unsqueeze(0)  # (1, D)
37

38
            # 5) UNet 前向
39
            noise_pred = unet(z_t, t, context)
40

41
            # 6) 损失
42
            loss = F.mse_loss(noise_pred, noise)
43

44
            # 7) 反向传播 (只更新 embedding)
45
            optimizer.zero_grad()
46
            loss.backward()
47
            optimizer.step()
48

49
        # 8) 保存 embedding
50
        saved_embed = concept_embed.weight[token_id].detach().cpu()
51
        return saved_embed
52

53
    return train

1
def ti_vs_dreambooth():
2
    """
3
    Textual Inversion vs DreamBooth 对比。
4
    """
5
    return {
6
        "训练参数": {
7
            "Textual Inversion": "~768K (仅 embedding)",
8
            "DreamBooth": "~860M (UNet) + ~123M (Text Encoder)",
9
        },
10
        "效果": {
11
            "Textual Inversion": "弱，只能捕捉浅层概念",
12
            "DreamBooth": "强，能捕捉主体的完整视觉特征",
13
        },
14
        "泛化能力": {
15
            "Textual Inversion": "有限，可能过拟合训练图像",
16
            "DreamBooth": "较好，能生成新姿态/背景的同主体",
17
        },
18
        "生成速度": {
19
            "Textual Inversion": "快 (embedding 已训练)",
20
            "DreamBooth": "快 (模型已微调)",
21
        },
22
        "组合能力": {
23
            "Textual Inversion": "好 (embedding 可组合)",
24
            "DreamBooth": "差 (模型权重直接叠加)",
25
        },
26
        "适用场景": {
27
            "Textual Inversion": "简单风格、图标、logo",
28
            "DreamBooth": "特定人物/宠物/产品",
29
        },
30
    }

1
┌─────────────────────────────────────────────────────────────┐
2
│  SD LoRA 架构                                               │
3
│                                                             │
4
│  原始权重 (冻结 ❄️):                                        │
5
│    W0 ∈ ℝ^{d×k}                                            │
6
│                                                             │
7
│  LoRA 适配器 (训练 🔥):                                     │
8
│    A ∈ ℝ^{r×k}  (初始化: 随机高斯)                        │
9
│    B ∈ ℝ^{d×r}  (初始化: 零)                              │
10
│                                                             │
11
│  前向传播:                                                  │
12
│    h = W0 x + BAx                                          │
13
│    或: h = (W0 + BA)x                                      │
14
│                                                             │
15
│  关键:                                                     │
16
│    - r << min(d, k) → 低秩                                  │
17
│    - 只更新 A 和 B，W0 冻结                                 │
18
│    - 最终权重 = W0 + BA                                    │
19
│                                                             │
20
└─────────────────────────────────────────────────────────────┘

1
def sd_lora_target_modules():
2
    """
3
    SD LoRA 的目标层配置。
4
    """
5
    return {
6
        "SD 1.5 / SD 2.x": {
7
            "UNet": {
8
                "推荐目标": [
9
                    "to_q", "to_k", "to_v", "to_out",  # Attention
10
                    "ff.net.0", "ff.net.2",              # FFN
11
                    "conv1", "conv2",                     # 下/上采样块
12
                ],
13
                "可选目标": [
14
                    "time_emb_proj",  # 时间嵌入投影
15
                    "norm1", "norm2",  # GroupNorm
16
                ],
17
            },
18
            "Text Encoder (CLIP)": {
19
                "推荐目标": [
20
                    "q_proj", "k_proj", "v_proj", "out_proj",  # Attention
21
                ],
22
                "说明": "Text Encoder 的 LoRA 对风格影响更大",
23
            },
24
            "VAE": {
25
                "推荐目标": ["一般不添加"],
26
                "原因": "VAE 已足够好，微调可能导致重建质量下降",
27
            },
28
        },
29
        "SDXL": {
30
            "UNet": {
31
                "目标层": "与 SD 1.5 相同",
32
                "额外": "mid_block 添加更多层",
33
            },
34
            "Text Encoder 1 (OpenCLIP)": {
35
                "推荐目标": ["q_proj", "v_proj"],
36
            },
37
            "Text Encoder 2 (CLIP)": {
38
                "推荐目标": ["q_proj", "v_proj"],
39
            },
40
            "重要": "SDXL LoRA 推荐同时训练双文本编码器",
41
        },
42
    }
43

44

45
def lora_rank_selection():
46
    """
47
    LoRA rank 选择指南。
48
    """
49
    return {
50
        "r=4 (极低)": {
51
            "参数量": "~0.1M",
52
            "适用": "简单风格、微弱效果",
53
            "质量": "较差",
54
        },
55
        "r=8": {
56
            "参数量": "~0.2M",
57
            "适用": "快速实验",
58
            "质量": "一般",
59
        },
60
        "r=16 (推荐)": {
61
            "参数量": "~0.4M",
62
            "适用": "大多数场景",
63
            "质量": "良好",
64
        },
65
        "r=32": {
66
            "参数量": "~0.8M",
67
            "适用": "复杂风格、人物",
68
            "质量": "很好",
69
        },
70
        "r=64": {
71
            "参数量": "~1.6M",
72
            "适用": "高质量要求",
73
            "质量": "优秀",
74
        },
75
        "r=128+": {
76
            "参数量": ">3M",
77
            "适用": "接近全量微调效果",
78
            "质量": "最佳",
79
            "注意": "文件变大，组合困难",
80
        },
81
    }

1
import torch
2
import torch.nn as nn
3
import torch.nn.functional as F
4
from typing import Dict, List, Optional
5

6

7
# ============ LoRA 层实现 ============
8
class LoRALinear(nn.Module):
9
    """
10
    LoRA 线性层。
11
    在原始线性层旁边添加低秩适配器。
12
    """
13
    def __init__(self, in_features, out_features, rank=4, alpha=1.0):
14
        super().__init__()
15
        self.rank = rank
16
        self.alpha = alpha
17
        self.scaling = alpha / rank
18

19
        # 原始权重 (冻结)
20
        self.weight = None  # 将在 forward 时使用
21

22
        # LoRA 适配器 (可训练)
23
        self.lora_A = nn.Parameter(torch.randn(rank, in_features) * 0.01)
24
        self.lora_B = nn.Parameter(torch.zeros(out_features, rank))
25

26
    def forward(self, x, weight):
27
        """
28
        参数:
29
            x: (B, *, in_features)
30
            weight: 原始线性层的权重
31
        """
32
        # 原始输出
33
        base_out = F.linear(x, weight)
34

35
        # LoRA 输出: h = Wx + (α/r) * BAx
36
        lora_out = (self.lora_B @ self.lora_A) * self.scaling
37
        return base_out + F.linear(x, lora_out)
38

39

40
class LoRAConv2d(nn.Module):
41
    """
42
    LoRA 卷积层。
43
    用于 SD UNet 的卷积层。
44
    """
45
    def __init__(self, in_channels, out_channels, kernel_size=3,
46
                 stride=1, padding=1, rank=4, alpha=1.0):
47
        super().__init__()
48
        self.rank = rank
49
        self.alpha = alpha
50
        self.scaling = alpha / rank
51

52
        # 原始权重 (冻结)
53
        self.weight = None
54

55
        # LoRA 适配器
56
        self.lora_A = nn.Parameter(torch.randn(rank, in_channels * kernel_size * kernel_size) * 0.01)
57
        self.lora_B = nn.Parameter(torch.zeros(out_channels * kernel_size * kernel_size, rank))
58

59
    def forward(self, x, weight, stride=1, padding=1):
60
        """
61
        参数:
62
            x: (B, C, H, W)
63
            weight: 原始卷积权重
64
        """
65
        # 原始输出
66
        base_out = F.conv2d(x, weight, stride=stride, padding=padding)
67

68
        # LoRA 输出
69
        lora_weight = (self.lora_B @ self.lora_A).view(
70
            weight.shape[0], weight.shape[1], 3, 3
71
        )
72
        lora_out = F.conv2d(x, lora_weight * self.scaling, stride=stride, padding=padding)
73

74
        return base_out + lora_out
75

76

77
# ============ SD LoRA 模型 ============
78
class SDLoRAModel(nn.Module):
79
    """
80
    带 LoRA 的 Stable Diffusion 模型。
81
    """
82

83
    def __init__(self, sd_model, rank=4, alpha=1.0, target_modules=None):
84
        super().__init__()
85
        self.sd_model = sd_model
86
        self.rank = rank
87
        self.alpha = alpha
88

89
        # 冻结原始模型
90
        for param in sd_model.parameters():
91
            param.requires_grad_(False)
92

93
        # 创建 LoRA 层
94
        self.lora_layers = nn.ModuleDict()
95
        self._inject_lora(target_modules or DEFAULT_SD_LORA_TARGETS)
96

97
    def _inject_lora(self, targets):
98
        """
99
        向目标层注入 LoRA。
100
        """
101
        for name, module in self.sd_model.named_modules():
102
            for target in targets:
103
                if target in name and isinstance(module, (nn.Linear, nn.Conv2d)):
104
                    self._create_lora_layer(name, module)
105

106
    def _create_lora_layer(self, name, module):
107
        """
108
        为指定层创建 LoRA。
109
        """
110
        if isinstance(module, nn.Linear):
111
            lora = LoRALinear(
112
                module.in_features, module.out_features,
113
                rank=self.rank, alpha=self.alpha
114
            )
115
        elif isinstance(module, nn.Conv2d):
116
            lora = LoRAConv2d(
117
                module.in_channels, module.out_channels,
118
                module.kernel_size, module.stride, module.padding,
119
                rank=self.rank, alpha=self.alpha
120
            )
121
        self.lora_layers[name] = lora
122

123
    def forward(self, x_t, t, context):
124
        """
125
        前向传播。
126
        LoRA 权重在 forward 中应用，不改变原始模型。
127
        """
128
        # 遍历所有层，手动应用 LoRA
129
        for name, module in self.sd_model.named_modules():
130
            if name in self.lora_layers:
131
                lora = self.lora_layers[name]
132
                if isinstance(module, nn.Linear):
133
                    original_forward = module.forward
134
                    module.forward = lambda x, m=module, l=lora: \
135
                        l(x, m.weight)
136
                elif isinstance(module, nn.Conv2d):
137
                    original_forward = module.forward
138
                    module.forward = lambda x, m=module, l=lora: \
139
                        l(x, m.weight, m.stride, m.padding)
140

141
        # 标准 SD 前向
142
        return self.sd_model(x_t, t, context)
143

144
    def merge_weights(self):
145
        """
146
        合并 LoRA 权重到原始模型。
147
        合并后模型等价，但不再需要 LoRA 层。
148
        """
149
        for name, module in self.sd_model.named_modules():
150
            if name in self.lora_layers:
151
                lora = self.lora_layers[name]
152
                if isinstance(module, nn.Linear):
153
                    # W_new = W_old + (α/r) * B @ A
154
                    delta = (lora.lora_B @ lora.lora_A) * lora.scaling
155
                    module.weight.data = module.weight.data + delta
156
                elif isinstance(module, nn.Conv2d):
157
                    delta = (lora.lora_B @ lora.lora_A).view(
158
                        module.weight.shape
159
                    ) * lora.scaling
160
                    module.weight.data = module.weight.data + delta
161

162
    def save_lora(self, path):
163
        """
164
        保存 LoRA 权重。
165
        """
166
        lora_state = {
167
            name: {
168
                "lora_A": layer.lora_A.data.cpu(),
169
                "lora_B": layer.lora_B.data.cpu(),
170
                "alpha": layer.alpha,
171
                "rank": layer.rank,
172
            }
173
            for name, layer in self.lora_layers.items()
174
        }
175
        torch.save(lora_state, path)
176

177
    @classmethod
178
    def load_lora(cls, sd_model, path, rank=4, alpha=1.0):
179
        """
180
        加载 LoRA 权重。
181
        """
182
        lora_state = torch.load(path, map_location="cpu")
183
        model = cls(sd_model, rank, alpha)
184

185
        for name, state in lora_state.items():
186
            if name in model.lora_layers:
187
                model.lora_layers[name].lora_A.data = state["lora_A"]
188
                model.lora_layers[name].lora_B.data = state["lora_B"]
189

190
        return model

1
def train_sd_lora(sd_model, vae, text_encoder, images, prompts,
2
                  rank=4, alpha=1.0, lr=1e-4, epochs=5):
3
    """
4
    SD LoRA 训练循环。
5
    """
6
    # 1) 创建 LoRA 模型
7
    lora_model = SDLoRAModel(sd_model, rank=rank, alpha=alpha)
8
    lora_model.train()
9

10
    # 2) 只训练 LoRA 参数
11
    optimizer = torch.optim.AdamW(
12
        lora_model.lora_layers.parameters(),
13
        lr=lr,
14
        weight_decay=0.01,
15
    )
16

17
    # 3) 冻结 VAE 和 Text Encoder
18
    vae.eval()
19
    text_encoder.eval()
20

21
    for epoch in range(epochs):
22
        for img, prompt in tqdm(zip(images, prompts), total=len(images)):
23
            # 图像 → 潜空间
24
            with torch.no_grad():
25
                z0 = vae.encode(img.unsqueeze(0))  # (1, 4, 64, 64)
26

27
            # 采样时间步
28
            t = torch.randint(0, 1000, (1,))
29

30
            # 加噪
31
            noise = torch.randn_like(z0)
32
            alpha_bar = get_diffusion_schedule(t)
33
            z_t = alpha_bar.sqrt() * z0 + (1 - alpha_bar).sqrt() * noise
34

35
            # 文本编码
36
            with torch.no_grad():
37
                context = text_encoder(prompt)  # (1, seq_len, 768)
38

39
            # UNet 前向 (带 LoRA)
40
            noise_pred = lora_model(z_t, t, context)
41

42
            # 损失
43
            loss = F.mse_loss(noise_pred, noise)
44

45
            # 反向传播
46
            optimizer.zero_grad()
47
            loss.backward()
48
            optimizer.step()
49

50
        print(f"Epoch {epoch + 1}: Loss = {loss.item():.4f}")
51

52
    # 4) 保存 LoRA
53
    lora_model.save_lora("my_style_lora.safetensors")
54

55
    return lora_model
56

57

58
DEFAULT_SD_LORA_TARGETS = [
59
    # UNet Attention
60
    "unet.up_blocks.0.attentions.0.transformer_blocks.0.attn1.to_q",
61
    "unet.up_blocks.0.attentions.0.transformer_blocks.0.attn1.to_k",
62
    "unet.up_blocks.0.attentions.0.transformer_blocks.0.attn1.to_v",
63
    "unet.up_blocks.0.attentions.0.transformer_blocks.0.attn1.to_out.0",
64
    "unet.up_blocks.0.attentions.0.transformer_blocks.0.attn2.to_q",
65
    "unet.up_blocks.0.attentions.0.transformer_blocks.0.attn2.to_k",
66
    "unet.up_blocks.0.attentions.0.transformer_blocks.0.attn2.to_v",
67
    "unet.up_blocks.0.attentions.0.transformer_blocks.0.attn2.to_out.0",
68
    # UNet FFN
69
    "unet.up_blocks.0.attentions.0.transformer_blocks.0.ff.net.0",
70
    "unet.up_blocks.0.attentions.0.transformer_blocks.0.ff.net.2",
71
    # Text Encoder
72
    "text_encoder.blocks.0.attn.q_proj",
73
    "text_encoder.blocks.0.attn.k_proj",
74
    "text_encoder.blocks.0.attn.v_proj",
75
]

1
LyCORIS 变体:
2
  ├── LoCon (LoRA on Convolution)
3
  │     在卷积层也添加 LoRA（SD LoRA 只在 attention 层）
4
  │
5
  ├── LoHA (LoRA with Hadamard Product)
6
  │     用 Hadamard 积替代矩阵乘法: ΔW = H ⊙ (B @ A)
7
  │     效果: 更紧凑的表示，效果更好
8
  │
9
  ├── LoKR (LoRA with Kronecker Product)
10
  │     用 Kronecker 积替代矩阵乘法
11
  │     效果: 保留更多结构信息
12
  │
13
  ├── DiP (Decoupled LoRA Pre-trained)
14
  │     分解为方向 + 幅度两部分
15
  │
16
  ├── GLora (Group LoRA)
17
  │     按组应用 LoRA，减少参数量
18
  │
19
  └── CompFAB (Comprehensive Fine-Adapter)
20
        结合多种技术的复合适配器

1
class LoCon(nn.Module):
2
    """
3
    LoCon: 在卷积层也应用 LoRA。
4
    SD LoRA 默认只在 attention 层，LoCon 扩展到 conv 层。
5
    """
6
    def __init__(self, in_channels, out_channels, kernel_size=3,
7
                 rank=4, alpha=1.0):
8
        super().__init__()
9
        self.rank = rank
10
        self.alpha = alpha
11
        self.scaling = alpha / rank
12

13
        # LoRA on weight (与 LoRA Linear 相同)
14
        self.lora_A = nn.Parameter(torch.randn(rank, in_channels * kernel_size * kernel_size) * 0.01)
15
        self.lora_B = nn.Parameter(torch.zeros(out_channels * kernel_size * kernel_size, rank))
16

17
        # LoRA on bias (可选)
18
        if hasattr(nn.Conv2d, 'bias') and nn.Conv2d.bias is not None:
19
            self.lora_bias = nn.Parameter(torch.zeros(out_channels))
20
        else:
21
            self.lora_bias = None
22

23
    def forward(self, x, weight, bias=None):
24
        # 原始输出
25
        out = F.conv2d(x, weight, bias=bias, padding=1)
26

27
        # LoRA 调整
28
        delta = (self.lora_B @ self.lora_A).view(
29
            weight.shape[0], weight.shape[1], 3, 3
30
        ) * self.scaling
31
        out = out + F.conv2d(x, delta)
32

33
        if self.lora_bias is not None:
34
            out = out + self.lora_bias.view(1, -1, 1, 1)
35

36
        return out

1
class LoHA(nn.Module):
2
    """
3
    LoHA: 使用 Hadamard 积的 LoRA。
4
    ΔW = H ⊙ (B @ A)
5
    其中 H 是可学习的 Hadamard 矩阵参数。
6
    """
7
    def __init__(self, in_features, out_features, rank=4, alpha=1.0):
8
        super().__init__()
9
        self.rank = rank
10
        self.alpha = alpha
11
        self.scaling = alpha / rank
12

13
        # 低秩矩阵
14
        self.lora_A = nn.Parameter(torch.randn(rank, in_features) * 0.01)
15
        self.lora_B = nn.Parameter(torch.randn(out_features, rank) * 0.01)
16

17
        # Hadamard 矩阵 (对角矩阵，用向量表示)
18
        self.h = nn.Parameter(torch.ones(out_features, rank))
19

20
    def forward(self, x, weight):
21
        # 原始输出
22
        base_out = F.linear(x, weight)
23

24
        # LoHA: ΔW = H ⊙ (B @ A)
25
        # H 是 Hadamard (逐元素乘法)
26
        delta = self.h * (self.lora_B @ self.lora_A)
27
        lora_out = F.linear(x, delta * self.scaling)
28

29
        return base_out + lora_out

1
def lycoris_variants():
2
    """
3
    LyCORIS 各变体对比。
4
    """
5
    return {
6
        "LoRA (标准)": {
7
            "公式": "ΔW = B @ A",
8
            "参数量": "2 × d × r",
9
            "效果": "好",
10
            "适用": "通用",
11
            "代表": "标准 SD LoRA",
12
        },
13
        "LoCon": {
14
            "公式": "LoRA + Conv LoRA",
15
            "参数量": "比 LoRA 多 ~20%",
16
            "效果": "更好（细节更丰富）",
17
            "适用": "风格、纹理",
18
            "代表": "社区广泛使用",
19
        },
20
        "LoHA": {
21
            "公式": "ΔW = H ⊙ (B @ A)",
22
            "参数量": "比 LoRA 多 ~50% (含 H)",
23
            "效果": "更好（表示更紧凑）",
24
            "适用": "风格、概念",
25
            "代表": "Kohaku 系列",
26
        },
27
        "LoKR": {
28
            "公式": "ΔW = Kronecker(B, K) @ A",
29
            "参数量": "与 rank 成线性关系",
30
            "效果": "保留结构",
31
            "适用": "结构化变化",
32
            "代表": "学术研究",
33
        },
34
        "DiP+": {
35
            "公式": "ΔW = D ⊙ M",
36
            "参数量": "d + r",
37
            "效果": "效率高",
38
            "适用": "轻量适配",
39
            "代表": "轻量化需求",
40
        },
41
    }

1
def sdxl_differences():
2
    """
3
    SDXL vs SD 1.5 的关键差异。
4
    """
5
    return {
6
        "双文本编码器": {
7
            "SD 1.5": "1 个 CLIP ViT-L/14 (768d)",
8
            "SDXL": "2 个编码器: OpenCLIP ViT-L/14 (768d) + OpenCLIP ViT-G/14 (1280d)",
9
            "LoRA 影响": "需要同时在两个编码器上应用 LoRA",
10
        },
11
        "更大的 UNet": {
12
            "SD 1.5": "~860M 参数",
13
            "SDXL": "~3.5B 参数",
14
            "LoRA 影响": "更大的微调空间，但也需要更高的 rank",
15
        },
16
        "更高的分辨率": {
17
            "SD 1.5": "512×512 (latent 64×64)",
18
            "SDXL": "1024×1024 (latent 128×128)",
19
            "LoRA 影响": "高分辨率生成效果更好，但 LoRA 效果更明显",
20
        },
21
        "Refiner": {
22
            "SD 1.5": "无",
23
            "SDXL": "额外的 refiner 模型",
24
            "LoRA 影响": "一般只训练 base UNet，refiner 可单独微调",
25
        },
26
    }
27

28

29
def sdxl_lora_targets():
30
    """
31
    SDXL LoRA 推荐的 target modules。
32
    """
33
    return {
34
        "UNet (必需)": [
35
            "to_q", "to_k", "to_v", "to_out",  # 所有 attention 层
36
            "ff.net.0", "ff.net.2",              # FFN
37
            "conv1", "conv2",                     # 卷积
38
            "time_emb_proj",                      # 时间嵌入
39
        ],
40
        "Text Encoder 1 (OpenCLIP ViT-L)": [
41
            "q_proj", "k_proj", "v_proj", "out_proj",
42
        ],
43
        "Text Encoder 2 (OpenCLIP ViT-G)": [
44
            "q_proj", "k_proj", "v_proj", "out_proj",
45
        ],
46
        "训练建议": {
47
            "最少": "只训练 UNet (可获得 80% 效果)",
48
            "推荐": "UNet + TE1 (最佳性价比)",
49
            "完整": "UNet + TE1 + TE2 (最高质量)",
50
        },
51
    }

1
def sdxl_lora_config():
2
    """
3
    SDXL LoRA 训练推荐配置。
4
    """
5
    return {
6
        "rank": {
7
            "UNet": "32-64 (SDXL UNet 更大，需要更高 rank)",
8
            "TE1": "16-32",
9
            "TE2": "8-16",
10
        },
11
        "alpha": {
12
            "UNet": "64 (与 rank 相等或 2x)",
13
            "TE": "rank (或 1.0)",
14
        },
15
        "学习率": {
16
            "UNet": "1e-4",
17
            "TE": "5e-5 (比 UNet 低)",
18
        },
19
        "batch_size": "1-4 (SDXL 显存需求更大)",
20
        "resolution": "1024×1024 (原生)",
21
        "epoch": "5-10",
22
        "训练时间": "约 2-4 小时 (A100 80GB)",
23
    }
24

25

26
def sdxl_vram():
27
    """
28
    SDXL LoRA 显存需求。
29
    """
30
    return {
31
        "SDXL + LoRA r=32": {
32
            "batch=1": "~24GB",
33
            "batch=2": "~40GB",
34
            "batch=4": "~70GB",
35
        },
36
        "优化技巧": {
37
            "梯度检查点": "减少 ~30% 显存",
38
            "CPU 卸载": "减少 ~40% 显存 (速度变慢)",
39
            "混合精度": "FP16 训练，节省 ~50% 显存",
40
        },
41
    }

1
Civitai 生态:
2
  ├── 模型类型:
3
  │     ├── Checkpoint (完整模型): ~2-7GB
4
  │     ├── LoRA (适配器): ~100MB - 500MB
5
  │     ├── Textual Inversion (embedding): ~100KB - 1MB
6
  │     ├── ControlNet (条件控制): ~1-2GB
7
  │     └── VAE (变分自编码器): ~300MB
8
  │
9
  ├── 搜索和发现:
10
  │     ├── 风格分类 (anime, realistic, art style)
11
  │     ├── 主题分类 (character, clothing, pose)
12
  │     ├── 评分和下载量
13
  │     └── 社区讨论
14
  │
15
  └── 使用方式:
16
        直接下载 .safetensors 文件
17
        在 WebUI / ComfyUI 中加载
18
        通过 API 访问 (civitai.com/api)

1
def comfyui_lora_workflow():
2
    """
3
    ComfyUI 中加载 LoRA 的节点配置。
4
    """
5
    return {
6
        "节点顺序": [
7
            "1. Load Checkpoint: 加载 SD 模型 (SD 1.5 / SDXL)",
8
            "2. Load LoRA: 加载 .safetensors 文件，设置 strength",
9
            "3. CLIP Text Encode: 输入正面提示词",
10
            "4. CLIP Text Encode (neg): 输入负面提示词",
11
            "5. KSampler: 设置采样参数",
12
            "6. VAE Decode: 解码潜变量到图像",
13
        ],
14
        "LoRA strength 参数": {
15
            "strength_model": "对模型权重的影响 (1.0 = 完整应用)",
16
            "strength_clip": "对 CLIP Text Encoder 的影响",
17
            "常见设置": "0.5-0.8 (太强会失真)",
18
        },
19
        "多 LoRA 叠加": {
20
            "方法": "多个 Load LoRA 节点串联",
21
            "注意": "叠加太多会导致冲突和 artifacts",
22
            "技巧": "先用低 strength 尝试叠加",
23
        },
24
    }

1
def lora_file_format():
2
    """
3
    LoRA 文件格式 (.safetensors)。
4
    """
5
    return {
6
        "格式": "safetensors (安全的 pickle 替代)",
7
        "内容": {
8
            "lora_weight": "LoRA 权重矩阵 (A 和 B)",
9
            "metadata": {
10
                "base_model": "基础模型名称",
11
                "rank": "LoRA rank",
12
                "alpha": "alpha 值",
13
                "train_steps": "训练步数",
14
                "trained_by": "训练者",
15
            },
16
        },
17
        "文件大小参考": {
18
            "r=4": "~0.5MB",
19
            "r=8": "~1MB",
20
            "r=16": "~2-3MB",
21
            "r=32": "~5-10MB",
22
            "r=64": "~10-20MB",
23
            "r=128": "~20-50MB",
24
        },
25
        "与 ckpt 的对比": {
26
            "SD 1.5": "~4GB",
27
            "SDXL": "~7GB",
28
            "LoRA (r=32)": "~5-10MB (压缩率 ~400-1000x)",
29
        },
30
    }
31

32

33
def safetensors_save():
34
    """
35
    保存为 safetensors 格式。
36
    """
37
    from safetensors.torch import save_file
38

39
    def save_lora_safetensors(lora_layers, path, metadata=None):
40
        """
41
        保存 LoRA 为 safetensors 格式。
42
        """
43
        tensors = {}
44
        for name, layer in lora_layers.items():
45
            tensors[f"{name}.lora_A"] = layer.lora_A.data.float()
46
            tensors[f"{name}.lora_B"] = layer.lora_B.data.float()
47
            tensors[f"{name}.alpha"] = torch.tensor(layer.alpha)
48
            tensors[f"{name}.rank"] = torch.tensor(layer.rank)
49

50
        save_file(tensors, path, metadata=metadata or {})
51
        print(f"Saved LoRA to {path}")