swiglu 激活函数学习笔记

几种激活函数函数对比：

https://zhuanlan.zhihu.com/p/696571171

pytorch 实现 可视化代码：

import torch
import torch.nn.functional as F
import numpy as np
import cv2def swiglu(x):x1, x2 = x.chunk(2, dim=-1)return F.silu(x1) * x2x_vals = torch.linspace(-5, 5, steps=300).unsqueeze(1)
x_input = torch.cat([x_vals, x_vals], dim=1)
y_vals = swiglu(x_input).squeeze().detach().numpy()w, h = 640, 480
img = np.ones((h, w, 3), dtype=np.uint8) * 255x_raw = x_vals.squeeze().numpy()
x_plot = ((x_raw + 5) / 10) * (w - 40) + 20
y_plot = ((y_vals - y_vals.min()) / (y_vals.max() - y_vals.min())) * (h - 40)
y_plot = h - y_plot - 20cv2.line(img, (20, h - 20), (w - 20, h - 20), (0, 0, 0), 2)
cv2.line(img, (20, 20), (20, h - 20), (0, 0, 0), 2)for i in range(-5, 6):x_pos = int(((i + 5) / 10) * (w - 40) + 20)cv2.line(img, (x_pos, h - 25), (x_pos, h - 15), (0, 0, 0), 1)cv2.putText(img, str(i), (x_pos - 10, h - 5), cv2.FONT_HERSHEY_SIMPLEX, 0.4, (0, 0, 0), 1)num_yticks = 5
for i in range(num_yticks + 1):y_val = y_vals.min() + (y_vals.max() - y_vals.min()) * i / num_yticksy_pos = int(h - ((i / num_yticks) * (h - 40)) - 20)cv2.line(img, (15, y_pos), (25, y_pos), (0, 0, 0), 1)cv2.putText(img, f"{y_val:.2f}", (0, y_pos + 5), cv2.FONT_HERSHEY_SIMPLEX, 0.4, (0, 0, 0), 1)for i in range(1, len(x_plot)):pt1 = (int(x_plot[i - 1]), int(y_plot[i - 1]))pt2 = (int(x_plot[i]), int(y_plot[i]))cv2.line(img, pt1, pt2, (255, 0, 0), 2)cv2.putText(img, "SwiGLU Activation", (30, 40), cv2.FONT_HERSHEY_SIMPLEX, 1, (50, 50, 50), 2)cv2.imshow("SwiGLU", img)
cv2.waitKey(0)
cv2.destroyAllWindows()