39 lines
942 B
Python
39 lines
942 B
Python
import torch
|
|
import torch.nn as nn
|
|
|
|
class Model(nn.Module):
|
|
"""
|
|
Simple model that performs RMSNorm.
|
|
"""
|
|
def __init__(self, dim, eps=1e-6):
|
|
super(Model, self).__init__()
|
|
self.eps = eps
|
|
self.weight = torch.nn.Parameter(torch.ones(dim))
|
|
|
|
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
|
"""
|
|
Applies RMSNorm activation to the input tensor.
|
|
|
|
Args:
|
|
x (torch.Tensor): Input tensor of shape (B, L, D).
|
|
|
|
Returns:
|
|
torch.Tensor: Output tensor with RMSNorm applied, same shape as input.
|
|
"""
|
|
dtype = x.dtype
|
|
x = x.float()
|
|
variance = x.pow(2).mean(-1, keepdim=True)
|
|
x = x * torch.rsqrt(variance + self.eps)
|
|
return (x * self.weight).to(dtype)
|
|
|
|
batch_size = 16
|
|
seq_len = 512
|
|
dim = 4096
|
|
|
|
def get_inputs():
|
|
x = torch.randn(batch_size, seq_len, dim)
|
|
return [x]
|
|
|
|
def get_init_inputs():
|
|
return [dim]
|