codeTransformer

Using Pytorch to Build Transformer

Encoder

首先是一个norm函数

norm里面做残差，会输入(x 和 淡粉色z1，残差值)，输出一个值粉红色z1

Basic Knowledge about How to use Pytorch

Example:

using pytorch to build a simple linear regression

import numpy as np
import torch
import torch.nn as nn

x = np.arange(1, 12, dtype = np.float32).reshape(-1,1)
y = 2 * x + 3

class LinearRegressionModel(nn.Module):
    def __init__(self, input_dim, output_dim):
        super().__init__()
        self.linear = nn.Linear(input_dim, output_dim)

    def forward(self, inp):
        out = self.linear(inp)
        return out


regression_model = LinearRegressionModel(input_dim = 1, output_dim = 1)

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
regression_model.to(device)

epochs = 1000
learning_rate = 0.01
optimizer = torch.optim.SGD(regression_model.parameters(), learning_rate)
criterion = nn.MSELoss()

for epoch in range(epochs):
    inputs = torch.from_numpy(x).to(device)
    labels = torch.from_numpy(y).to(device)

    # train
    optimizer.zero_grad()
    outputs = regression_model(inputs)
    loss = criterion(outputs, labels)
    loss.backward()
    optimizer.step()

    if epoch % 50 == 0:
        print("epoch:", epoch, "loss:", loss.item())


# print(regression_model)
'''
torch.from_numpy(x): put the data formation from numpy into tensor
data.numpy(): get the results
require_grad: use gradient descent to update the weights
'''
# predict = regression_model(torch.from_numpy(x).requires_grad_()).data.numpy()  # 通过训练好的模型预测结果
predict = regression_model(torch.from_numpy(x).requires_grad_(False)).detach().numpy()  # 通过训练好的模型预测结果


# another way
# tensor_x = torch.from_numpy(x).float().to(device)
# tensor_x.requires_grad_(True)
# predict = regression_model(tensor_x).detach().numpy()

print(y)
print(predict)