[Python] The problem of MLP output converging into one

There was a problem while training the model for my high school performance evaluation. I used the area of the fan shape to draw the relationship between the angle and the cos value and put it as a loss function. That's why the loss function is a little complicated. but It can be differentiated, and it's not negative, so i thought that it works properly. But the output of the model always converges to some number. I gave a penalty, reduced the normalization layer, scaled the input through log, set the initialization of the model, or also introduced a scheduler to solve this problem. but the result was same.

set.py
​

class Model(nn.Module):
def __init__(self):
super().__init__()
self.model = nn.Sequential(
nn.Linear(1, 128),
nn.LeakyReLU(negative_slope=0.01),
nn.LayerNorm(128),

nn.Linear(128, 512),
nn.LeakyReLU(),

nn.Linear(512, 512),
nn.LeakyReLU(),

nn.Linear(512, 128),
nn.LeakyReLU(),
nn.LayerNorm(128),

nn.Linear(128, 1),
nn.Sigmoid()
)


loss_fuction

class Compute_fucs():
def __init__(self):
self.grid = torch.linspace(0, 1, steps=10000000, dtype=torch.float64)
self.scale = 1
def compute_circle(self, cos):
cos = cos.to('cuda')
starts = cos
xs = starts + (self.grid.to('cuda') * (1 - starts))
length = (self.grid[1]-self.grid[0])*(1 - starts)
length = length.to('cuda')
ys = torch.sqrt(torch.clamp(1 - xs**2, min = 1e-6))
result = length*ys
return result.sum(dim=1)

def compute_triangle(self, cos):
cos = cos.to('cuda')
return cos*torch.sqrt(torch.clamp(1 - cos**2,min = 1e-6))

def loss_fn(self, output, target,panerty):
cos = output

out1 = self.compute_circle(cos)
out2 = self.compute_triangle(cos)

output = 2*out1 + out2
target = target*(torch.pi / 180)

loss = torch.mean((target*self.scale - output*self.scale)**2)
return loss + panerty


trainer

class Modeltrainer():
def __init__(self,model = None, opt = None, best_loss = None):
self.model = copy.deepcopy(model)
self._init_weights()
self.model = self.model.to('cuda')
self.opt = opt if opt is not None else torch.optim.Adam(model.parameters(), lr=0.001)
self.compute = Compute_fucs()
self.loss_fn = self.compute.loss_fn
self.best_loss = best_loss if best_loss is not None else 1e30
self.best_state = None
self.opt_best_state = None
self.scheduler = CyclicLR(self.opt,
base_lr=0.0001,
max_lr=0.006,
step_size_up=113,
mode='exp_range',
gamma=0.99994
)

def _init_weights(self):
for m in self.model.modules():
if isinstance(m, nn.Linear) and m.out_features != 1:
nn.init.kaiming_normal_(
m.weight,
a=0.01,
mode='fan_in',
nonlinearity='leaky_relu'
)
if m.bias is not None:
nn.init.constant_(m.bias, 0)
if isinstance(m, nn.Linear) and m.out_features == 1:
nn.init.xavier_uniform_(m.weight, gain=nn.init.calculate_gain('sigmoid'))
if m.bias is not None:
nn.init.constant_(m.bias, 0)

def is_decrease(self,y,x,creat_graph=None):
dy_dx = torch.autograd.grad(y.sum(), x,create_graph=creat_graph)[0]
dec = torch.tensor(0.0, dtype=torch.float64, device='cuda')
if (dy_dx > 0).any() == True:
dec += torch.sum(dy_dx[dy_dx > 0])

if (dy_dx < -1).any() == True:
dec += torch.sum(torch.abs(dy_dx[dy_dx < -1]))

return dec




def train(self,x,y,num_epochs):

self.best_state = copy.deepcopy(self.model.state_dict())
self.opt_best_state = copy.deepcopy(self.opt.state_dict())
finished = False
eval_loss = []
dataset = TensorDataset(x, y)
g = torch.Generator()
g.manual_seed(42)
train_loader = DataLoader(dataset, batch_size=8, shuffle=True, generator=g)
for epoch in range(num_epochs):

loop = tqdm(train_loader, desc=f"Epoch {epoch+1}/{num_epochs}",leave=False)
for x, y in loop:
x = x.to('cuda')
y = y.to('cuda')

self.opt.zero_grad()
output = self.model(x)
panerty = self.is_decrease(output,x,creat_graph=True)
loss = self.loss_fn(output, y, panerty)
loss.backward(retain_graph=True)
self.opt.step()

loop.set_postfix(loss=loss.item())
self.scheduler.step()

train_model.py
​

data = torch.linspace(0, 90, steps=901, dtype=torch.float64)
data = data[:,None]
datax = torch.log(data+1).requires_grad_(True)
model = set.Model().double()
opt = torch.optim.Adam(model.parameters(), lr=0.001)
try:
trainer.train(datax,data,num_epochs=10**5)
finally:
torch.save( {'model_state_dict': trainer.best_state,
'optimizer_state_dict': trainer.opt_best_state,
'best_loss': trainer.best_loss}, './model.pth')


These are the codes that I think are necessary to reproduce the problem. The performance evaluation has already been passed somehow, but I want to solve this problem

Continue reading...