I define a simple network and train it.
import numpy as np
import matplotlib.pyplot as plt
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.optim.lr_scheduler import MultiStepLR
from torch.utils.data import random_split, Dataset, DataLoader


class BaseNet(nn.Module):
    def __init__(self):
        super().__init__()
        self.net = nn.Sequential(
            nn.BatchNorm1d(num_features=1, momentum=0.1, eps=2e-5),
            nn.Conv1d(in_channels=1, out_channels=4, kernel_size=1, bias=False),
            nn.BatchNorm1d(num_features=4, momentum=0.1, eps=2e-5),
            nn.PReLU(num_parameters=4),
            nn.Conv1d(in_channels=4, out_channels=16, kernel_size=3, bias=False, padding=1),
            nn.BatchNorm1d(num_features=16, momentum=0.1, eps=2e-5),
            nn.PReLU(num_parameters=16),
            nn.Conv1d(in_channels=16, out_channels=4, kernel_size=3, bias=False, padding=1),
            nn.BatchNorm1d(num_features=4, momentum=0.1, eps=2e-5),
            nn.PReLU(num_parameters=4),
            nn.Conv1d(in_channels=4, out_channels=1, kernel_size=1, bias=False),
        )

    def forward(self, x):
        return self.net(x)


class DataSet(Dataset):
    def __init__(self, x, y):
        super().__init__()
        self.x = torch.tensor(x).float()
        self.y = torch.tensor(y).float()

    def __len__(self):
        return self.x.shape[0]

    def __getitem__(self, item):
        return self.x[item], self.y[item]


def train(train_data, val_data):
    def ridiculous_val(n, v):
        with torch.no_grad():
            n = n.eval()
            m = list()
            for a, b in v:
                p = n(a.cuda())
                m.append(F.l1_loss(p, b.cuda()).item())
            return np.mean(m)

    net = BaseNet()
    net = net.cuda()
    opt = optim.SGD(net.parameters(), lr=0.1, momentum=0.9, weight_decay=0.9)
    scheduler = MultiStepLR(opt, [500, 800, 1100, 1400, 1700], gamma=0.1)
    loss = nn.L1Loss()
    min_loss = np.float("inf")
    n_batch = 0
    net = net.train()
    for epoch in range(0, 99999):
        for x, y in train_data:
            x, y = x.cuda(), y.cuda()

            opt.zero_grad()
            pred = net(x)
            loss_val = loss(y, pred)
            loss_val.backward()
            opt.step()
            print("[{}],  {}".format(epoch, loss_val.item()))
            n_batch += 1

            if n_batch % 100 == 0:
                _loss = ridiculous_val(net, val_data)
                print("min ", _loss, min_loss)
                if _loss < min_loss:
                    min_loss = _loss
                    print("saving...", min_loss)
                    torch.save(net.state_dict(), "net.pth")
                net = net.train()
        scheduler.step(epoch)


def val(some_data):
    net = BaseNet()
    net = net.cuda()
    state_dict = torch.load("net.pth")
    net.load_state_dict(state_dict)
    pred = np.empty(some_data.shape)
    for i, x in enumerate(some_data):
        pred[i, ...] = net(torch.from_numpy(x[None, :, :]).float().cuda()).cpu().detach().numpy().squeeze()
    return pred


if __name__ == '__main__':
    dst = np.load("diff.npy")  # shape (12000, 512)
    src = np.load("src.npy")  # shape (12000, 512)
    src, dst = src[:, None, :], dst[:, None, :]

    data = DataSet(src, dst)
    train_len, val_len = int(len(data) * 0.5), int(len(data) * 0.2)
    test_len = len(data) - train_len - val_len
    train_set, val_set, test_set = random_split(data, [train_len, val_len, test_len])
    train_set = DataLoader(train_set, batch_size=1024, shuffle=True, pin_memory=True, drop_last=False)
    val_set = DataLoader(val_set, batch_size=1024, shuffle=False, pin_memory=True, drop_last=False)

    train(train_set, val_set)

I observed that
epoch 33
[param for param in net.parameters()][2].abs().max() is 1e-6
[param for param in net.parameters()][2].grad.abs().max() is 7e-6.
loss is 0.263
epoch 67
[param for param in net.parameters()][2].abs().max() is 6e-11
[param for param in net.parameters()][2].grad.abs().max() is 2e-11
loss is 0.264
epoch 134
[param for param in net.parameters()][2].abs().max() is 3e-20
[param for param in net.parameters()][2].grad.abs().max() is 2e-20
loss is 0.264
After epoch 324, both of parameters and grad are 0, loss is 0.263.
Does it mean that this network think that function f(x) = 0 is the best mapping function for my data?
Or I do something wrong?
Here are two examples of my data, specifically, they are the output of another image encode network.
 
Maybe this is a stupid question, but I am really looking forward to getting some suggestions, thanks.