I do not know how to appreciate you. I was really frustrated before you help me.
I would appreciate it if you help me about this 2 new problems:
This is my code (input and its label is just a toy example):
# Besmelahel vahedel ghahar
# Khodaya be omide to

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim

class convZ(torch.autograd.Function):

    # Note that both forward and backward are @staticmethods
    @staticmethod
    # bias is an optional argument
    def forward(ctx, input, weight, bias=None):
        ctx.save_for_backward(input, weight, bias)
        output = torch.empty(len(input[1]) - 1)

        start_col_indx = 0
        end_col_indx = 2

        out_row_inx = 0
        out_col_indx = 0

        for i in range(len(input[1]) - 1):
            # print(torch.mm(input[:, start_col_indx:end_col_indx], weight))
            conv_mul = input[:, start_col_indx:end_col_indx] * weight
            start_col_indx += 1
            end_col_indx += 1
            conv_sum = torch.sum(conv_mul)
            output[out_col_indx] = conv_sum
            out_col_indx += 1

        # print(output)


        # print(weight.t())
        # output = input.mm(weight.t())
        # output = input + weight
        if bias is not None:
            output += bias.unsqueeze(0).expand_as(output)
        return output

    # This function has only a single output, so it gets only one gradient
    @staticmethod
    def backward(ctx, grad_output):
        # This is a pattern that is very convenient - at the top of backward
        # unpack saved_tensors and initialize all gradients w.r.t. inputs to
        # None. Thanks to the fact that additional trailing Nones are
        # ignored, the return statement is simple even when the function has
        # optional inputs.
        input, weight, bias = ctx.saved_tensors
        grad_input = grad_weight = grad_bias = None

        # These needs_input_grad checks are optional and there only to
        # improve efficiency. If you want to make your code simpler, you can
        # skip them. Returning gradients for inputs that don't require it is
        # not an error.
        if ctx.needs_input_grad[0]:
            grad_input = grad_output.mm(weight)
        if ctx.needs_input_grad[1]:
            grad_weight = grad_output.t().mm(input)
        if bias is not None and ctx.needs_input_grad[2]:
            grad_bias = grad_output.sum(0).squeeze(0)

        return grad_input, grad_weight, grad_bias


class MyConvZ(nn.Module):
    def __init__(self):
        super(MyConvZ, self).__init__()
        self.fn = convZ.apply
        # weight tensor = out_channels× in_channels/groups ×kH×kW
        self.weight = nn.Parameter(torch.randn(1, 1, 2, 2)) # when groups=1

    def forward(self, x):
        x = self.fn(x, self.weight)
        return x


class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.convZ = MyConvZ()
        self.pool = nn.AvgPool1d(2)

    def forward(self, x):
        x = F.relu(self.convZ(x))
        temp2 = []
        for i in range(len(x)):
            temp2.append(x[i])
        temp3 = [[temp2]]
        temp4 = torch.Tensor(temp3)
        x = self.pool(temp4)
        return x


if __name__ == '__main__':

    # input tensor = minibatch×in_channels×iH×iW
    input1 = torch.Tensor([[1, 0, 0, 1], [0, 1, 0, 1]]) #(1, 1, 2, 4)
    label1 = torch.LongTensor([0.])

    net = Net()

    # 3. Define a Loss function and optimizer
    criterion = nn.CrossEntropyLoss()
    optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)

    # zero the parameter gradients
    optimizer.zero_grad()

    # forward + backward + optimize
    outputs = net(input1)
    temp1 = []
    temp1.append(outputs[0])
    temp2 = torch.Tensor([temp1])
    loss = criterion(temp2, label1)
    loss.backward()
    optimizer.step()

    # print statistics
    running_loss = 0.0
    running_loss += loss.item()


print('Finished Training')



Since you and I make a custom layer, parameter of this layer customized, too, like. self.weight = nn.Parameter(torch.randn(1, 1, 2, 2))
If we have a next layer from standard pytorch after this layer, parameters would be considered automatically in addition to our customized parameter?


After running this program, I received this error:
RuntimeError: element 0 of tensors does not require grad and does not have a grad_fn


Many thanks before all