say full attention matrix size is N and block size is block_N
attention_W  = torch.zeros(N,N)
attention_W_diag1 = torch.randn(block_N,block_N).requires_grad_(True)
attention_W_diag2 = torch.randn(block_N,block_N).requires_grad_(True)
attention_W_off_diag = torch.randn(block_N,block_N).requires_grad_(True)
attention_W[:block_N,:block_N] = attention_W_diag1
attention_W[block_N:,block_N:] = attention_W_diag2
attention_W[block_N:,:block_N] = attention_W_off_diag
attention_W[:block_N,block_N:] = attention_W_off_diag