I ended up using this code:
import torch
layers = [
    {"type": "Linear", "in_features": 10, "out_features": 1024},
    {"type": "ReLU"},
    {"type": "View", "shape": (-1, 16, 64)},
    {"type": "Conv1d", "in_channels": 16, "out_channels": 32, "kernel_size": 3},
    {"type": "ReLU"},
    {"type": "Conv1d", "in_channels": 32, "out_channels": 64, "kernel_size": 3},
    {"type": "ReLU"}
    # ...
    ]

class View(nn.Module):
    def __init__(self, shape):
        super(View, self).__init__()
        self.shape = shape
        
    def forward(self, x):
        return x.view(*self.shape)

def build_model(config):
    layer_list = []
    for layer_config in config:
        layer_type = layer_config.pop('type')
        if layer_type == "View":
            layer = View(**layer_config)
        else:
            layer = getattr(torch.nn, layer_type)(**layer_config)
        layer_list.append(layer)
    return nn.Sequential(*layer_list)

model = build_model(layers)

Unfortunately it is really not as clean as before