You can have a look at my code snippet for generating CNN archs. It’s for 1-D stuff so you’ll need to just adapt accordingly. I’m mostly a java/c++ programmer so full disclaimer this stuff isn’t the most pythonic.
def get_random_arch_config(window_size: int,
                           param_budget: int,
                           trials_before_bailing: int, 
                           num_layers: int) -> list:
    """    
    :param window_size: 
        the L dimension of the input tensor, used to be sure the 
        candidate architectures don't pool too much and end up with 
        < 1 length outputs 
    :param param_budget: 
        the allowable number of estimated parameters to use in the configuration. 
        Candidate configurations with more than this budget are ignored and retried. 
    :param trials_before_bailing:
        how many random configurations to try, from which to
        choose the best configuration.
        If no random configs are acceptable, bail out and we'll
        use the default
    :return:
        a list of conv_config configurations for conv blocks, each its own list with the format:
             [out_channel_log2, kernel_size, pool_stride]
    """

    rand = Random()
    best_conv_configs = None
    best_budget = 1e7
    num_trials = -1
    remaining_budget = param_budget

    # find the best (in terms of using most parameters up to the budget limit)
    # of 'trials_before_bailing' random architectures
    for s in range(trials_before_bailing):
        layer_configs = []

        remaining_budget = param_budget
        in_chans = 1
        comp_input_size = window_size

        for layer_num in range(num_layers):
            kernel_size = round(1.5 ** rand.randint(0, 10))
            out_channels_log2 = rand.randint(4, 11)
            out_chans = 2 ** out_channels_log2
            stride = util.kernel_size_to_stride(kernel_size, comp_input_size)

            # final layer needs to do 1x1 convolutions and no more reduction
            if layer_num == num_layers - 1:
                kernel_size = 1
                stride = 1
                out_chans = window_size
                out_channels_log2 = round(math.log(window_size, 2))

            # enforces the constraint of ending up with input_size=1 by last layer
            if layer_num == num_layers - 2:
                stride = 1
                pool_stride = comp_input_size
            else:
                pool_stride = 1 if comp_input_size == 1 else 2 ** rand.randint(0, 2)

            padding = math.ceil(kernel_size - stride) // 2

            total_ops = util.calc_conv_ops(comp_input_size, in_chans, out_chans, kernel_size, stride)
            remaining_budget -= total_ops

            temp_input_size = comp_input_size
            comp_input_size = util.calc_output_size(comp_input_size, kernel_size, stride, pool_stride, padding)

            if comp_input_size <= 0 or remaining_budget < 0:
                break

            layer = {"in_ch": in_chans, "out_ch_log2": out_channels_log2, "in_size": temp_input_size,
                     "out_size": comp_input_size, "kernel_size": kernel_size, "pool_stride": pool_stride}

            layer_configs.append(layer)
            in_chans = out_chans

        if comp_input_size <= 0:
            continue

        if 0 < remaining_budget < best_budget:
            best_conv_configs = layer_configs
            best_budget = remaining_budget
            num_trials = s

        # return early if it's this optimal
        if 0 < remaining_budget < param_budget * 0.1:
            break

    conv_configs = []
    if best_conv_configs:
        for config in best_conv_configs:
            conv_configs.append([config["out_ch_log2"], config["kernel_size"], config["pool_stride"]])
    return conv_configs
My crossover function is simple but it might save you time:

def crossover_config(rand: Random, 
                     window_size: int, 
                     parent_1: dict, 
                     parent_2: dict, 
                     param_budget: int) -> list:
    viable = False
    new_conv_config = []
    cost = param_budget + 1
    tries = 0
    p1_conv_blocks: List[List] = parent_1["conv_config"]
    p2_conv_blocks: List[List] = parent_2["conv_config"]

    while cost > param_budget or not viable:
        new_conv_config = []
        sht = p1_conv_blocks if len(p1_conv_blocks) < len(p2_conv_blocks) else p2_conv_blocks
        lon = p1_conv_blocks if len(p1_conv_blocks) >= len(p2_conv_blocks) else p2_conv_blocks

        for i in range(len(sht)):
            source = p1_conv_blocks[i] if rand.random() < 0.5 else p2_conv_blocks[i]
            new_conv_config.append(copy.deepcopy(source))


        # give the child the average depth of the parents        
        for i in range((len(lon) - len(sht)) // 2):
            new_conv_config.append(copy.deepcopy(lon[i]))

        cost, viable = util.calc_config_param_cost_and_viability(window_size, new_conv_config)

        # bail and choose the first parent's config
        if not viable and tries > 100:
            new_conv_config = copy.deepcopy(p1_conv_blocks)
            break

        tries += 1
    return new_conv_config
Some ideas:

make a utility function for checking the validity of your network, and use this whenever you mutate / crossover to check that the newly created architecture is still viable.
You probably want to write a hash function for your architecture config to check whether it’s the same as another one in your population (if it is, you’ll just duplicate effort in running it).
Inherit conv layer’s input channels param from the output channels param of the previous conv layer (you definitely don’t want to randomize that or only a small fraction of architectures will be self consistent)
Make liberal use of copy.deepcopy(xyz) when mutating / doing crossover – you don’t want to be changing other configurations by accident.

Make your own mutate operators like

delete conv-block
duplicate last block
random adjustment to kernel size, pool stride, out channels

Also, if you come up with good improvements – I’d like to know! I had trouble generating configurations longer than, say 14 blocks deep, due to the sparsity of viable random configurations. I considered adding some notion of “ambition” as the layers are generated, reducing this quantity if large amounts of the param budget were used early.