Well, I did some tests and select() does not seem so cheap, here is what I tested:
from torch.nn.functional import relu
import torch

def complex_relu(input_r,input_i):
    return relu(input_r), relu(input_i)

def complex_relu2(complex_input):
    return torch.stack((relu(complex_input[...,0]), relu(complex_input[...,1])), dim = -1)


def complex_relu3(complex_input):
    complex_input[...,0] = relu(complex_input.select(-1, 0))
    complex_input[...,1] = relu(complex_input.select(-1, 1))
    return complex_input

def complex_relu4(complex_input):
    return torch.stack((relu(complex_input.select(-1, 0)), relu(complex_input[...,1])), dim = -1)

device = torch.device("cuda:0" )

n = 1000
X_r = torch.randn(n,n).to(device)
X_i = torch.randn(n,n).to(device)
X = torch.stack((X_r,X_i),dim=-1).to(device)

%timeit complex_relu(X_r,X_i)

33.1 µs ± 6.04 ns per loop (mean ± std. dev. of 7 runs, 10000 loops each)
%timeit complex_relu2(X)

148 µs ± 32.6 ns per loop (mean ± std. dev. of 7 runs, 10000 loops each)
%timeit complex_relu3(X)

130 µs ± 12.5 ns per loop (mean ± std. dev. of 7 runs, 10000 loops each)
%timeit complex_relu4(X)

148 µs ± 33.1 ns per loop (mean ± std. dev. of 7 runs, 10000 loops each)
So, any thoughts on how to have the two layer system to be as fast as the two separate arguments?