Hi Patrick, thanks. In fact you can implement a pointwise convolution directly in Linear, since a pointwise convolution is in fact just a Linear operation:
import time

import torch
from torch import nn



N = 128
seq_len = 5
embedding_size = 32

input = torch.rand(N, embedding_size, seq_len)

c1 = nn.Conv1d(embedding_size, embedding_size, kernel_size=1, padding=0, bias=False)
start_time = time.time()
for it in range(500):
    output = c1(input)
print('c1 time', time.time() - start_time)
print('output[0,:,0]', output[0,:,0])

# use linear...

c2 = nn.Linear(embedding_size, embedding_size, bias=False)
c2.weight.data[:] = c1.weight.data.view(embedding_size, embedding_size)
start_time = time.time()
for it in range(500):
    output = c2(input.transpose(-2, -1)).transpose(-1, -2)
print('c2 time', time.time() - start_time)
print('output[0,:,0]', output[0,:,0])

(Edit: note that I wrote this code after posting the question)