I guess the cast is related to this line of code 58 in interpolate, which was added in this PR 18 to fix tracing.
PS: I could run the code without modifying anything or building from source:
class AsppPooling(nn.Module):
    def __init__(self, in_channels, out_channels, norm_layer):
        super(AsppPooling, self).__init__()
        self.gap = nn.Sequential(nn.AdaptiveAvgPool2d(1),
                                 nn.Conv2d(in_channels, out_channels, 1, bias=False),
                                 norm_layer(out_channels),
                                 nn.ReLU(True))

    def forward(self, x):
        _, _, h, w = x.size()
        pool = self.gap(x)
        return F.interpolate(pool, (h,w))


model = AsppPooling(3, 3, nn.BatchNorm2d)
torch.onnx.export(model, torch.randn(2, 3, 2, 2), 'tmp.onnx', verbose=True)