I had to use a specific size image and resize/scale was not an option. So, I used a code like:
class CustomInceptionV3(models.Inception3):
def init(self, model_orig, num_classes):
super(CustomInceptionV3, self).init()
num_feats = model_orig.fc.in_features
self.fc = nn.Linear(num_feats, num_classes)
self.aux_logits = model_orig.aux_logits
    self.Conv2d_1a_3x3 = model_orig.Conv2d_1a_3x3
    self.Conv2d_2a_3x3 = model_orig.Conv2d_2a_3x3
    self.Conv2d_2b_3x3 = model_orig.Conv2d_2b_3x3

    self.Conv2d_3b_1x1 = model_orig.Conv2d_3b_1x1
    self.Conv2d_4a_3x3 = model_orig.Conv2d_4a_3x3

    self.Mixed_5b = model_orig.Mixed_5b
    self.Mixed_5c = model_orig.Mixed_5c
    self.Mixed_5d = model_orig.Mixed_5d
    self.Mixed_6a = model_orig.Mixed_6a
    self.Mixed_6b = model_orig.Mixed_6b
    self.Mixed_6c = model_orig.Mixed_6c
    self.Mixed_6d = model_orig.Mixed_6d
    self.Mixed_6e = model_orig.Mixed_6e
    self.Mixed_7a = model_orig.Mixed_7a
    self.Mixed_7b = model_orig.Mixed_7b
    self.Mixed_7c = model_orig.Mixed_7c

def forward(self, x):

    if self.transform_input:
        x = x.clone()
        x[:, 0] = x[:, 0] * (0.229 / 0.5) + (0.485 - 0.5) / 0.5
        x[:, 1] = x[:, 1] * (0.224 / 0.5) + (0.456 - 0.5) / 0.5
        x[:, 2] = x[:, 2] * (0.225 / 0.5) + (0.406 - 0.5) / 0.5
    x = self.Conv2d_1a_3x3(x)
    x = self.Conv2d_2a_3x3(x)
    x = self.Conv2d_2b_3x3(x)
    x = F.max_pool2d(x, kernel_size=3, stride=2)
    x = self.Conv2d_3b_1x1(x)
    x = self.Conv2d_4a_3x3(x)
    x = F.max_pool2d(x, kernel_size=3, stride=2)
    x = self.Mixed_5b(x)
    x = self.Mixed_5c(x)
    x = self.Mixed_5d(x)
    x = self.Mixed_6a(x)
    x = self.Mixed_6b(x)
    x = self.Mixed_6c(x)
    x = self.Mixed_6d(x)
    x = self.Mixed_6e(x)
    if self.training and self.aux_logits:
        aux = self.AuxLogits(x)
    x = self.Mixed_7a(x)
    x = self.Mixed_7b(x)
    x = self.Mixed_7c(x)
    x = F.adaptive_avg_pool2d(x, 1)
    x = x.view(x.size(0), -1)
    x = self.fc(x)
    if self.training and self.aux_logits:
        return x, aux
    return x

class CustomInceptionAux(nn.Module):
def __init__(self, in_channels, num_classes):
    super(CustomInceptionAux, self).__init__()
    self.conv0 = BasicConv2d(in_channels, 128, kernel_size=1)
    self.conv1 = BasicConv2d(128, 768, kernel_size=5)
    self.conv1.stddev = 0.01
    num_feats = 768
    self.fc = nn.Linear(num_feats, num_classes)

def forward(self, x):
    x = F.adaptive_avg_pool2d(x, 5)
    x = self.conv0(x)
    x = self.conv1(x)
    x = x.view(x.size(0), -1)
    x = self.fc(x)
    return x

class BasicConv2d(nn.Module):
def __init__(self, in_channels, out_channels, **kwargs):
    super(BasicConv2d, self).__init__()
    self.conv = nn.Conv2d(in_channels, out_channels, bias=False, **kwargs)
    self.bn = nn.BatchNorm2d(out_channels, eps=0.001)

def forward(self, x):
    x = self.conv(x)
    x = self.bn(x)
    return F.relu(x, inplace=True)

model_orig = torchvision.models.inception_v3(pretrained=True)
model = CustomInceptionV3(model_orig, num_classes=4)
model.AuxLogits = CustomInceptionAux(768, 4)
model.AuxLogits.conv0.conv.weight.data = model_orig.AuxLogits.conv0.conv.weight.data
model.AuxLogits.conv0.bn.data = model_orig.AuxLogits.conv0.bn.weight.data
model.AuxLogits.conv1.conv.weight.data = model_orig.AuxLogits.conv1.conv.weight.data
model.AuxLogits.conv1.bn.data = model_orig.AuxLogits.conv1.bn.weight.data
model.eval()