Hi @ptrblck, I guess @Jaideep_Valani is currently involved in the PANDA Kaggle competition. Just like him, I tried to implement a transfer learning algorithm for EfficientNetB0 (or B3 in his case).
My code can be found here: which is almost the same as the aforementioned but has some additional lines for completion:
class Model(nn.Module):
    def __init__(self, n=6, pre=True):
        super().__init__()
        
        # Load model backbone
        model = EfficientNet.from_pretrained('efficientnet-b0')
        
        # runs through the pretrained efficientnet, remove BN and FC layer
        self.enc = nn.Sequential(*list(model.children())[:-2])
        print(self.enc)
        
        # Replace the network transfer learning part with my own:
        nc = list(model.children())[-1].in_features
        self.head = nn.Sequential(AdaptiveConcatPool2d(),
                                  Flatten(),
                                  nn.Linear(2*nc,512), 
                                  Mish(),
                                  nn.BatchNorm1d(512), 
                                  nn.Dropout(0.5),
                                  nn.Linear(512,n))
        
        
    def forward(self, *x):
        #...

        x = self.enc(x)
       
        # ...
        
        x = self.head(x)
      
        return x

Strangely, I get an NotImplementedError at x = self.enc(x)
You can find the error message here:
---------------------------------------------------------------------------
NotImplementedError                       Traceback (most recent call last)
/opt/conda/lib/python3.6/site-packages/fastai/basic_train.py in fit(epochs, learn, callbacks, metrics)
    100                 xb, yb = cb_handler.on_batch_begin(xb, yb)
--> 101                 loss = loss_batch(learn.model, xb, yb, learn.loss_func, learn.opt, cb_handler)
    102                 if cb_handler.on_batch_end(loss): break

/opt/conda/lib/python3.6/site-packages/fastai/basic_train.py in loss_batch(model, xb, yb, loss_func, opt, cb_handler)
     25     if not is_listy(yb): yb = [yb]
---> 26     out = model(*xb)
     27     out = cb_handler.on_loss_begin(out)

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    531         else:
--> 532             result = self.forward(*input, **kwargs)
    533         for hook in self._forward_hooks.values():

<ipython-input-14-a64347871882> in forward(self, *x)
     29         
---> 30         x = self.enc(x)
     31 

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    531         else:
--> 532             result = self.forward(*input, **kwargs)
    533         for hook in self._forward_hooks.values():

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/container.py in forward(self, input)
     99         for module in self:
--> 100             input = module(input)
    101         return input

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    531         else:
--> 532             result = self.forward(*input, **kwargs)
    533         for hook in self._forward_hooks.values():

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py in forward(self, *input)
     95         """
---> 96         raise NotImplementedError
     97 

NotImplementedError: