I don’t think the problem is with ReLU layers as such,
If we will execute the below,

import torch
import torch.nn as nn

class SimpleTwoTower(nn.Module):
    
    def __init__(self, n_items, n_users, ln):
        super(SimpleTwoTower, self).__init__()
        
        self.ln = ln
        self.item_emb = nn.Embedding(num_embeddings=n_items, embedding_dim=self.ln[0])
        self.user_emb = nn.Embedding(num_embeddings=n_users, embedding_dim=self.ln[0])
       
        
        self.item_layers = [] #nn.ModuleList()
        self.user_layers = [] #nn.ModuleList()
        
        for i, n in enumerate(ln[0:-1]):
            m = int(ln[i+1])
            self.item_layers.append(nn.Linear(n, m, bias=True))
            self.item_layers.append(nn.ReLU())
            
            self.user_layers.append(nn.Linear(n, m, bias=True))
            self.user_layers.append(nn.ReLU())
            
            
        self.item_layers = nn.Sequential(*self.item_layers)
        self.user_layers = nn.Sequential(*self.user_layers)
        
        self.dot = torch.matmul
        self.sigmoid = nn.Sigmoid()

    def forward(self, items, users):
        
        item_emb = self.item_emb(items)
        user_emb = self.user_emb(users)
        
        item_emb = self.item_layers(item_emb)
        user_emb = self.user_layers(user_emb)

        dp = self.dot(user_emb, item_emb.t())
        return self.sigmoid(dp)


# let's create our model here using the snippet as-is
SimpleTwoTower(10000, 1000,  [2**10, 2**8, 2**6])

"""
# output will be something like this

SimpleTwoTower(
  (item_emb): Embedding(10000, 1024)
  (user_emb): Embedding(1000, 1024)
  (item_layers): Sequential(
    (0): Linear(in_features=1024, out_features=256, bias=True)
    (1): ReLU()
    (2): Linear(in_features=256, out_features=64, bias=True)
    (3): ReLU() **#### <<<<<<< isn't this an issue?**
  )
  (user_layers): Sequential(
    (0): Linear(in_features=1024, out_features=256, bias=True)
    (1): ReLU()
    (2): Linear(in_features=256, out_features=64, bias=True)
    (3): ReLU() **#### <<<<<<< isn't this an issue?**
  )
  (sigmoid): Sigmoid()
)
"""

So, Do we really want to have that ReLU as well? From what i know about the Two-tower model, we will use the raw_output from the last linear layer as the feature vector (i.e. latent vec etc)
Also, Do we want to use embedding layers as well?
cc @ptrblck @Mimi_Lazarova
Please correct my understanding!
Be Safe,
Best,
Aditya.