Summary

.gitignore

@@ -0,0 +1,171 @@
+# Compiled source #
+###################
+*.com
+*.class
+*.dll
+*.exe
+*.o
+*.so
+*.pyc
+
+# Packages #
+############
+# it's better to unpack these files and commit the raw source
+# git has its own built in compression methods
+*.7z
+*.dmg
+*.gz
+*.iso
+*.rar
+#*.tar
+*.zip
+
+# Logs and databases #
+######################
+*.log
+*.sqlite
+
+# OS generated files #
+######################
+.DS_Store
+ehthumbs.db
+Icon
+Thumbs.db
+.tmtags
+.idea
+.vscode
+tags
+vendor.tags
+tmtagsHistory
+*.sublime-project
+*.sublime-workspace
+.bundle
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+node_modules/
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# Custom
+*.mp4

FER_dinamic_LSTM_Aff-Wild2.pth → FER_dinamic_LSTM_Aff-Wild2.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e053fa829d3cf550d4d3372338b4a96e79a2a016fa8e04b218a83e436a439331
-size 11591051
+oid sha256:21b0cabebe7bd0257ca8aaa991efc7546c6f46fd4d17f759d33abbb859abdacc
+size 11569812

FER_dinamic_LSTM_CREMA-D.pth → FER_dinamic_LSTM_CREMA-D.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1fb0aa416759d17e2f58b31b757a6ee9fae1e5749c7382598824533a97a49150
-size 11590417
+oid sha256:5486a9b4816fb86c4fcbbbc7e8b6506c9e66fc6db25404ed492da119330b86ee
+size 11569208

FER_dinamic_LSTM_IEMOCAP.pth → FER_dinamic_LSTM_IEMOCAP.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4efe957cd4d3ffac0f283528dc7c0631b2cad9958c7c244555b43faaccfa84d0
-size 11590417
+oid sha256:0cd1561a72f9de26c315bb857f03e8946635db047e0dbea52bb0276610f19751
+size 11569208

FER_dinamic_LSTM_RAMAS.pth → FER_dinamic_LSTM_RAMAS.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:adb2d8325ecd266c9f5cb1bab5fa02758e9c7a90a719205ccdd6531a65fa8697
-size 11590359
+oid sha256:ba1a49032311f91580eff67732bbb0a7077f1382c8a65e5d0fca01b1ad09ba37
+size 11569180

FER_dinamic_LSTM_RAVDESS.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b8eb7e702d4be62bba48dd54addd53698c95fd94ff8293fb53fd8d59ab22248
+size 11569208

FER_dinamic_LSTM_RAVDESS.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:43bc117590334f5f64465d3dd80c894baafe80b83911959cf403cba41d2bbf54
-size 11590417

FER_dinamic_LSTM_SAVEE.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa651fe5a937837610dea81fdf4e0079e1ebda07f28657007bcbc985faf25fc5
+size 11569180

FER_dinamic_LSTM_SAVEE.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:c7b89232ffa9fefaeaca64f3d8dc6271065f8b6cd56fe5a32e76bc93a8138669
-size 11590359

FER_static_ResNet50_AffectNet.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8274190b5be4355bd2f07b59f593fcdb294f9d7c563bfa9ac9e5ea06c10692d2
+size 98562934

FER_static_ResNet50_AffectNet.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:790f76fe4d443953b3b71a6899bdd981742e8e1b954da48483d3eea6c3c717a1
-size 98631726

run_webcam.ipynb

@@ -17,10 +17,167 @@
     "warnings.simplefilter(\"ignore\", UserWarning)\n",
     "\n",
     "import torch\n",
+    "import torch.nn as  nn\n",
+    "import torch.nn.functional as F\n",
     "from PIL import Image\n",
     "from torchvision import transforms"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "a0907155",
+   "metadata": {},
+   "source": [
+    "#### Model architectures"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f67038e3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class Bottleneck(nn.Module):\n",
+    "    expansion = 4\n",
+    "    def __init__(self, in_channels, out_channels, i_downsample=None, stride=1):\n",
+    "        super(Bottleneck, self).__init__()\n",
+    "        \n",
+    "        self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=stride, padding=0, bias=False)\n",
+    "        self.batch_norm1 = nn.BatchNorm2d(out_channels, eps=0.001, momentum=0.99)\n",
+    "        \n",
+    "        self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, padding='same', bias=False)\n",
+    "        self.batch_norm2 = nn.BatchNorm2d(out_channels, eps=0.001, momentum=0.99)\n",
+    "        \n",
+    "        self.conv3 = nn.Conv2d(out_channels, out_channels*self.expansion, kernel_size=1, stride=1, padding=0, bias=False)\n",
+    "        self.batch_norm3 = nn.BatchNorm2d(out_channels*self.expansion, eps=0.001, momentum=0.99)\n",
+    "        \n",
+    "        self.i_downsample = i_downsample\n",
+    "        self.stride = stride\n",
+    "        self.relu = nn.ReLU()\n",
+    "        \n",
+    "    def forward(self, x):\n",
+    "        identity = x.clone()\n",
+    "        x = self.relu(self.batch_norm1(self.conv1(x)))\n",
+    "        \n",
+    "        x = self.relu(self.batch_norm2(self.conv2(x)))\n",
+    "        \n",
+    "        x = self.conv3(x)\n",
+    "        x = self.batch_norm3(x)\n",
+    "        \n",
+    "        #downsample if needed\n",
+    "        if self.i_downsample is not None:\n",
+    "            identity = self.i_downsample(identity)\n",
+    "        #add identity\n",
+    "        x+=identity\n",
+    "        x=self.relu(x)\n",
+    "        \n",
+    "        return x\n",
+    "\n",
+    "class Conv2dSame(torch.nn.Conv2d):\n",
+    "\n",
+    "    def calc_same_pad(self, i: int, k: int, s: int, d: int) -> int:\n",
+    "        return max((math.ceil(i / s) - 1) * s + (k - 1) * d + 1 - i, 0)\n",
+    "\n",
+    "    def forward(self, x: torch.Tensor) -> torch.Tensor:\n",
+    "        ih, iw = x.size()[-2:]\n",
+    "\n",
+    "        pad_h = self.calc_same_pad(i=ih, k=self.kernel_size[0], s=self.stride[0], d=self.dilation[0])\n",
+    "        pad_w = self.calc_same_pad(i=iw, k=self.kernel_size[1], s=self.stride[1], d=self.dilation[1])\n",
+    "\n",
+    "        if pad_h > 0 or pad_w > 0:\n",
+    "            x = F.pad(\n",
+    "                x, [pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2]\n",
+    "            )\n",
+    "        return F.conv2d(\n",
+    "            x,\n",
+    "            self.weight,\n",
+    "            self.bias,\n",
+    "            self.stride,\n",
+    "            self.padding,\n",
+    "            self.dilation,\n",
+    "            self.groups,\n",
+    "        )\n",
+    "\n",
+    "class ResNet(nn.Module):\n",
+    "    def __init__(self, ResBlock, layer_list, num_classes, num_channels=3):\n",
+    "        super(ResNet, self).__init__()\n",
+    "        self.in_channels = 64\n",
+    "\n",
+    "        self.conv_layer_s2_same = Conv2dSame(num_channels, 64, 7, stride=2, groups=1, bias=False)\n",
+    "        self.batch_norm1 = nn.BatchNorm2d(64, eps=0.001, momentum=0.99)\n",
+    "        self.relu = nn.ReLU()\n",
+    "        self.max_pool = nn.MaxPool2d(kernel_size = 3, stride=2)\n",
+    "        \n",
+    "        self.layer1 = self._make_layer(ResBlock, layer_list[0], planes=64, stride=1)\n",
+    "        self.layer2 = self._make_layer(ResBlock, layer_list[1], planes=128, stride=2)\n",
+    "        self.layer3 = self._make_layer(ResBlock, layer_list[2], planes=256, stride=2)\n",
+    "        self.layer4 = self._make_layer(ResBlock, layer_list[3], planes=512, stride=2)\n",
+    "        \n",
+    "        self.avgpool = nn.AdaptiveAvgPool2d((1,1))\n",
+    "        self.fc1 = nn.Linear(512*ResBlock.expansion, 512)\n",
+    "        self.relu1 = nn.ReLU()\n",
+    "        self.fc2 = nn.Linear(512, num_classes)\n",
+    "\n",
+    "    def extract_features(self, x):\n",
+    "        x = self.relu(self.batch_norm1(self.conv_layer_s2_same(x)))\n",
+    "        x = self.max_pool(x)\n",
+    "        # print(x.shape)\n",
+    "        x = self.layer1(x)\n",
+    "        x = self.layer2(x)\n",
+    "        x = self.layer3(x)\n",
+    "        x = self.layer4(x)\n",
+    "        \n",
+    "        x = self.avgpool(x)\n",
+    "        x = x.reshape(x.shape[0], -1)\n",
+    "        x = self.fc1(x)\n",
+    "        return x\n",
+    "        \n",
+    "    def forward(self, x):\n",
+    "        x = self.extract_features(x)\n",
+    "        x = self.relu1(x)\n",
+    "        x = self.fc2(x)\n",
+    "        return x\n",
+    "        \n",
+    "    def _make_layer(self, ResBlock, blocks, planes, stride=1):\n",
+    "        ii_downsample = None\n",
+    "        layers = []\n",
+    "        \n",
+    "        if stride != 1 or self.in_channels != planes*ResBlock.expansion:\n",
+    "            ii_downsample = nn.Sequential(\n",
+    "                nn.Conv2d(self.in_channels, planes*ResBlock.expansion, kernel_size=1, stride=stride, bias=False, padding=0),\n",
+    "                nn.BatchNorm2d(planes*ResBlock.expansion, eps=0.001, momentum=0.99)\n",
+    "            )\n",
+    "            \n",
+    "        layers.append(ResBlock(self.in_channels, planes, i_downsample=ii_downsample, stride=stride))\n",
+    "        self.in_channels = planes*ResBlock.expansion\n",
+    "        \n",
+    "        for i in range(blocks-1):\n",
+    "            layers.append(ResBlock(self.in_channels, planes))\n",
+    "            \n",
+    "        return nn.Sequential(*layers)\n",
+    "        \n",
+    "def ResNet50(num_classes, channels=3):\n",
+    "    return ResNet(Bottleneck, [3,4,6,3], num_classes, channels)\n",
+    "\n",
+    "\n",
+    "class LSTMPyTorch(nn.Module):\n",
+    "    def __init__(self):\n",
+    "        super(LSTMPyTorch, self).__init__()\n",
+    "        \n",
+    "        self.lstm1 = nn.LSTM(input_size=512, hidden_size=512, batch_first=True, bidirectional=False)\n",
+    "        self.lstm2 = nn.LSTM(input_size=512, hidden_size=256, batch_first=True, bidirectional=False)\n",
+    "        self.fc = nn.Linear(256, 7)\n",
+    "        self.softmax = nn.Softmax(dim=1)\n",
+    "\n",
+    "    def forward(self, x):\n",
+    "        x, _ = self.lstm1(x)\n",
+    "        x, _ = self.lstm2(x)        \n",
+    "        x = self.fc(x[:, -1, :])\n",
+    "        x = self.softmax(x)\n",
+    "        return x"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "fcbcf9fa-a7cc-4d4c-b723-6d7efd49b94b",
@@ -177,7 +334,7 @@
    "source": [
     "mp_face_mesh = mp.solutions.face_mesh\n",
     "\n",
-    "name_backbone_model = 'FER_static_ResNet50_AffectNet.pth'\n",
+    "name_backbone_model = 'FER_static_ResNet50_AffectNet.pt'\n",
     "# name_LSTM_model = 'IEMOCAP'\n",
     "# name_LSTM_model = 'CREMA-D'\n",
     "# name_LSTM_model = 'RAMAS'\n",
@@ -186,12 +343,16 @@
     "name_LSTM_model = 'Aff-Wild2'\n",
     "\n",
     "# torch\n",
-    "pth_backbone_model = torch.jit.load(name_backbone_model)\n",
+    "\n",
+    "pth_backbone_model = ResNet50(7, channels=3)\n",
+    "pth_backbone_model.load_state_dict(torch.load(name_backbone_model))\n",
     "pth_backbone_model.eval()\n",
     "\n",
-    "pth_LSTM_model = torch.jit.load('FER_dinamic_LSTM_{0}.pth'.format(name_LSTM_model))\n",
+    "pth_LSTM_model = LSTMPyTorch()\n",
+    "pth_LSTM_model.load_state_dict(torch.load('FER_dinamic_LSTM_{0}.pt'.format(name_LSTM_model)))\n",
     "pth_LSTM_model.eval()\n",
     "\n",
+    "\n",
     "DICT_EMO = {0: 'Neutral', 1: 'Happiness', 2: 'Sadness', 3: 'Surprise', 4: 'Fear', 5: 'Disgust', 6: 'Anger'}\n",
     "\n",
     "cap = cv2.VideoCapture(0)\n",
@@ -220,7 +381,7 @@
     "        frame_copy = cv2.cvtColor(frame_copy, cv2.COLOR_BGR2RGB)\n",
     "        results = face_mesh.process(frame_copy)\n",
     "        frame_copy.flags.writeable = True\n",
-    "         \n",
+    "\n",
     "        if results.multi_face_landmarks:\n",
     "            for fl in results.multi_face_landmarks:\n",
     "                startX, startY, endX, endY  = get_box(fl, w, h)\n",