add application

flaskr/Dockerfile

@@ -0,0 +1,33 @@
+# Use a Debian-based Python image
+FROM python:3.8-slim
+
+# Update packages and install the necessary system dependencies
+RUN apt-get update && apt-get install -y \
+    git \
+    gcc \
+    libhdf5-dev \
+    pkg-config \
+    g++ \
+    libgl1-mesa-glx \  
+    libglib2.0-0 \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install PyTorch and torchvision (or other packages as needed)
+RUN pip install torch torchvision
+
+# Set the working directory in the container
+WORKDIR /app
+
+# Copy your application code to the container
+COPY app/ ./
+COPY requirements.txt requirements.txt
+
+# Install Python dependencies from requirements.txt
+RUN pip install -r requirements.txt
+
+
+
+# Define the command to run your app using CMD which should be overridden when using the container with different commands
+CMD ["python", "app.py", "--address", "0.0.0.0", "--port", "7860", "--allow-websocket-origin", "ddnb338-lumos-app.hf.space"]
+
+

flaskr/app/app.py

@@ -0,0 +1,78 @@
+import os
+import base64
+from flask import Flask, render_template,request
+import config
+#local imports
+from detection_model_run import run_detection
+from helper import preprocess_keypoints
+from classification_model_run import run_classification
+from generate_light import  generate_new_image
+from show_points import display_keypoints
+
+app = Flask(__name__)
+app.config.from_object(config)
+
+UPLOAD_FOLDER = 'captures'  # Define the directory to save uploaded images
+
+# Ensure the upload directory exists
+if not os.path.exists(UPLOAD_FOLDER):
+    os.makedirs(UPLOAD_FOLDER)
+
+app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
+@app.route('/')
+def index():
+    return render_template("index.html")
+
+
+def run_model_evaluation(image_path, useGan=False, imageID=0):
+    # run keypoint detection on image from camera 
+   
+    if useGan:
+        new_image_path = generate_new_image(image_path, app.config['GAN_MODEL_WEIGHTS_PATH'], imageID = imageID)
+        keypoints= run_detection(new_image_path, app.config['POSE_MODEL_WEIGHTS_PATH_GAN'])
+        if isinstance(keypoints, str):
+            display_keypoints(keypoints, ganImage = True, imageID = imageID)
+            return 'No keypoints detected'
+        display_keypoints(keypoints, ganImage = True, imageID = imageID)
+    else:
+        keypoints = run_detection(image_path, app.config['POSE_MODEL_WEIGHTS_PATH_NOGAN'])
+        if isinstance(keypoints, str):
+            display_keypoints(keypoints, ganImage = False, imageID = imageID)
+            return 'No keypoints detected'
+        display_keypoints(keypoints, ganImage = False, imageID = imageID)
+    #preprocess the keypoints for classification
+    input_array = preprocess_keypoints(keypoints)
+    predicted_class = run_classification(input_array)
+    categoryOrder = ['basketball', 'bowling', 'boxing', 'football', 'golf', 'hacky sack',
+       'rowing, stationary', 'skateboarding', 'skiing, downhill', 'soccer',
+       'softball, general',
+       'tennis, hitting balls, non-game play, moderate effort']
+
+    return categoryOrder[predicted_class]
+   
+
+@app.route('/upload', methods=['POST'])
+def upload():
+    data_url = request.json.get('image_data')
+    useGAN = request.json.get('use_model_gan')
+    imageID = request.json.get('unique_ID')
+    if data_url:
+        # Remove header from base64 encoded image
+        img_data = data_url.split(',')[1]
+        
+        # Save the image to a file
+        image_path = os.path.join(app.config['UPLOAD_FOLDER'], f'image_{imageID}.png')
+        with open( image_path,'wb') as f:
+            f.write(base64.b64decode(img_data))
+
+        if useGAN == False:
+            answer = run_model_evaluation(image_path, useGan = False, imageID = imageID)
+        else:
+            answer = run_model_evaluation(image_path, useGan = True, imageID= imageID)
+        return answer
+    return 'No image data received.'
+
+
+if __name__=="__main__":
+    app.run(debug = True, port=5001, host = '0.0.0.0')
+

flaskr/app/classification_model_run.py

@@ -0,0 +1,45 @@
+import torch
+from torch.utils.data import Dataset, DataLoader
+device = 'cuda' if torch.cuda.is_available() else 'cpu'
+import torch.nn as nn
+import torch.nn.functional as F
+import numpy as np
+
+class PoseNet(nn.Module):
+  def __init__(self, input_size, hidden_size_1, num_classes):
+    super().__init__()
+    self.layer1 = nn.Linear(input_size, hidden_size_1)
+    self.relu = nn.ReLU()
+    self.layer2 = nn.Linear(hidden_size_1, num_classes)
+    self.dropout = nn.Dropout(p=0.3)
+    self.bn1 = nn.BatchNorm1d(100)
+    self.bn2 = nn.BatchNorm1d(num_classes)
+  def forward(self, x):
+    x = self.layer1(x)
+    x = self.relu(x)
+    x = self.dropout(x)
+    x = self.layer2(x)
+    return x
+
+def run_classification(input_array):
+    # Load the model architecture
+    model = PoseNet(32, 120, 12) #32, 120, 12
+    
+    # Load the saved weights
+    model.load_state_dict(torch.load('final_models/finalClassification.pth'))  #changed
+
+    input_tensor = torch.from_numpy(input_array).to(device)
+    input_tensor = input_tensor.unsqueeze(0)  # Add batch dimension
+
+    # Set the model to evaluation mode
+    model.eval()
+
+    # Perform prediction
+    with torch.no_grad():
+        output = model.forward(input_tensor)
+        probabilities = F.softmax(output, dim=-1)
+        print("probabilities", probabilities)
+        #predicted_class = torch.argmax(probabilities, dim=1).item()
+        predicted_class = np.argmax(probabilities.cpu().numpy(),axis=-1)
+
+    return predicted_class[0][0]

flaskr/app/config.py

@@ -0,0 +1,3 @@
+POSE_MODEL_WEIGHTS_PATH_GAN = 'final_models/finalPose_vGAN_v2.pth'
+POSE_MODEL_WEIGHTS_PATH_NOGAN = 'final_models/finalPose.pth'
+GAN_MODEL_WEIGHTS_PATH = 'final_models/gen_model_v2.h5'

flaskr/app/detection_model_run.py

@@ -0,0 +1,37 @@
+from detectron2.config import get_cfg
+from detectron2.engine import DefaultPredictor
+import cv2
+from detectron2 import model_zoo
+
+def build_config(weights_path):
+    cfg = get_cfg()
+    # Update the config with the model weights
+    cfg.merge_from_file(model_zoo.get_config_file("COCO-Keypoints/keypoint_rcnn_R_50_FPN_3x.yaml"))
+    cfg.MODEL.WEIGHTS = weights_path 
+    cfg.MODEL.ROI_KEYPOINT_HEAD.NUM_KEYPOINTS = 16
+    cfg.MODEL.DEVICE = 'cpu'
+    cfg.MODEL.ROI_HEADS.NUM_CLASSES = 1  # person
+    cfg.MODEL.RETINANET.NUM_CLASSES = 1
+
+    return cfg
+
+
+def run_detection(image_path, weights_path):
+    cfg = build_config(weights_path)
+    # Create the predictor
+    predictor = DefaultPredictor(cfg)
+    image = cv2.imread(image_path)
+    # Perform prediction
+    outputs = predictor(image)
+    
+    instances = outputs["instances"]
+    #Get the index of the bounding box with the highest confidence score
+    if len(instances) == 0:
+        return 'no keypoints'
+    else:
+        highest_confidence_index = instances.scores.argmax()
+
+    # Retrieve the keypoints associated with the highest confidence bounding box
+    highest_confidence_keypoints = instances.pred_keypoints[highest_confidence_index].cpu().numpy()
+
+    return highest_confidence_keypoints

flaskr/app/final_models/finalClassification.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4781d81b0fd7974d83fa3e4a14b94d3f6bf41fb545b62e8caf229e6207e0c455
+size 28100

flaskr/app/final_models/finalPose.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5574dc2d8aa75e6afb8abf100d974eacf4d6f2360f22670e97d20667aad5ed9
+size 472677452

flaskr/app/final_models/finalPose_vGAN_v2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:040b1bc4ecd9e55a57346b78f0b952cf35acf9c314b22d765ed78bae85e55df2
+size 624722920

flaskr/app/final_models/gen_model_v2.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8420e128c7f2c485193642def551b9b1af19b2d778dea3734a64c659ab2a49fd
+size 23588688

flaskr/app/generate_light.py

@@ -0,0 +1,29 @@
+import numpy as np
+import cv2, os
+from PIL import Image
+import tensorflow as tf
+from keras.preprocessing import image
+from tensorflow.keras.models import load_model
+from tensorflow.keras.preprocessing.image import img_to_array, save_img, array_to_img
+
+
+def generate_new_image(image_path, model_path, imageID = 0):
+    # Processing Image
+    img = cv2.imread(image_path)
+    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
+    original_height, original_width = img.shape[:2]
+    img_arr = (img_to_array(img) - 127.5) / 127.5
+    resized = cv2.resize(img_arr, (256, 256), interpolation=cv2.INTER_AREA)
+    ready_img = np.expand_dims(resized, axis=0)
+
+    # Loading Model
+    model = load_model(model_path)
+
+    # Prdicting Image
+    pred = model.predict(ready_img)
+    pred = (cv2.medianBlur(pred[0], 1) + 1) / 2
+    pred = cv2.resize(pred, (original_width, original_height))
+    pred = array_to_img(pred)
+    final_image_path = f"captures/new_image_{imageID}.png"
+    save_img(final_image_path, pred)
+    return final_image_path

flaskr/app/helper.py

@@ -0,0 +1,98 @@
+import numpy as np 
+import pandas as pd
+from scipy.spatial.distance import euclidean
+
+def normalize(row, lenBody_x, lenBody_y):
+    row_df = pd.DataFrame(row).transpose()
+    excluded_columns = ['center_X','center_Y']
+
+    for column in row_df.columns:
+        if column not in excluded_columns:
+            if (column.find('_X') != -1) and ((row_df[column]!=-1).all()):
+                row_df[column] = (row_df[column] - row_df['center_X'])/lenBody_x
+            if (column.find('_Y') != -1) and ((row_df[column]!=-1).all()):
+                row_df[column] = (row_df[column] - row_df['center_Y'])/lenBody_y
+    return row_df.squeeze()
+
+def calculate_distance(point1, point2):
+    if -1 in point1 or -1 in point2:
+        return -1
+    return euclidean(point1, point2)
+
+def point_to_point_distance(df):
+    df_new = pd.DataFrame()
+    df_new['r ankle'] = df.apply(lambda row: (row['r ankle_X'], row['r ankle_Y']), axis=1)
+    df_new['r knee'] = df.apply(lambda row: (row['r knee_X'], row['r knee_Y']), axis=1)
+    df_new['r hip'] = df.apply(lambda row: (row['r hip_X'], row['r hip_Y']), axis=1)
+    df_new['l hip'] = df.apply(lambda row: (row['l hip_X'], row['l hip_Y']), axis=1)
+    df_new['l knee'] = df.apply(lambda row: (row['l knee_X'], row['l knee_Y']), axis=1)
+    df_new['l ankle'] = df.apply(lambda row: (row['l ankle_X'], row['l ankle_Y']), axis=1)
+    df_new['pelvis'] = df.apply(lambda row: (row['pelvis_X'], row['pelvis_Y']), axis=1)
+    df_new['thorax'] = df.apply(lambda row: (row['thorax_X'], row['thorax_Y']), axis=1)
+    df_new['upper neck'] = df.apply(lambda row: (row['upper neck_X'], row['upper neck_Y']), axis=1)
+    df_new['head top'] = df.apply(lambda row: (row['head top_X'], row['head top_Y']), axis=1)
+    df_new['r wrist'] = df.apply(lambda row: (row['r wrist_X'], row['r wrist_Y']), axis=1)
+    df_new['r elbow'] = df.apply(lambda row: (row['r elbow_X'], row['l elbow_Y']), axis=1)
+    df_new['r shoulder'] = df.apply(lambda row: (row['r shoulder_X'], row['r shoulder_Y']), axis=1)
+    df_new['l shoulder'] = df.apply(lambda row: (row['l shoulder_X'], row['l shoulder_Y']), axis=1)
+    df_new['l elbow'] = df.apply(lambda row: (row['l elbow_X'], row['l elbow_Y']), axis=1)
+    df_new['l wrist'] = df.apply(lambda row: (row['l wrist_X'], row['l wrist_Y']), axis=1)
+
+    distances_list = []
+
+    # Iterate over each row
+    for index, row in df_new.iterrows():
+        check = []
+        # Dictionary to store distances for the current row
+        row_distances = {}
+        # Iterate over each column pair
+        for column1 in df_new.columns:
+            for column2 in df_new.columns:
+                tupletoAdd = (column2, column1)
+                check.append(tupletoAdd)
+                if column1 != column2 and (column1, column2) not in check:
+                    # Calculate distance between current pair of columns
+                    distance = calculate_distance(row[column1], row[column2])
+                    # Construct the name for the new distance column
+                    distance_column_name = f'{column1}_{column2}_distance'
+                    # Store the distance in the dictionary
+                    row_distances[distance_column_name] = distance
+        # Append the distances for the current row to the list
+        distances_list.append(row_distances)
+
+    # Create a DataFrame from the list of dictionaries
+    distances_df = pd.DataFrame(distances_list)
+    return distances_df
+
+def preprocess_keypoints(keypoints):
+    x_values = [keypoint[0] for keypoint in keypoints]
+    y_values = [keypoint[1] for keypoint in keypoints]
+
+    lenBody_x = max(x_values) - min(x_values)
+    lenBody_y = max(y_values) - min(y_values)
+    labels= ['r ankle','r knee', 'r hip', 'l hip', 'l knee', 'l ankle', 'pelvis', 'thorax', 'upper neck', 'head top', 'r wrist', 'r elbow', 'r shoulder', 'l shoulder', 'l elbow', 'l wrist']
+    new_labels = []
+    for label in labels:
+        new_labels.extend([f"{label}_X", f"{label}_Y"])
+    new_keypoints = []
+    for keypoint in keypoints:
+        if keypoint[2]<0.05:    #might change threshold
+            new_keypoints.extend([-1, -1])
+        else:
+            new_keypoints.extend([keypoint[0], keypoint[1]])
+    
+
+    df = pd.DataFrame([new_keypoints], columns = new_labels)
+    #find center
+    df['center_X'] = (df['r hip_X'] + df['l hip_X'])/2
+    df['center_Y'] = (df['r hip_Y'] + df['l hip_Y']) / 2
+
+    #normalize the values
+    df_normalized = df.apply(lambda row: normalize(row, lenBody_x, lenBody_y), axis=1)
+    df_final = df_normalized.drop(columns = ['center_X', 'center_Y'])
+
+    #df_to_return = point_to_point_distance(df_final)
+    return df_final.to_numpy().astype(np.float32)
+
+
+

flaskr/app/show_points.py

@@ -0,0 +1,22 @@
+import numpy as np
+import cv2
+
+def display_keypoints(keypoints, ganImage = False, imageID = 0):
+
+    if ganImage:
+        image = cv2.imread(f"captures/new_image_{imageID}.png")
+    else:
+        image = cv2.imread(f"captures/image_{imageID}.png")  
+    
+    if isinstance(keypoints, str):
+        cv2.imwrite(f'static/keypoint_image_{imageID}.jpg', image)
+        return
+
+    print(keypoints)
+    # Iterate over each keypoint and draw a circle on the image
+    for kp in keypoints:
+        x, y, c = kp
+        if c>0.07:
+            cv2.circle(image, (int(x), int(y)), 5, (0, 255, 0), -1)  # Draw a green circle
+
+    cv2.imwrite(f'static/keypoint_image_{imageID}.jpg', image)

flaskr/app/templates/index.html

@@ -0,0 +1,240 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Let's Guess That Pose</title>
+    <style>
+        body {
+            background-color: #000;
+            display: flex;
+            flex-direction: column;
+            align-items: center;
+            justify-content: center;
+            height: 100vh;
+            margin: 0;
+        }
+
+        h1 {
+            font-family: 'Comic Sans MS', cursive, sans-serif;
+            color: #FF69B4; /* Pink */
+            font-size: 72px; /* Bigger font size */
+            text-align: center; /* Center align */
+            margin-bottom: 20px; /* Some spacing under the title */
+        }
+
+        button {
+            font-size: 24px; /* Adjust button font size */
+            background-color: #FF0066; /* Dark pink background color */
+            color: white; /* White lettering */
+            padding: 10px 20px; /* Padding for the button */
+            border: none; /* Remove border */
+            border-radius: 5px; /* Add border radius */
+            cursor: pointer; /* Change cursor on hover */
+            transition: background-color 0.3s; /* Smooth transition */
+        }
+
+        button:hover {
+            background-color: #E6005C; /* Darker pink on hover */
+        }
+
+        #main-container {
+            color:white;
+            display: flex; /* Use flexbox layout */
+            align-items: flex-start; /* Align items at the start of the container */
+            justify-content: center; /* Center align child elements horizontally */
+            margin-top: 20px; /* Add margin space */
+            width: 100%;
+        }
+
+        #buttons-container {
+            display: flex; /* Use flexbox layout */
+            justify-content: center; /* Center child elements horizontally */
+            margin-bottom: 20px; /* Add margin at the bottom */
+            align-items: center;
+        }
+
+        #video-container,
+        #modified-image-container {
+            margin: 0 20px; /* Add horizontal margin space */
+        }
+        #video-container video{
+            width: 100%; /* Ensure the video fills its container */
+            height: 100%;
+        }
+
+        #video-container video,
+        #modified-image-container img {
+            height: auto; /* Maintain aspect ratio */
+            display: block; /* Ensure images fill their container */
+            margin-bottom: 10px; /* Add space between the video and the image */
+        }
+        #modified-image-container img{
+            width: 400px;
+        }
+
+        #countdown {
+            font-size: 48px;
+            color: pink;
+            display: none; /* Initially hide the countdown */
+        }
+        .switch {
+            position: relative;
+            display: inline-block;
+            width: 60px;
+            height: 34px;
+        }
+
+        /* Hide default HTML checkbox */
+        .switch input {
+            opacity: 0;
+            width: 0;
+            height: 0;
+        }
+
+        /* The slider */
+        .slider {
+            position: absolute;
+            cursor: pointer;
+            top: 0;
+            left: 0;
+            right: 0;
+            bottom: 0;
+            background-color: #ccc;
+            transition: .4s;
+            border-radius: 34px;
+        }
+
+        .slider:before {
+            position: absolute;
+            content: "";
+            height: 26px;
+            width: 26px;
+            left: 4px;
+            bottom: 4px;
+            background-color: white;
+            transition: .4s;
+            border-radius: 50%;
+        }
+
+        input:checked + .slider {
+            background-color: #2196F3;
+        }
+
+        input:focus + .slider {
+            box-shadow: 0 0 1px #2196F3;
+        }
+
+        input:checked + .slider:before {
+            transform: translateX(26px);
+        }
+
+        /* Rounded sliders */
+        .slider.round {
+            border-radius: 34px;
+        }
+
+        .slider.round:before {
+            border-radius: 50%;
+        }
+    </style>
+</head>
+<body>
+    <h1>Let's Guess That Pose</h1>
+    <div id ="buttons-container">
+        <button id="start">Start</button>
+        <label class="switch">
+            <input type="checkbox" id = "model-toggle" onchange="toggleCheckbox()">
+            <span class="slider round"></span>
+        </label>
+    </div>
+    <div id="main-container">
+        <div id="video-container">
+            <video width="800" height="800" id="video" autoplay></video>
+            <canvas width="800" height="800" id="canvas" style="display:none;"></canvas>
+            <div id="countdown">10</div>
+        </div>
+        <div id="modified-image-container">
+            <img id="modifiedImage" src="/static/IMG_7330.GIF" alt="Original Image">
+            <div id="guess"></div>
+        </div>
+
+    </div>
+    <script>
+        const video = document.getElementById('video');
+        const canvas = document.getElementById('canvas');
+        const startButton = document.getElementById('start');
+        const countdownElement = document.getElementById('countdown');
+        const guessElement = document.getElementById('guess');
+        const keypointImage = document.getElementById('modifiedImage');
+
+        let countdown = 10;
+        let useModelGAN = false;
+        let imageID = 0;
+
+        // Access the camera
+        navigator.mediaDevices.getUserMedia({ video: true })
+            .then((stream) => {
+                video.srcObject = stream;
+            })
+            .catch((err) => {
+                console.error('Error accessing the camera: ', err);
+            });
+
+        function captureImage() {
+            const context = canvas.getContext('2d');
+            context.drawImage(video, 0, 0, canvas.width, canvas.height);
+            const imageData = canvas.toDataURL('image/png');
+
+            // Send image data to Flask route for processing
+            fetch('/upload', {
+                method: 'POST',
+                body: JSON.stringify({ image_data: imageData,  use_model_gan: useModelGAN, unique_ID: imageID }),
+                headers: {
+                    'Content-Type': 'application/json'
+                }
+            })
+            .then(response => response.text())
+            .then(data => {
+                //if (data != 'No keypoints detected'){
+                    //keypointImage.src = `static/keypoint_image_${imageID}.jpg`;
+                   // }
+                keypointImage.src = `static/keypoint_image_${imageID}.jpg`;
+                guessElement.textContent = `Prediction: ${data}`;
+            })
+            .catch(error => console.error('Error sending image data: ', error));
+        }
+
+        function startCountdown() {
+            countdownElement.style.display = 'block'; // Show the countdown
+            if (countdown > 0) {
+                countdownElement.textContent = countdown;
+                setTimeout(startCountdown, 1000);  //will call startCountdown again after 1000 milliseconds
+                countdown--;
+            } else {
+                captureImage();
+                countdown = 5;
+                countdownElement.style.display = 'none';
+            }
+        }
+
+        function toggleCheckbox() {
+            useModelGAN = document.getElementById('model-toggle').checked;
+        }
+
+        startButton.addEventListener('click', () => {
+            keypointImage.src = '/static/IMG_7330.GIF';
+            guessElement.textContent = '';
+            startCountdown();
+            imageID++;
+        });
+    </script>
+</body>
+</html>
+
+
+
+
+
+
+

flaskr/requirements.txt

@@ -0,0 +1,9 @@
+git+https://github.com/facebookresearch/detectron2.git
+opencv-python
+torch
+torchvision
+torchaudio
+tensorflow
+keras
+pandas
+Flask