Spaces:

ilhamap
/

live-vision

Sleeping

App Files Files Community

ilhamap commited on Feb 1, 2024

Commit

2474dcb

verified ·

1 Parent(s): ff8e5ca

Upload 3 files

Browse files

Files changed (3) hide show

public/index.html +34 -0
public/sketch.js +444 -0
public/style.css +137 -0

public/index.html ADDED Viewed

	@@ -0,0 +1,34 @@

+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <script src="https://cdn.socket.io/4.5.4/socket.io.min.js" integrity="sha384-/KNQL8Nu5gCHLqwqfQjA689Hhoqgi2S84SNUxC3roTe4EhJ9AfLkp8QiQcU8AMzI" crossorigin="anonymous"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/1.4.0/p5.js"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/1.4.0/addons/p5.sound.min.js"></script>
+    <link rel="stylesheet" type="text/css" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.3.0/css/all.min.css" crossorigin="anonymous">
+    <link rel="stylesheet" type="text/css" href="style.css">
+    <meta charset="utf-8" />
+  </head>
+  <body>
+    <div id="main-container">
+      <div id="title">
+        <h2>Google Gemini Pro Vision - live vision</h2>
+        <h3>1. turn on your camera - 2. start live vision</h3>
+      </div>
+      <div id="appcontent-ctn">
+        <div id="left-ctn">
+          <div id="canvas-container"></div>
+          <div id="buttons-container"></div>
+          <div id="checkbox-rear"></div>
+        </div>
+        <div id="right-ctn">
+          <div id="instructions-container"></div>
+          <div id="loading-div"></div>
+          <div id="vision-text-container"></div>
+        </div>
+      </div>
+    </div>
+    <script src="./sketch.js"></script>
+  </body>
+</html>

public/sketch.js ADDED Viewed

	@@ -0,0 +1,444 @@

+let myCanvas
+const arrows = []
+let current_points = []
+let t_size
+let button
+let isOnCanvas = false;
+let socket;
+let live = false;
+let timer;
+let capture;
+let instructions;
+let current_instructions;
+let messages_list = []
+let timerWaiting;
+let waiting = false;
+let active_rearcam = false;
+let camConfig = "user";
+let isOnMobile;
+let camIsON = false;
+let device_orientation;
+let graphicsToSend;
+function isMobileDevice() {
+    return (typeof window.orientation !== "undefined") || (navigator.userAgent.indexOf('IEMobile') !== -1);
+}
+//-----------------------------------------------------
+//-----------------------------------------------------
+function preload(){
+    isOnMobile = isMobileDevice()
+    console.log("is on mobile ? " + isOnMobile)
+    if(isOnMobile){
+      device_orientation = screen.orientation.type;
+    }
+    // Socket events handlers
+    socket = io();
+    // client-side
+    socket.on("connect", () => {
+        console.log(socket.id);
+    });
+    socket.on("hello", (arg) => {
+        let api_data_test = arg[1]
+        console.log(api_data_test)
+    });
+    socket.on("api_error", (arg) => {
+        let message = arg
+        console.log(message)
+    });
+    // ————————
+}
+function call_api(socket, instructions){
+    console.log("Calling API ... ")
+    waiting = true;
+    show_loading()
+    //cnv_data = myCanvas.elt.toDataURL('image/png');
+    graphicsToSend.loadPixels()
+    cnv_data = graphicsToSend.canvas.toDataURL('image/png');
+    let data = [
+        cnv_data,
+        instructions
+    ]
+    socket.emit("ask_api", data);
+}
+function windowResized() {
+    if(windowWidth <= 500){
+        resizeCanvas(380, 380*320/576);
+    } else {
+        resizeCanvas(576, 320);
+    }
+}
+function setup() {
+    screen.orientation.addEventListener("change", function(e) {
+      device_orientation = screen.orientation.type
+      console.log(device_orientation)
+      if(camIsON){
+                if(capture){
+                    capture.remove()
+                    capture = undefined
+                  }
+                  if(active_rearcam == true){
+                    let new_constraints = {
+                          audio: false,
+                          video: {
+                            facingMode:  {
+                              exact: "environment"
+                            }
+                          }
+                        };
+                    capture = createCapture(new_constraints)
+                    capture.hide()
+                  } else {
+                    let new_constraints = {
+                          audio: false,
+                          video: {
+                            facingMode:  "user"
+                          }
+                          //video: {
+                            //facingMode: "user"
+                          //}
+                        };
+                    capture = createCapture(new_constraints)
+                    capture.hide()
+                  }
+                }
+    });
+    pixelDensity(1)
+    if(windowWidth <= 500){
+        myCanvas = createCanvas(380, 380*320/576);
+    } else {
+        myCanvas = createCanvas(576, 320);
+    }
+    background(220)
+    myCanvas.id('myCanvas')
+    myCanvas.parent("canvas-container")
+    graphicsToSend = createGraphics(myCanvas.width, myCanvas.height)
+    socket.on("api_response", (response) => {
+        waiting = false;
+        clearTimeout(timerWaiting)
+        vision_text = response
+        messages_list.push(vision_text)
+        if(live === true){
+            loadingDiv = document.getElementById("loading-div")
+            loadingDiv.innerHTML = "_"
+            display_messages()
+            console.log("Sending new requestion in 3 seconds")
+            timer = setTimeout(() => {
+                console.log(current_instructions)
+                call_api(socket, current_instructions)
+            }, 3000)
+        }
+    });
+    // Watch if cursor is above canvas or not
+    let getCanvas = document.getElementById('myCanvas');
+    getCanvas.addEventListener("pointerdown", (e) => {
+        //console.log("pointerDown");
+        getCanvas.setPointerCapture(e.pointerId);
+        isOnCanvas = true;
+    }, false);
+    getCanvas.addEventListener("pointerup", (e) => {
+        //console.log("pointerUp");
+        if (isOnCanvas) {
+        getCanvas.releasePointerCapture(e.pointerId);
+        isOnCanvas = false;
+        }
+    }, false);
+    //—————————
+    // Buttons
+    start_capture_button=createButton("turn on cam")
+    start_capture_button.mousePressed(() => {
+        camIsON = true;
+        if(!active_rearcam){
+            camConfig = "user"
+        } else {
+            camConfig = {
+                exact: "environment"
+            }
+        }
+        let constraints = {
+            audio: false,
+            video: {
+              facingMode:  camConfig
+            }
+          };
+        capture = createCapture(constraints);
+        capture.hide();
+    })
+    if(isOnMobile){
+        checkbox_rearcam = createCheckbox("switch to rear cam");
+        checkbox_rearcam.parent("checkbox-rear");
+        checkbox_rearcam.mousePressed( () => {
+            if(active_rearcam == false){
+                active_rearcam = true;
+            } else {
+                active_rearcam = false
+            }
+            if(camIsON){
+                if(capture){
+                    capture.remove()
+                    capture = undefined
+                  }
+                  if(active_rearcam == true){
+                    let new_constraints = {
+                          audio: false,
+                          video: {
+                            facingMode:  {
+                              exact: "environment"
+                            }
+                          }
+                        };
+                    capture = createCapture(new_constraints)
+                    capture.hide()
+                  } else {
+                    let new_constraints = {
+                          audio: false,
+                          video: {
+                            facingMode:  "user"
+                          }
+                          //video: {
+                            //facingMode: "user"
+                          //}
+                        };
+                    capture = createCapture(new_constraints)
+                    capture.hide()
+                  }
+                }
+            })
+    }
+    start_button = createButton('start live vision');
+    start_button.mousePressed(() => {
+        live = true;
+        current_instructions = input_instructions.value()
+        console.log("Live in ON")
+        console.log(current_instructions)
+        call_api(socket, current_instructions)
+    })
+    stop_button = createButton('stop all');
+    stop_button.mousePressed(() => {
+        live = false;
+        waiting = false;
+        // Abort the timer
+        clearTimeout(timer);
+        clearTimeout(timerWaiting)
+        loadingDiv.innerHTML = "_"
+        if(capture){
+            capture.remove();
+            capture = undefined
+            camIsON = false;
+        }
+        console.log("live is OFF")
+        //redraw()
+    })
+    input_instructions = createInput("What is happening ? ")
+    change_instructions = createButton("change instructions")
+    change_instructions.mousePressed(() => {
+        current_instructions = input_instructions.value()
+    })
+    start_capture_button.parent("buttons-container")
+    start_button.parent("buttons-container")
+    stop_button.parent("buttons-container")
+    input_instructions.parent("instructions-container")
+    change_instructions.parent("instructions-container")
+    loadingDiv = document.getElementById("loading-div")
+}
+function draw() {
+    background(220);
+    textAlign(CENTER);
+    text('turn on your webcam', width/2, height/2);
+    if(capture != undefined){
+      if(!active_rearcam){
+        if(isOnMobile){
+          if(device_orientation == "portrait" || device_orientation == "portrait-primary" || device_orientation == "portrait-secondary"){
+            graphicsToSend.image(capture,(myCanvas.width/2)-(capture.width/2), -(capture.height/4))
+            push()
+          //move image by the width of image to the left
+            translate(myCanvas.width, 0);
+          //then scale push()
+          //move image by the width of image to the left
+          //to flip the image
+            scale(-1, 1);
+            image(capture,(myCanvas.width/2)-(capture.width/2), -(capture.height/4))
+            pop()
+          } else {
+            graphicsToSend.image(capture, 0, 0)
+            push()
+          //move image by the width of image to the left
+          translate(myCanvas.width, 0);
+          //then scale push()
+          //move image by the width of image to the left
+          //to flip the image
+            scale(-1, 1);
+            image(capture, 0, 0)
+            pop()
+          }
+        } else {
+          graphicsToSend.image(capture, 0, 0)
+          push()
+          //move image by the width of image to the left
+          translate(myCanvas.width, 0);
+          //then scale push()
+          //move image by the width of image to the left
+          //to flip the image
+          scale(-1, 1);
+          image(capture, 0, 0)
+          pop()
+        }
+      } else {
+          if(isOnMobile){
+            if(device_orientation == "portrait" || device_orientation == "portrait-primary" || device_orientation == "portrait-secondary"){
+              graphicsToSend.image(capture,(myCanvas.width/2)-(capture.width/2), -(capture.height/4))
+              image(capture,(myCanvas.width/2)-(capture.width/2), -(capture.height/4))
+            } else {
+                graphicsToSend.image(capture, 0, 0)
+                image(capture, 0, 0)
+            }
+          } else {
+            graphicsToSend.image(capture, 0, 0)
+            image(capture, 0, 0)
+          }
+      }
+    }
+}
+function display_messages(){
+    visionDiv = document.getElementById("vision-text-container")
+    visionDiv.innerHTML = ""
+    //for(i=0; i < messages_list.length - 1; i++){
+    //    newNode = document.createElement('div');
+    //    newNode.classList.add('text-msg')
+    //    newNode.innerHTML = messages_list[i];
+    //    visionDiv.appendChild(newNode);
+    //}
+    newNode_last = document.createElement('div');
+    newNode_last.classList.add('text-msg')
+    visionDiv.appendChild(newNode_last);
+    const text_msg = messages_list[messages_list.length - 1]
+    const words = text_msg.split(" "); // splits the text into an array of words
+    // Function that returns a promise which resolves after a specified number of milliseconds
+    function wait(ms) {
+        return new Promise((resolve) => setTimeout(resolve, ms));
+    }
+    async function streamText(words) {
+        for (let word of words) {
+            newNode_last.innerHTML += " " + word;
+            //visionDiv.scrollTop = visionDiv.scrollHeight;
+            await wait(30); // Wait for 1 second before logging next word
+        }
+    }
+    streamText(words);
+}
+function clean(){
+    if(waiting == true){
+        loadingDiv.innerHTML = ""
+        timerWaiting = setTimeout(show_loading, 500)
+    } else {
+        loadingDiv.innerHTML = "_"
+    }
+}
+function show_loading(){
+    if(waiting == true){
+        loadingDiv.innerHTML = ""
+        loading_text = ". . ."
+    } else {
+        loading_text = "_"
+    }
+    const dots = loading_text.split(" "); // splits the text into an array of words
+    // Function that returns a promise which resolves after a specified number of milliseconds
+    function wait(ms) {
+        if(waiting == true){
+            return new Promise((resolve) => resolveTimer = setTimeout(resolve, ms));
+        } else {
+            clearTimeout(resolveTimer)
+            loadingDiv.innerHTML = "_"
+        }
+    }
+    async function streamDots(words) {
+        if(waiting == true){
+            for (let word of words) {
+                //console.log(word);
+                loadingDiv.innerHTML += " " + word;
+                //visionDiv.scrollTop = visionDiv.scrollHeight;
+                await wait(500); // Wait for 1 second before logging next word
+            }
+            //console.log("ENLEVE")
+            //timerWaiting = setTimeout(show_loading, 500)
+            clean()
+        } else {
+            clearTimeout(timerWaiting)
+            //loadingDiv.innerHTML = "_"
+        }
+    }
+    streamDots(dots);
+}

public/style.css ADDED Viewed

	@@ -0,0 +1,137 @@

+body {
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    height: 100vh;
+    margin: 0;
+    color: #3E3E3E;
+    overflow: scroll;
+}
+div#main-container {
+    display: flex;
+    flex-direction: column;
+    flex-wrap: nowrap;
+    height: 100vh;
+    align-items: center;
+    row-gap: 40px;
+}
+div#title {
+    font-family: sans-serif;
+    text-align: center;
+}
+div#appcontent-ctn {
+    display: flex;
+    column-gap: 30px;
+    flex-wrap: wrap;
+    justify-content: center;
+    row-gap: 4px;
+}
+div#left-ctn {
+    display: flex;
+    flex-direction: column;
+    row-gap: 4px;
+}
+div#right-ctn {
+    display: flex;
+    flex-direction: column;
+    row-gap: 4px;
+    overflow: scroll;
+    height: 320px;
+}
+div#checkbox-rear {
+    width: 100%;
+}
+div#checkbox-rear > div {
+    display: flex;
+    flex-direction: row;
+    justify-content: flex-start;
+    width: 100%;
+    align-items: center;
+    height: 30px;
+    font-family: sans-serif;
+}
+div#buttons-container {
+    display: flex;
+    justify-content: flex-start;
+    flex-direction: row;
+    width: 100%;
+    height: 40px;
+    gap: 4px;
+}
+div#instructions-container {
+    display: flex;
+    flex-direction: row;
+    justify-content: flex-start;
+    width: 100%;
+    height: 40px;
+    gap: 4px;
+}
+div#canvas-container {
+    margin: 0;
+}
+canvas#myCanvas {
+    border-radius: 4px;
+}
+#buttons-container > button {
+    cursor: pointer;
+    flex: 1;
+}
+div#instructions-container > input[type="text"] {
+    flex: 3;
+    padding: 0 10px;
+    font-size: large;
+}
+div#instructions-container > button {
+    cursor: pointer;
+    flex: 1;
+}
+div#loading-div {
+    font-size: 40px;
+    width: 100%;
+    height: 40px;
+}
+div#vision-text-container {
+    width: 576px;
+    font-family: sans-serif;
+    display: flex;
+    flex-direction: column;
+    flex: 1;
+    overflow: scroll;
+}
+.text-msg {
+    font-size: 0.8em;
+    line-height: 1.4em;
+}
+.text-msg:last-child{
+    font-size: 1.2em;
+    line-height: 1.3em;
+    font-weight: bold;
+}
+div#vision-text-container > div {
+    margin: 10px 0;
+    padding-bottom: 10px;
+}
+@media screen and (max-width: 500px) {
+    div#vision-text-container {
+        width: 380px;
+    }
+  }