Upload 36 files

src/context/VLMContext.tsx

@@ -67,7 +67,7 @@ export const VLMProvider: React.FC<React.PropsWithChildren> = ({ children }) =>
   );
 
   const runInference = useCallback(
-    async (video: HTMLVideoElement, instruction: string, onTextUpdate?: (text: string) => void): Promise<string> => {
+    async (media: HTMLVideoElement | HTMLImageElement, instruction: string, onTextUpdate?: (text: string) => void): Promise<string> => {
       if (inferenceLock.current) {
         console.log("Inference already running, skipping frame");
         return ""; // Return empty string to signal a skip
@@ -83,13 +83,25 @@ export const VLMProvider: React.FC<React.PropsWithChildren> = ({ children }) =>
       }
       const canvas = canvasRef.current;
 
-      canvas.width = video.videoWidth;
-      canvas.height = video.videoHeight;
+      // Support both video and image
+      let width = 0;
+      let height = 0;
+      if (media instanceof HTMLVideoElement) {
+        width = media.videoWidth;
+        height = media.videoHeight;
+      } else if (media instanceof HTMLImageElement) {
+        width = media.naturalWidth;
+        height = media.naturalHeight;
+      } else {
+        throw new Error("Unsupported media type");
+      }
+      canvas.width = width;
+      canvas.height = height;
 
       const ctx = canvas.getContext("2d", { willReadFrequently: true });
       if (!ctx) throw new Error("Could not get canvas context");
 
-      ctx.drawImage(video, 0, 0);
+      ctx.drawImage(media, 0, 0, width, height);
 
       const frame = ctx.getImageData(0, 0, canvas.width, canvas.height);
       const rawImg = new RawImage(frame.data, frame.width, frame.height, 4);

src/types/vlm.ts

@@ -4,7 +4,7 @@ export type VLMContextValue = {
   error: string | null;
   loadModel: (onProgress?: (msg: string) => void) => Promise<void>;
   runInference: (
-    video: HTMLVideoElement,
+    media: HTMLVideoElement | HTMLImageElement,
     instruction: string,
     onTextUpdate?: (text: string) => void,
   ) => Promise<string>;