Upload 36 files

src/components/MultiSourceCaptioningView.tsx

@@ -33,12 +33,13 @@ export default function MultiSourceCaptioningView() {
   const [debugOutput, setDebugOutput] = useState<string>("");
   const [canvasDims, setCanvasDims] = useState<{w:number,h:number}|null>(null);
   const [videoDims, setVideoDims] = useState<{w:number,h:number}|null>(null);
+  const [inferenceStatus, setInferenceStatus] = useState<string>("");
 
   const videoRef = useRef<HTMLVideoElement | null>(null);
   const canvasRef = useRef<HTMLCanvasElement | null>(null);
   const imageRef = useRef<HTMLImageElement | null>(null);
   const webcamStreamRef = useRef<MediaStream | null>(null);
-  const { isLoaded, runInference } = useVLMContext();
+  const { isLoaded, isLoading, error: modelError, runInference } = useVLMContext();
 
   // Webcam setup and teardown (unchanged)
   useEffect(() => {
@@ -91,6 +92,7 @@ export default function MultiSourceCaptioningView() {
       try {
         setProcessing(true);
         setError(null);
+        setInferenceStatus("Running inference...");
         const fakeVideo = {
           videoWidth: canvas.width,
           videoHeight: canvas.height,
@@ -98,11 +100,14 @@ export default function MultiSourceCaptioningView() {
         } as unknown as HTMLVideoElement;
         const result = await runInference(fakeVideo, prompt);
         setDebugOutput(result);
+        setInferenceStatus("Inference complete.");
         ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
         const boxes = extractJsonFromMarkdown(result) || [];
+        if (boxes.length === 0) setInferenceStatus("No boxes detected or model output invalid.");
         drawBoundingBoxesOnCanvas(ctx, boxes);
       } catch (e) {
         setError(e instanceof Error ? e.message : String(e));
+        setInferenceStatus("Inference error: " + (e instanceof Error ? e.message : String(e)));
       } finally {
         setProcessing(false);
       }
@@ -132,6 +137,7 @@ export default function MultiSourceCaptioningView() {
       try {
         setProcessing(true);
         setError(null);
+        setInferenceStatus("Running inference...");
         const fakeVideo = {
           videoWidth: canvas.width,
           videoHeight: canvas.height,
@@ -139,11 +145,14 @@ export default function MultiSourceCaptioningView() {
         } as unknown as HTMLVideoElement;
         const result = await runInference(fakeVideo, prompt);
         setDebugOutput(result);
+        setInferenceStatus("Inference complete.");
         ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
         const boxes = extractJsonFromMarkdown(result) || [];
+        if (boxes.length === 0) setInferenceStatus("No boxes detected or model output invalid.");
         drawBoundingBoxesOnCanvas(ctx, boxes);
       } catch (e) {
         setError(e instanceof Error ? e.message : String(e));
+        setInferenceStatus("Inference error: " + (e instanceof Error ? e.message : String(e)));
       } finally {
         setProcessing(false);
       }
@@ -171,6 +180,7 @@ export default function MultiSourceCaptioningView() {
     try {
       setProcessing(true);
       setError(null);
+      setInferenceStatus("Running inference...");
       const fakeVideo = {
         videoWidth: canvas.width,
         videoHeight: canvas.height,
@@ -178,12 +188,15 @@ export default function MultiSourceCaptioningView() {
       } as unknown as HTMLVideoElement;
       const result = await runInference(fakeVideo, prompt);
       setDebugOutput(result);
+      setInferenceStatus("Inference complete.");
       ctx.drawImage(img, 0, 0, canvas.width, canvas.height);
       const boxes = extractJsonFromMarkdown(result) || [];
+      if (boxes.length === 0) setInferenceStatus("No boxes detected or model output invalid.");
       drawBoundingBoxesOnCanvas(ctx, boxes);
       setImageProcessed(true);
     } catch (e) {
       setError(e instanceof Error ? e.message : String(e));
+      setInferenceStatus("Inference error: " + (e instanceof Error ? e.message : String(e)));
     } finally {
       setProcessing(false);
     }
@@ -206,6 +219,7 @@ export default function MultiSourceCaptioningView() {
       try {
         setProcessing(true);
         setError(null);
+        setInferenceStatus("Running inference...");
         const fakeVideo = {
           videoWidth: canvas.width,
           videoHeight: canvas.height,
@@ -213,11 +227,14 @@ export default function MultiSourceCaptioningView() {
         } as unknown as HTMLVideoElement;
         const result = await runInference(fakeVideo, prompt);
         setDebugOutput(result);
+        setInferenceStatus("Inference complete.");
         ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
         const boxes = extractJsonFromMarkdown(result) || [];
+        if (boxes.length === 0) setInferenceStatus("No boxes detected or model output invalid.");
         drawBoundingBoxesOnCanvas(ctx, boxes);
       } catch (e) {
         setError(e instanceof Error ? e.message : String(e));
+        setInferenceStatus("Inference error: " + (e instanceof Error ? e.message : String(e)));
       } finally {
         setProcessing(false);
       }
@@ -247,6 +264,7 @@ export default function MultiSourceCaptioningView() {
       try {
         setProcessing(true);
         setError(null);
+        setInferenceStatus("Running inference...");
         const fakeVideo = {
           videoWidth: canvas.width,
           videoHeight: canvas.height,
@@ -254,11 +272,14 @@ export default function MultiSourceCaptioningView() {
         } as unknown as HTMLVideoElement;
         const result = await runInference(fakeVideo, prompt);
         setDebugOutput(result);
+        setInferenceStatus("Inference complete.");
         ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
         const boxes = extractJsonFromMarkdown(result) || [];
+        if (boxes.length === 0) setInferenceStatus("No boxes detected or model output invalid.");
         drawBoundingBoxesOnCanvas(ctx, boxes);
       } catch (e) {
         setError(e instanceof Error ? e.message : String(e));
+        setInferenceStatus("Inference error: " + (e instanceof Error ? e.message : String(e)));
       } finally {
         setProcessing(false);
       }
@@ -314,6 +335,10 @@ export default function MultiSourceCaptioningView() {
 
   return (
     <div className="absolute inset-0 text-white">
+      <div className="fixed top-0 left-0 w-full bg-gray-900 text-white text-center py-2 z-50">
+        {isLoading ? "Loading model..." : isLoaded ? "Model loaded" : modelError ? `Model error: ${modelError}` : "Model not loaded"}
+      </div>
+      <div className="text-center text-sm text-blue-300 mt-2">{inferenceStatus}</div>
       <div className="flex flex-col items-center justify-center h-full w-full">
         {/* Mode Selector */}
         <div className="mb-6">