Upload 49 files

.gitattributes

@@ -54,3 +54,24 @@ dist/examples/6.jpg filter=lfs diff=lfs merge=lfs -text
 dist/examples/7.jpg filter=lfs diff=lfs merge=lfs -text
 dist/examples/8.jpg filter=lfs diff=lfs merge=lfs -text
 dist/examples/9.jpg filter=lfs diff=lfs merge=lfs -text
+frontend/src/assets/examples/1.jpg filter=lfs diff=lfs merge=lfs -text
+frontend/src/assets/examples/10.jpg filter=lfs diff=lfs merge=lfs -text
+frontend/src/assets/examples/11.jpg filter=lfs diff=lfs merge=lfs -text
+frontend/src/assets/examples/12.jpg filter=lfs diff=lfs merge=lfs -text
+frontend/src/assets/examples/13.jpg filter=lfs diff=lfs merge=lfs -text
+frontend/src/assets/examples/14.jpg filter=lfs diff=lfs merge=lfs -text
+frontend/src/assets/examples/15.jpg filter=lfs diff=lfs merge=lfs -text
+frontend/src/assets/examples/16.jpg filter=lfs diff=lfs merge=lfs -text
+frontend/src/assets/examples/17.jpg filter=lfs diff=lfs merge=lfs -text
+frontend/src/assets/examples/18.jpg filter=lfs diff=lfs merge=lfs -text
+frontend/src/assets/examples/19.jpg filter=lfs diff=lfs merge=lfs -text
+frontend/src/assets/examples/2.jpg filter=lfs diff=lfs merge=lfs -text
+frontend/src/assets/examples/20.jpg filter=lfs diff=lfs merge=lfs -text
+frontend/src/assets/examples/21.jpg filter=lfs diff=lfs merge=lfs -text
+frontend/src/assets/examples/3.jpg filter=lfs diff=lfs merge=lfs -text
+frontend/src/assets/examples/4.jpg filter=lfs diff=lfs merge=lfs -text
+frontend/src/assets/examples/5.jpg filter=lfs diff=lfs merge=lfs -text
+frontend/src/assets/examples/6.jpg filter=lfs diff=lfs merge=lfs -text
+frontend/src/assets/examples/7.jpg filter=lfs diff=lfs merge=lfs -text
+frontend/src/assets/examples/8.jpg filter=lfs diff=lfs merge=lfs -text
+frontend/src/assets/examples/9.jpg filter=lfs diff=lfs merge=lfs -text

frontend/README.md

@@ -0,0 +1,126 @@
+## Segment Anything Simple Web demo
+
+This **front-end only** React based web demo shows how to load a fixed image and corresponding `.npy` file of the SAM image embedding, and run the SAM ONNX model in the browser using Web Assembly with mulithreading enabled by `SharedArrayBuffer`, Web Worker, and SIMD128.
+
+<img src="https://github.com/facebookresearch/segment-anything/raw/main/assets/minidemo.gif" width="500"/>
+
+## Run the app
+
+Install Yarn
+
+```
+npm install --g yarn
+```
+
+Build and run:
+
+```
+yarn && yarn start
+```
+
+Navigate to [`http://localhost:8081/`](http://localhost:8081/)
+
+Move your cursor around to see the mask prediction update in real time.
+
+## Export the image embedding
+
+In the [ONNX Model Example notebook](https://github.com/facebookresearch/segment-anything/blob/main/notebooks/onnx_model_example.ipynb) upload the image of your choice and generate and save corresponding embedding.
+
+Initialize the predictor:
+
+```python
+checkpoint = "sam_vit_h_4b8939.pth"
+model_type = "vit_h"
+sam = sam_model_registry[model_type](checkpoint=checkpoint)
+sam.to(device='cuda')
+predictor = SamPredictor(sam)
+```
+
+Set the new image and export the embedding:
+
+```
+image = cv2.imread('src/assets/dogs.jpg')
+predictor.set_image(image)
+image_embedding = predictor.get_image_embedding().cpu().numpy()
+np.save("dogs_embedding.npy", image_embedding)
+```
+
+Save the new image and embedding in `src/assets/data`.
+
+## Export the ONNX model
+
+You also need to export the quantized ONNX model from the [ONNX Model Example notebook](https://github.com/facebookresearch/segment-anything/blob/main/notebooks/onnx_model_example.ipynb).
+
+Run the cell in the notebook which saves the `sam_onnx_quantized_example.onnx` file, download it and copy it to the path `/model/sam_onnx_quantized_example.onnx`.
+
+Here is a snippet of the export/quantization code:
+
+```
+onnx_model_path = "sam_onnx_example.onnx"
+onnx_model_quantized_path = "sam_onnx_quantized_example.onnx"
+quantize_dynamic(
+    model_input=onnx_model_path,
+    model_output=onnx_model_quantized_path,
+    optimize_model=True,
+    per_channel=False,
+    reduce_range=False,
+    weight_type=QuantType.QUInt8,
+)
+```
+
+**NOTE: if you change the ONNX model by using a new checkpoint you need to also re-export the embedding.**
+
+## Update the image, embedding, model in the app
+
+Update the following file paths at the top of`App.tsx`:
+
+```py
+const IMAGE_PATH = "/assets/data/dogs.jpg";
+const IMAGE_EMBEDDING = "/assets/data/dogs_embedding.npy";
+const MODEL_DIR = "/model/sam_onnx_quantized_example.onnx";
+```
+
+## ONNX multithreading with SharedArrayBuffer
+
+To use multithreading, the appropriate headers need to be set to create a cross origin isolation state which will enable use of `SharedArrayBuffer` (see this [blog post](https://cloudblogs.microsoft.com/opensource/2021/09/02/onnx-runtime-web-running-your-machine-learning-model-in-browser/) for more details)
+
+The headers below are set in `configs/webpack/dev.js`:
+
+```js
+headers: {
+    "Cross-Origin-Opener-Policy": "same-origin",
+    "Cross-Origin-Embedder-Policy": "credentialless",
+}
+```
+
+## Structure of the app
+
+**`App.tsx`**
+
+- Initializes ONNX model
+- Loads image embedding and image
+- Runs the ONNX model based on input prompts
+
+**`Stage.tsx`**
+
+- Handles mouse move interaction to update the ONNX model prompt
+
+**`Tool.tsx`**
+
+- Renders the image and the mask prediction
+
+**`helpers/maskUtils.tsx`**
+
+- Conversion of ONNX model output from array to an HTMLImageElement
+
+**`helpers/onnxModelAPI.tsx`**
+
+- Formats the inputs for the ONNX model
+
+**`helpers/scaleHelper.tsx`**
+
+- Handles image scaling logic for SAM (longest size 1024)
+
+**`hooks/`**
+
+- Handle shared state for the app

frontend/configs/webpack/common.js

@@ -0,0 +1,85 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// All rights reserved.
+
+// This source code is licensed under the license found in the
+// LICENSE file in the root directory of this source tree.
+
+const { resolve } = require("path");
+const HtmlWebpackPlugin = require("html-webpack-plugin");
+const FriendlyErrorsWebpackPlugin = require("friendly-errors-webpack-plugin");
+const CopyPlugin = require("copy-webpack-plugin");
+const webpack = require("webpack");
+
+module.exports = {
+  entry: "./src/index.tsx",
+  resolve: {
+    extensions: [".js", ".jsx", ".ts", ".tsx"],
+    fallback: { 'process/browser': require.resolve('process/browser'), }
+  },
+  output: {
+    path: resolve(__dirname, "dist"),
+  },
+  module: {
+    rules: [
+      {
+        test: /\.mjs$/,
+        include: /node_modules/,
+        type: "javascript/auto",
+        resolve: {
+          fullySpecified: false,
+        },
+      },
+      {
+        test: [/\.jsx?$/, /\.tsx?$/],
+        use: ["ts-loader"],
+        exclude: /node_modules/,
+      },
+      {
+        test: /\.css$/,
+        use: ["style-loader", "css-loader"],
+      },
+      {
+        test: /\.(scss|sass)$/,
+        use: ["style-loader", "css-loader", "postcss-loader"],
+      },
+      {
+        test: /\.(jpe?g|png|gif|svg)$/i,
+        use: [
+          "file-loader?hash=sha512&digest=hex&name=img/[contenthash].[ext]",
+          "image-webpack-loader?bypassOnDebug&optipng.optimizationLevel=7&gifsicle.interlaced=false",
+        ],
+      },
+      {
+        test: /\.(woff|woff2|ttf)$/,
+        use: {
+          loader: "url-loader",
+        },
+      },
+    ],
+  },
+  plugins: [
+    new CopyPlugin({
+      patterns: [
+        {
+          from: "node_modules/onnxruntime-web/dist/*.wasm",
+          to: "[name][ext]",
+        },
+        {
+          from: "model",
+          to: "model",
+        },
+        {
+          from: "src/assets/examples",
+          to: "examples",
+        },
+      ],
+    }),
+    new HtmlWebpackPlugin({
+      template: "./src/assets/index.html",
+    }),
+    new FriendlyErrorsWebpackPlugin(),
+    new webpack.ProvidePlugin({
+      process: "process/browser",
+    }),
+  ],
+};

frontend/configs/webpack/dev.js

@@ -0,0 +1,25 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// All rights reserved.
+
+// This source code is licensed under the license found in the
+// LICENSE file in the root directory of this source tree.
+
+// development config
+const { merge } = require("webpack-merge");
+const commonConfig = require("./common");
+
+module.exports = merge(commonConfig, {
+  mode: "development",
+  devServer: {
+    hot: true, // enable HMR on the server
+    open: true,
+    // These headers enable the cross origin isolation state
+    // needed to enable use of SharedArrayBuffer for ONNX 
+    // multithreading. 
+    headers: {
+      "Cross-Origin-Opener-Policy": "same-origin",
+      "Cross-Origin-Embedder-Policy": "credentialless",
+    },
+  },
+  devtool: "cheap-module-source-map",
+});

frontend/configs/webpack/prod.js

@@ -0,0 +1,22 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// All rights reserved.
+
+// This source code is licensed under the license found in the
+// LICENSE file in the root directory of this source tree.
+
+// production config
+const { merge } = require("webpack-merge");
+const { resolve } = require("path");
+const Dotenv = require("dotenv-webpack");
+const commonConfig = require("./common");
+
+module.exports = merge(commonConfig, {
+  mode: "production",
+  output: {
+    filename: "js/bundle.[contenthash].min.js",
+    path: resolve(__dirname, "../../dist"),
+    publicPath: "/",
+  },
+  devtool: "source-map",
+  plugins: [new Dotenv()],
+});

frontend/model/sam_onnx_example.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76edead1afcb7a3ed7672939bf375aeb64b57f943769ee286fa02f0a382b3393
+size 16501766

frontend/model/sam_onnx_quantized_example.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b134e7a1a9779862761aa207bb91dd2bdbf2c1182486dc216c71dbba1b109819
+size 8824322

frontend/package.json

@@ -0,0 +1,64 @@
+{
+  "name": "segment-anything-mini-demo",
+  "version": "0.1.0",
+  "license": "MIT",
+  "scripts": {
+    "build": "yarn run clean-dist && webpack --config=configs/webpack/prod.js && mv dist/*.wasm dist/js && rsync -r --delete dist ../",
+    "clean-dist": "rimraf dist/*",
+    "lint": "eslint './src/**/*.{js,ts,tsx}' --quiet",
+    "start": "yarn run start-dev",
+    "test": "yarn run start-model-test",
+    "start-dev": "webpack serve --config=configs/webpack/dev.js"
+  },
+  "devDependencies": {
+    "@babel/core": "^7.18.13",
+    "@babel/preset-env": "^7.18.10",
+    "@babel/preset-react": "^7.18.6",
+    "@babel/preset-typescript": "^7.18.6",
+    "@pmmmwh/react-refresh-webpack-plugin": "^0.5.7",
+    "@testing-library/react": "^13.3.0",
+    "@types/node": "^18.7.13",
+    "@types/react": "^18.0.17",
+    "@types/react-dom": "^18.0.6",
+    "@types/underscore": "^1.11.4",
+    "@typescript-eslint/eslint-plugin": "^5.35.1",
+    "@typescript-eslint/parser": "^5.35.1",
+    "babel-loader": "^8.2.5",
+    "copy-webpack-plugin": "^11.0.0",
+    "css-loader": "^6.7.1",
+    "dotenv": "^16.0.2",
+    "dotenv-webpack": "^8.0.1",
+    "eslint": "^8.22.0",
+    "eslint-plugin-react": "^7.31.0",
+    "file-loader": "^6.2.0",
+    "fork-ts-checker-webpack-plugin": "^7.2.13",
+    "friendly-errors-webpack-plugin": "^1.7.0",
+    "html-webpack-plugin": "^5.5.0",
+    "image-webpack-loader": "^8.1.0",
+    "postcss-loader": "^7.0.1",
+    "postcss-preset-env": "^7.8.0",
+    "process": "^0.11.10",
+    "rimraf": "^3.0.2",
+    "sass": "^1.54.5",
+    "sass-loader": "^13.0.2",
+    "style-loader": "^3.3.1",
+    "tailwindcss": "^3.1.8",
+    "ts-loader": "^9.3.1",
+    "typescript": "^4.8.2",
+    "webpack": "^5.74.0",
+    "webpack-cli": "^4.10.0",
+    "webpack-dev-server": "^4.10.0",
+    "webpack-dotenv-plugin": "^2.1.0",
+    "webpack-merge": "^5.8.0"
+  },
+  "dependencies": {
+    "@gradio/client": "^1.7.1",
+    "npyjs": "^0.4.0",
+    "onnxruntime-web": "1.14.0",
+    "react": "^18.2.0",
+    "react-dom": "^18.2.0",
+    "react-refresh": "^0.14.0",
+    "underscore": "^1.13.6",
+    "axios": "^1.6.7"
+  }
+}

frontend/postcss.config.js

@@ -0,0 +1,10 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// All rights reserved.
+
+// This source code is licensed under the license found in the
+// LICENSE file in the root directory of this source tree.
+
+const tailwindcss = require("tailwindcss");
+module.exports = {
+  plugins: ["postcss-preset-env", 'tailwindcss/nesting', tailwindcss],
+};

frontend/src/App.tsx

@@ -0,0 +1,306 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// All rights reserved.
+
+// This source code is licensed under the license found in the
+// LICENSE file in the root directory of this source tree.
+
+import { InferenceSession, Tensor } from "onnxruntime-web";
+import React, { useContext, useEffect, useState, useRef } from "react";
+import axios from "axios";
+import "./assets/scss/App.scss";
+import { handleImageScale } from "./components/helpers/scaleHelper";
+import { modelScaleProps, QueueStatus } from "./components/helpers/Interfaces";
+import { onnxMaskToImage, arrayToImageData, imageDataToURL } from "./components/helpers/maskUtils";
+import { modelData } from "./components/helpers/onnxModelAPI";
+import Stage, { DescriptionState } from "./components/Stage";
+import AppContext from "./components/hooks/createContext";
+import { imageToSamEmbedding } from "./services/maskApi";
+import LoadingOverlay from "./components/LoadingOverlay";
+import ErrorModal from './components/ErrorModal';
+import QueueStatusIndicator from "./components/QueueStatusIndicator";
+
+const ort = require("onnxruntime-web");
+
+// Define image and model paths
+const MODEL_DIR = "/model/sam_onnx_quantized_example.onnx";
+
+const App = () => {
+  const {
+    clicks: [clicks, setClicks],
+    image: [image, setImage],
+    maskImg: [maskImg, setMaskImg],
+    maskImgData: [maskImgData, setMaskImgData],
+    isClicked: [isClicked, setIsClicked]
+  } = useContext(AppContext)!;
+  const [model, setModel] = useState<InferenceSession | null>(null);
+  const [tensor, setTensor] = useState<Tensor | null>(null);
+  const [modelScale, setModelScale] = useState<modelScaleProps | null>(null);
+  const [isLoading, setIsLoading] = useState<boolean>(false);
+  const [error, setError] = useState<string | null>(null);
+  const [descriptionState, setDescriptionState] = useState<DescriptionState>({
+    state: 'ready',
+    description: ''
+  });
+  const [queueStatus, setQueueStatus] = useState<QueueStatus>({ inQueue: false });
+
+  // Initialize the ONNX model
+  useEffect(() => {
+    const initModel = async () => {
+      try {
+        if (MODEL_DIR === undefined) return;
+        const URL: string = MODEL_DIR;
+        const model = await InferenceSession.create(URL);
+        setModel(model);
+      } catch (e) {
+        console.log(e);
+      }
+    };
+    initModel();
+  }, []);
+
+  const handleImageUpload = async (event: React.ChangeEvent<HTMLInputElement>) => {
+    const file = event.target.files?.[0];
+    if (!file) return;
+
+    try {
+      const url = URL.createObjectURL(file);
+      await loadImage(new URL(url));
+    } catch (error) {
+      setError('Failed to load image. Please try again with a different image.');
+      console.error('Error loading image:', error);
+    }
+  };
+
+  const loadImage = async (url: URL) => {
+    try {
+      setIsLoading(true);
+      const img = new Image();
+      img.src = url.href;
+      img.onload = async () => {
+        const { height, width, samScale } = handleImageScale(img);
+        setModelScale({
+          height: height,
+          width: width,
+          samScale: samScale,
+        });
+        img.width = width;
+        img.height = height;
+        setImage(img);
+
+        // After image is loaded, fetch its embedding from Gradio
+        await fetchImageEmbedding(img);
+        setIsLoading(false);
+      };
+    } catch (error) {
+      console.log(error);
+      setIsLoading(false);
+    }
+  };
+
+  const fetchImageEmbedding = async (img: HTMLImageElement) => {
+    try {
+      // Create a canvas to convert the image to base64
+      const canvas = document.createElement('canvas');
+      canvas.width = img.width;
+      canvas.height = img.height;
+      const ctx = canvas.getContext('2d');
+      ctx?.drawImage(img, 0, 0);
+      
+      // Convert image to base64 data URL and extract the base64 string
+      const base64Image = canvas.toDataURL('image/jpeg').split(',')[1];
+
+      // Make request to Gradio API
+      const samEmbedding = await imageToSamEmbedding(
+        base64Image,
+        (status: QueueStatus) => {
+          setQueueStatus(status);
+        }
+      );
+
+      // Convert base64 embedding back to array buffer
+      const binaryString = window.atob(samEmbedding);
+      const len = binaryString.length;
+      const bytes = new Uint8Array(len);
+      for (let i = 0; i < len; i++) {
+        bytes[i] = binaryString.charCodeAt(i);
+      }
+
+      // Create tensor from the embedding
+      const embedding = new ort.Tensor(
+        'float32',
+        new Float32Array(bytes.buffer),  // Convert to Float32Array
+        [1, 256, 64, 64] // SAM embedding shape
+      );
+      setTensor(embedding);
+    } catch (error) {
+      setQueueStatus({ inQueue: false }); // Reset queue status on error
+      let errorMessage = 'Failed to process image. Please try again.';
+      if (axios.isAxiosError(error)) {
+        errorMessage = error.response?.data?.message || errorMessage;
+      }
+      setError(errorMessage);
+      console.error('Error fetching embedding:', error);
+    }
+  };
+
+  useEffect(() => {
+    const handleMaskUpdate = async () => {
+      await runONNX();
+    };
+    handleMaskUpdate();
+  }, [clicks]);
+
+  const runONNX = async () => {
+    try {
+      // Don't run if already described or is describing
+      if (descriptionState.state !== 'ready') return;
+      
+      console.log('Running ONNX model with:', {
+        modelLoaded: model !== null,
+        hasClicks: clicks !== null,
+        hasTensor: tensor !== null,
+        hasModelScale: modelScale !== null
+      });
+
+      if (
+        model === null ||
+        clicks === null ||
+        tensor === null ||
+        modelScale === null
+      ) {
+        console.log('Missing required inputs, returning early');
+        return;
+      }
+      else {
+        console.log('Preparing model feeds with:', {
+          clicks,
+          tensorShape: tensor.dims,
+          modelScale
+        });
+
+        const feeds = modelData({
+          clicks,
+          tensor,
+          modelScale,
+        });
+
+        if (feeds === undefined) {
+          console.log('Model feeds undefined, returning early');
+          return;
+        }
+
+        console.log('Running model with feeds:', feeds);
+        const results = await model.run(feeds);
+        console.log('Model run complete, got results:', results);
+
+        const output = results[model.outputNames[0]];
+        console.log('Processing output with dims:', output.dims);
+
+        // Calculate and log the mask area (number of non-zero values)
+        const maskArray = Array.from(output.data as Uint8Array);
+        const maskArea = maskArray.filter(val => val > 0).length;
+        console.log('Mask area (number of non-zero pixels):', maskArea);
+
+        // Double check that the state is ready before processing the mask since the state may have changed
+        if (descriptionState.state !== 'ready') return;
+        // If clicked, we only handle the first mask (note that mask will be cleared after clicking before handling to let us know if it's the first mask).
+        if (isClicked && maskImgData != null) return;
+        if (maskArea > 0) {
+          setMaskImg(onnxMaskToImage(output.data, output.dims[2], output.dims[3], false));
+          setMaskImgData(imageDataToURL(arrayToImageData(output.data, output.dims[2], output.dims[3], true)));
+        } else {
+          console.warn('No mask area detected, clearing mask');
+          setMaskImg(null);
+          // setMaskImgData(null);
+        }
+        
+        console.log('Mask processing complete');
+      }
+    } catch (e) {
+      setError('Failed to process the image. Please try again.');
+      console.error('Error running ONNX model:', e);
+    }
+  };
+
+  const handleNewRegion = () => {
+    setDescriptionState({
+      state: 'ready',
+      description: ''
+    } as DescriptionState);
+    setMaskImg(null);
+    // setMaskImgData(null);
+    setIsClicked(false);
+  };
+
+  const handleCopyDescription = () => {
+    navigator.clipboard.writeText(descriptionState.description);
+  };
+
+  const handleReset = () => {
+    // Clear all states
+    setDescriptionState({
+      state: 'ready',
+      description: ''
+    } as DescriptionState);
+    setMaskImg(null);
+    // setMaskImgData(null);
+    setImage(null);
+    setClicks(null);
+    setIsClicked(false);
+  };
+
+  return (
+    <div className="flex flex-col h-screen">
+      {isLoading && <LoadingOverlay />}
+      {error && <ErrorModal message={error} onClose={() => setError(null)} />}
+      <QueueStatusIndicator queueStatus={queueStatus} />
+      <div className="flex-1">
+        <Stage 
+          onImageUpload={handleImageUpload} 
+          descriptionState={descriptionState}
+          setDescriptionState={setDescriptionState}
+          queueStatus={queueStatus}
+          setQueueStatus={setQueueStatus}
+        />
+      </div>
+      <div className="description-container">
+        <div className={`description-box ${descriptionState.state !== 'described' ? descriptionState.state : ''}`}>
+          {descriptionState.description ? (
+            descriptionState.description + (descriptionState.state === 'describing' ? '...' : '')
+          ) : descriptionState.state === 'describing' ? (
+            <em>Describing the region... (this may take a while if compute resources are busy)</em>
+          ) : (
+            image ? (
+              <em>Click on the image to describe the region</em>
+            ) : (
+              <em>Upload an image to describe the region</em>
+            )
+          )}
+        </div>
+        <div className="description-controls">
+          <button 
+            onClick={handleCopyDescription}
+            disabled={descriptionState.state !== 'described'}
+          >
+            Copy description
+          </button>
+          <button 
+            onClick={handleNewRegion}
+            disabled={descriptionState.state !== 'described'}
+          >
+            Describe a new region
+          </button>
+          <button 
+            onClick={handleReset}
+            className="reset-button"
+            disabled={descriptionState.state === 'describing' || !image}
+          >
+            Try a new image
+          </button>
+        </div>
+      </div>
+    </div>
+  );
+};
+
+export default App;

frontend/src/assets/index.html

@@ -0,0 +1,18 @@
+<!DOCTYPE html>
+<html lang="en" dir="ltr" prefix="og: https://ogp.me/ns#" class="w-full h-full">
+  <head>
+    <meta charset="utf-8" />
+    <meta
+      name="viewport"
+      content="width=device-width, initial-scale=1, shrink-to-fit=no"
+    />
+    <title>Describe Anything Demo</title>
+
+    <!--  Meta Tags -->
+    <meta property="og:type" content="website" />
+    <meta property="og:title" content="Describe Anything Demo" />
+  </head>
+  <body class="w-full h-full">
+    <div id="root" class="w-full h-full"></div>
+  </body>
+</html>

frontend/src/assets/scss/App.scss

@@ -0,0 +1,99 @@
+@tailwind base;
+@tailwind components;
+@tailwind utilities;
+
+.fixed {
+  position: fixed;
+}
+
+.inset-0 {
+  top: 0;
+  right: 0;
+  bottom: 0;
+  left: 0;
+}
+
+.bg-opacity-75 {
+  --tw-bg-opacity: 0.75;
+}
+
+.z-50 {
+  z-index: 50;
+}
+
+.description-container {
+  margin: 20px;
+  display: flex;
+  gap: 20px;
+  height: 140px;
+}
+
+.description-box {
+  flex: 1;
+  background-color: #f5f5f5;
+  border-radius: 4px;
+  padding: 15px;
+  margin-bottom: 0;
+  color: #333;
+  overflow-y: auto;
+  
+  &.describing, &.ready {
+    background-color: #e9ecef;
+    color: #6c757d;
+    font-style: italic;
+  }
+}
+
+.description-controls {
+  display: flex;
+  flex-direction: column;
+  justify-content: space-between;
+  width: 200px;
+  gap: 10px;
+  
+  button {
+    padding: 8px 16px;
+    border: none;
+    border-radius: 4px;
+    background-color: #007bff;
+    color: white;
+    cursor: pointer;
+    white-space: nowrap;
+    
+    &:disabled {
+      background-color: #cccccc;
+      cursor: not-allowed;
+    }
+    
+    &:hover:not(:disabled) {
+      background-color: #0056b3;
+    }
+
+    &.reset-button {
+        background-color: #007bff;
+      
+      &:hover:not(:disabled) {
+        background-color: #0056b3;
+      }
+      
+      &:disabled {
+        background-color: #cccccc;
+      }
+    }
+  }
+}
+
+#root {
+  display: flex;
+  flex-direction: column;
+  height: 100vh;
+}
+
+.stage-container {
+  flex: 1;
+  min-height: 0;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  overflow: hidden;
+}

frontend/src/components/ErrorModal.tsx

@@ -0,0 +1,32 @@
+import React from 'react';
+
+interface ErrorModalProps {
+  message: string;
+  onClose: () => void;
+}
+
+const ErrorModal: React.FC<ErrorModalProps> = ({ message, onClose }) => {
+  return (
+    <div className="fixed inset-0 bg-black bg-opacity-50 flex items-center justify-center z-50">
+      <div className="bg-white p-6 rounded-lg shadow-xl max-w-md w-full mx-4">
+        <div className="flex flex-col items-center">
+          <div className="bg-red-100 p-4 rounded-full mb-4">
+            <svg className="w-6 h-6 text-red-600" fill="none" stroke="currentColor" viewBox="0 0 24 24">
+              <path strokeLinecap="round" strokeLinejoin="round" strokeWidth="2" d="M6 18L18 6M6 6l12 12" />
+            </svg>
+          </div>
+          <h3 className="text-lg font-semibold text-gray-900 mb-2">Error</h3>
+          <p className="text-gray-600 text-center mb-6">{message}</p>
+          <button
+            onClick={onClose}
+            className="bg-red-600 text-white px-4 py-2 rounded hover:bg-red-700 transition-colors"
+          >
+            Close
+          </button>
+        </div>
+      </div>
+    </div>
+  );
+};
+
+export default ErrorModal; 

frontend/src/components/LoadingOverlay.tsx

@@ -0,0 +1,30 @@
+import React from 'react';
+
+const LoadingOverlay: React.FC = () => {
+  return (
+    <div className="fixed inset-0 bg-gray-500 bg-opacity-75 flex items-center justify-center z-50">
+      <div className="bg-white p-8 rounded-lg shadow-xl flex flex-col items-center">
+        <svg width="54" height="54" viewBox="0 0 54 54" fill="none" xmlns="http://www.w3.org/2000/svg" className="w-16 h-16 mb-4">
+          <path d="M5.92017 41.0562L27.0002 48.0802" stroke="#1C2B33" strokeWidth="2" strokeMiterlimit="10"/>
+          <path d="M5.92017 12.9438L27.0002 26.9998" stroke="#1C2B33" strokeWidth="2" strokeMiterlimit="10"/>
+          <path d="M27 5.91992L48.08 26.9999" stroke="#1C2B33" strokeWidth="2" strokeMiterlimit="10"/>
+          <path d="M5.92017 41.0559L27.0002 5.91992" stroke="#1C2B33" strokeWidth="2" strokeMiterlimit="10"/>
+          <path d="M27 48.08L48.08 27" stroke="#1C2B33" strokeWidth="2" strokeMiterlimit="10"/>
+          <path d="M27 27H48.08" stroke="#1C2B33" strokeWidth="2" strokeMiterlimit="10"/>
+          <path d="M5.92017 12.9439L27.0002 5.91992" stroke="#1C2B33" strokeWidth="2" strokeMiterlimit="10"/>
+          <path d="M5.92017 41.056L27.0002 27" stroke="#1C2B33" strokeWidth="2" strokeMiterlimit="10"/>
+          <path d="M5.92017 12.9438L27.0002 48.0798" stroke="#1C2B33" strokeWidth="2" strokeMiterlimit="10"/>
+          <path d="M26.9998 31.9201C29.7171 31.9201 31.9198 29.7173 31.9198 27.0001C31.9198 24.2828 29.7171 22.0801 26.9998 22.0801C24.2826 22.0801 22.0798 24.2828 22.0798 27.0001C22.0798 29.7173 24.2826 31.9201 26.9998 31.9201Z" fill="white" stroke="#1C2B33" strokeWidth="2" strokeMiterlimit="10"/>
+          <path d="M5.92 17.8639C8.63724 17.8639 10.84 15.6612 10.84 12.9439C10.84 10.2267 8.63724 8.02393 5.92 8.02393C3.20276 8.02393 1 10.2267 1 12.9439C1 15.6612 3.20276 17.8639 5.92 17.8639Z" fill="white" stroke="#1C2B33" strokeWidth="2" strokeMiterlimit="10"/>
+          <path d="M5.92 45.9757C8.63724 45.9757 10.84 43.773 10.84 41.0557C10.84 38.3385 8.63724 36.1357 5.92 36.1357C3.20276 36.1357 1 38.3385 1 41.0557C1 43.773 3.20276 45.9757 5.92 45.9757Z" fill="white" stroke="#1C2B33" strokeWidth="2" strokeMiterlimit="10"/>
+          <path d="M48.0806 31.9201C50.7979 31.9201 53.0006 29.7173 53.0006 27.0001C53.0006 24.2828 50.7979 22.0801 48.0806 22.0801C45.3634 22.0801 43.1606 24.2828 43.1606 27.0001C43.1606 29.7173 45.3634 31.9201 48.0806 31.9201Z" fill="white" stroke="#1C2B33" strokeWidth="2" strokeMiterlimit="10"/>
+          <path d="M26.9998 53.0002C29.7171 53.0002 31.9198 50.7974 31.9198 48.0802C31.9198 45.3629 29.7171 43.1602 26.9998 43.1602C24.2826 43.1602 22.0798 45.3629 22.0798 48.0802C22.0798 50.7974 24.2826 53.0002 26.9998 53.0002Z" fill="white" stroke="#1C2B33" strokeWidth="2" strokeMiterlimit="10"/>
+          <path d="M26.9998 10.84C29.7171 10.84 31.9198 8.63724 31.9198 5.92C31.9198 3.20276 29.7171 1 26.9998 1C24.2826 1 22.0798 3.20276 22.0798 5.92C22.0798 8.63724 24.2826 10.84 26.9998 10.84Z" fill="white" stroke="#1C2B33" strokeWidth="2" strokeMiterlimit="10"/>
+        </svg>
+        <p className="text-lg font-semibold text-gray-800">Loading image embedding...</p>
+      </div>
+    </div>
+  );
+};
+
+export default LoadingOverlay; 

frontend/src/components/QueueStatusIndicator.tsx

@@ -0,0 +1,29 @@
+import React from 'react';
+import { QueueStatus } from './helpers/Interfaces';
+
+interface QueueStatusIndicatorProps {
+  queueStatus: QueueStatus;
+}
+
+const QueueStatusIndicator: React.FC<QueueStatusIndicatorProps> = ({ queueStatus }) => {
+  if (!queueStatus.inQueue) return null;
+
+  return (
+    <div className="fixed top-4 right-4 bg-white rounded-lg shadow-lg p-4 z-50">
+      <div className="flex flex-col gap-2">
+        {queueStatus.rank === 0 ? (
+          <p className="text-sm">You're next in line! ({queueStatus.queueSize} total in queue)</p>
+        ) : (
+          <p className="text-sm">Queue position: {queueStatus.rank! + 1} of {queueStatus.queueSize}</p>
+        )}
+        {queueStatus.rankEta && (
+          <p className="text-sm text-gray-600">
+            Estimated wait: {Math.ceil(queueStatus.rankEta)} seconds
+          </p>
+        )}
+      </div>
+    </div>
+  );
+};
+
+export default QueueStatusIndicator; 

frontend/src/components/Stage.tsx

@@ -0,0 +1,343 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// All rights reserved.
+
+// This source code is licensed under the license found in the
+// LICENSE file in the root directory of this source tree.
+
+import React, { useContext, useState, useEffect } from "react";
+import * as _ from "underscore";
+import Tool from "./Tool";
+import { modelInputProps, QueueStatus } from "./helpers/Interfaces";
+import AppContext from "./hooks/createContext";
+// import { describeMask } from '../services/maskApi';
+
+interface DescriptionState {
+  state: string; // 'ready', 'describing', 'described'
+  description: string;
+}
+
+interface StageProps {
+  onImageUpload: (event: React.ChangeEvent<HTMLInputElement>) => Promise<void>;
+  descriptionState: DescriptionState;
+  setDescriptionState: React.Dispatch<React.SetStateAction<DescriptionState>>;
+  queueStatus: QueueStatus;
+  setQueueStatus: (status: QueueStatus) => void;
+}
+
+const EXAMPLE_IMAGES = Array.from({ length: 21 }, (_, i) => `/examples/${i + 1}.jpg`);
+const BREAKPOINT_MEDIUM = 2100;
+const BREAKPOINT_SMALL = 1100;
+
+const Stage = ({ onImageUpload, descriptionState, setDescriptionState, queueStatus, setQueueStatus }: StageProps) => {
+  const {
+    clicks: [, setClicks],
+    image: [image],
+    maskImg: [maskImg],
+    maskImgData: [maskImgData]
+  } = useContext(AppContext)!;
+
+  const [isDragging, setIsDragging] = useState(false);
+  const [currentPage, setCurrentPage] = useState(1);
+  const [imagesPerPage, setImagesPerPage] = useState(8);
+
+  useEffect(() => {
+    const handleResize = () => {
+      if (window.innerWidth < BREAKPOINT_SMALL) {
+        setImagesPerPage(1);
+      } else if (window.innerWidth < BREAKPOINT_MEDIUM) {
+        setImagesPerPage(4);
+      } else {
+        setImagesPerPage(8);
+      }
+    };
+
+    // Set initial value
+    handleResize();
+
+    // Add event listener
+    window.addEventListener('resize', handleResize);
+
+    // Cleanup
+    return () => window.removeEventListener('resize', handleResize);
+  }, []);
+
+  const getClick = (x: number, y: number): modelInputProps => {
+    const clickType = 1;
+    return { x, y, clickType };
+  };
+
+  const handleMouseMove = _.throttle((e: any) => {
+    if (descriptionState.state !== 'ready') return;
+    if (e.clientX === undefined || e.clientY === undefined) {
+      console.warn('Mouse move event does not contain clientX or clientY');
+      return;
+    }
+    let el = e.nativeEvent.target;
+    const rect = el.getBoundingClientRect();
+    
+    // Calculate the actual dimensions of the contained image
+    const containerAspectRatio = el.offsetWidth / el.offsetHeight;
+    const imageAspectRatio = image ? image.width / image.height : 1;
+    
+    let renderedWidth, renderedHeight;
+    if (containerAspectRatio > imageAspectRatio) {
+      // Image is constrained by height
+      renderedHeight = el.offsetHeight;
+      renderedWidth = renderedHeight * imageAspectRatio;
+    } else {
+      // Image is constrained by width
+      renderedWidth = el.offsetWidth;
+      renderedHeight = renderedWidth / imageAspectRatio;
+    }
+
+    // Calculate the empty space offset
+    const offsetX = (el.offsetWidth - renderedWidth) / 2;
+    const offsetY = (el.offsetHeight - renderedHeight) / 2;
+
+    // Get click position relative to the actual image
+    let x = e.clientX - rect.left - offsetX;
+    let y = e.clientY - rect.top - offsetY;
+
+    // Convert to original image coordinates
+    const scaleX = image ? image.width / renderedWidth : 1;
+    const scaleY = image ? image.height / renderedHeight : 1;
+    x *= scaleX;
+    y *= scaleY;
+
+    // Ensure coordinates are within bounds
+    if (image) {
+      x = Math.max(0, Math.min(x, image.width));
+      y = Math.max(0, Math.min(y, image.height));
+    }
+
+    const click = getClick(x, y);
+    if (click) {
+      setClicks([click]);
+    }
+  }, 15);
+
+  const handleDragEnter = (e: React.DragEvent) => {
+    e.preventDefault();
+    e.stopPropagation();
+    setIsDragging(true);
+  };
+
+  const handleDragLeave = (e: React.DragEvent) => {
+    e.preventDefault();
+    e.stopPropagation();
+    setIsDragging(false);
+  };
+
+  const handleDragOver = (e: React.DragEvent) => {
+    e.preventDefault();
+    e.stopPropagation();
+  };
+
+  const handleDrop = async (e: React.DragEvent) => {
+    e.preventDefault();
+    e.stopPropagation();
+    setIsDragging(false);
+
+    const files = e.dataTransfer.files;
+    if (files && files[0]) {
+      const file = files[0];
+      // Cast to unknown first, then to the desired type
+      const syntheticEvent = {
+        target: {
+          files: [file]
+        }
+      } as unknown as React.ChangeEvent<HTMLInputElement>;
+      
+      onImageUpload(syntheticEvent);
+    }
+  };
+
+  const flexCenterClasses = "flex items-center justify-center";
+  
+  // const handleDescribeMask = async () => {
+  //   if (!maskImg || !maskImgData || !image) {
+  //     console.warn('No mask or image available to describe');
+  //     return;
+  //   }
+
+  //   try {
+  //     const canvas = document.createElement('canvas');
+  //     canvas.width = image.width;
+  //     canvas.height = image.height;
+  //     const ctx = canvas.getContext('2d');
+  //     ctx?.drawImage(image, 0, 0);
+  //     const imageBase64 = canvas.toDataURL('image/jpeg').split(',')[1];
+  //     const maskBase64 = maskImgData.split(',')[1];
+
+  //     const result = await describeMask(maskBase64, imageBase64);
+  //     console.log('Mask description:', result.description);
+      
+  //     alert("Mask description: " + result.description);
+  //   } catch (error) {
+  //     console.error('Failed to describe mask:', error);
+  //   }
+  // };
+
+  return (
+    <div 
+      className={`flex flex-col w-full h-full relative`}
+      onDragEnter={handleDragEnter}
+      onDragOver={handleDragOver}
+      onDragLeave={handleDragLeave}
+      onDrop={handleDrop}
+    >
+      {/* Title and Description */}
+      <div className="w-full px-8 mb-8 flex flex-col justify-center mt-4">
+        <div className="flex flex-col sm:flex-row justify-between items-center gap-4">
+          <h1 className="text-3xl font-bold text-center sm:text-left"><a href="/">Describe Anything Model Demo</a></h1>
+          <div className="flex flex-wrap justify-center gap-4 sm:space-x-8 text-lg font-medium">
+            <a href="https://describe-anything.github.io/" target="_blank" rel="noopener noreferrer" className="text-gray-600 hover:text-gray-800">Project Page</a>
+            <a href="https://github.com/NVlabs/describe-anything?tab=readme-ov-file#simple-gradio-demo-for-detailed-localized-video-descriptions" target="_blank" rel="noopener noreferrer" className="text-gray-600 hover:text-gray-800">DAM for video</a>
+          </div>
+        </div>
+        <div className="border-b border-gray-300 mt-4 mb-4"></div>
+        {!image && <div className="space-y-4 text-gray-600 text-left">
+          <p>Describe Anything Model (DAM) takes in a region of an image or a video in the form of points/boxes/scribbles/masks and outputs detailed descriptions to the region. For videos, it is sufficient to supply annotation on any frame.</p>
+          <p>This demo supports DAM model that takes points on images as queries. For other use cases, please refer to the <a href="" className="text-gray-600 hover:text-gray-800 underline">inference scripts and video demo</a> for more details.</p>
+        </div>}
+      </div>
+
+      {/* Main Content Area */}
+      <div className={`flex items-center justify-center flex-grow overflow-hidden`}>
+        {/* Main Stage */}
+        <div 
+          className={`${flexCenterClasses} relative w-full h-full max-h-[calc(100vh-300px)] px-8 ${
+            isDragging ? 'border-4 border-dashed border-blue-500 bg-blue-50' : ''
+          }`}
+        >
+          {image ? (
+            <>
+              <Tool 
+                handleMouseMove={handleMouseMove} 
+                descriptionState={descriptionState}
+                setDescriptionState={setDescriptionState}
+                queueStatus={queueStatus}
+                setQueueStatus={setQueueStatus}
+              />
+            </>
+          ) : (
+            <>
+              <div className="flex flex-col items-center gap-6 w-full h-full">
+                <div className="flex-1" />
+                
+                <div className="text-gray-500 text-lg">
+                  {isDragging ? 'Drop image here' : 'Upload your own image'}
+                </div>
+                <div className="flex gap-4 mb-8">
+                  <input
+                    type="file"
+                    id="imageUpload"
+                    accept="image/*"
+                    onChange={onImageUpload}
+                    className="hidden"
+                  />
+                  <label
+                    htmlFor="imageUpload"
+                    className="bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-4 rounded cursor-pointer"
+                  >
+                    Upload Image
+                  </label>
+                </div>
+
+                <div className="text-gray-500 text-lg">
+                  or choose an example image below
+                </div>
+                
+                <div className="relative w-full max-w-[2200px]">
+                  {/* Left Arrow */}
+                  <button
+                    onClick={() => setCurrentPage(prev => Math.max(prev - 1, 1))}
+                    disabled={currentPage === 1}
+                    className={`absolute left-0 top-1/2 -translate-y-1/2 z-10 p-4 ${
+                      currentPage === 1 
+                        ? 'text-gray-300 cursor-not-allowed' 
+                        : 'text-gray-600 hover:text-gray-800'
+                    }`}
+                  >
+                    <svg xmlns="http://www.w3.org/2000/svg" className="h-8 w-8" fill="none" viewBox="0 0 24 24" stroke="currentColor">
+                      <path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M15 19l-7-7 7-7" />
+                    </svg>
+                  </button>
+
+                  {/* Example Images */}
+                  <div className="flex flex-wrap justify-center gap-8 px-16">
+                    {EXAMPLE_IMAGES.slice(
+                      (currentPage - 1) * imagesPerPage,
+                      currentPage * imagesPerPage
+                    ).map((src, index) => (
+                      <img
+                        key={index}
+                        src={src}
+                        alt={`Example ${index + 1}`}
+                        className="w-[200px] h-[150px] object-cover rounded-sm cursor-pointer hover:opacity-80 transition-opacity"
+                        onClick={() => {
+                          fetch(src)
+                            .then(res => res.blob())
+                            .then(blob => {
+                              const file = new File([blob], `example-${index + 1}.jpg`, { type: 'image/jpeg' });
+                              const syntheticEvent = {
+                                target: {
+                                  files: [file]
+                                }
+                              } as unknown as React.ChangeEvent<HTMLInputElement>;
+                              
+                              onImageUpload(syntheticEvent);
+                            });
+                        }}
+                      />
+                    ))}
+                  </div>
+
+                  {/* Right Arrow */}
+                  <button
+                    onClick={() => setCurrentPage(prev => Math.min(prev + 1, Math.ceil(EXAMPLE_IMAGES.length / imagesPerPage)))}
+                    disabled={currentPage === Math.ceil(EXAMPLE_IMAGES.length / imagesPerPage)}
+                    className={`absolute right-0 top-1/2 -translate-y-1/2 z-10 p-4 ${
+                      currentPage === Math.ceil(EXAMPLE_IMAGES.length / imagesPerPage)
+                        ? 'text-gray-300 cursor-not-allowed'
+                        : 'text-gray-600 hover:text-gray-800'
+                    }`}
+                  >
+                    <svg xmlns="http://www.w3.org/2000/svg" className="h-8 w-8" fill="none" viewBox="0 0 24 24" stroke="currentColor">
+                      <path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M9 5l7 7-7 7" />
+                    </svg>
+                  </button>
+
+                  {/* Page Indicator */}
+                  {/* <div className="w-full text-center mt-4 text-gray-600">
+                    Page {currentPage} of {Math.ceil(EXAMPLE_IMAGES.length / imagesPerPage)}
+                  </div> */}
+                </div>
+
+                <div className="flex-1" /> {/* Bottom spacer */}
+                {/* Image Credits */}
+                {!image && (
+                <div className="pl-5 pr-5 text-gray-500 text-sm">
+                  Image credit for example images: {' '}
+                  <a 
+                    href="https://segment-anything.com/terms" 
+                    target="_blank" 
+                    className="text-gray-600 hover:text-gray-800 underline"
+                  >
+                    Segment Anything Materials
+                  </a>
+                  {' '}(CC BY-SA 4.0)
+                </div>
+                )}
+              </div>
+            </>
+          )}
+        </div>
+      </div>
+      
+    </div>
+  );
+};
+
+export default Stage;
+export type { DescriptionState };

frontend/src/components/Tool.tsx

@@ -0,0 +1,182 @@
+import React, { useContext, useEffect, useState } from "react";
+import AppContext from "./hooks/createContext";
+import { ToolProps, QueueStatus } from "./helpers/Interfaces";
+import * as _ from "underscore";
+import { describeMask, describeMaskWithoutStreaming } from "../services/maskApi";
+import ErrorModal from './ErrorModal';
+import { DescriptionState } from "./Stage";
+
+const prompt = "<image>\nDescribe the masked region in detail.";
+
+const Tool = ({ 
+  handleMouseMove, 
+  descriptionState, 
+  setDescriptionState,
+  queueStatus,
+  setQueueStatus 
+}: ToolProps) => {
+  console.log("Tool handleMouseMove");
+  const {
+    image: [image],
+    maskImg: [maskImg, setMaskImg],
+    maskImgData: [maskImgData, setMaskImgData],
+    isClicked: [isClicked, setIsClicked]
+  } = useContext(AppContext)!;
+
+  const [shouldFitToWidth, setShouldFitToWidth] = useState(true);
+  const bodyEl = document.body;
+  const fitToPage = () => {
+    if (!image) return;
+    const maxWidth = window.innerWidth - 64; // Account for padding (32px on each side)
+    const maxHeight = window.innerHeight - 200; // Account for header and some padding
+    const imageAspectRatio = image.width / image.height;
+    const containerAspectRatio = maxWidth / maxHeight;
+    
+    setShouldFitToWidth(
+      imageAspectRatio > containerAspectRatio || 
+      image.width > maxWidth
+    );
+  };
+  const resizeObserver = new ResizeObserver((entries) => {
+    for (const entry of entries) {
+      if (entry.target === bodyEl) {
+        fitToPage();
+      }
+    }
+  });
+  useEffect(() => {
+    fitToPage();
+    resizeObserver.observe(bodyEl);
+    return () => {
+      resizeObserver.unobserve(bodyEl);
+    };
+  }, [image]);
+
+  const imageClasses = "";
+  const maskImageClasses = `absolute opacity-40 pointer-events-none`;
+
+  const [error, setError] = useState<string | null>(null);
+  const [useStreaming, setUseStreaming] = useState(true);
+
+  useEffect(() => {
+      if (!isClicked || !maskImg || !maskImgData || !image || descriptionState.state !== 'ready') {
+        console.log("Not ready to call model, isClicked:", isClicked, "maskImg:", maskImg !== null, "maskImgData:", maskImgData !== null, "image:", image !== null, "descriptionState.state:", descriptionState.state);
+        return;
+      }
+
+      try {
+        setDescriptionState({
+          state: 'describing',
+          description: ''
+        } as DescriptionState);
+
+        const canvas = document.createElement('canvas');
+        canvas.width = image.width;
+        canvas.height = image.height;
+        const ctx = canvas.getContext('2d');
+        ctx?.drawImage(image, 0, 0);
+        const imageBase64 = canvas.toDataURL('image/jpeg').split(',')[1];
+        const maskBase64 = maskImgData.split(',')[1];
+
+        const describeMaskWithFallback = async (useStreamingInFunction: boolean) => {
+          try {
+            let result;
+            console.log("useStreaming", useStreaming, "useStreamingInFunction", useStreamingInFunction);
+            if (useStreamingInFunction) {
+              result = await describeMask(
+                maskBase64, 
+                imageBase64,
+                prompt,
+                (streamResult: string) => {
+                  setDescriptionState({
+                    state: 'describing',
+                    description: streamResult
+                  } as DescriptionState);
+                },
+                (status: QueueStatus) => {
+                  setQueueStatus(status);
+                }
+              );
+            } else {
+              result = await describeMaskWithoutStreaming(
+                maskBase64,
+                imageBase64,
+                prompt
+              );
+            }
+            
+            setDescriptionState({
+              state: 'described',
+              description: result
+            } as DescriptionState);
+            setQueueStatus({ inQueue: false });
+            setIsClicked(false);
+          } catch (error) {
+            if (useStreaming) {
+              console.log("Error describing mask, switching to non-streaming", error);
+              setUseStreaming(false);
+              describeMaskWithFallback(false);
+            } else {
+              setError('Failed to generate description. Please try again.');
+              setDescriptionState({
+                state: 'ready',
+                description: ''
+              } as DescriptionState);
+              setIsClicked(false);
+              console.error('Failed to describe mask:', error);
+            }
+          }
+        };
+
+        describeMaskWithFallback(useStreaming);
+
+      } catch (error) {
+        setIsClicked(false);
+        setError('Failed to generate description. Please try again.');
+        setDescriptionState({
+          state: 'ready',
+          description: ''
+        } as DescriptionState);
+        console.error('Failed to describe mask:', error);
+      }
+  }, [maskImgData]);
+
+  const handleClick = async (e: React.MouseEvent<HTMLImageElement>) => {
+    if (descriptionState.state !== 'ready') return;
+    
+    setMaskImg(null);
+    setMaskImgData(null);
+    setIsClicked(true);
+    handleMouseMove(e);
+  };
+
+  return (
+    <>
+      {error && <ErrorModal message={error} onClose={() => setError(null)} />}
+      <div className="relative flex items-center justify-center w-full h-full">
+        {image && (
+          <img
+            onMouseMove={handleMouseMove}
+            onMouseLeave={() => _.defer(() => (descriptionState.state === 'ready' && !isClicked) ? setMaskImg(null) : undefined)}
+            onTouchStart={handleMouseMove}
+            onClick={handleClick}
+            src={image.src}
+            className={`${
+              shouldFitToWidth ? "w-full" : "h-full"
+            } ${imageClasses} object-contain max-h-full max-w-full`}
+          ></img>
+        )}
+        {maskImg && (
+          <img
+            src={maskImg.src}
+            className={`${
+              shouldFitToWidth ? "w-full" : "h-full"
+            } ${maskImageClasses} object-contain max-h-full max-w-full`}
+          ></img>
+        )}
+      </div>
+    </>
+  );
+};
+
+export default Tool;

frontend/src/components/helpers/Interfaces.tsx

@@ -0,0 +1,47 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// All rights reserved.
+
+// This source code is licensed under the license found in the
+// LICENSE file in the root directory of this source tree.
+
+import { Tensor } from "onnxruntime-web";
+import { DescriptionState } from "../Stage";
+
+export interface modelScaleProps {
+  samScale: number;
+  height: number;
+  width: number;
+}
+
+export interface modelInputProps {
+  x: number;
+  y: number;
+  clickType: number;
+}
+
+export interface modeDataProps {
+  clicks?: Array<modelInputProps>;
+  tensor: Tensor;
+  modelScale: modelScaleProps;
+}
+
+export interface ToolProps {
+  handleMouseMove: (e: any) => void;
+  descriptionState: DescriptionState;
+  setDescriptionState: (value: DescriptionState) => void;
+  queueStatus: QueueStatus;
+  setQueueStatus: (value: QueueStatus) => void;
+}
+
+export interface StageProps {
+  onImageUpload: (event: React.ChangeEvent<HTMLInputElement>) => void;
+  descriptionState: DescriptionState;
+  setDescriptionState: (value: DescriptionState) => void;
+}
+
+export interface QueueStatus {
+  inQueue: boolean;
+  rank?: number;
+  queueSize?: number;
+  rankEta?: number | null;
+}

frontend/src/components/helpers/imageUtils.tsx

@@ -0,0 +1,21 @@
+import { Buffer } from 'buffer';
+
+export const base64ToImage = async (base64String: string): Promise<HTMLImageElement> => {
+  return new Promise((resolve, reject) => {
+    const img = new Image();
+    img.onload = () => resolve(img);
+    img.onerror = reject;
+    img.src = base64String.startsWith('data:') ? 
+      base64String : 
+      `data:image/png;base64,${base64String}`;
+  });
+};
+
+export const imageToBase64 = (img: HTMLImageElement): string => {
+  const canvas = document.createElement('canvas');
+  canvas.width = img.width;
+  canvas.height = img.height;
+  const ctx = canvas.getContext('2d');
+  ctx?.drawImage(img, 0, 0);
+  return canvas.toDataURL('image/png');
+}; 

frontend/src/components/helpers/maskUtils.tsx

@@ -0,0 +1,65 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// All rights reserved.
+
+// This source code is licensed under the license found in the
+// LICENSE file in the root directory of this source tree.
+
+// Convert the onnx model mask prediction to ImageData
+function arrayToImageData(input: any, width: number, height: number, binary: boolean) {
+  let [r, g, b, a] = [0, 114, 189, 255]; // the masks's blue color
+  let [r_bg, g_bg, b_bg, a_bg] = [0, 0, 0, 0]; // the background's white color
+  if (binary) {
+    [r, g, b, a] = [255, 255, 255, 255]; // black and white
+    [r_bg, g_bg, b_bg, a_bg] = [0, 0, 0, 255]; // black and white
+  }
+  
+  const arr = new Uint8ClampedArray(4 * width * height).fill(0);
+  for (let i = 0; i < input.length; i++) {
+
+    // Threshold the onnx model mask prediction at 0.0
+    // This is equivalent to thresholding the mask using predictor.model.mask_threshold
+    // in python
+    if (input[i] > 0.0) {
+      arr[4 * i + 0] = r;
+      arr[4 * i + 1] = g;
+      arr[4 * i + 2] = b;
+      arr[4 * i + 3] = a;
+    } else if (binary){
+      arr[4 * i + 0] = r_bg;
+      arr[4 * i + 1] = g_bg;
+      arr[4 * i + 2] = b_bg;
+      arr[4 * i + 3] = a_bg;
+    }
+  }
+  return new ImageData(arr, height, width);
+}
+
+// Use a Canvas element to produce an image from ImageData
+function imageDataToImage(imageData: ImageData) {
+  const canvas = imageDataToCanvas(imageData);
+  const image = new Image();
+  image.src = canvas.toDataURL();
+  return image;
+}
+
+function imageDataToURL(imageData: ImageData) {
+  const canvas = imageDataToCanvas(imageData);
+  return canvas.toDataURL();
+}
+
+// Canvas elements can be created from ImageData
+function imageDataToCanvas(imageData: ImageData) {
+  const canvas = document.createElement("canvas");
+  const ctx = canvas.getContext("2d");
+  canvas.width = imageData.width;
+  canvas.height = imageData.height;
+  ctx?.putImageData(imageData, 0, 0);
+  return canvas;
+}
+
+// Convert the onnx model mask output to an HTMLImageElement
+function onnxMaskToImage(input: any, width: number, height: number, binary: boolean) {
+  return imageDataToImage(arrayToImageData(input, width, height, binary));
+}
+
+export { arrayToImageData, imageDataToImage, onnxMaskToImage, imageDataToURL };

frontend/src/components/helpers/onnxModelAPI.tsx

@@ -0,0 +1,71 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// All rights reserved.
+
+// This source code is licensed under the license found in the
+// LICENSE file in the root directory of this source tree.
+
+import { Tensor } from "onnxruntime-web";
+import { modeDataProps } from "./Interfaces";
+
+const modelData = ({ clicks, tensor, modelScale }: modeDataProps) => {
+  const imageEmbedding = tensor;
+  let pointCoords;
+  let pointLabels;
+  let pointCoordsTensor;
+  let pointLabelsTensor;
+
+  // Check there are input click prompts
+  if (clicks) {
+    let n = clicks.length;
+
+    // If there is no box input, a single padding point with 
+    // label -1 and coordinates (0.0, 0.0) should be concatenated
+    // so initialize the array to support (n + 1) points.
+    pointCoords = new Float32Array(2 * (n + 1));
+    pointLabels = new Float32Array(n + 1);
+
+    // Add clicks and scale to what SAM expects
+    for (let i = 0; i < n; i++) {
+      pointCoords[2 * i] = clicks[i].x * modelScale.samScale;
+      pointCoords[2 * i + 1] = clicks[i].y * modelScale.samScale;
+      pointLabels[i] = clicks[i].clickType;
+    }
+
+    // Add in the extra point/label when only clicks and no box
+    // The extra point is at (0, 0) with label -1
+    pointCoords[2 * n] = 0.0;
+    pointCoords[2 * n + 1] = 0.0;
+    pointLabels[n] = -1.0;
+
+    // Create the tensor
+    pointCoordsTensor = new Tensor("float32", pointCoords, [1, n + 1, 2]);
+    pointLabelsTensor = new Tensor("float32", pointLabels, [1, n + 1]);
+  }
+  const imageSizeTensor = new Tensor("float32", [
+    modelScale.height,
+    modelScale.width,
+  ]);
+
+  if (pointCoordsTensor === undefined || pointLabelsTensor === undefined)
+    return;
+
+  // There is no previous mask, so default to an empty tensor
+  const maskInput = new Tensor(
+    "float32",
+    new Float32Array(256 * 256),
+    [1, 1, 256, 256]
+  );
+  // There is no previous mask, so default to 0
+  const hasMaskInput = new Tensor("float32", [0]);
+
+  return {
+    image_embeddings: imageEmbedding,
+    point_coords: pointCoordsTensor,
+    point_labels: pointLabelsTensor,
+    orig_im_size: imageSizeTensor,
+    mask_input: maskInput,
+    has_mask_input: hasMaskInput,
+  };
+};
+
+export { modelData };

frontend/src/components/helpers/scaleHelper.tsx

@@ -0,0 +1,18 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// All rights reserved.
+
+// This source code is licensed under the license found in the
+// LICENSE file in the root directory of this source tree.
+
+
+// Helper function for handling image scaling needed for SAM
+const handleImageScale = (image: HTMLImageElement) => {
+  // Input images to SAM must be resized so the longest side is 1024
+  const LONG_SIDE_LENGTH = 1024;
+  let w = image.naturalWidth;
+  let h = image.naturalHeight;
+  const samScale = LONG_SIDE_LENGTH / Math.max(h, w);
+  return { height: h, width: w, samScale };
+};
+
+export { handleImageScale };

frontend/src/components/hooks/context.tsx

@@ -0,0 +1,35 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// All rights reserved.
+
+// This source code is licensed under the license found in the
+// LICENSE file in the root directory of this source tree.
+
+import React, { useState } from "react";
+import { modelInputProps } from "../helpers/Interfaces";
+import AppContext from "./createContext";
+
+const AppContextProvider = (props: {
+  children: React.ReactElement<any, string | React.JSXElementConstructor<any>>;
+}) => {
+  const [clicks, setClicks] = useState<Array<modelInputProps> | null>(null);
+  const [image, setImage] = useState<HTMLImageElement | null>(null);
+  const [maskImg, setMaskImg] = useState<HTMLImageElement | null>(null);
+  const [maskImgData, setMaskImgData] = useState<string | null>(null);
+  const [isClicked, setIsClicked] = useState<boolean>(false);
+
+  return (
+    <AppContext.Provider
+      value={{
+        clicks: [clicks, setClicks],
+        image: [image, setImage],
+        maskImg: [maskImg, setMaskImg],
+        maskImgData: [maskImgData, setMaskImgData],
+        isClicked: [isClicked, setIsClicked],
+      }}
+    >
+      {props.children}
+    </AppContext.Provider>
+  );
+};
+
+export default AppContextProvider;

frontend/src/components/hooks/createContext.tsx

@@ -0,0 +1,35 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// All rights reserved.
+
+// This source code is licensed under the license found in the
+// LICENSE file in the root directory of this source tree.
+
+import { createContext } from "react";
+import { modelInputProps } from "../helpers/Interfaces";
+
+interface contextProps {
+  clicks: [
+    clicks: modelInputProps[] | null,
+    setClicks: (e: modelInputProps[] | null) => void
+  ];
+  image: [
+    image: HTMLImageElement | null,
+    setImage: (e: HTMLImageElement | null) => void
+  ];
+  maskImg: [
+    maskImg: HTMLImageElement | null,
+    setMaskImg: (e: HTMLImageElement | null) => void
+  ];
+  maskImgData: [
+    maskImgData: string | null,
+    setMaskImgData: (e: string | null) => void
+  ];
+  isClicked: [
+    isClicked: boolean,
+    setIsClicked: (e: boolean) => void
+  ];
+}
+
+const AppContext = createContext<contextProps | null>(null);
+
+export default AppContext;

frontend/src/index.tsx

@@ -0,0 +1,17 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// All rights reserved.
+
+// This source code is licensed under the license found in the
+// LICENSE file in the root directory of this source tree.
+
+import * as React from "react";
+import { createRoot } from "react-dom/client";
+import AppContextProvider from "./components/hooks/context";
+import App from "./App";
+const container = document.getElementById("root");
+const root = createRoot(container!);
+root.render(
+  <AppContextProvider>
+    <App/>
+  </AppContextProvider>
+);

frontend/src/services/maskApi.tsx

@@ -0,0 +1,211 @@
+import axios from 'axios';
+import * as _ from 'underscore';
+
+const API_URL = process.env.NODE_ENV === 'development' ? 'http://localhost:7860/gradio_api' : '/gradio_api';
+
+export const describeMaskWithoutStreaming = _.throttle(async (
+  maskBase64: string,
+  imageBase64: string,
+  query: string
+): Promise<string> => {
+  try {
+    const response = await axios.post(`${API_URL}/run/describe_without_streaming`, {
+      data: [imageBase64, maskBase64, query],
+    });
+    
+    console.log("response", response.data);
+    return response.data.data[0];
+  } catch (error) {
+    console.error('Error describing mask:', error);
+    throw error;
+  }
+}, 100);
+
+export const describeMask = _.throttle(async (
+  maskBase64: string,
+  imageBase64: string,
+  query: string,
+  onStreamUpdate: (token: string) => void,
+  onQueueUpdate?: (status: {
+    inQueue: boolean,
+    rank?: number,
+    queueSize?: number,
+    rankEta?: number | null
+  }) => void
+): Promise<string> => {
+  console.log("describeMask");
+  const initiateResponse = await axios.post(`${API_URL}/call/describe`, {
+    data: [imageBase64, maskBase64, query],
+  });
+  
+  const eventId = initiateResponse.data.event_id;
+  
+  const response = await axios.get(`${API_URL}/queue/data?session_hash=${eventId}`, {
+    headers: {
+      'Accept': 'text/event-stream',
+    },
+    responseType: 'stream',
+    adapter: 'fetch',
+  });
+
+  const stream = response.data;
+  const reader = stream.pipeThrough(new TextDecoderStream()).getReader();
+
+  let result = '';
+  let partialMessage = '';
+
+  while (true) {
+    const { value, done } = await reader.read();
+    if (done) {
+      return result;
+    }
+
+    // Concatenate with any previous partial message
+    const currentData = partialMessage + value;
+    const lines = currentData.split('\n');
+    
+    // Save the last line if it's incomplete
+    partialMessage = lines[lines.length - 1];
+    
+    // Process all complete lines except the last one
+    let eventType = '';
+    for (let i = 0; i < lines.length - 1; i++) {
+      const line = lines[i];
+      if (line.startsWith('event: ')) {
+        eventType = line.slice(7); // Remove 'event: ' prefix
+        console.log('Event message', line);
+      } else if (line.startsWith('data: ')) {
+        const eventData = line.slice(6); // Remove 'data: ' prefix
+        try {
+          let data = JSON.parse(eventData);
+          if (data['msg']) {
+            eventType = data['msg'];
+            if (eventType === 'process_generating') {
+              eventType = 'generating';
+              data = data['output']['data'];
+            } else if (eventType === 'process_completed') {
+              eventType = 'complete';
+              data = data['output']['data'];
+            }
+          }
+          
+          if (eventType === 'estimation' && onQueueUpdate) {
+            onQueueUpdate({
+              inQueue: true,
+              rank: data.rank,
+              queueSize: data.queue_size,
+              rankEta: data.rank_eta
+            });
+          } else if (eventType === 'process_starts' && onQueueUpdate) {
+            onQueueUpdate({
+              inQueue: false
+            });
+          } else if ((eventType === 'generating' || eventType === 'complete') && data[0]) {
+            result = data[0];
+            onStreamUpdate(data[0]);
+            
+            if (eventType === 'complete') {
+              return result;
+            }
+          }
+        } catch (e) {
+          console.log('Error parsing SSE message:', e);
+        }
+      } else if (line !== '') {
+        console.log('Unknown message', line);
+      }
+    }
+  }
+}, 100);
+
+export const imageToSamEmbedding = _.throttle(async (
+  imageBase64: string,
+  onQueueUpdate?: (status: {
+    inQueue: boolean,
+    rank?: number,
+    queueSize?: number,
+    rankEta?: number | null
+  }) => void
+): Promise<string> => {
+  // First call to initiate the process
+  const initiateResponse = await axios.post(`${API_URL}/call/image_to_sam_embedding`, {
+    data: [imageBase64]
+  });
+  
+  const eventId = initiateResponse.data.event_id;
+  
+  // Get the stream for queue updates and results
+  const response = await axios.get(`${API_URL}/queue/data?session_hash=${eventId}`, {
+    headers: {
+      'Accept': 'text/event-stream',
+    },
+    responseType: 'stream',
+    adapter: 'fetch',
+  });
+
+  const stream = response.data;
+  const reader = stream.pipeThrough(new TextDecoderStream()).getReader();
+
+  let result = '';
+  let partialMessage = '';
+
+  while (true) {
+    const { value, done } = await reader.read();
+    if (done) {
+      return result;
+    }
+
+    // Concatenate with any previous partial message
+    const currentData = partialMessage + value;
+    const lines = currentData.split('\n');
+    
+    // Save the last line if it's incomplete (doesn't end with \n)
+    // The endpoint will send an empty line to indicate the end of a message, so it's ok to not process the partial message.
+    partialMessage = lines[lines.length - 1];
+    
+    // Process all complete lines except the last one
+    let eventType = '';
+    for (let i = 0; i < lines.length - 1; i++) {
+      const line = lines[i];
+      if (line.startsWith('event: ')) {
+        eventType = line.slice(7);
+      } else if (line.startsWith('data: ')) {
+        const eventData = line.slice(6);
+        try {
+          let data = JSON.parse(eventData);
+          if (data['msg']) {
+            eventType = data['msg'];
+            console.log("Event type:", eventType);
+            if (eventType === 'process_completed') {
+              eventType = 'complete';
+              data = data['output']['data'];
+            }
+          }
+          
+          if (eventType === 'estimation' && onQueueUpdate) {
+            onQueueUpdate({
+              inQueue: true,
+              rank: data.rank,
+              queueSize: data.queue_size,
+              rankEta: data.rank_eta
+            });
+          } else if (eventType === 'process_starts' && onQueueUpdate) {
+            onQueueUpdate({
+              inQueue: false
+            });
+          } else if (eventType === 'complete' && data[0]) {
+            result = data[0];
+            console.log("Result for image to sam embedding:", result);
+            return result;
+          } else {
+            console.log("Unknown event type:", eventType);
+          }
+        } catch (e) {
+          console.log('Error parsing SSE message:', e, 'Raw data:', eventData);
+        }
+      }
+    }
+  }
+}, 100);
+
+export { API_URL };

frontend/tailwind.config.js

@@ -0,0 +1,12 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// All rights reserved.
+
+// This source code is licensed under the license found in the
+// LICENSE file in the root directory of this source tree.
+
+/** @type {import('tailwindcss').Config} */
+module.exports = {
+  content: ["./src/**/*.{html,js,tsx}"],
+  theme: {},
+  plugins: [],
+};

frontend/tsconfig.json

@@ -0,0 +1,24 @@
+{
+  "compilerOptions": {
+    "lib": ["dom", "dom.iterable", "esnext"],
+    "allowJs": true,
+    "skipLibCheck": true,
+    "strict": true,
+    "forceConsistentCasingInFileNames": true,
+    "noEmit": false,
+    "esModuleInterop": true,
+    "module": "esnext",
+    "moduleResolution": "node",
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "jsx": "react",
+    "incremental": true,
+    "target": "ESNext",
+    "useDefineForClassFields": true,
+    "allowSyntheticDefaultImports": true,
+    "outDir": "./dist/",
+    "sourceMap": true
+  },
+  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", "src"],
+  "exclude": ["node_modules"]
+}