import os import subprocess import sys import shutil # ============ Configuration ============ REPO_URL = "https://github.com/facebookresearch/sam-3d-objects.git" REPO_DIR = "/home/user/app/sam-3d-objects" # ============ Install Dependencies & Setup ============ def patch_pyproject_toml(): """ Removes 'bpy==4.3.0' from pyproject.toml to prevent installation failures. """ print("Patching pyproject.toml to remove strict bpy dependency...") pyproject_path = os.path.join(REPO_DIR, "pyproject.toml") if os.path.exists(pyproject_path): with open(pyproject_path, "r") as f: content = f.read() # Remove dependency entries for bpy new_content = content.replace('"bpy==4.3.0",', '') new_content = new_content.replace("'bpy==4.3.0',", '') with open(pyproject_path, "w") as f: f.write(new_content) print("Patch applied successfully.") else: print(f"Warning: {pyproject_path} not found. Skipping patch.") def install_dependencies(): """ Installs the specific list of dependencies requested by the user individually. """ print("Starting manual installation sequence...") # 1. Basic PIP Upgrade env = os.environ.copy() env["PIP_EXTRA_INDEX_URL"] = "https://pypi.ngc.nvidia.com https://download.pytorch.org/whl/cu121" env["PIP_FIND_LINKS"] = "https://nvidia-kaolin.s3.us-east-2.amazonaws.com/torch-2.5.1_cu121.html" env["CUDA_HOME"] = "/usr/local/cuda" env["MAX_JOBS"] = "4" subprocess.run([sys.executable, "-m", "pip", "install", "--upgrade", "pip"], env=env, check=True) # 2. User Defined Package List packages = [ # MUST INSTALL FIRST: PyTorch (required for building flash_attn, pytorch3d, etc.) "torch", "torchvision", # Core ML & Hydra "hydra-core", "hydra-submitit-launcher", "omegaconf", "numpy", "einops", "einops-exts", "timm", "diffusers", "transformers", "accelerate", "safetensors", # Testing & Dev Tools "pytest", "pipdeptree", "findpydeps", "lovely_tensors", "autoflake", "black", "flake8", "usort", # Visualization & UI "seaborn", "gradio", "tensorboard", "wandb", "polyscope", # 3D & Graphics "open3d", "pyrender", "point-cloud-utils", "pymeshfix", "xatlas", "panda3d-gltf", "fvcore", "roma", "smplx", "OpenEXR", "imath", # Video & Audio "av", "decord", "librosa", # CUDA & GPU "cuda-python", "nvidia-cuda-nvcc-cu12", "nvidia-pyindex", "spconv-cu121", "xformers", "torchaudio", # ML Optimization "auto_gptq", "bitsandbytes", "peft", "optimum", "optree", "lightning", "sentence-transformers", # Data & Serialization "h5py", "fastavro", "jsonlines", "jsonpickle", "orjson", "simplejson", "webdataset", # Image Processing "opencv-python", "scikit-image", "pycocotools", "ftfy", # Web & Networking "Flask", "Werkzeug", "hdfs", "httplib2", "PySocks", "gdown", # Utilities "astor", "async-timeout", "colorama", "deprecation", "easydict", "exceptiongroup", "fasteners", "loguru", "objsize", "randomname", "rootutils", "Rtree", "tomli", # JSON Schema & URI "fqdn", "isoduration", "jsonpointer", "uri-template", "webcolors", # Graph & Docs "igraph", "pydot", "pdoc3", # Jupyter & Misc "jupyter", "dataclasses", "crcmod", "conda-pack", "pip-system-certs", "python-pycg", "pymongo", "sagemaker", "mosaicml-streaming", "bpy", # Git installs (pytorch3d requires torch, so it's near the end) "git+https://github.com/nerfstudio-project/gsplat.git", "git+https://github.com/facebookresearch/pytorch3d.git", "git+https://github.com/microsoft/MoGe.git", "utils3d", ] # Packages that require special handling - install with --no-build-isolation # so they can find torch during build special_packages = [ "flash_attn", "kaolin", ] for pkg in packages: print(f"----------------------------------------") print(f"Installing: {pkg}") print(f"----------------------------------------") try: cmd = [sys.executable, "-m", "pip", "install", pkg] subprocess.run(cmd, env=env, check=True) except subprocess.CalledProcessError as e: print(f"❌ Failed to install {pkg}. Continuing to next package...") # Install packages that need --no-build-isolation (require torch at build time) for pkg in special_packages: print(f"----------------------------------------") print(f"Installing (no-build-isolation): {pkg}") print(f"----------------------------------------") try: cmd = [sys.executable, "-m", "pip", "install", "--no-build-isolation", pkg] subprocess.run(cmd, env=env, check=True) except subprocess.CalledProcessError as e: print(f"❌ Failed to install {pkg}. Continuing to next package...") # 3. Clone & Install Main Repo (SAM 3D Objects) if not os.path.exists(REPO_DIR): print(f"Cloning repository to {REPO_DIR}...") subprocess.run(["git", "clone", REPO_URL, REPO_DIR], check=True) os.chdir(REPO_DIR) patch_pyproject_toml() print("Installing sam-3d-objects in editable mode...") subprocess.run([sys.executable, "-m", "pip", "install", "--no-deps", "-e", "."], env=env, check=True) # 4. Apply Hydra Patch patch_script = os.path.join(REPO_DIR, "patching", "hydra") if os.path.exists(patch_script): print("Applying Hydra patch...") subprocess.run(["chmod", "+x", patch_script], check=True) subprocess.run([patch_script], check=True) # Run installation install_dependencies() # Add repo to Python path if REPO_DIR not in sys.path: sys.path.insert(0, REPO_DIR) # Set environment variables required for runtime os.environ["CUDA_HOME"] = "/usr/local/cuda" os.environ["LIDRA_SKIP_INIT"] = "true" os.environ["PYTORCH3D_NO_CUDA_CHECK"] = "1" # ============ Imports ============ import spaces import builtins from typing import Optional, List, Callable from copy import deepcopy import gradio as gr import numpy as np from PIL import Image import torch import math from omegaconf import OmegaConf, DictConfig, ListConfig from hydra.utils import instantiate, get_method # Lazy imports placeholder _sam3d_imported = False _pipeline = None # ============ Security / Config Filters ============ WHITELIST_FILTERS = [ lambda target: target.split(".", 1)[0] in {"sam3d_objects", "torch", "torchvision", "moge"}, ] BLACKLIST_FILTERS = [ lambda target: get_method(target) in { builtins.exec, builtins.eval, builtins.__import__, os.kill, os.system, os.putenv, os.remove, os.removedirs, os.rmdir, os.fchdir, os.setuid, os.fork, os.forkpty, os.killpg, os.rename, os.renames, os.truncate, os.replace, os.unlink, os.fchmod, os.fchown, os.chmod, os.chown, os.chroot, os.lchown, os.getcwd, os.chdir, shutil.rmtree, shutil.move, shutil.chown, subprocess.Popen, builtins.help, }, ] def check_target(target: str, whitelist_filters: List[Callable], blacklist_filters: List[Callable]): if any(filt(target) for filt in whitelist_filters): if not any(filt(target) for filt in blacklist_filters): return raise RuntimeError(f"target '{target}' is not allowed") def check_hydra_safety(config: DictConfig, whitelist_filters: List[Callable], blacklist_filters: List[Callable]): to_check = [config] while to_check: node = to_check.pop() if isinstance(node, DictConfig): to_check.extend(list(node.values())) if "_target_" in node: check_target(node["_target_"], whitelist_filters, blacklist_filters) elif isinstance(node, ListConfig): to_check.extend(list(node)) # ============ Lazy Loading & Model Logic ============ def lazy_import_sam3d(): """Import sam3d modules lazily after GPU is available.""" global _sam3d_imported if not _sam3d_imported: global utils3d, sam3d_objects, InferencePipelinePointMap, render_utils, SceneVisualizer global quaternion_multiply, quaternion_invert try: import utils3d as _utils3d utils3d = _utils3d import sam3d_objects as _sam3d_objects sam3d_objects = _sam3d_objects from sam3d_objects.pipeline.inference_pipeline_pointmap import InferencePipelinePointMap as _IPP InferencePipelinePointMap = _IPP from sam3d_objects.model.backbone.tdfy_dit.utils import render_utils as _ru render_utils = _ru from sam3d_objects.utils.visualization import SceneVisualizer as _SV SceneVisualizer = _SV from pytorch3d.transforms import quaternion_multiply as _qm, quaternion_invert as _qi quaternion_multiply, quaternion_invert = _qm, _qi _sam3d_imported = True except ImportError as e: print(f"Failed to import SAM 3D modules: {e}") print("Ensure the installation step completed successfully.") subprocess.run([sys.executable, "-m", "pipdeptree"]) raise def load_pipeline(config_file: str): """Load the inference pipeline (call inside GPU context).""" global _pipeline if _pipeline is None: lazy_import_sam3d() config = OmegaConf.load(config_file) config.rendering_engine = "pytorch3d" config.compile_model = False config.workspace_dir = os.path.dirname(config_file) check_hydra_safety(config, WHITELIST_FILTERS, BLACKLIST_FILTERS) _pipeline = instantiate(config) return _pipeline def merge_mask_to_rgba(image, mask): mask = mask.astype(np.uint8) * 255 mask = mask[..., None] return np.concatenate([image[..., :3], mask], axis=-1) @spaces.GPU(duration=120) def run_inference(image: np.ndarray, mask: np.ndarray, config_file: str, seed: Optional[int] = None, pointmap=None) -> dict: """GPU-decorated inference function for ZeroGPU.""" global _pipeline _pipeline = load_pipeline(config_file) if hasattr(_pipeline, 'to'): _pipeline.to('cuda') rgba_image = merge_mask_to_rgba(image, mask) return _pipeline.run( rgba_image, None, seed, stage1_only=False, with_mesh_postprocess=False, with_texture_baking=False, with_layout_postprocess=True, use_vertex_color=True, stage1_inference_steps=None, pointmap=pointmap, ) # ============ Gradio Interface ============ CONFIG_FILE = os.path.join(REPO_DIR, "configs/inference.yaml") def process_image(input_image, input_mask, seed): if input_image is None: return None, "Please provide an input image" if input_mask is None: return None, "Please provide an object mask" image = np.array(input_image) mask = np.array(input_mask.convert("L")) > 127 seed_val = int(seed) if seed else None try: result = run_inference(image, mask, CONFIG_FILE, seed_val) if "gaussian" in result and result["gaussian"]: ply_path = "/tmp/output.ply" result["gaussian"][0].save_ply(ply_path) return ply_path, "✅ Inference complete!" return None, "⚠️ No 3D output generated" except Exception as e: import traceback traceback.print_exc() return None, f"❌ Error: {str(e)}" with gr.Blocks(title="SAM 3D Objects", theme=gr.themes.Soft()) as demo: gr.Markdown(""" # 🎨 SAM 3D Objects - Single Image to 3D Upload an image and a mask to generate a 3D Gaussian Splat model. **Note:** First inference may take longer due to model loading. """) with gr.Row(): with gr.Column(scale=1): input_image = gr.Image(label="Input Image", type="pil") input_mask = gr.Image(label="Object Mask (white = object)", type="pil") seed = gr.Number(label="Seed (optional)", value=42, precision=0) run_btn = gr.Button("🚀 Generate 3D", variant="primary", size="lg") with gr.Column(scale=1): output_model = gr.Model3D(label="3D Output", clear_color=[0.1, 0.1, 0.1, 1.0]) status = gr.Textbox(label="Status", interactive=False) run_btn.click(fn=process_image, inputs=[input_image, input_mask, seed], outputs=[output_model, status]) if __name__ == "__main__": demo.launch()