Update builder.sh

builder.sh

@@ -1,9 +1,10 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-echo "🚀 Builder (Apex + Q8 + FlashAttn + layer_norm) — runtime, GPU visível, cache persistente"
+echo "🚀 Builder (Apex + Q8) — runtime, GPU visível, cache persistente"
 
 # ===== Persistência e caches =====
+# Prioriza /data (HF Spaces) e mantém compatibilidade com /app
 if [ -d /data ]; then
   export HF_HOME="${HF_HOME:-/data/.cache/huggingface}"
   export TORCH_HOME="${TORCH_HOME:-/data/.cache/torch}"
@@ -13,7 +14,10 @@ else
 fi
 export HF_HUB_CACHE="${HF_HUB_CACHE:-$HF_HOME/hub}"
 mkdir -p "$HF_HOME" "$HF_HUB_CACHE" "$TORCH_HOME"
-mkdir -p /app/.cache && ln -sf "$HF_HOME" /app/.cache/huggingface
+
+# Symlink de compatibilidade (se scripts esperarem /app/.cache/huggingface)
+mkdir -p /app/.cache
+ln -sf "$HF_HOME" /app/.cache/huggingface
 
 # ===== Repositório de wheels no Hub =====
 export SELF_HF_REPO_ID="${SELF_HF_REPO_ID:-carlex3321/aduc-sdr}"
@@ -27,195 +31,174 @@ mkdir -p /app/wheels /app/cuda_cache /app/wheels/src
 chmod -R 777 /app/wheels || true
 export CUDA_CACHE_PATH="/app/cuda_cache"
 
-# ===== Dependências mínimas =====
-python - <<'PY'
-import sys, subprocess
-def ensure(p):
-    try: __import__(p)
-    except Exception: subprocess.check_call([sys.executable, "-m", "pip", "install", "-U", p])
-for p in ["pip","setuptools","wheel","packaging","ninja","huggingface_hub"]:
-    ensure(p)
-PY
+# Licença (NVIDIA NGC) se presente
+if [ -f "/NGC-DL-CONTAINER-LICENSE" ]; then
+  cp -f /NGC-DL-CONTAINER-LICENSE /app/wheels/NGC-DL-CONTAINER-LICENSE || true
+fi
 
-# ===== Detectar Python, Torch e CUDA =====
-export PY_TAG="$(python -c 'import sys;print(f"cp{sys.version_info[0]}{sys.version_info[1]}")')"
-export TORCH_VER="$(python - <<'PY'
-import torch, re
-v = torch.__version__.split('+')[0]
-print(re.sub(r'[^0-9a-zA-Z\.-]', '', v))
-PY
-)"
-export CU_TAG="cu$(python - <<'PY'
-import torch
-print((torch.version.cuda or "").replace(".","") or "unknown")
+# ===== Dependências mínimas de build =====
+python -m pip install -v -U pip build setuptools wheel hatchling hatch-vcs scikit-build-core cmake ninja packaging "huggingface_hub[hf_transfer]" || true
+
+# ===== Tags de ambiente (Python/CUDA/Torch) =====
+PY_TAG="$(python -c 'import sys; print(f"cp{sys.version_info[0]}{sys.version_info[1]}")' 2>/dev/null || echo cp310)"
+TORCH_VER="$(python - <<'PY'
+try:
+  import torch, re
+  v = torch.__version__
+  print(re.sub(r'\+.*$', '', v))
+except Exception:
+  print("unknown")
 PY
 )"
-echo "[env] PY_TAG=$PY_TAG TORCH_VER=$TORCH_VER CU_TAG=$CU_TAG"
-
-log() { echo "[$(date +%H:%M:%S)] $*"; }
-
-# ===== Downloader silencioso: retorna só o caminho =====
-install_from_hf () {
-  # $1: prefixo do nome do pacote (.whl) a procurar
-  local NAME="$1"
-  echo "[hub] buscando wheel '$NAME' em ${SELF_HF_REPO_ID} (py=${PY_TAG} cu=${CU_TAG} torch=${TORCH_VER})"
-  python - "$NAME" <<'PY' 2>/dev/null || exit 0
-from huggingface_hub import HfApi, HfFolder
-import os, re, sys
-name = sys.argv[1]
-repo = os.environ.get("SELF_HF_REPO_ID") or ""
-py = os.environ.get("PY_TAG") or ""
-cu = os.environ.get("CU_TAG") or ""
-tv = os.environ.get("TORCH_VER") or ""
-api = HfApi(token=os.getenv("HF_TOKEN") or HfFolder.get_token())
+CU_TAG="$(python - <<'PY'
 try:
-    files = api.list_repo_files(repo_id=repo, repo_type="model")
+  import torch
+  cu = getattr(torch.version, "cuda", None)
+  print("cu"+cu.replace(".","")) if cu else print("")
 except Exception:
-    sys.exit(0)
-def pick(patts):
-    c=[]
-    for f in files:
-        if f.endswith(".whl") and any(p.search(f) for p in patts):
-            c.append(f)
-    if not c: return None
-    c.sort(reverse=True)
-    return c[0]
-p_strict=[re.compile(rf"{re.escape(name)}.*{re.escape(py)}.*{re.escape(cu)}.*{re.escape(tv)}.*\.whl$")]
-p_relax =[re.compile(rf"{re.escape(name)}.*{re.escape(py)}.*{re.escape(cu)}.*\.whl$")]
-target = pick(p_strict) or pick(p_relax)
-if not target: sys.exit(0)
-local = api.hf_hub_download(repo_id=repo, filename=target, repo_type="model", local_dir="/app/wheels")
-print(local)
+  print("")
 PY
-}
-
+)"
+echo "[env] PY_TAG=${PY_TAG} TORCH_VER=${TORCH_VER} CU_TAG=${CU_TAG}"
 
 # ===== Checkers =====
-check_apex () { python - <<'PY'
-import importlib, sys
+check_apex() {
+python - <<'PY'
 try:
-    import apex
-    importlib.import_module("fused_layer_norm_cuda")
-    sys.exit(0)
+  from apex.normalization import FusedLayerNorm, FusedRMSNorm
+  import importlib; importlib.import_module("fused_layer_norm_cuda")
+  ok = True
 except Exception:
-    sys.exit(1)
+  ok = False
+raise SystemExit(0 if ok else 1)
 PY
 }
-check_q8 () { python - <<'PY'
-import importlib, sys
-for m in ("ltx_q8_kernels","q8_kernels"):
-    try: importlib.import_module(m); sys.exit(0)
-    except Exception: pass
-sys.exit(1)
+
+check_q8() {
+python - <<'PY'
+import importlib.util
+spec = importlib.util.find_spec("ltx_q8_kernels") or importlib.util.find_spec("q8_kernels")
+raise SystemExit(0 if spec else 1)
 PY
 }
-check_flashattn () { python - <<'PY'
-import importlib, sys
+
+# ===== Download de wheels do Hub =====
+install_from_hf () {
+  local PKG="$1" # 'apex' ou 'q8_kernels'
+  echo "[hub] Buscando wheel de ${PKG} em ${SELF_HF_REPO_ID} (py=${PY_TAG}, cu=${CU_TAG})"
+  python - "$PKG" "$PY_TAG" "$CU_TAG" <<'PY' || exit 0
+import os, sys
+from huggingface_hub import HfApi, hf_hub_download, HfFolder
+
+pkg, py_tag, cu_tag = sys.argv[1], sys.argv[2], sys.argv[3]
+repo = os.environ.get("SELF_HF_REPO_ID","carlex3321/aduc-sdr")
+api = HfApi(token=os.getenv("HF_TOKEN") or HfFolder.get_token())
+
 try:
-    importlib.import_module("flash_attn._C")
-    sys.exit(0)
+  files = api.list_repo_files(repo_id=repo, repo_type="model")
 except Exception:
-    sys.exit(1)
-PY
-}
-check_layernorm () { python - <<'PY'
-import importlib, sys
-try:
-    importlib.import_module("layer_norm_cuda")
-    sys.exit(0)
-except Exception:
-    sys.exit(1)
+  raise SystemExit(0)
+
+cands = [f for f in files if f.endswith(".whl") and f.rsplit("/",1)[-1].startswith(pkg+"-") and py_tag in f]
+pref = [f for f in cands if cu_tag and cu_tag in f] or cands
+if not pref:
+  raise SystemExit(0)
+target = sorted(pref, reverse=True)[0]
+print(target)
+path = hf_hub_download(repo_id=repo, filename=target, repo_type="model", local_dir="/app/wheels")
+print(path)
 PY
 }
 
-# ===== Apex =====
-APEX_REPO="${APEX_REPO:-https://github.com/NVIDIA/apex.git}"
-APEX_COMMIT="${APEX_COMMIT:-master}"
+# ===== Builders =====
 build_apex () {
   local SRC="/app/wheels/src/apex"
-  rm -rf "$SRC"
-  git clone --filter=blob:none "$APEX_REPO" "$SRC"
-  git -C "$SRC" checkout "$APEX_COMMIT" || true
-  export MAX_JOBS="${MAX_JOBS:-4}" APEX_CPP_EXT=1 APEX_CUDA_EXT=1
-  log "[build] apex — MAX_JOBS=$MAX_JOBS"
+  echo "[build] Fonte Apex em ${SRC}"
+  if [ -d "$SRC/.git" ]; then
+    git -C "$SRC" fetch --all -p || true
+    git -C "$SRC" reset --hard HEAD || true
+    git -C "$SRC" clean -fdx || true
+  else
+    rm -rf "$SRC"
+    git clone --depth 1 https://github.com/NVIDIA/apex "$SRC"
+  fi
+
+  echo "[build] Compilando Apex -> wheel"
+  export APEX_CPP_EXT=1 APEX_CUDA_EXT=1 APEX_ALL_CONTRIB_EXT=0
   python -m pip wheel --no-build-isolation --no-deps "$SRC" -w /app/wheels || true
+
   local W="$(ls -t /app/wheels/apex-*.whl 2>/dev/null | head -n1 || true)"
-  if [ -n "$W" ]; then python -m pip install -U --no-deps --no-build-isolation "$W" || true
-  else (cd "$SRC" && python -m pip install -U --no-build-isolation .) || true
-  fi
-}
-ensure_apex () {
-  log "[flow] === apex ==="
-  if check_apex; then log "[flow] apex OK"; return 0; fi
-  local HFW="$(install_from_hf "apex" || true)"
-  if [ -n "${HFW:-}" ]; then
-    python -m pip install -U --no-build-isolation "$HFW" || true
-    if check_apex; then log "[flow] apex via Hub OK"; return 0; fi
+  if [ -n "${W}" ]; then
+    python -m pip install -U --no-deps "${W}" || true
+    echo "[build] Apex instalado da wheel: ${W}"
+  else
+    echo "[build] Nenhuma wheel Apex gerada; instalando do source"
+    python -m pip install --no-build-isolation "$SRC" || true
   fi
-  build_apex
-  if check_apex; then log "[flow] apex build OK"; return 0; fi
-  log "[flow] apex falhou"; return 1
 }
 
-# ===== Q8 =====
-Q8_REPO="${Q8_REPO:-https://github.com/lynix94/llm-q8-kernels.git}"
-Q8_COMMIT="${Q8_COMMIT:-main}"
+Q8_REPO="${Q8_REPO:-https://github.com/Lightricks/LTX-Video-Q8-Kernels.git}"
+Q8_COMMIT="${Q8_COMMIT:-f3066edea210082799ca5a2bbf9ef0321c5dd8fc}"
 build_q8 () {
-  local SRC="/app/wheels/src/q8"
+  local SRC="/app/wheels/src/q8_kernels"
   rm -rf "$SRC"
   git clone --filter=blob:none "$Q8_REPO" "$SRC"
-  git -C "$SRC" checkout "$Q8_COMMIT" || true
-  export MAX_JOBS="${MAX_JOBS:-4}"
-  log "[build] q8 — MAX_JOBS=$MAX_JOBS"
-  python -m pip wheel --no-build-isolation --no-deps "$SRC" -w /app/wheels || true
-  local W="$(ls -t /app/wheels/*q8*kernels*-*.whl /app/wheels/*q8*kernel*-*.whl 2>/dev/null | head -n1 || true)"
-  if [ -n "$W" ]; then python -m pip install -U --no-deps --no-build-isolation "$W" || true
-  else (cd "$SRC" && python -m pip install -U --no-build-isolation .) || true
+  git -C "$SRC" checkout "$Q8_COMMIT"
+  git -C "$SRC" submodule update --init --recursive
+
+  echo "[build] Compilando Q8 Kernels -> wheel"
+  python -m pip wheel --no-build-isolation "$SRC" -w /app/wheels || true
+
+  local W="$(ls -t /app/wheels/q8_kernels-*.whl 2>/dev/null | head -n1 || true)"
+  if [ -n "${W}" ]; then
+    python -m pip install  -U --no-deps "${W}" || true
+    echo "[build] Q8 instalado da wheel: ${W}"
+  else
+    echo "[build] Nenhuma wheel q8_kernels gerada; instalando do source"
+    python -m pip install  --no-build-isolation "$SRC" || true
   fi
 }
-ensure_q8 () {
-  log "[flow] === q8 ==="
-  if check_q8; then log "[flow] q8 OK"; return 0; fi
-  local HFW="$(install_from_hf "q8" || true)"; [ -z "${HFW:-}" ] && HFW="$(install_from_hf "kernels" || true)"
-  if [ -n "${HFW:-}" ]; then
-    python -m pip install -U --no-build-isolation "$HFW" || true
-    if check_q8; then log "[flow] q8 via Hub OK"; return 0; fi
+
+# ===== Pipeline genérico =====
+ensure_pkg () {
+  local PKG="$1"       # apex | q8_kernels
+  local CHECK_FN="$2"  # check_apex | check_q8
+  local BUILD_FN="$3"  # build_apex | build_q8
+  echo "[flow] === ${PKG} ==="
+
+  if ${CHECK_FN}; then
+    echo "[flow] ${PKG}: já instalado (import OK)"
+    return 0
   fi
-  build_q8
-  if check_q8; then log "[flow] q8 build OK"; return 0; fi
-  log "[flow] q8 falhou"; return 1
-}
 
-# ===== FlashAttention =====
-FLASHATTN_REPO="${FLASHATTN_REPO:-https://github.com/Dao-AILab/flash-attention.git}"
-FLASHATTN_REF="${FLASHATTN_REF:-v2.8.3}"
-build_flashattn_repo () {
-  local SRC="/app/wheels/src/flash-attention"
-  rm -rf "$SRC"
-  git clone --filter=blob:none "$FLASHATTN_REPO" "$SRC"
-  git -C "$SRC" checkout "$FLASHATTN_REF" || true
-  git -C "$SRC" submodule update --init --recursive || true
-  export MAX_JOBS="${MAX_JOBS:-4}"
-  log "[build] flash-attn — MAX_JOBS=$MAX_JOBS"
-  python -m pip wheel --no-build-isolation --no-deps "$SRC" -w /app/wheels || true
-  local W="$(ls -t /app/wheels/flash_attn-*.whl 2>/dev/null | head -n1 || true)"
-  if [ -n "$W" ]; then python -m pip install -U --no-deps --no-build-isolation "$W" || true
-  else (cd "$SRC" && python -m pip install -U --no-build-isolation .) || true
+  echo "[flow] ${PKG}: tentando wheel do Hub (${SELF_HF_REPO_ID})"
+  HF_OUT="$(install_from_hf "$PKG" || true)"
+  if [ -n "${HF_OUT:-}" ]; then
+    WHEEL_PATH="$(printf "%s\n" "${HF_OUT}" | tail -n1)"
+    echo "[hub] Baixado: ${WHEEL_PATH}"
+    python -m pip install -U --no-build-isolation "${WHEEL_PATH}" || true
+    if ${CHECK_FN}; then
+      echo "[flow] ${PKG}: sucesso via Hub (${WHEEL_PATH})"
+      return 0
+    else
+      echo "[flow] ${PKG}: import falhou após wheel do Hub; compilando"
+    fi
+  else
+    echo "[hub] Nenhuma wheel compatível encontrada para ${PKG}"
   fi
-}
-ensure_flashattn () {
-  log "[flow] === flash-attn ==="
-  if check_flashattn; then log "[flow] flash-attn OK"; return 0; fi
-  local HF1="$(install_from_hf "flash_attn" || true)"
-  if [ -n "${HF1:-}" ]; then
-    python -m pip install -U --no-build-isolation "$HF1" || true
-    if check_flashattn; then log "[flow] flash-attn via Hub OK"; return 0; fi
+
+  echo "[flow] ${PKG}: compilando (fallback)"
+  ${BUILD_FN}
+  if ${CHECK_FN}; then
+    echo "[flow] ${PKG}: sucesso após compilação"
+    return 0
   fi
-  build_flashattn_repo
-  if check_flashattn; then log "[flow] flash-attn build OK"; return 0; fi
-  log "[flow] flash-attn falhou"; return 1
+
+  echo "[flow] ${PKG}: falhou após build; seguindo adiante"
+  return 1
 }
 
+
 # ===== layer_norm (wheel separada) =====
 check_layernorm () { python - <<'PY'
 import importlib, sys
@@ -281,34 +264,32 @@ ensure_layernorm () {
   log "[flow] layer_norm falhou"; return 1
 }
 
-# ===== Execução =====
-ensure_apex || true
-ensure_q8 || true
-ensure_flashattn || true
-ensure_layernorm || true
 
 
-# ===== Upload para HF =====
-echo "[hub] upload de wheels para $SELF_HF_REPO_ID"
+# ===== Execução: Apex e Q8 =====
+
+ensure_pkg "apex" check_apex build_apex || true
+ensure_pkg "q8_kernels" check_q8 build_q8 || true
+ensure_layernorm || true
+
+# ===== Upload das wheels geradas (opcional) =====
 python - <<'PY'
-import os, glob, json, time
-from huggingface_hub import HfApi, HfFolder, upload_file
-repo = os.environ.get("SELF_HF_REPO_ID")
-token = HfFolder.get_token()
-if not (repo and token):
-    print("[hub] token/repo ausentes; pulando upload"); raise SystemExit(0)
-paths = sorted(glob.glob("/app/wheels/*.whl"))
-py = os.environ.get("PY_TAG"); cu = os.environ.get("CU_TAG"); torchv = os.environ.get("TORCH_VER")
-stamp = time.strftime("%Y%m%d-%H%M%S")
-for p in paths:
-    dest = f"wheels/{py}-{cu}-torch{torchv}/" + os.path.basename(p)
-    upload_file(path_or_fileobj=p, path_in_repo=dest, repo_id=repo, repo_type="model")
-    print("[hub] uploaded:", dest)
-meta = {"py": py, "cuda": cu, "torch": torchv, "timestamp": stamp, "files": [os.path.basename(x) for x in paths]}
-upload_file(path_or_fileobj=bytes(json.dumps(meta, indent=2), "utf-8"),
-            path_in_repo=f"wheels/{py}-{cu}-torch{torchv}/manifest-{stamp}.json",
-            repo_id=repo, repo_type="model")
-print("[hub] manifest enviado")
+import os
+from huggingface_hub import HfApi, HfFolder
+repo=os.environ.get("SELF_HF_REPO_ID","carlex3321/aduc-sdr")
+token=os.getenv("HF_TOKEN") or HfFolder.get_token()
+if not token:
+  raise SystemExit("HF_TOKEN ausente; upload desabilitado")
+api=HfApi(token=token)
+api.upload_folder(
+  folder_path="/app/wheels",
+  repo_id=repo,
+  repo_type="model",
+  allow_patterns=["*.whl","NGC-DL-CONTAINER-LICENSE"],
+  ignore_patterns=["**/src/**","**/*.log","**/logs/**",".git/**"],
+)
+print("Upload concluído (wheels + licença).")
 PY
 
-echo "[done] builder-10 concluído"
+chmod -R 777 /app/wheels || true
+echo "✅ Builder finalizado."