update dep

app.py

@@ -32,18 +32,18 @@ import numpy as np
 import torch as th
 import torch.distributed as dist
 
-def install_dependency():
-    # install apex
-    subprocess.run(
-        f'FORCE_CUDA=1 {sys.executable} -m pip install -v --disable-pip-version-check --no-cache-dir --no-build-isolation --config-settings "--build-option=--cpp_ext" --config-settings "--build-option=--cuda_ext" git+https://github.com/NVIDIA/apex.git@master',
-        shell=True,
-    )
+# def install_dependency():
+#     # install apex
+#     subprocess.run(
+#         f'FORCE_CUDA=1 {sys.executable} -m pip install -v --disable-pip-version-check --no-cache-dir --no-build-isolation --config-settings "--build-option=--cpp_ext" --config-settings "--build-option=--cuda_ext" git+https://github.com/NVIDIA/apex.git@master',
+#         shell=True,
+#     )
 
 th.backends.cuda.matmul.allow_tf32 = True
 th.backends.cudnn.allow_tf32 = True
 th.backends.cudnn.enabled = True
 
-install_dependency()
+# install_dependency()
 
 from guided_diffusion import dist_util, logger
 from guided_diffusion.script_util import (

dit/dit_models_xformers.py

@@ -24,8 +24,12 @@ from pdb import set_trace as st
 from ldm.modules.attention import CrossAttention
 from vit.vision_transformer import MemEffAttention as Attention
 # import apex
-from apex.normalization import FusedRMSNorm as RMSNorm
-from apex.normalization import FusedLayerNorm as LayerNorm
+try:
+    from apex.normalization import FusedRMSNorm as RMSNorm
+    from apex.normalization import FusedLayerNorm as LayerNorm
+except:
+    from torch.nn import LayerNorm as LayerNorm
+    from .norm import RMSNorm
 
 # from torch.nn import LayerNorm
 # from xformers import triton

dit/norm.py

@@ -0,0 +1,18 @@
+import torch
+
+def rms_norm(x, weight=None, eps=1e-05):
+    output = x / torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + eps)
+    return output * weight if weight is not None else output
+
+class RMSNorm(torch.nn.Module):
+
+    def __init__(self, normalized_shape, eps=1e-05, weight=True, dtype=None, device=None):
+        super().__init__()
+        self.eps = eps
+        if weight:
+            self.weight = torch.nn.Parameter(torch.ones(normalized_shape, dtype=dtype, device=device))
+        else:
+            self.register_parameter('weight', None)
+
+    def forward(self, x):
+        return rms_norm(x.float(), self.weight, self.eps).to(dtype=x.dtype)

ldm/modules/attention.py

@@ -7,16 +7,17 @@ from einops import rearrange, repeat
 from pdb import set_trace as st
 
 from ldm.modules.diffusionmodules.util import checkpoint
-from apex.normalization import FusedLayerNorm as LayerNorm
+# from apex.normalization import FusedLayerNorm as LayerNorm
 
 
 # CrossAttn precision handling
 import os
 _ATTN_PRECISION = os.environ.get("ATTN_PRECISION", "fp32")
 from xformers.ops import MemoryEfficientAttentionFlashAttentionOp
-# from xformers.ops import RMSNorm, fmha, rope_padded
-# import apex
-from apex.normalization import FusedRMSNorm as RMSNorm
+try:
+    from apex.normalization import FusedRMSNorm as RMSNorm
+except:
+    from dit.norm import RMSNorm
 
 
 def exists(val):