Upload config

config.json

@@ -10,15 +10,10 @@
   "d_state": 16,
   "dt_rank": 48,
   "expand": 2,
-  "fused_add_norm": true,
   "initializer_range": 0.02,
   "model_type": "mamba",
   "n_layer": 24,
-  "norm_epsilon": 1e-05,
   "pad_vocab_size_multiple": 8,
-  "residual_in_fp32": true,
-  "rms_norm": true,
-  "ssm_cfg": {},
   "transformers_version": "4.37.2",
   "vocab_size": 50280
 }

configuration_mamba.py

@@ -17,14 +17,9 @@ class MambaConfig(PretrainedConfig):
         conv_bias=True,
         bias=False,
         n_layer=64,
-        norm_epsilon=1e-5,
         dt_rank: Union[int, str] = "auto",
         pad_vocab_size_multiple=8,
         initializer_range=0.02,
-        rms_norm: bool = True,
-        fused_add_norm: bool = True,
-        ssm_cfg={},
-        residual_in_fp32: bool = True,
         **kwargs,
     ):
         self.vocab_size = vocab_size
@@ -39,11 +34,6 @@ class MambaConfig(PretrainedConfig):
         self.dt_rank = dt_rank
         self.initializer_range = initializer_range
         self.bias = bias
-        self.ssm_cfg = ssm_cfg
-        self.norm_epsilon = norm_epsilon
-        self.rms_norm = rms_norm
-        self.residual_in_fp32 = residual_in_fp32
-        self.fused_add_norm = fused_add_norm
 
         if self.dt_rank == "auto":
             self.dt_rank = math.ceil(self.d_model / 16)