Use approximate GELU from PyTorch

modeling_norbert.py

@@ -8,7 +8,6 @@ from torch.utils import checkpoint
 
 from .configuration_norbert import NorbertConfig
 from transformers.modeling_utils import PreTrainedModel
-from transformers.activations import gelu_new
 from transformers.modeling_outputs import (
     MaskedLMOutput,
     MultipleChoiceModelOutput,
@@ -17,7 +16,6 @@ from transformers.modeling_outputs import (
     TokenClassifierOutput,
     BaseModelOutput
 )
-from transformers.pytorch_utils import softmax_backward_data
 
 
 class Encoder(nn.Module):
@@ -81,7 +79,7 @@ class EncoderLayer(nn.Module):
 class GeGLU(nn.Module):
     def forward(self, x):
         x, gate = x.chunk(2, dim=-1)
-        x = x * gelu_new(gate)
+        x = x * F.gelu(gate, approximate="tanh")
         return x