from transformers import RobertaForMaskedLM 
from transformers.modeling_longformer import LongformerSelfAttention
# from transformers import LongformerSelfAttention


class RobertaLongSelfAttention(LongformerSelfAttention):
    def forward(
        self,
        hidden_states,
        attention_mask=None,
        head_mask=None,
        encoder_hidden_states=None,
        encoder_attention_mask=None,
        output_attentions=False,
    ):
        return super().forward(hidden_states, attention_mask=attention_mask, output_attentions=output_attentions)

class RobertaLongForMaskedLM(RobertaForMaskedLM):
    def __init__(self, config):
        super().__init__(config)
        for i, layer in enumerate(self.roberta.encoder.layer):
            # replace the `modeling_bert.BertSelfAttention` object with `LongformerSelfAttention`
            layer.attention.self = RobertaLongSelfAttention(config, layer_id=i)

1535334f289ee5bd94162c70bd094ab574cb796b
<mask token>


class RobertaLongForMaskedLM(RobertaForMaskedLM):
    <mask token>

<mask token>


class RobertaLongSelfAttention(LongformerSelfAttention):
    <mask token>


class RobertaLongForMaskedLM(RobertaForMaskedLM):

    def __init__(self, config):
        super().__init__(config)
        for i, layer in enumerate(self.roberta.encoder.layer):
            layer.attention.self = RobertaLongSelfAttention(config, layer_id=i)

<mask token>


class RobertaLongSelfAttention(LongformerSelfAttention):

    def forward(self, hidden_states, attention_mask=None, head_mask=None,
        encoder_hidden_states=None, encoder_attention_mask=None,
        output_attentions=False):
        return super().forward(hidden_states, attention_mask=attention_mask,
            output_attentions=output_attentions)


class RobertaLongForMaskedLM(RobertaForMaskedLM):

    def __init__(self, config):
        super().__init__(config)
        for i, layer in enumerate(self.roberta.encoder.layer):
            layer.attention.self = RobertaLongSelfAttention(config, layer_id=i)

from transformers import RobertaForMaskedLM
from transformers.modeling_longformer import LongformerSelfAttention


class RobertaLongSelfAttention(LongformerSelfAttention):

    def forward(self, hidden_states, attention_mask=None, head_mask=None,
        encoder_hidden_states=None, encoder_attention_mask=None,
        output_attentions=False):
        return super().forward(hidden_states, attention_mask=attention_mask,
            output_attentions=output_attentions)


class RobertaLongForMaskedLM(RobertaForMaskedLM):

    def __init__(self, config):
        super().__init__(config)
        for i, layer in enumerate(self.roberta.encoder.layer):
            layer.attention.self = RobertaLongSelfAttention(config, layer_id=i)

from transformers import RobertaForMaskedLM 
from transformers.modeling_longformer import LongformerSelfAttention
# from transformers import LongformerSelfAttention


class RobertaLongSelfAttention(LongformerSelfAttention):
    def forward(
        self,
        hidden_states,
        attention_mask=None,
        head_mask=None,
        encoder_hidden_states=None,
        encoder_attention_mask=None,
        output_attentions=False,
    ):
        return super().forward(hidden_states, attention_mask=attention_mask, output_attentions=output_attentions)

class RobertaLongForMaskedLM(RobertaForMaskedLM):
    def __init__(self, config):
        super().__init__(config)
        for i, layer in enumerate(self.roberta.encoder.layer):
            # replace the `modeling_bert.BertSelfAttention` object with `LongformerSelfAttention`
            layer.attention.self = RobertaLongSelfAttention(config, layer_id=i)
