from allennlp.modules.token_embedders.bert_token_embedder import BertEmbedder, TokenEmbedder, PretrainedBertModel
from typing import List
import torch
from allennlp.modules.scalar_mix import ScalarMix


@TokenEmbedder.register("bert-pretrained-split-mix")
class PretrainedBertEmbedderSplitMix(BertEmbedder):
    def __init__(self, pretrained_model: str, requires_grad: bool = False, top_layer_only: bool = False,
                 scalar_mix_parameters: List[float] = None) -> None:

        model = PretrainedBertModel.load(pretrained_model)

        for param in model.parameters():
            param.requires_grad = requires_grad

        super().__init__(bert_model=model,
                         top_layer_only=top_layer_only,
                         scalar_mix_parameters=scalar_mix_parameters)

        # instead of one scalar mix, keep two!
        self._scalar_mix_1 = ScalarMix(model.config.num_hidden_layers,
                                          do_layer_norm=False,
                                          initial_scalar_parameters=scalar_mix_parameters,
                                          trainable=scalar_mix_parameters is None)

        self._scalar_mix_2 = ScalarMix(model.config.num_hidden_layers,
                                      do_layer_norm=False,
                                      initial_scalar_parameters=scalar_mix_parameters,
                                      trainable=scalar_mix_parameters is None)

        self.scalar_mix_id = {1: self._scalar_mix_1, 2: self._scalar_mix_2}

    def forward(self,
                input_ids: torch.LongTensor,
                offsets: torch.LongTensor = None,
                token_type_ids: torch.LongTensor = None,
                mix_id: int = 0):

        self._scalar_mix = self.scalar_mix_id[mix_id]  # decide which scalar mix to update
        return super().forward(input_ids, offsets, token_type_ids)

b12768ce4b8d85f107318ef92a51f41b7e7910b0
<mask token>

<mask token>


@TokenEmbedder.register('bert-pretrained-split-mix')
class PretrainedBertEmbedderSplitMix(BertEmbedder):

    def __init__(self, pretrained_model: str, requires_grad: bool=False,
        top_layer_only: bool=False, scalar_mix_parameters: List[float]=None
        ) ->None:
        model = PretrainedBertModel.load(pretrained_model)
        for param in model.parameters():
            param.requires_grad = requires_grad
        super().__init__(bert_model=model, top_layer_only=top_layer_only,
            scalar_mix_parameters=scalar_mix_parameters)
        self._scalar_mix_1 = ScalarMix(model.config.num_hidden_layers,
            do_layer_norm=False, initial_scalar_parameters=
            scalar_mix_parameters, trainable=scalar_mix_parameters is None)
        self._scalar_mix_2 = ScalarMix(model.config.num_hidden_layers,
            do_layer_norm=False, initial_scalar_parameters=
            scalar_mix_parameters, trainable=scalar_mix_parameters is None)
        self.scalar_mix_id = {(1): self._scalar_mix_1, (2): self._scalar_mix_2}
    <mask token>

<mask token>


@TokenEmbedder.register('bert-pretrained-split-mix')
class PretrainedBertEmbedderSplitMix(BertEmbedder):

    def __init__(self, pretrained_model: str, requires_grad: bool=False,
        top_layer_only: bool=False, scalar_mix_parameters: List[float]=None
        ) ->None:
        model = PretrainedBertModel.load(pretrained_model)
        for param in model.parameters():
            param.requires_grad = requires_grad
        super().__init__(bert_model=model, top_layer_only=top_layer_only,
            scalar_mix_parameters=scalar_mix_parameters)
        self._scalar_mix_1 = ScalarMix(model.config.num_hidden_layers,
            do_layer_norm=False, initial_scalar_parameters=
            scalar_mix_parameters, trainable=scalar_mix_parameters is None)
        self._scalar_mix_2 = ScalarMix(model.config.num_hidden_layers,
            do_layer_norm=False, initial_scalar_parameters=
            scalar_mix_parameters, trainable=scalar_mix_parameters is None)
        self.scalar_mix_id = {(1): self._scalar_mix_1, (2): self._scalar_mix_2}

    def forward(self, input_ids: torch.LongTensor, offsets: torch.
        LongTensor=None, token_type_ids: torch.LongTensor=None, mix_id: int=0):
        self._scalar_mix = self.scalar_mix_id[mix_id]
        return super().forward(input_ids, offsets, token_type_ids)

from allennlp.modules.token_embedders.bert_token_embedder import BertEmbedder, TokenEmbedder, PretrainedBertModel
from typing import List
import torch
from allennlp.modules.scalar_mix import ScalarMix


@TokenEmbedder.register('bert-pretrained-split-mix')
class PretrainedBertEmbedderSplitMix(BertEmbedder):

    def __init__(self, pretrained_model: str, requires_grad: bool=False,
        top_layer_only: bool=False, scalar_mix_parameters: List[float]=None
        ) ->None:
        model = PretrainedBertModel.load(pretrained_model)
        for param in model.parameters():
            param.requires_grad = requires_grad
        super().__init__(bert_model=model, top_layer_only=top_layer_only,
            scalar_mix_parameters=scalar_mix_parameters)
        self._scalar_mix_1 = ScalarMix(model.config.num_hidden_layers,
            do_layer_norm=False, initial_scalar_parameters=
            scalar_mix_parameters, trainable=scalar_mix_parameters is None)
        self._scalar_mix_2 = ScalarMix(model.config.num_hidden_layers,
            do_layer_norm=False, initial_scalar_parameters=
            scalar_mix_parameters, trainable=scalar_mix_parameters is None)
        self.scalar_mix_id = {(1): self._scalar_mix_1, (2): self._scalar_mix_2}

    def forward(self, input_ids: torch.LongTensor, offsets: torch.
        LongTensor=None, token_type_ids: torch.LongTensor=None, mix_id: int=0):
        self._scalar_mix = self.scalar_mix_id[mix_id]
        return super().forward(input_ids, offsets, token_type_ids)

from allennlp.modules.token_embedders.bert_token_embedder import BertEmbedder, TokenEmbedder, PretrainedBertModel
from typing import List
import torch
from allennlp.modules.scalar_mix import ScalarMix


@TokenEmbedder.register("bert-pretrained-split-mix")
class PretrainedBertEmbedderSplitMix(BertEmbedder):
    def __init__(self, pretrained_model: str, requires_grad: bool = False, top_layer_only: bool = False,
                 scalar_mix_parameters: List[float] = None) -> None:

        model = PretrainedBertModel.load(pretrained_model)

        for param in model.parameters():
            param.requires_grad = requires_grad

        super().__init__(bert_model=model,
                         top_layer_only=top_layer_only,
                         scalar_mix_parameters=scalar_mix_parameters)

        # instead of one scalar mix, keep two!
        self._scalar_mix_1 = ScalarMix(model.config.num_hidden_layers,
                                          do_layer_norm=False,
                                          initial_scalar_parameters=scalar_mix_parameters,
                                          trainable=scalar_mix_parameters is None)

        self._scalar_mix_2 = ScalarMix(model.config.num_hidden_layers,
                                      do_layer_norm=False,
                                      initial_scalar_parameters=scalar_mix_parameters,
                                      trainable=scalar_mix_parameters is None)

        self.scalar_mix_id = {1: self._scalar_mix_1, 2: self._scalar_mix_2}

    def forward(self,
                input_ids: torch.LongTensor,
                offsets: torch.LongTensor = None,
                token_type_ids: torch.LongTensor = None,
                mix_id: int = 0):

        self._scalar_mix = self.scalar_mix_id[mix_id]  # decide which scalar mix to update
        return super().forward(input_ids, offsets, token_type_ids)
