Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,8 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+dataset/train/train.en filter=lfs diff=lfs merge=lfs -text
+dataset/train/train.vi filter=lfs diff=lfs merge=lfs -text
+dataset_convert/train/train.en filter=lfs diff=lfs merge=lfs -text
+dataset_convert/train/train.vi filter=lfs diff=lfs merge=lfs -text
+saved_models/en_vi_translation.keras filter=lfs diff=lfs merge=lfs -text

dataset/train/train.en

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c26dfeed74b6bf3752f5ca552f2412456f0de153f7c804df8717931fb3a5c78a
+size 13603614

dataset/train/train.vi

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:707206edf2dc0280273952c7b70544ea8a1363aa69aaeb9d70514b888dc3067d
+size 18074646

dataset_convert/train/train.en

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c26dfeed74b6bf3752f5ca552f2412456f0de153f7c804df8717931fb3a5c78a
+size 13603614

dataset_convert/train/train.vi

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28fe7bf65585138135caa5d35b3426fcd37748a8c392608132decf36ae275d89
+size 19722027

en-vi-translation-transformer-tensorflow.log

@@ -0,0 +1,5 @@
+[{"stream_name":"stderr","time":5.45447475,"data":"/usr/local/lib/python3.10/dist-packages/traitlets/traitlets.py:2915: FutureWarning: --Exporter.preprocessors=[\"nbconvert.preprocessors.ExtractOutputPreprocessor\"] for containers is deprecated in traitlets 5.0. You can pass `--Exporter.preprocessors item` ... multiple times to add items to a list.\n"}
+,{"stream_name":"stderr","time":5.454559784,"data":"  warn(\n"}
+,{"stream_name":"stderr","time":5.528515974,"data":"[NbConvertApp] Converting notebook __notebook__.ipynb to html\n"}
+,{"stream_name":"stderr","time":8.239659837,"data":"[NbConvertApp] Writing 448192 bytes to __results__.html\n"}
+]

main.py

@@ -0,0 +1,104 @@
+import tensorflow as tf
+from translator import Translator
+from utils import tokenizer_utils
+from utils.preprocessing import input_processing, output_processing
+from models.transformer import Transformer
+from models.encoder import Encoder
+from models.decoder import Decoder
+from models.layers import EncoderLayer, DecoderLayer, MultiHeadAttention, point_wise_feed_forward_network
+from models.utils import masked_loss, masked_accuracy
+import argparse
+
+def main(sentences: list, model: tf.keras.Model, en_tokenizer, vi_tokenizer) -> None:
+    """
+    Translates input English sentences to Vietnamese using a pre-trained model.
+
+    Args:
+        sentences (list): List of English sentences to translate.
+        model (tf.keras.Model): The pre-trained translation model.
+        en_tokenizer: English tokenizer.
+        vi_tokenizer: Vietnamese tokenizer.
+    """
+    # Initialize the translator with tokenizers and the model
+    translator = Translator(en_tokenizer, vi_tokenizer, model)
+
+    # Process and translate each sentence
+    for sentence in sentences:
+        processed_sentence = input_processing(sentence)
+        translated_text = translator(processed_sentence)
+        translated_text = output_processing(translated_text)
+
+        # Display the input and translated text
+        print("Input:", processed_sentence)
+        print("Translated:", translated_text)
+        print("-" * 50)
+
+if __name__ == "__main__":
+    # Set up argument parser
+    parser = argparse.ArgumentParser(
+        description="Translate English sentences to Vietnamese using a pre-trained transformer model.",
+        epilog="Example: python translate.py --sentence 'Hello, world!' --sentence 'The sun is shining.'"
+    )
+    parser.add_argument(
+        "--sentence",
+        type=str,
+        nargs="*",
+        default=[
+            (
+                "For at least six centuries, residents along a lake in the mountains of central Japan "
+                "have marked the depth of winter by celebrating the return of a natural phenomenon "
+                "once revered as the trail of a wandering god."
+            )
+        ],
+        help="One or more English sentences to translate (default: provided example sentence)"
+    )
+    parser.add_argument(
+        "--model_path",
+        type=str,
+        default="saved_models/en_vi_translation.keras",
+        help="Path to the pre-trained model file (default: saved_models/en_vi_translation.keras)"
+    )
+
+    # Parse arguments
+    args = parser.parse_args()
+
+    # Define custom objects required for loading the model
+    custom_objects = {
+        "Transformer": Transformer,
+        "Encoder": Encoder,
+        "Decoder": Decoder,
+        "EncoderLayer": EncoderLayer,
+        "DecoderLayer": DecoderLayer,
+        "MultiHeadAttention": MultiHeadAttention,
+        "point_wise_feed_forward_network": point_wise_feed_forward_network,
+        "masked_loss": masked_loss,
+        "masked_accuracy": masked_accuracy,
+    }
+
+    # Load the pre-trained model once
+    print("Loading model from:", args.model_path)
+    loaded_model = tf.keras.models.load_model(
+        args.model_path, custom_objects=custom_objects
+    )
+    print("Model loaded successfully.")
+
+    # Load English and Vietnamese tokenizers once
+    en_tokenizer, vi_tokenizer = tokenizer_utils.load_tokenizers()
+
+    # Run the translation for all provided sentences
+    main(sentences=args.sentence, model=loaded_model, en_tokenizer=en_tokenizer, vi_tokenizer=vi_tokenizer)
+
+    # Interactive loop for additional translations
+    while True:
+        choice = input("Would you like to translate another sentence? (Y/n): ").strip().lower()
+        if choice in ['no', 'n', 'quit', 'q']:
+            print("Exiting the program.")
+            break
+        elif choice in ['yes', 'y']:
+            new_sentence = input("Enter an English sentence to translate: ").strip()
+            if new_sentence:
+                main(sentences=[new_sentence], model=loaded_model, en_tokenizer=en_tokenizer, vi_tokenizer=vi_tokenizer)
+            else:
+                print("No sentence provided. Please try again.")
+        else:
+            print("Invalid input. Please enter 'y' or 'n'.")

models/decoder.py

@@ -0,0 +1,53 @@
+import tensorflow as tf
+from .layers import DecoderLayer
+
+@tf.keras.utils.register_keras_serializable()
+class Decoder(tf.keras.layers.Layer):
+    def __init__(self, num_layers, d_model, num_heads, dff, target_vocab_size, 
+                 max_tokens, dropout_rate, **kwargs):
+        super(Decoder, self).__init__(**kwargs)
+        self.d_model = d_model
+        self.num_layers = num_layers
+        self.embedding = tf.keras.layers.Embedding(target_vocab_size, d_model)
+        self.pos_encoding = self.positional_encoding(max_tokens, d_model)
+        self.dec_layers = [DecoderLayer(d_model, num_heads, dff, dropout_rate) 
+                          for _ in range(num_layers)]
+        self.dropout = tf.keras.layers.Dropout(dropout_rate)
+
+    def call(self, x, enc_output, training=None, look_ahead_mask=None, padding_mask=None):
+        seq_len = tf.shape(x)[1]
+        x = self.embedding(x)
+        x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))
+        x += self.pos_encoding[:, :seq_len, :]
+        x = self.dropout(x, training=training)
+        for i in range(self.num_layers):
+            x = self.dec_layers[i](x, enc_output, training=training, 
+                                  look_ahead_mask=look_ahead_mask, 
+                                  padding_mask=padding_mask)
+        return x
+
+    def positional_encoding(self, max_len, d_model):
+        angle_rads = self.get_angles(tf.range(max_len, dtype=tf.float32)[:, tf.newaxis],
+                                    tf.range(d_model, dtype=tf.float32)[tf.newaxis, :], 
+                                    d_model)
+        sines = tf.math.sin(angle_rads[:, 0::2])
+        cosines = tf.math.cos(angle_rads[:, 1::2])
+        pos_encoding = tf.concat([sines, cosines], axis=-1)
+        return pos_encoding[tf.newaxis, ...]
+
+    def get_angles(self, pos, i, d_model):
+        angle_rates = 1 / tf.pow(10000, (2 * (i // 2)) / tf.cast(d_model, tf.float32))
+        return pos * angle_rates
+
+    def get_config(self):
+        config = super().get_config()
+        config.update({
+            'num_layers': self.num_layers,
+            'd_model': self.d_model,
+            'num_heads': self.num_heads,
+            'dff': self.dff,
+            'target_vocab_size': self.embedding.input_dim,
+            'max_tokens': self.pos_encoding.shape[1],
+            'dropout_rate': self.dropout.rate
+        })
+        return config

models/encoder.py

@@ -0,0 +1,51 @@
+import tensorflow as tf
+from .layers import EncoderLayer
+
+@tf.keras.utils.register_keras_serializable()
+class Encoder(tf.keras.layers.Layer):
+    def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size, 
+                 max_tokens, dropout_rate, **kwargs):
+        super(Encoder, self).__init__(**kwargs)
+        self.d_model = d_model
+        self.num_layers = num_layers
+        self.embedding = tf.keras.layers.Embedding(input_vocab_size, d_model)
+        self.pos_encoding = self.positional_encoding(max_tokens, d_model)
+        self.enc_layers = [EncoderLayer(d_model, num_heads, dff, dropout_rate) 
+                          for _ in range(num_layers)]
+        self.dropout = tf.keras.layers.Dropout(dropout_rate)
+
+    def call(self, x, training=None, mask=None):
+        seq_len = tf.shape(x)[1]
+        x = self.embedding(x)
+        x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))
+        x += self.pos_encoding[:, :seq_len, :]
+        x = self.dropout(x, training=training)
+        for i in range(self.num_layers):
+            x = self.enc_layers[i](x, training=training, mask=mask)
+        return x
+
+    def positional_encoding(self, max_len, d_model):
+        angle_rads = self.get_angles(tf.range(max_len, dtype=tf.float32)[:, tf.newaxis],
+                                    tf.range(d_model, dtype=tf.float32)[tf.newaxis, :], 
+                                    d_model)
+        sines = tf.math.sin(angle_rads[:, 0::2])
+        cosines = tf.math.cos(angle_rads[:, 1::2])
+        pos_encoding = tf.concat([sines, cosines], axis=-1)
+        return pos_encoding[tf.newaxis, ...]
+
+    def get_angles(self, pos, i, d_model):
+        angle_rates = 1 / tf.pow(10000, (2 * (i // 2)) / tf.cast(d_model, tf.float32))
+        return pos * angle_rates
+
+    def get_config(self):
+        config = super().get_config()
+        config.update({
+            'num_layers': self.num_layers,
+            'd_model': self.d_model,
+            'num_heads': self.num_heads,
+            'dff': self.dff,
+            'input_vocab_size': self.embedding.input_dim,
+            'max_tokens': self.pos_encoding.shape[1],
+            'dropout_rate': self.dropout.rate
+        })
+        return config

models/layers.py

@@ -0,0 +1,123 @@
+import tensorflow as tf
+from tensorflow.keras.layers import Layer, Dense, Dropout, LayerNormalization
+
+@tf.keras.utils.register_keras_serializable()
+class EncoderLayer(Layer):
+    def __init__(self, d_model, num_heads, dff, dropout_rate, **kwargs):
+        super(EncoderLayer, self).__init__(**kwargs)
+        self.mha = MultiHeadAttention(d_model, num_heads)
+        self.ffn = point_wise_feed_forward_network(d_model, dff)
+        self.layernorm1 = LayerNormalization(epsilon=1e-6)
+        self.layernorm2 = LayerNormalization(epsilon=1e-6)
+        self.dropout1 = Dropout(dropout_rate)
+        self.dropout2 = Dropout(dropout_rate)
+
+    def call(self, x, training=None, mask=None):
+        attn_output, _ = self.mha(x, x, x, mask)
+        attn_output = self.dropout1(attn_output, training=training)
+        out1 = self.layernorm1(x + attn_output)
+        ffn_output = self.ffn(out1)
+        ffn_output = self.dropout2(ffn_output, training=training)
+        out2 = self.layernorm2(out1 + ffn_output)
+        return out2
+
+    def get_config(self):
+        config = super().get_config()
+        config.update({
+            'd_model': self.mha.d_model,
+            'num_heads': self.mha.num_heads,
+            'dff': self.ffn.layers[0].units,
+            'dropout_rate': self.dropout1.rate
+        })
+        return config
+
+@tf.keras.utils.register_keras_serializable()
+class DecoderLayer(Layer):
+    def __init__(self, d_model, num_heads, dff, dropout_rate, **kwargs):
+        super(DecoderLayer, self).__init__(**kwargs)
+        self.mha1 = MultiHeadAttention(d_model, num_heads)
+        self.mha2 = MultiHeadAttention(d_model, num_heads)
+        self.ffn = point_wise_feed_forward_network(d_model, dff)
+        self.layernorm1 = LayerNormalization(epsilon=1e-6)
+        self.layernorm2 = LayerNormalization(epsilon=1e-6)
+        self.layernorm3 = LayerNormalization(epsilon=1e-6)
+        self.dropout1 = Dropout(dropout_rate)
+        self.dropout2 = Dropout(dropout_rate)
+        self.dropout3 = Dropout(dropout_rate)
+
+    def call(self, x, enc_output, training=None, look_ahead_mask=None, padding_mask=None):
+        attn1, _ = self.mha1(x, x, x, look_ahead_mask)
+        attn1 = self.dropout1(attn1, training=training)
+        out1 = self.layernorm1(x + attn1)
+        attn2, _ = self.mha2(enc_output, enc_output, out1, padding_mask)
+        attn2 = self.dropout2(attn2, training=training)
+        out2 = self.layernorm2(out1 + attn2)
+        ffn_output = self.ffn(out2)
+        ffn_output = self.dropout3(ffn_output, training=training)
+        out3 = self.layernorm3(out2 + ffn_output)
+        return out3
+
+    def get_config(self):
+        config = super().get_config()
+        config.update({
+            'd_model': self.mha1.d_model,
+            'num_heads': self.mha1.num_heads,
+            'dff': self.ffn.layers[0].units,
+            'dropout_rate': self.dropout1.rate
+        })
+        return config
+
+@tf.keras.utils.register_keras_serializable()
+class MultiHeadAttention(Layer):
+    def __init__(self, d_model, num_heads, **kwargs):
+        super(MultiHeadAttention, self).__init__(**kwargs)
+        self.num_heads = num_heads
+        self.d_model = d_model
+        assert d_model % num_heads == 0
+        self.depth = d_model // num_heads
+        self.wq = Dense(d_model)
+        self.wk = Dense(d_model)
+        self.wv = Dense(d_model)
+        self.dense = Dense(d_model)
+
+    def split_heads(self, x, batch_size):
+        x = tf.reshape(x, (batch_size, -1, self.num_heads, self.depth))
+        return tf.transpose(x, perm=[0, 2, 1, 3])
+
+    def call(self, v, k, q, mask=None):
+        batch_size = tf.shape(q)[0]
+        q = self.wq(q)
+        k = self.wk(k)
+        v = self.wv(v)
+        q = self.split_heads(q, batch_size)
+        k = self.split_heads(k, batch_size)
+        v = self.split_heads(v, batch_size)
+        scaled_attention, _ = self.scaled_dot_product_attention(q, k, v, mask)
+        scaled_attention = tf.transpose(scaled_attention, perm=[0, 2, 1, 3])
+        concat_attention = tf.reshape(scaled_attention, (batch_size, -1, self.d_model))
+        output = self.dense(concat_attention)
+        return output, _
+
+    def scaled_dot_product_attention(self, q, k, v, mask):
+        matmul_qk = tf.matmul(q, k, transpose_b=True)
+        dk = tf.cast(tf.shape(k)[-1], tf.float32)
+        scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)
+        if mask is not None:
+            scaled_attention_logits += (mask * -1e9)
+        attention_weights = tf.nn.softmax(scaled_attention_logits, axis=-1)
+        output = tf.matmul(attention_weights, v)
+        return output, attention_weights
+
+    def get_config(self):
+        config = super().get_config()
+        config.update({
+            'd_model': self.d_model,
+            'num_heads': self.num_heads
+        })
+        return config
+
+def point_wise_feed_forward_network(d_model, dff):
+    return tf.keras.Sequential([
+        Dense(dff, activation='relu'),
+        Dense(d_model)
+    ])

models/transformer.py

@@ -0,0 +1,58 @@
+import tensorflow as tf
+from .encoder import Encoder
+from .decoder import Decoder
+from tensorflow.keras.layers import Dense
+
+@tf.keras.utils.register_keras_serializable()
+class Transformer(tf.keras.Model):
+    def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size, 
+                 target_vocab_size, max_tokens, dropout_rate=0.1, **kwargs):
+        super(Transformer, self).__init__(**kwargs)
+        self.num_layers = num_layers
+        self.d_model = d_model
+        self.num_heads = num_heads
+        self.dff = dff
+        self.input_vocab_size = input_vocab_size
+        self.target_vocab_size = target_vocab_size
+        self.max_tokens = max_tokens
+        self.dropout_rate = dropout_rate
+        
+        self.encoder = Encoder(num_layers, d_model, num_heads, dff, 
+                             input_vocab_size, max_tokens, dropout_rate)
+        self.decoder = Decoder(num_layers, d_model, num_heads, dff, 
+                              target_vocab_size, max_tokens, dropout_rate)
+        self.final_layer = Dense(target_vocab_size)
+
+    def call(self, inputs, training=None):
+        enc_input, dec_input = inputs
+        enc_padding_mask = self.create_padding_mask(enc_input)
+        look_ahead_mask = self.create_look_ahead_mask(tf.shape(dec_input)[1])
+        dec_padding_mask = self.create_padding_mask(enc_input)
+        enc_output = self.encoder(enc_input, training=training, mask=enc_padding_mask)
+        dec_output = self.decoder(dec_input, enc_output, training=training, 
+                                 look_ahead_mask=look_ahead_mask, 
+                                 padding_mask=dec_padding_mask)
+        final_output = self.final_layer(dec_output)
+        return final_output
+
+    def create_padding_mask(self, seq):
+        mask = tf.cast(tf.math.equal(seq, 0), tf.float32)
+        return mask[:, tf.newaxis, tf.newaxis, :]
+
+    def create_look_ahead_mask(self, size):
+        mask = 1 - tf.linalg.band_part(tf.ones((size, size)), -1, 0)
+        return mask
+
+    def get_config(self):
+        config = super().get_config()
+        config.update({
+            'num_layers': self.num_layers,
+            'd_model': self.d_model,
+            'num_heads': self.num_heads,
+            'dff': self.dff,
+            'input_vocab_size': self.input_vocab_size,
+            'target_vocab_size': self.target_vocab_size,
+            'max_tokens': self.max_tokens,
+            'dropout_rate': self.dropout_rate
+        })
+        return config

models/utils.py

@@ -0,0 +1,23 @@
+import tensorflow as tf
+
+@tf.keras.utils.register_keras_serializable()
+def masked_loss(label, pred):
+    mask = label != 0
+    loss_object = tf.keras.losses.SparseCategoricalCrossentropy(
+        from_logits=True, reduction='none')
+    loss = loss_object(label, pred)
+    mask = tf.cast(mask, dtype=loss.dtype)
+    loss *= mask
+    loss = tf.reduce_sum(loss)/tf.reduce_sum(mask)
+    return loss
+
+@tf.keras.utils.register_keras_serializable()
+def masked_accuracy(label, pred):
+    pred = tf.argmax(pred, axis=2)
+    label = tf.cast(label, pred.dtype)
+    match = label == pred
+    mask = label != 0
+    match = match & mask
+    match = tf.cast(match, dtype=tf.float32)
+    mask = tf.cast(mask, dtype=tf.float32)
+    return tf.reduce_sum(match)/tf.reduce_sum(mask)

requirements.txt

@@ -0,0 +1,6 @@
+pandas==2.2.3
+matplotlib==3.7.5
+tensorflow_datasets==4.9.7
+tensorflow==2.17.1
+tensorflow-text==2.17.0
+underthesea==6.8.4

saved_models/backup_weights/latest.weights.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4e1e9122a5896689ccb4cb169ff3b2edf9e5474655468fd638b68cb31159eb4
+size 368283712

saved_models/backup_weights/training_metadata.json

@@ -0,0 +1 @@
+{"epoch": 50, "batch": 0}

saved_models/en_vi_translation.keras

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:85b96fdac99f5d2b1fb6f6823cb6c5691fd2ff339d8a966f0b0cc6da52cc33fd
+size 368250218

tokenizers/en_tokenizer.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d172551a4da648a35629de06593b3051ce32598c3166464de46fc4a1f6f90981
+size 3973568

tokenizers/vi_tokenizer.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42a96a2d1b932a52603083dff973a0eba79998c5e2c2717e54f5c6eeaf84b8fa
+size 5835250

translator.py

@@ -0,0 +1,44 @@
+import tensorflow as tf
+import numpy as np
+from models.transformer import Transformer
+from models.encoder import Encoder
+from models.decoder import Decoder
+from models.layers import EncoderLayer, DecoderLayer, MultiHeadAttention, point_wise_feed_forward_network
+from models.utils import masked_loss, masked_accuracy
+
+class Translator(tf.Module):
+    def __init__(self, en_tokenizer, vi_tokenizer, transformer, max_tokens=512):
+        self.tokenizers_en = en_tokenizer
+        self.tokenizers_vi = vi_tokenizer
+        self.transformer = transformer
+        self.max_tokens = max_tokens
+
+    def __call__(self, sentence, max_length=None):
+        if max_length is None:
+            max_length = self.max_tokens
+            
+        sentence = self.tokenizers_en.texts_to_sequences([sentence])
+        sentence = sentence[0] + np.zeros(self.max_tokens - len(sentence[0]), 
+                                       dtype=np.int32).tolist()
+        sentence = tf.convert_to_tensor([sentence])
+        encoder_input = sentence
+
+        start = self.tokenizers_vi.texts_to_sequences(["<sos>"])[0]
+        end = self.tokenizers_vi.texts_to_sequences(["<eos>"])[0]
+
+        output_array = tf.TensorArray(dtype=tf.int32, size=0, dynamic_size=True)
+        output_array = output_array.write(0, start)
+
+        for i in tf.range(max_length):
+            output = tf.transpose(output_array.stack())
+            predictions = self.transformer([encoder_input, output], training=False)
+            predictions = predictions[:, -1:, :]
+            predicted_id = tf.argmax(predictions, axis=-1, output_type=tf.int32)
+            output_array = output_array.write(i+1, predicted_id[0])
+
+            if predicted_id == end:
+                break
+
+        output = tf.transpose(output_array.stack())
+        text = self.tokenizers_vi.sequences_to_texts(output.numpy().tolist())[0]
+        return text

utils/preprocessing.py

@@ -0,0 +1,19 @@
+import string
+
+def input_processing(in_string):
+
+    punct_marks = string.punctuation
+    for mark in punct_marks:
+        if mark in in_string:
+            in_string = in_string.replace(mark, " " + mark)
+    in_string = in_string.replace("scholl", "school")
+    in_string = in_string.strip()
+    
+    return in_string
+
+def output_processing(in_string):
+
+    in_string = in_string.replace("_", " ").replace('<sos>', '').replace('<eos>', '').strip()
+    in_string = in_string[0].upper() + in_string[1:]
+
+    return in_string

utils/tokenizer_utils.py

@@ -0,0 +1,14 @@
+import pickle
+import tensorflow as tf
+
+def load_tokenizers(en_path='tokenizers/en_tokenizer.pkl', 
+                   vi_path='tokenizers/vi_tokenizer.pkl'):
+    with open(en_path, 'rb') as f:
+        en_tokenizer = pickle.load(f)
+    with open(vi_path, 'rb') as f:
+        vi_tokenizer = pickle.load(f)
+    
+    en_tokenizer = tf.keras.preprocessing.text.tokenizer_from_json(en_tokenizer)
+    vi_tokenizer = tf.keras.preprocessing.text.tokenizer_from_json(vi_tokenizer)
+    
+    return en_tokenizer, vi_tokenizer