Upload mini sentiment transformer

README.md

@@ -0,0 +1,80 @@
+---
+language: en
+license: mit
+library_name: transformers
+tags:
+- sentiment-analysis
+- text-classification
+- transformers
+- mini-transformer
+datasets:
+- glue/sst2
+model-index:
+- name: mini-sentiment-transformer
+  results:
+  - task:
+      type: text-classification
+      name: Sentiment Analysis
+    dataset:
+      name: SST-2
+      type: glue
+      args: sst2
+    metrics:
+    - type: accuracy
+      value: 0.8154
+      name: Validation Accuracy
+---
+
+# Mini Sentiment Transformer
+
+This is a tiny transformer model for sentiment analysis, created as a learning project to understand transformer architecture. It's much smaller than BERT or DistilBERT, with only around 4,188,802 parameters.
+
+## Model Details
+
+- Developed by: leorigasaki54
+- Type: Text Classification (Sentiment Analysis)
+- Language: English
+- Training Data: SST-2 (Stanford Sentiment Treebank)
+- Size: 4,188,802 parameters (4.19M)
+- Architecture:
+  - 2 transformer encoder layers
+  - 2 attention heads per layer
+  - 128 embedding dimensions
+  - 256 feed-forward dimensions
+
+## Usage
+
+```python
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
+import torch.nn.functional as F
+
+# Load tokenizer and model
+tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")  # We use DistilBERT tokenizer
+model = AutoModelForSequenceClassification.from_pretrained("leorigasaki54/mini-sentiment-transformer")
+
+# Prepare input
+text = "I really enjoyed this movie!"
+inputs = tokenizer(text, return_tensors="pt", padding=True, truncation=True, max_length=64)
+
+# Make prediction
+with torch.no_grad():
+    outputs = model(**inputs)
+    probabilities = F.softmax(outputs.logits, dim=-1)
+    prediction = torch.argmax(probabilities, dim=-1).item()
+
+sentiment = "Positive" if prediction == 1 else "Negative"
+confidence = probabilities[0][prediction].item()
+
+print(f"Sentiment: {sentiment} (confidence: {confidence:.4f})")
+```
+
+## Limitations
+
+- This is a minimal implementation meant for educational purposes
+- Performance may be lower than larger models like BERT or DistilBERT
+- The model has been trained only on movie reviews and may not generalize well to other domains
+- Limited to English language text only
+
+## Training
+
+The model was trained on the SST-2 dataset for 5 epochs using Adam optimizer with a learning rate of 5e-5.

checkpoint-126/config.json

@@ -0,0 +1,25 @@
+{
+  "architectures": [
+    "BertForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 512,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 2,
+  "num_hidden_layers": 2,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "problem_type": "single_label_classification",
+  "torch_dtype": "float32",
+  "transformers_version": "4.54.1",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 30522
+}

checkpoint-126/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4506b237f1050833cf91481441ea8a1df63a263334521dc2aa6842b56b34e141
+size 17549312

checkpoint-126/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b2cbf47f37585b1382e08c469ff83a1b52b06c0ed22d1260c825a579e88cc2ef
+size 35123898

checkpoint-126/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e14576e5536188efec313f809265425da6e5c56074fb0b274dfe0222bbb6bf43
+size 14244

checkpoint-126/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a30776912918b6f7493dde431aac45fc2965212e97f718858f7ba1f4fe1e5e15
+size 1064

checkpoint-126/trainer_state.json

@@ -0,0 +1,58 @@
+{
+  "best_global_step": 126,
+  "best_metric": 0.6766817569732666,
+  "best_model_checkpoint": "./mini-sentiment-model/checkpoint-126",
+  "epoch": 2.0,
+  "eval_steps": 500,
+  "global_step": 126,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 1.0,
+      "eval_accuracy": 0.54,
+      "eval_f1": 0.6993464052287581,
+      "eval_loss": 0.681846022605896,
+      "eval_precision": 0.5376884422110553,
+      "eval_recall": 1.0,
+      "eval_runtime": 0.1526,
+      "eval_samples_per_second": 1310.261,
+      "eval_steps_per_second": 85.167,
+      "step": 63
+    },
+    {
+      "epoch": 2.0,
+      "eval_accuracy": 0.57,
+      "eval_f1": 0.7094594594594594,
+      "eval_loss": 0.6766817569732666,
+      "eval_precision": 0.5555555555555556,
+      "eval_recall": 0.9813084112149533,
+      "eval_runtime": 0.1029,
+      "eval_samples_per_second": 1943.287,
+      "eval_steps_per_second": 126.314,
+      "step": 126
+    }
+  ],
+  "logging_steps": 500,
+  "max_steps": 189,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 635243520000.0,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-126/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e791125ba3e22840402f92bd40cab4a742f51fb9913759fe4f66250ab01752be
+size 5368

checkpoint-189/config.json

@@ -0,0 +1,25 @@
+{
+  "architectures": [
+    "BertForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 512,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 2,
+  "num_hidden_layers": 2,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "problem_type": "single_label_classification",
+  "torch_dtype": "float32",
+  "transformers_version": "4.54.1",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 30522
+}

checkpoint-189/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83c3b3706aee4f4ea7b0e8810fc3b535c3d10cdf331932742559bb0e5eb2ede1
+size 17549312

checkpoint-189/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5164d35ec81d9efbe548d69284069bb83499db2ab4e6068ba7fc9faa116050ca
+size 35123898

checkpoint-189/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6048021907f08141fd2b6a404cf4620a36db966f1f3ce745c9e4f2b13b28e9dd
+size 14244

checkpoint-189/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd9834cb37dd37c0bb479c91093a4be5fae04ef7c14da536f96b6acfdec15663
+size 1064

checkpoint-189/trainer_state.json

@@ -0,0 +1,70 @@
+{
+  "best_global_step": 189,
+  "best_metric": 0.6738117933273315,
+  "best_model_checkpoint": "./mini-sentiment-model/checkpoint-189",
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 189,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 1.0,
+      "eval_accuracy": 0.54,
+      "eval_f1": 0.6993464052287581,
+      "eval_loss": 0.681846022605896,
+      "eval_precision": 0.5376884422110553,
+      "eval_recall": 1.0,
+      "eval_runtime": 0.1526,
+      "eval_samples_per_second": 1310.261,
+      "eval_steps_per_second": 85.167,
+      "step": 63
+    },
+    {
+      "epoch": 2.0,
+      "eval_accuracy": 0.57,
+      "eval_f1": 0.7094594594594594,
+      "eval_loss": 0.6766817569732666,
+      "eval_precision": 0.5555555555555556,
+      "eval_recall": 0.9813084112149533,
+      "eval_runtime": 0.1029,
+      "eval_samples_per_second": 1943.287,
+      "eval_steps_per_second": 126.314,
+      "step": 126
+    },
+    {
+      "epoch": 3.0,
+      "eval_accuracy": 0.595,
+      "eval_f1": 0.7197231833910035,
+      "eval_loss": 0.6738117933273315,
+      "eval_precision": 0.5714285714285714,
+      "eval_recall": 0.9719626168224299,
+      "eval_runtime": 0.2443,
+      "eval_samples_per_second": 818.694,
+      "eval_steps_per_second": 53.215,
+      "step": 189
+    }
+  ],
+  "logging_steps": 500,
+  "max_steps": 189,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 952865280000.0,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-189/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e791125ba3e22840402f92bd40cab4a742f51fb9913759fe4f66250ab01752be
+size 5368

checkpoint-63/config.json

@@ -0,0 +1,25 @@
+{
+  "architectures": [
+    "BertForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 512,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 2,
+  "num_hidden_layers": 2,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "problem_type": "single_label_classification",
+  "torch_dtype": "float32",
+  "transformers_version": "4.54.1",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 30522
+}

checkpoint-63/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:379ff04d2e9de33d5ecd3f470def81b52a37a15dc136c95dfd0414ae2b982603
+size 17549312

checkpoint-63/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a2ccf39cfd807f8e97f497d888cf45872bdf13c60bf29d0150abb8dee9a5922
+size 35123898

checkpoint-63/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ee3244d1fb8d2938b76e0da0926eed0036f4636ced09415eb94b79dac5736c7
+size 14244

checkpoint-63/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1f2899150945e7607692260e26159240946ed36221da68ea6e9bb708e62775d
+size 1064

checkpoint-63/trainer_state.json

@@ -0,0 +1,46 @@
+{
+  "best_global_step": 63,
+  "best_metric": 0.681846022605896,
+  "best_model_checkpoint": "./mini-sentiment-model/checkpoint-63",
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 63,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 1.0,
+      "eval_accuracy": 0.54,
+      "eval_f1": 0.6993464052287581,
+      "eval_loss": 0.681846022605896,
+      "eval_precision": 0.5376884422110553,
+      "eval_recall": 1.0,
+      "eval_runtime": 0.1526,
+      "eval_samples_per_second": 1310.261,
+      "eval_steps_per_second": 85.167,
+      "step": 63
+    }
+  ],
+  "logging_steps": 500,
+  "max_steps": 189,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 317621760000.0,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-63/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e791125ba3e22840402f92bd40cab4a742f51fb9913759fe4f66250ab01752be
+size 5368

config.json

@@ -0,0 +1,15 @@
+{
+  "vocab_size": 30522,
+  "d_model": 128,
+  "num_heads": 2,
+  "num_layers": 2,
+  "d_ff": 256,
+  "num_classes": 2,
+  "max_length": 64,
+  "model_type": "mini-sentiment-transformer",
+  "architectures": [
+    "MiniSentimentTransformer"
+  ],
+  "tokenizer_class": "AutoTokenizer",
+  "transformers_version": "4.30.0"
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7d74915b6bb603d563a5b7a5be748c18174f7c38d4f54420154fe96ba4ad703
+size 16800334

runs/Aug01_11-34-03_5f8061b8deee/events.out.tfevents.1754048063.5f8061b8deee.5518.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2940d53128736f5e2997f126e59fb6dfb48930ad33d428aed747b5688162de4
+size 6652

runs/Aug01_11-34-03_5f8061b8deee/events.out.tfevents.1754048289.5f8061b8deee.5518.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa76d23aa58e2d8edd4ce9bd526769c6fc489aacabcb7ee778ee1f12344fce45
+size 560

tokenizer_config.json

@@ -0,0 +1,7 @@
+{
+  "base_tokenizer": "distilbert-base-uncased",
+  "max_length": 64,
+  "model_max_length": 64,
+  "padding_side": "right",
+  "truncation_side": "right"
+}