Initial commit

adjacency_matrix/graph_extended_comments.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0508863549ac3faea223be7d93bef5ec24b70af65124223fec485e1021b0f3e
+size 829003020

config.json

@@ -1,9 +1,10 @@
 {
   "attention_probs_dropout_prob": 0.1,
+  "bert_model": "readerbench/RoBERT-base",
   "classifier_dropout": null,
   "do_lower_case": 1,
   "do_remove_accents": 0,
-  "gcn_adj_matrix": "adjacency_matrix/graph_dataset_comments.pkl",
+  "gcn_adj_matrix": "adjacency_matrix/graph_extended_comments.pkl",
   "gcn_embedding_dim": 32,
   "gradient_checkpointing": false,
   "hidden_act": "gelu",
@@ -33,7 +34,7 @@
   "pad_token_id": 0,
   "position_embedding_type": "absolute",
   "tf_threshold": 0.0,
-  "transformers_version": "4.30.2",
+  "transformers_version": "4.31.0",
   "type_vocab_size": 2,
   "use_cache": true,
   "vocab_size": 37788,

modeling_vcgn.py

@@ -1,4 +1,6 @@
+from typing import List, Union
 import torch
+import torch.nn.functional as F
 from transformers import PreTrainedModel, BertTokenizer
 from transformers.utils import is_remote_url, download_url
 from pathlib import Path
@@ -49,6 +51,9 @@ def get_torch_gcn(gcn_vocab_adj_tf, gcn_vocab_adj,gcn_config:VGCNConfig):
         adj = gcn_vocab_adj_list[i]
         adj = normalize_adj(adj)
         norm_gcn_vocab_adj_list.append(sparse_scipy2torch(adj.tocoo()))
+    
+    for t in norm_gcn_vocab_adj_list:
+        t.requires_grad = False
 
     del gcn_vocab_adj_list
 
@@ -66,7 +71,8 @@ class VCGNModelForTextClassification(PreTrainedModel):
         self.remove_stop_words = False
         self.tokenizer = None
         self.norm_gcn_vocab_adj_list = None
-    
+        self.gcn_vocab_size = config.vocab_size
+           
 
         self.load_adj_matrix(config.gcn_adj_matrix)
 
@@ -80,26 +86,97 @@ class VCGNModelForTextClassification(PreTrainedModel):
         )
 
     def load_adj_matrix(self, adj_matrix):
+        filename = None
         if Path(adj_matrix).is_file():
-            #load file
-            gcn_vocab_adj_tf, gcn_vocab_adj, adj_config = pkl.load(open(adj_matrix, 'rb'))
-        if is_remote_url(adj_matrix):
-            resolved_archive_file = download_url(adj_matrix)
+            filename = Path(adj_matrix)
+            #load file           
+        elif (Path(__file__).parent / Path(adj_matrix)).is_file():
+            filename = Path(__file__).parent / Path(adj_matrix)
+        elif is_remote_url(adj_matrix):
+            filename = download_url(adj_matrix)
+
+
+        gcn_vocab_adj_tf, gcn_vocab_adj, adj_config = pkl.load(open(filename, 'rb'))
+
 
         self.pre_trained_model_name = adj_config['bert_model']
         self.remove_stop_words = adj_config['remove_stop_words']
         self.tokenizer =  BertTokenizer.from_pretrained(self.pre_trained_model_name)
-        self.norm_gcn_vocab_adj_list = get_torch_gcn(gcn_vocab_adj_tf, gcn_vocab_adj, self.config)
-       
+        self.norm_gcn_vocab_adj_list =  get_torch_gcn(gcn_vocab_adj_tf, gcn_vocab_adj, self.config)
+
+    def _prep_batch(self, batch: torch.Tensor):
+
+        vocab_size = self.tokenizer.vocab_size
+
+        batch_gcn_swop_eye = F.one_hot(batch, vocab_size).float().to(self.device) # shape (batch_size, seq_len, vocab_size)
+        batch_gcn_swop_eye = batch_gcn_swop_eye.transpose(1,2) # shape (batch_size,  vocab_size, seq_len)
+                                                                                  # set all [PAD] tokens to 0
+        batch_gcn_swop_eye[:, self.tokenizer.pad_token_id, :] = 0
+        batch_gcn_swop_eye[:, self.tokenizer.cls_token_id, :] = 0
+        batch_gcn_swop_eye[:, self.tokenizer.sep_token_id, :] = 0
+
+        batch_gcn_swop_eye = F.pad(batch_gcn_swop_eye,(0,self.config.gcn_embedding_dim,0,0,0,0),value=0)
+
+        batch = F.pad(batch, (0, self.config.gcn_embedding_dim), 'constant', 0)
+        
+        #fill gcn tokens with [SEP]
+        mask = torch.zeros(batch.shape[0], batch.shape[1] + 1, dtype=batch.dtype, device=self.device)
+        mask2 = torch.zeros(batch.shape[0], batch.shape[1] + 1, dtype=batch.dtype, device=self.device)
+
+        pos_start = (batch==self.tokenizer.pad_token_id).int().argmax(1)
+
+        mask[(torch.arange(batch.shape[0]), pos_start)] = 1
+        mask2[(torch.arange(batch.shape[0]), pos_start+self.config.gcn_embedding_dim)] = 1
+
+        mask = mask.cumsum(1)[:, :-1].bool()
+        mask2 = mask2.cumsum(1)[:, :-1].bool()
 
-    def forward(self, tensor, labels=None):
-        logits = self.model(tensor)
+        mask = mask & ~mask2
+
+        batch.masked_fill_(mask, self.tokenizer.sep_token_id)
+
+        return batch, batch_gcn_swop_eye
+    
+    def text_to_batch(self, text: Union[List[str], str]):
+        if isinstance(text, str):
+            text = [text]
+        encoded = self.tokenizer.batch_encode_plus(text, padding=True, truncation=True, return_tensors='pt', max_length=self.config.max_seq_len-self.config.gcn_embedding_dim)
+        return encoded['input_ids'].to(self.device)
+
+    def forward(self, input:Union[torch.Tensor, List[str], str], labels=None):
+
+        if not isinstance(input, torch.Tensor):
+            input = self.text_to_batch(input)
+        
+        input, batch_gcn_swop_eye = self._prep_batch(input)
+
+        segment_ids = torch.zeros_like(input).int().to(self.device)
+        input_mask = (input>0).int().to(self.device)
+
+
+        logits = self.model(batch_gcn_swop_eye, input, segment_ids, input_mask )
         if labels is not None:
             loss = torch.nn.cross_entropy(logits, labels)
             return {"loss": loss, "logits": logits}
         return {"logits": logits}
 
-
+    def predict(self, text: Union[List[str], str], as_dict=True):
+        with torch.no_grad():
+            logits = self.forward(text)['logits']
+            if as_dict:
+                label_id = torch.argmax(logits, dim=1).cpu().numpy()
+                label = [self.config.id2label[l] for l in label_id]
+                return {
+                    "logits": logits,
+                    "label_id": label_id,
+                    "label": label,
+                    }
+            else:
+                return torch.argmax(logits, dim=1).cpu().numpy()
+        
+    @property
+    def device(self):
+        return next(self.parameters()).device
 
 import torch
 import torch.nn as nn
@@ -130,7 +207,13 @@ class VocabGraphConvolution(nn.Module):
     """
     def __init__(self,adj_matrix,voc_dim, num_adj, hid_dim, out_dim, dropout_rate=0.2):
         super(VocabGraphConvolution, self).__init__()
-        self.adj_matrix=adj_matrix
+        if type(adj_matrix) is not list:
+            self.adj_matrix=adj_matrix
+        else:
+            self.adj_matrix=torch.nn.ParameterList([torch.nn.Parameter(x) for x in adj_matrix])
+            for p in self.adj_matrix:
+                p.requires_grad=False
+
         self.voc_dim=voc_dim
         self.num_adj=num_adj
         self.hid_dim=hid_dim
@@ -147,7 +230,7 @@ class VocabGraphConvolution(nn.Module):
 
     def reset_parameters(self):
         for n,p in self.named_parameters():
-            if n.startswith('W') or n.startswith('a') or n in ('W','a','dense'):
+            if n.startswith('W') :
                 init.kaiming_uniform_(p, a=math.sqrt(5))
 
     def forward(self, X_dv, add_linear_mapping_term=False):

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5b23581bedc6271217c0910a5676cfbb76a36b8b707a8f8f4171986cc6e5d8dd
-size 479695719
+oid sha256:2dd4760540bf1667e77b45ab271e0a87376a97ecb0ea7ab669391e45a5606820
+size 481615461