sanity check

tasks/audio.py

@@ -132,11 +132,11 @@ async def evaluate_audio(request: AudioEvaluationRequest):
     
     return results
 
-if __name__ == "__main__":
-    sample_request = AudioEvaluationRequest(
-        dataset_name="rfcx/frugalai",  # Replace with actual dataset name
-        test_size=0.2,  # Example values
-        test_seed=42
-    )
-#
-    asyncio.run(evaluate_audio(sample_request))
+# if __name__ == "__main__":
+#     sample_request = AudioEvaluationRequest(
+#         dataset_name="rfcx/frugalai",  # Replace with actual dataset name
+#         test_size=0.2,  # Example values
+#         test_seed=42
+#     )
+# #
+#     asyncio.run(evaluate_audio(sample_request))

tasks/models/frugal_2025-02-01/frugal_kan_features_2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3fbc9f7a73a40a99863fbbf70e244598d2594e451f01737812b553e354541c2
+size 614523

tasks/run.py

@@ -2,7 +2,7 @@ from torch.utils.data import DataLoader
 from .utils.data import FFTDataset, SplitDataset
 from datasets import load_dataset
 from .utils.train import Trainer, XGBoostTrainer
-from .utils.models import CNNKan, KanEncoder, CNNKanFeaturesEncoder
+from .utils.models import CNNKan, KanEncoder, CNNKanFeaturesEncoder, CNNFeaturesEncoder
 from .utils.data_utils import *
 from huggingface_hub import login
 import yaml
@@ -13,6 +13,42 @@ import pandas as pd
 import seaborn as sns
 import matplotlib.pyplot as plt
 from collections import OrderedDict
+import xgboost as xgb
+from tqdm import tqdm
+from sklearn.metrics import accuracy_score, classification_report, roc_auc_score
+from sklearn.model_selection import train_test_split
+import warnings
+
+warnings.filterwarnings("ignore")
+
+
+
+def create_dataframe(ds, save_name='train'):
+    try:
+        df = pd.read_csv(f"tasks/utils/dfs/{save_name}.csv")
+    except FileNotFoundError:
+        data = []
+
+        # Iterate over the dataset
+        pbar = tqdm(enumerate(ds))
+        for i, batch in pbar:
+            label = batch['label']
+            features = batch['audio']['features']
+
+            # Flatten the nested dictionary structure
+            feature_dict = {'label': label}
+            for k, v in features.items():
+                if isinstance(v, dict):
+                    for sub_k, sub_v in v.items():
+                        feature_dict[f"{k}_{sub_k}"] = sub_v[0].item()  # Aggregate (e.g., mean)
+            data.append(feature_dict)
+        # Convert to DataFrame
+        df = pd.DataFrame(data)
+        print(os.getcwd())
+        df.to_csv(f"tasks/utils/dfs/{save_name}.csv", index=False)
+    X = df.drop(columns=['label'])
+    y = df['label']
+    return X, y
 
 # local_rank = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 current_date = datetime.date.today().strftime("%Y-%m-%d")
@@ -37,18 +73,62 @@ with open("../logs//token.txt", "r") as f:
 local_rank = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 login(api_key)
 dataset = load_dataset("rfcx/frugalai", streaming=True)
+full_ds = FFTDataset(dataset["train"], features=True)
 
-train_ds = SplitDataset(FFTDataset(dataset["train"]), is_train=True)
+train_ds = SplitDataset(FFTDataset(dataset["train"], features=True), is_train=True)
   
 train_dl = DataLoader(train_ds, batch_size=data_args.batch_size, collate_fn=collate_fn)
 
-val_ds = SplitDataset(FFTDataset(dataset["train"]), is_train=False)
+val_ds = SplitDataset(FFTDataset(dataset["train"], features=True), is_train=False)
     
 val_dl = DataLoader(val_ds,batch_size=data_args.batch_size, collate_fn=collate_fn)
 
-test_ds = FFTDataset(dataset["test"])
+test_ds = FFTDataset(dataset["test"], features=True)
 test_dl = DataLoader(test_ds,batch_size=data_args.batch_size, collate_fn=collate_fn)
 
+
+x,y = create_dataframe(full_ds, save_name='train_val')
+print(x.shape)
+x_train, x_val, y_train, y_val = train_test_split(x, y, test_size=0.2, random_state=42)
+
+evals_result = {}
+num_boost_round = 1000  # Set a large number of boosting rounds
+
+# Watchlist to monitor performance on train and validation data
+
+dtrain = xgb.DMatrix(x_train, label=y_train)
+dval = xgb.DMatrix(x_val, label=y_val)
+watchlist = [(dtrain, 'train'), (dval, 'eval')]
+params = {
+            'objective': 'binary:logistic',
+            'eval_metric': 'logloss',
+            **boost_args.get_dict()
+        }
+# Train the model
+xgb_model = xgb.train(
+    params,
+    dtrain,
+    num_boost_round=num_boost_round,
+    evals=watchlist,
+    early_stopping_rounds=10,  # Early stopping after 10 rounds with no improvement
+    evals_result=evals_result,
+    verbose_eval=False  # Show evaluation results for each iteration
+)
+
+xgb_pred = xgb_model.predict(dval, output_margin=False)  # Take probability of class 1
+# xgb_pred = torch.tensor(xgb_pred, dtype=torch.float32, device=x.device).unsqueeze(1)
+y_pred = (xgb_pred >= 0.5).astype(int)
+
+# Get the number of trees in the trained model
+
+accuracy = accuracy_score(y_val, y_pred)
+roc_auc = roc_auc_score(y_val, y_pred)
+
+print(f'Accuracy: {accuracy:.4f}')
+print(f'ROC AUC Score: {roc_auc:.4f}')
+num_xgb_features = xgb_model.best_iteration + 1
+print(num_xgb_features)
+
 # data = []
 #
 # # Iterate over the dataset
@@ -92,7 +172,8 @@ test_dl = DataLoader(test_ds,batch_size=data_args.batch_size, collate_fn=collate
 # model = DualEncoder(model_args, model_args_f, conformer_args)
 # model = FasterKAN([18000,64,64,16,1])
 # model = CNNKan(model_args, conformer_args, kan_args.get_dict())
-model = CNNKanFeaturesEncoder(model_args, mlp_args, kan_args.get_dict())
+# model = CNNKanFeaturesEncoder(xgb_model, model_args, kan_args.get_dict())
+model = CNNFeaturesEncoder(xgb_model,model_args)
 # model.kan.speed()
 # model = KanEncoder(kan_args.get_dict())
 model = model.to(local_rank)

tasks/utils/data.py

@@ -52,11 +52,16 @@ class SplitDataset(IterableDataset):
 
 
 class FFTDataset(IterableDataset):
-    def __init__(self, original_dataset, max_len=72000, orig_sample_rate=12000, target_sample_rate=3000):
+    def __init__(self, original_dataset,
+                 max_len=72000,
+                 orig_sample_rate=12000,
+                 target_sample_rate=3000,
+                 features=False):
         self.dataset = original_dataset
         self.resampler = T.Resample(orig_freq=orig_sample_rate, new_freq=target_sample_rate)
         self.target_sample_rate = target_sample_rate
         self.max_len = max_len
+        self.features = features
 
 
     def normalize_audio(self, audio):
@@ -89,8 +94,10 @@ class FFTDataset(IterableDataset):
             fft_data = fft(audio_data)
             magnitude = torch.abs(fft_data)
             phase = torch.angle(fft_data)
-            features = compute_all_features(audio_data, sample_rate=self.target_sample_rate)
-            features_arr = torch.tensor([v for _, v in features['frequency_domain'].items()])
+            if self.features:
+                features = compute_all_features(audio_data, sample_rate=self.target_sample_rate)
+                # features_arr = torch.tensor([v for _, v in features['frequency_domain'].items()])
+                item['audio']['features'] = features
             magnitude_centered = fftshift(magnitude)
             phase_centered = fftshift(phase)
             # cwt = features['cwt_power']
@@ -103,7 +110,6 @@ class FFTDataset(IterableDataset):
             # item['audio']['cwt_mag'] = torch.nan_to_num(cwt, 0)
             item['audio']['array'] = torch.nan_to_num(audio_data, 0)
             # item['audio']['features'] = features
-            item['audio']['features_arr'] = torch.nan_to_num(features_arr, 0)
             yield item
 
 

tasks/utils/data_utils.py

@@ -5,10 +5,10 @@ from torch.nn.utils.rnn import pad_sequence
 
 def collate_fn(batch):
     # Extract audio arrays and FFT data from the batch of dictionaries
-    audio_arrays = [torch.tensor(item['audio']['array']) for item in batch]
-    fft_arrays = [torch.tensor(item['audio']['fft_mag']) for item in batch]
+    audio_arrays = [item['audio']['array'] for item in batch]
+    fft_arrays = [item['audio']['fft_mag'] for item in batch]
     # cwt_arrays = [torch.tensor(item['audio']['cwt_mag']) for item in batch]
-    # features = [item['audio']['features'] for item in batch]
+    features = [item['audio']['features'] for item in batch]
     # features_arr = torch.stack([item['audio']['features_arr'] for item in batch])
     labels = [torch.tensor(item['label']) for item in batch]
     
@@ -22,7 +22,7 @@ def collate_fn(batch):
         'audio': {
             'array': padded_audio,
             'fft_mag': padded_fft,
-            # 'features': features,
+            'features': features,
             # 'features_arr': features_arr,
             # 'cwt_mag': padded_cwt,
         },

tasks/utils/models.py

@@ -3,6 +3,10 @@ import torch.nn as nn
 from .Modules.conformer import ConformerEncoder, ConformerDecoder
 from .Modules.mhsa_pro import RotaryEmbedding, ContinuousRotaryEmbedding
 from .kan.fasterkan import FasterKAN
+import numpy as np
+import xgboost as xgb
+import pandas as pd
+
 
 
 class Sine(nn.Module):
@@ -161,6 +165,46 @@ class DualEncoder(nn.Module):
         logits = torch.cat([x1, x2], dim=-1)
         return self.regressor(logits).squeeze()
 
+class CNNFeaturesEncoder(nn.Module):
+    def __init__(self, xgb_model, args, mlp_hidden=64):
+        super().__init__()
+        self.xgb_model = xgb_model
+        self.best_xgb_features = xgb_model.best_iteration + 1
+        self.backbone = CNNEncoder(args)
+        self.total_features = self.best_xgb_features + args.encoder_dims[-1]
+        self.mlp = nn.Sequential(
+            nn.Linear(self.total_features, mlp_hidden),
+            nn.BatchNorm1d(mlp_hidden),
+            nn.SiLU(),
+            nn.Linear(mlp_hidden, mlp_hidden),
+            nn.BatchNorm1d(mlp_hidden),
+            nn.SiLU(),
+            nn.Linear(mlp_hidden, 1),
+        )
+
+    def _create_features_data(self, features):
+        # Handle batch processing
+        batch_size = len(features)
+        data = []
+
+        # Iterate through each item in the batch
+        for batch_idx in range(batch_size):
+            feature_dict = {}
+            for k, v in features[batch_idx].items():
+                feature_dict[f"frequency_domain_{k}"] = v[0].item()
+            data.append(feature_dict)
+
+        return pd.DataFrame(data)
+    def forward(self, x: torch.Tensor, f) -> torch.Tensor:
+        x = self.backbone(x)
+        x = x.mean(dim=-1)
+        f_np = self._create_features_data(f)
+        dtest = xgb.DMatrix(f_np)  # Convert input to DMatrix
+        xgb_features = self.xgb_model.predict(dtest, pred_leaf=True).astype(np.float32)
+        xgb_features = torch.tensor(xgb_features, dtype=torch.float32, device=x.device)
+        x_f = torch.cat([x, xgb_features[:, :self.best_xgb_features]], dim=1)
+        return self.mlp(x_f)
+
 class CNNKan(nn.Module):
     def __init__(self, args, conformer_args, kan_args):
         super().__init__()
@@ -173,18 +217,35 @@ class CNNKan(nn.Module):
         return self.kan(x)
 
 class CNNKanFeaturesEncoder(nn.Module):
-    def __init__(self, args, mlp_args, kan_args):
+    def __init__(self, xgb_model, args,  kan_args):
         super().__init__()
+        self.xgb_model = xgb_model
+        self.best_xgb_features = xgb_model.best_iteration + 1
         self.backbone = CNNEncoder(args)
-        self.mlp = MLPEncoder(mlp_args)
-        kan_args['layers_hidden'][0] += self.mlp.output_dim
+        kan_args['layers_hidden'][0] += self.best_xgb_features
         self.kan = FasterKAN(**kan_args)
 
-    def forward(self, x: torch.Tensor, f: torch.Tensor) -> torch.Tensor:
+    def _create_features_data(self, features):
+        # Handle batch processing
+        batch_size = len(features)
+        data = []
+
+        # Iterate through each item in the batch
+        for batch_idx in range(batch_size):
+            feature_dict = {}
+            for k, v in features[batch_idx].items():
+                feature_dict[f"frequency_domain_{k}"] = v[0].item()
+            data.append(feature_dict)
+
+        return pd.DataFrame(data)
+    def forward(self, x: torch.Tensor, f) -> torch.Tensor:
         x = self.backbone(x)
         x = x.mean(dim=1)
-        f = self.mlp(f)
-        x_f = torch.cat([x, f], dim=-1)
+        f_np = self._create_features_data(f)
+        dtest = xgb.DMatrix(f_np)  # Convert input to DMatrix
+        xgb_features = self.xgb_model.predict(dtest, pred_leaf=True).astype(np.float32)
+        xgb_features = torch.tensor(xgb_features, dtype=torch.float32, device=x.device)
+        x_f = torch.cat([x, xgb_features[:, :self.best_xgb_features]], dim=1)
         return self.kan(x_f)
 
 class KanEncoder(nn.Module):

tasks/utils/train.py

@@ -226,14 +226,14 @@ class Trainer(object):
     
     def train_batch(self, batch, batch_idx, device):
         x, fft, y = batch['audio']['array'], batch['audio']['fft_mag'], batch['label']
-        # features = batch['audio']['features_arr'].to(device).float()
+        features = batch['audio']['features']
         # cwt = batch['audio']['cwt_mag']
         x = x.to(device).float()
         fft = fft.to(device).float()
         # cwt = cwt.to(device).float()
         y = y.to(device).float()
         x_fft = torch.cat((x.unsqueeze(dim=1), fft.unsqueeze(dim=1)), dim=1)
-        y_pred = self.model(x_fft).squeeze()
+        y_pred = self.model(x_fft, features).squeeze()
         loss = self.criterion(y_pred, y)
         loss.backward()
         self.optimizer.step()
@@ -267,13 +267,15 @@ class Trainer(object):
 
     def eval_batch(self, batch, batch_idx, device):
         x, fft, y = batch['audio']['array'], batch['audio']['fft_mag'], batch['label']
+        features = batch['audio']['features']
+
         # features = batch['audio']['features_arr'].to(device).float()
         x = x.to(device).float()
         fft = fft.to(device).float()
         x_fft = torch.cat((x.unsqueeze(dim=1), fft.unsqueeze(dim=1)), dim=1)
         y = y.to(device).float()
         with torch.no_grad():
-            y_pred = self.model(x_fft).squeeze()
+            y_pred = self.model(x_fft, features).squeeze()
         loss = self.criterion(y_pred.squeeze(), y)
         probs = torch.sigmoid(y_pred)
         cls_pred = (probs > 0.5).float()

tasks/utils/transforms.py

@@ -156,7 +156,6 @@ def compute_time_domain_features(audio, sample_rate, frame_length=2048, hop_leng
 
     return features
 
-
 def compute_frequency_domain_features(audio, sample_rate, n_fft=2048, hop_length=512):
     """
     Compute frequency-domain features from audio signal.
@@ -175,7 +174,6 @@ def compute_frequency_domain_features(audio, sample_rate, n_fft=2048, hop_length
             sr=sample_rate,
             n_fft=n_fft,
             hop_length=hop_length,
-
         )
         features['spectral_centroid'] = torch.FloatTensor([spectral_centroids.max()])
     except Exception as e:
@@ -188,7 +186,6 @@ def compute_frequency_domain_features(audio, sample_rate, n_fft=2048, hop_length
             sr=sample_rate,
             n_fft=n_fft,
             hop_length=hop_length,
-
         )
         features['spectral_rolloff'] = torch.FloatTensor([spectral_rolloff.max()])
     except Exception as e:
@@ -205,6 +202,7 @@ def compute_frequency_domain_features(audio, sample_rate, n_fft=2048, hop_length
         features['spectral_bandwidth'] = torch.FloatTensor([spectral_bandwidth.max()])
     except Exception as e:
         features['spectral_bandwidth'] = torch.FloatTensor([np.nan])
+
     # 4. Spectral Contrast
     try:
         spectral_contrast = librosa.feature.spectral_contrast(
@@ -240,6 +238,77 @@ def compute_frequency_domain_features(audio, sample_rate, n_fft=2048, hop_length
     except Exception as e:
         features['spectral_flux'] = torch.FloatTensor([np.nan])
 
+    # 7. MFCCs (Mel-Frequency Cepstral Coefficients)
+    try:
+        mfccs = librosa.feature.mfcc(
+            y=audio_np,
+            sr=sample_rate,
+            n_mfcc=13,  # Number of MFCCs to compute
+            n_fft=n_fft,
+            hop_length=hop_length
+        )
+        features['mfcc_mean'] = torch.FloatTensor([mfccs.mean()])
+    except Exception as e:
+        features['mfcc_mean'] = torch.FloatTensor([np.nan])
+
+    # 8. Chroma Features
+    try:
+        chroma = librosa.feature.chroma_stft(
+            y=audio_np,
+            sr=sample_rate,
+            n_fft=n_fft,
+            hop_length=hop_length
+        )
+        features['chroma_mean'] = torch.FloatTensor([chroma.mean()])
+    except Exception as e:
+        features['chroma_mean'] = torch.FloatTensor([np.nan])
+
+    # 9. Spectral Kurtosis
+    try:
+        spectral_kurtosis = librosa.feature.spectral_kurtosis(
+            y=audio_np,
+            sr=sample_rate,
+            n_fft=n_fft,
+            hop_length=hop_length
+        )
+        features['spectral_kurtosis'] = torch.FloatTensor([spectral_kurtosis.mean()])
+    except Exception as e:
+        features['spectral_kurtosis'] = torch.FloatTensor([np.nan])
+
+    # 10. Spectral Skewness
+    try:
+        spectral_skewness = librosa.feature.spectral_skewness(
+            y=audio_np,
+            sr=sample_rate,
+            n_fft=n_fft,
+            hop_length=hop_length
+        )
+        features['spectral_skewness'] = torch.FloatTensor([spectral_skewness.mean()])
+    except Exception as e:
+        features['spectral_skewness'] = torch.FloatTensor([np.nan])
+
+    # 11. Spectral Slope
+    try:
+        spectral_slope = librosa.feature.spectral_slope(
+            y=audio_np,
+            sr=sample_rate,
+            n_fft=n_fft,
+            hop_length=hop_length
+        )
+        features['spectral_slope'] = torch.FloatTensor([spectral_slope.mean()])
+    except Exception as e:
+        features['spectral_slope'] = torch.FloatTensor([np.nan])
+
+    # 12. Tonnetz (Tonal Centroid Features)
+    try:
+        tonnetz = librosa.feature.tonnetz(
+            y=audio_np,
+            sr=sample_rate
+        )
+        features['tonnetz_mean'] = torch.FloatTensor([tonnetz.mean()])
+    except Exception as e:
+        features['tonnetz_mean'] = torch.FloatTensor([np.nan])
+
     return features
 
 
@@ -267,6 +336,4 @@ def compute_all_features(audio, sample_rate, wavelet='db1', decompos_level=4):
     # features['time_domain'] = compute_time_domain_features(audio, sample_rate)
 
     # Frequency domain features
-    features['frequency_domain'] = compute_frequency_domain_features(audio, sample_rate)
-
-    return features
+    return compute_frequency_domain_features(audio, sample_rate)