import os
import pandas as pd
import torch
import torch.nn as nn
from torchvision import datasets,models,transforms
from torch.utils.tensorboard import SummaryWriter
import time, os, copy

image_transforms = { 
    'train': transforms.Compose([
        transforms.RandomResizedCrop(size=256, scale=(0.8, 1.0)),
        transforms.RandomRotation(degrees=15),
        transforms.CenterCrop(size=224),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406],
                             [0.229, 0.224, 0.225])
    ]),
    'valid': transforms.Compose([
        transforms.Resize(size=256),
        transforms.CenterCrop(size=224),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406],
                             [0.229, 0.224, 0.225])
    ])
}

train_path='/home/abhinav/kaggle/intel_image_classification/input/train/'
val_path='/home/abhinav/kaggle/intel_image_classification/input/val/'

saving_path={
    'mobile_net':"/home/abhinav/kaggle/intel_image_classification/models/mobilenetv2.pth",
    'resnet50':"/home/abhinav/kaggle/intel_image_classification/models/resnet50.pth",
    'SpinalNet_ResNet':"/home/abhinav/kaggle/intel_image_classification/models/SpinalNet_ResNet.pth"
}



dataset = {
    'train': datasets.ImageFolder(root=train_path, transform=image_transforms['train']),
    'valid': datasets.ImageFolder(root=val_path, transform=image_transforms['valid'])
}
 
dataset_sizes = {
    'train':len(dataset['train']),
    'valid':len(dataset['valid'])
}

dataloaders = {
    'train':torch.utils.data.DataLoader(dataset['train'], batch_size=32, shuffle=True),
    'valid':torch.utils.data.DataLoader(dataset['valid'], batch_size=32, shuffle=True,)
}

class_names = dataset['train'].classes
print("Classes:", class_names)
 
print("Training-set size:",dataset_sizes['train'],
      "\nValidation-set size:", dataset_sizes['valid'])

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

print(device," is available")

# print("\nLoading mobilenetv2 ...\n")
# model_ft = models.mobilenet_v2(pretrained=True)    
# for params in list(model_ft.parameters())[0:-5]:
#     params.requires_grad = False

# model_ft = models.resnet50(pretrained=True)    
# for params in list(model_ft.parameters())[0:-5]:
#     params.requires_grad = False

# # num_ftrs=model_ft.classifier[-1].in_features
# num_ftrs = model_ft.fc.in_features
# model_ft.fc=nn.Sequential(
#     nn.Dropout(p=0.2, inplace=False),
#     nn.Linear(in_features=num_ftrs, out_features=6, bias=True)
#     )    
# model_ft = model_ft.to(device)


model_ft = models.resnet50(pretrained=True)
num_ftrs = model_ft.fc.in_features

half_in_size = round(num_ftrs/2)
layer_width = 12 #Small for Resnet, large for VGG
Num_class=6

class SpinalNet_ResNet(nn.Module):
    def __init__(self):
        super(SpinalNet_ResNet, self).__init__()
        
        self.fc_spinal_layer1 = nn.Sequential(
            #nn.Dropout(p = 0.5), 
            nn.Linear(half_in_size, layer_width),
            #nn.BatchNorm1d(layer_width), 
            nn.ReLU(inplace=True),)
        self.fc_spinal_layer2 = nn.Sequential(
            #nn.Dropout(p = 0.5), 
            nn.Linear(half_in_size+layer_width, layer_width),
            #nn.BatchNorm1d(layer_width), 
            nn.ReLU(inplace=True),)
        self.fc_spinal_layer3 = nn.Sequential(
            #nn.Dropout(p = 0.5), 
            nn.Linear(half_in_size+layer_width, layer_width),
            #nn.BatchNorm1d(layer_width), 
            nn.ReLU(inplace=True),)
        self.fc_spinal_layer4 = nn.Sequential(
            #nn.Dropout(p = 0.5), 
            nn.Linear(half_in_size+layer_width, layer_width),
            #nn.BatchNorm1d(layer_width), 
            nn.ReLU(inplace=True),)
        self.fc_out = nn.Sequential(
            #nn.Dropout(p = 0.5), 
            nn.Linear(layer_width*4, Num_class),)
        
    def forward(self, x):
        x1 = self.fc_spinal_layer1(x[:, 0:half_in_size])
        x2 = self.fc_spinal_layer2(torch.cat([ x[:,half_in_size:2*half_in_size], x1], dim=1))
        x3 = self.fc_spinal_layer3(torch.cat([ x[:,0:half_in_size], x2], dim=1))
        x4 = self.fc_spinal_layer4(torch.cat([ x[:,half_in_size:2*half_in_size], x3], dim=1))
        
        
        x = torch.cat([x1, x2], dim=1)
        x = torch.cat([x, x3], dim=1)
        x = torch.cat([x, x4], dim=1)

        
        x = self.fc_out(x)
        return x

model_ft.fc=SpinalNet_ResNet()
model_ft.to(device)


# print('Model Summary:-\n')
for num, (name, param) in enumerate(model_ft.named_parameters()):
    print(num, name, param.requires_grad )
# summary(model_ft, input_size=(3, 224, 224))
print(model_ft)

criterion = nn.CrossEntropyLoss()

optimizer_ft = torch.optim.SGD(model_ft.parameters(), lr=0.001, momentum=0.9)

exp_lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer_ft, step_size=7, gamma=0.1)

print("\nTraining:-\n")
def train_model(model, criterion, optimizer, scheduler, num_epochs=30):
    since = time.time()

    best_model_wts = copy.deepcopy(model.state_dict())
    best_acc = 0.0

    # Tensorboard summary
    writer = SummaryWriter()
    
    for epoch in range(num_epochs):
        print('Epoch {}/{}'.format(epoch, num_epochs - 1))
        print('-' * 10)

        # Each epoch has a training and validation phase
        for phase in ['train', 'valid']:
            if phase == 'train':
                model.train()  # Set model to training mode
            else:
                model.eval()   # Set model to evaluate mode

            running_loss = 0.0
            running_corrects = 0

            # Iterate over data.
            for inputs, labels in dataloaders[phase]:
                inputs = inputs.to(device, non_blocking=True)
                labels = labels.to(device, non_blocking=True)

                # zero the parameter gradients
                optimizer.zero_grad()

                # forward
                # track history if only in train
                with torch.set_grad_enabled(phase == 'train'):
                    outputs = model(inputs)
                    _, preds = torch.max(outputs, 1)
                    loss = criterion(outputs, labels)

                    # backward + optimize only if in training phase
                    if phase == 'train':
                        loss.backward()
                        optimizer.step()

                # statistics
                running_loss += loss.item() * inputs.size(0)
                running_corrects += torch.sum(preds == labels.data)
            if phase == 'train':
                scheduler.step()

            epoch_loss = running_loss / dataset_sizes[phase]
            epoch_acc = running_corrects.double() / dataset_sizes[phase]

            print('{} Loss: {:.4f} Acc: {:.4f}'.format(
                phase, epoch_loss, epoch_acc))

            # Record training loss and accuracy for each phase
            if phase == 'train':
                writer.add_scalar('Train/Loss', epoch_loss, epoch)
                writer.add_scalar('Train/Accuracy', epoch_acc, epoch)
                writer.flush()
            else:
                writer.add_scalar('Valid/Loss', epoch_loss, epoch)
                writer.add_scalar('Valid/Accuracy', epoch_acc, epoch)
                writer.flush()

            # deep copy the model
            if phase == 'valid' and epoch_acc > best_acc:
                best_acc = epoch_acc
                best_model_wts = copy.deepcopy(model.state_dict())

        print()

    time_elapsed = time.time() - since
    print('Training complete in {:.0f}m {:.0f}s'.format(
        time_elapsed // 60, time_elapsed % 60))
    print('Best val Acc: {:4f}'.format(best_acc))

    # load best model weights
    model.load_state_dict(best_model_wts)
    return model


model_ft = train_model(model_ft, criterion, optimizer_ft, exp_lr_scheduler,
                       num_epochs=10)
# Save the entire model
print("\nSaving the model...")
torch.save(model_ft, saving_path['SpinalNet_ResNet'])

976d468cd39493e7410ca9ce30e217b1345b6a25
<mask token>


class SpinalNet_ResNet(nn.Module):

    def __init__(self):
        super(SpinalNet_ResNet, self).__init__()
        self.fc_spinal_layer1 = nn.Sequential(nn.Linear(half_in_size,
            layer_width), nn.ReLU(inplace=True))
        self.fc_spinal_layer2 = nn.Sequential(nn.Linear(half_in_size +
            layer_width, layer_width), nn.ReLU(inplace=True))
        self.fc_spinal_layer3 = nn.Sequential(nn.Linear(half_in_size +
            layer_width, layer_width), nn.ReLU(inplace=True))
        self.fc_spinal_layer4 = nn.Sequential(nn.Linear(half_in_size +
            layer_width, layer_width), nn.ReLU(inplace=True))
        self.fc_out = nn.Sequential(nn.Linear(layer_width * 4, Num_class))

    def forward(self, x):
        x1 = self.fc_spinal_layer1(x[:, 0:half_in_size])
        x2 = self.fc_spinal_layer2(torch.cat([x[:, half_in_size:2 *
            half_in_size], x1], dim=1))
        x3 = self.fc_spinal_layer3(torch.cat([x[:, 0:half_in_size], x2], dim=1)
            )
        x4 = self.fc_spinal_layer4(torch.cat([x[:, half_in_size:2 *
            half_in_size], x3], dim=1))
        x = torch.cat([x1, x2], dim=1)
        x = torch.cat([x, x3], dim=1)
        x = torch.cat([x, x4], dim=1)
        x = self.fc_out(x)
        return x


<mask token>


def train_model(model, criterion, optimizer, scheduler, num_epochs=30):
    since = time.time()
    best_model_wts = copy.deepcopy(model.state_dict())
    best_acc = 0.0
    writer = SummaryWriter()
    for epoch in range(num_epochs):
        print('Epoch {}/{}'.format(epoch, num_epochs - 1))
        print('-' * 10)
        for phase in ['train', 'valid']:
            if phase == 'train':
                model.train()
            else:
                model.eval()
            running_loss = 0.0
            running_corrects = 0
            for inputs, labels in dataloaders[phase]:
                inputs = inputs.to(device, non_blocking=True)
                labels = labels.to(device, non_blocking=True)
                optimizer.zero_grad()
                with torch.set_grad_enabled(phase == 'train'):
                    outputs = model(inputs)
                    _, preds = torch.max(outputs, 1)
                    loss = criterion(outputs, labels)
                    if phase == 'train':
                        loss.backward()
                        optimizer.step()
                running_loss += loss.item() * inputs.size(0)
                running_corrects += torch.sum(preds == labels.data)
            if phase == 'train':
                scheduler.step()
            epoch_loss = running_loss / dataset_sizes[phase]
            epoch_acc = running_corrects.double() / dataset_sizes[phase]
            print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss,
                epoch_acc))
            if phase == 'train':
                writer.add_scalar('Train/Loss', epoch_loss, epoch)
                writer.add_scalar('Train/Accuracy', epoch_acc, epoch)
                writer.flush()
            else:
                writer.add_scalar('Valid/Loss', epoch_loss, epoch)
                writer.add_scalar('Valid/Accuracy', epoch_acc, epoch)
                writer.flush()
            if phase == 'valid' and epoch_acc > best_acc:
                best_acc = epoch_acc
                best_model_wts = copy.deepcopy(model.state_dict())
        print()
    time_elapsed = time.time() - since
    print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60,
        time_elapsed % 60))
    print('Best val Acc: {:4f}'.format(best_acc))
    model.load_state_dict(best_model_wts)
    return model


<mask token>

<mask token>
print('Classes:', class_names)
print('Training-set size:', dataset_sizes['train'],
    """
Validation-set size:""", dataset_sizes['valid'])
<mask token>
print(device, ' is available')
<mask token>


class SpinalNet_ResNet(nn.Module):

    def __init__(self):
        super(SpinalNet_ResNet, self).__init__()
        self.fc_spinal_layer1 = nn.Sequential(nn.Linear(half_in_size,
            layer_width), nn.ReLU(inplace=True))
        self.fc_spinal_layer2 = nn.Sequential(nn.Linear(half_in_size +
            layer_width, layer_width), nn.ReLU(inplace=True))
        self.fc_spinal_layer3 = nn.Sequential(nn.Linear(half_in_size +
            layer_width, layer_width), nn.ReLU(inplace=True))
        self.fc_spinal_layer4 = nn.Sequential(nn.Linear(half_in_size +
            layer_width, layer_width), nn.ReLU(inplace=True))
        self.fc_out = nn.Sequential(nn.Linear(layer_width * 4, Num_class))

    def forward(self, x):
        x1 = self.fc_spinal_layer1(x[:, 0:half_in_size])
        x2 = self.fc_spinal_layer2(torch.cat([x[:, half_in_size:2 *
            half_in_size], x1], dim=1))
        x3 = self.fc_spinal_layer3(torch.cat([x[:, 0:half_in_size], x2], dim=1)
            )
        x4 = self.fc_spinal_layer4(torch.cat([x[:, half_in_size:2 *
            half_in_size], x3], dim=1))
        x = torch.cat([x1, x2], dim=1)
        x = torch.cat([x, x3], dim=1)
        x = torch.cat([x, x4], dim=1)
        x = self.fc_out(x)
        return x


<mask token>
model_ft.to(device)
for num, (name, param) in enumerate(model_ft.named_parameters()):
    print(num, name, param.requires_grad)
print(model_ft)
<mask token>
print('\nTraining:-\n')


def train_model(model, criterion, optimizer, scheduler, num_epochs=30):
    since = time.time()
    best_model_wts = copy.deepcopy(model.state_dict())
    best_acc = 0.0
    writer = SummaryWriter()
    for epoch in range(num_epochs):
        print('Epoch {}/{}'.format(epoch, num_epochs - 1))
        print('-' * 10)
        for phase in ['train', 'valid']:
            if phase == 'train':
                model.train()
            else:
                model.eval()
            running_loss = 0.0
            running_corrects = 0
            for inputs, labels in dataloaders[phase]:
                inputs = inputs.to(device, non_blocking=True)
                labels = labels.to(device, non_blocking=True)
                optimizer.zero_grad()
                with torch.set_grad_enabled(phase == 'train'):
                    outputs = model(inputs)
                    _, preds = torch.max(outputs, 1)
                    loss = criterion(outputs, labels)
                    if phase == 'train':
                        loss.backward()
                        optimizer.step()
                running_loss += loss.item() * inputs.size(0)
                running_corrects += torch.sum(preds == labels.data)
            if phase == 'train':
                scheduler.step()
            epoch_loss = running_loss / dataset_sizes[phase]
            epoch_acc = running_corrects.double() / dataset_sizes[phase]
            print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss,
                epoch_acc))
            if phase == 'train':
                writer.add_scalar('Train/Loss', epoch_loss, epoch)
                writer.add_scalar('Train/Accuracy', epoch_acc, epoch)
                writer.flush()
            else:
                writer.add_scalar('Valid/Loss', epoch_loss, epoch)
                writer.add_scalar('Valid/Accuracy', epoch_acc, epoch)
                writer.flush()
            if phase == 'valid' and epoch_acc > best_acc:
                best_acc = epoch_acc
                best_model_wts = copy.deepcopy(model.state_dict())
        print()
    time_elapsed = time.time() - since
    print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60,
        time_elapsed % 60))
    print('Best val Acc: {:4f}'.format(best_acc))
    model.load_state_dict(best_model_wts)
    return model


<mask token>
print("""
Saving the model...""")
torch.save(model_ft, saving_path['SpinalNet_ResNet'])

<mask token>
image_transforms = {'train': transforms.Compose([transforms.
    RandomResizedCrop(size=256, scale=(0.8, 1.0)), transforms.
    RandomRotation(degrees=15), transforms.CenterCrop(size=224), transforms
    .ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224,
    0.225])]), 'valid': transforms.Compose([transforms.Resize(size=256),
    transforms.CenterCrop(size=224), transforms.ToTensor(), transforms.
    Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])}
train_path = '/home/abhinav/kaggle/intel_image_classification/input/train/'
val_path = '/home/abhinav/kaggle/intel_image_classification/input/val/'
saving_path = {'mobile_net':
    '/home/abhinav/kaggle/intel_image_classification/models/mobilenetv2.pth',
    'resnet50':
    '/home/abhinav/kaggle/intel_image_classification/models/resnet50.pth',
    'SpinalNet_ResNet':
    '/home/abhinav/kaggle/intel_image_classification/models/SpinalNet_ResNet.pth'
    }
dataset = {'train': datasets.ImageFolder(root=train_path, transform=
    image_transforms['train']), 'valid': datasets.ImageFolder(root=val_path,
    transform=image_transforms['valid'])}
dataset_sizes = {'train': len(dataset['train']), 'valid': len(dataset['valid'])
    }
dataloaders = {'train': torch.utils.data.DataLoader(dataset['train'],
    batch_size=32, shuffle=True), 'valid': torch.utils.data.DataLoader(
    dataset['valid'], batch_size=32, shuffle=True)}
class_names = dataset['train'].classes
print('Classes:', class_names)
print('Training-set size:', dataset_sizes['train'],
    """
Validation-set size:""", dataset_sizes['valid'])
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
print(device, ' is available')
model_ft = models.resnet50(pretrained=True)
num_ftrs = model_ft.fc.in_features
half_in_size = round(num_ftrs / 2)
layer_width = 12
Num_class = 6


class SpinalNet_ResNet(nn.Module):

    def __init__(self):
        super(SpinalNet_ResNet, self).__init__()
        self.fc_spinal_layer1 = nn.Sequential(nn.Linear(half_in_size,
            layer_width), nn.ReLU(inplace=True))
        self.fc_spinal_layer2 = nn.Sequential(nn.Linear(half_in_size +
            layer_width, layer_width), nn.ReLU(inplace=True))
        self.fc_spinal_layer3 = nn.Sequential(nn.Linear(half_in_size +
            layer_width, layer_width), nn.ReLU(inplace=True))
        self.fc_spinal_layer4 = nn.Sequential(nn.Linear(half_in_size +
            layer_width, layer_width), nn.ReLU(inplace=True))
        self.fc_out = nn.Sequential(nn.Linear(layer_width * 4, Num_class))

    def forward(self, x):
        x1 = self.fc_spinal_layer1(x[:, 0:half_in_size])
        x2 = self.fc_spinal_layer2(torch.cat([x[:, half_in_size:2 *
            half_in_size], x1], dim=1))
        x3 = self.fc_spinal_layer3(torch.cat([x[:, 0:half_in_size], x2], dim=1)
            )
        x4 = self.fc_spinal_layer4(torch.cat([x[:, half_in_size:2 *
            half_in_size], x3], dim=1))
        x = torch.cat([x1, x2], dim=1)
        x = torch.cat([x, x3], dim=1)
        x = torch.cat([x, x4], dim=1)
        x = self.fc_out(x)
        return x


model_ft.fc = SpinalNet_ResNet()
model_ft.to(device)
for num, (name, param) in enumerate(model_ft.named_parameters()):
    print(num, name, param.requires_grad)
print(model_ft)
criterion = nn.CrossEntropyLoss()
optimizer_ft = torch.optim.SGD(model_ft.parameters(), lr=0.001, momentum=0.9)
exp_lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer_ft, step_size=
    7, gamma=0.1)
print('\nTraining:-\n')


def train_model(model, criterion, optimizer, scheduler, num_epochs=30):
    since = time.time()
    best_model_wts = copy.deepcopy(model.state_dict())
    best_acc = 0.0
    writer = SummaryWriter()
    for epoch in range(num_epochs):
        print('Epoch {}/{}'.format(epoch, num_epochs - 1))
        print('-' * 10)
        for phase in ['train', 'valid']:
            if phase == 'train':
                model.train()
            else:
                model.eval()
            running_loss = 0.0
            running_corrects = 0
            for inputs, labels in dataloaders[phase]:
                inputs = inputs.to(device, non_blocking=True)
                labels = labels.to(device, non_blocking=True)
                optimizer.zero_grad()
                with torch.set_grad_enabled(phase == 'train'):
                    outputs = model(inputs)
                    _, preds = torch.max(outputs, 1)
                    loss = criterion(outputs, labels)
                    if phase == 'train':
                        loss.backward()
                        optimizer.step()
                running_loss += loss.item() * inputs.size(0)
                running_corrects += torch.sum(preds == labels.data)
            if phase == 'train':
                scheduler.step()
            epoch_loss = running_loss / dataset_sizes[phase]
            epoch_acc = running_corrects.double() / dataset_sizes[phase]
            print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss,
                epoch_acc))
            if phase == 'train':
                writer.add_scalar('Train/Loss', epoch_loss, epoch)
                writer.add_scalar('Train/Accuracy', epoch_acc, epoch)
                writer.flush()
            else:
                writer.add_scalar('Valid/Loss', epoch_loss, epoch)
                writer.add_scalar('Valid/Accuracy', epoch_acc, epoch)
                writer.flush()
            if phase == 'valid' and epoch_acc > best_acc:
                best_acc = epoch_acc
                best_model_wts = copy.deepcopy(model.state_dict())
        print()
    time_elapsed = time.time() - since
    print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60,
        time_elapsed % 60))
    print('Best val Acc: {:4f}'.format(best_acc))
    model.load_state_dict(best_model_wts)
    return model


model_ft = train_model(model_ft, criterion, optimizer_ft, exp_lr_scheduler,
    num_epochs=10)
print("""
Saving the model...""")
torch.save(model_ft, saving_path['SpinalNet_ResNet'])

import os
import pandas as pd
import torch
import torch.nn as nn
from torchvision import datasets, models, transforms
from torch.utils.tensorboard import SummaryWriter
import time, os, copy
image_transforms = {'train': transforms.Compose([transforms.
    RandomResizedCrop(size=256, scale=(0.8, 1.0)), transforms.
    RandomRotation(degrees=15), transforms.CenterCrop(size=224), transforms
    .ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224,
    0.225])]), 'valid': transforms.Compose([transforms.Resize(size=256),
    transforms.CenterCrop(size=224), transforms.ToTensor(), transforms.
    Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])}
train_path = '/home/abhinav/kaggle/intel_image_classification/input/train/'
val_path = '/home/abhinav/kaggle/intel_image_classification/input/val/'
saving_path = {'mobile_net':
    '/home/abhinav/kaggle/intel_image_classification/models/mobilenetv2.pth',
    'resnet50':
    '/home/abhinav/kaggle/intel_image_classification/models/resnet50.pth',
    'SpinalNet_ResNet':
    '/home/abhinav/kaggle/intel_image_classification/models/SpinalNet_ResNet.pth'
    }
dataset = {'train': datasets.ImageFolder(root=train_path, transform=
    image_transforms['train']), 'valid': datasets.ImageFolder(root=val_path,
    transform=image_transforms['valid'])}
dataset_sizes = {'train': len(dataset['train']), 'valid': len(dataset['valid'])
    }
dataloaders = {'train': torch.utils.data.DataLoader(dataset['train'],
    batch_size=32, shuffle=True), 'valid': torch.utils.data.DataLoader(
    dataset['valid'], batch_size=32, shuffle=True)}
class_names = dataset['train'].classes
print('Classes:', class_names)
print('Training-set size:', dataset_sizes['train'],
    """
Validation-set size:""", dataset_sizes['valid'])
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
print(device, ' is available')
model_ft = models.resnet50(pretrained=True)
num_ftrs = model_ft.fc.in_features
half_in_size = round(num_ftrs / 2)
layer_width = 12
Num_class = 6


class SpinalNet_ResNet(nn.Module):

    def __init__(self):
        super(SpinalNet_ResNet, self).__init__()
        self.fc_spinal_layer1 = nn.Sequential(nn.Linear(half_in_size,
            layer_width), nn.ReLU(inplace=True))
        self.fc_spinal_layer2 = nn.Sequential(nn.Linear(half_in_size +
            layer_width, layer_width), nn.ReLU(inplace=True))
        self.fc_spinal_layer3 = nn.Sequential(nn.Linear(half_in_size +
            layer_width, layer_width), nn.ReLU(inplace=True))
        self.fc_spinal_layer4 = nn.Sequential(nn.Linear(half_in_size +
            layer_width, layer_width), nn.ReLU(inplace=True))
        self.fc_out = nn.Sequential(nn.Linear(layer_width * 4, Num_class))

    def forward(self, x):
        x1 = self.fc_spinal_layer1(x[:, 0:half_in_size])
        x2 = self.fc_spinal_layer2(torch.cat([x[:, half_in_size:2 *
            half_in_size], x1], dim=1))
        x3 = self.fc_spinal_layer3(torch.cat([x[:, 0:half_in_size], x2], dim=1)
            )
        x4 = self.fc_spinal_layer4(torch.cat([x[:, half_in_size:2 *
            half_in_size], x3], dim=1))
        x = torch.cat([x1, x2], dim=1)
        x = torch.cat([x, x3], dim=1)
        x = torch.cat([x, x4], dim=1)
        x = self.fc_out(x)
        return x


model_ft.fc = SpinalNet_ResNet()
model_ft.to(device)
for num, (name, param) in enumerate(model_ft.named_parameters()):
    print(num, name, param.requires_grad)
print(model_ft)
criterion = nn.CrossEntropyLoss()
optimizer_ft = torch.optim.SGD(model_ft.parameters(), lr=0.001, momentum=0.9)
exp_lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer_ft, step_size=
    7, gamma=0.1)
print('\nTraining:-\n')


def train_model(model, criterion, optimizer, scheduler, num_epochs=30):
    since = time.time()
    best_model_wts = copy.deepcopy(model.state_dict())
    best_acc = 0.0
    writer = SummaryWriter()
    for epoch in range(num_epochs):
        print('Epoch {}/{}'.format(epoch, num_epochs - 1))
        print('-' * 10)
        for phase in ['train', 'valid']:
            if phase == 'train':
                model.train()
            else:
                model.eval()
            running_loss = 0.0
            running_corrects = 0
            for inputs, labels in dataloaders[phase]:
                inputs = inputs.to(device, non_blocking=True)
                labels = labels.to(device, non_blocking=True)
                optimizer.zero_grad()
                with torch.set_grad_enabled(phase == 'train'):
                    outputs = model(inputs)
                    _, preds = torch.max(outputs, 1)
                    loss = criterion(outputs, labels)
                    if phase == 'train':
                        loss.backward()
                        optimizer.step()
                running_loss += loss.item() * inputs.size(0)
                running_corrects += torch.sum(preds == labels.data)
            if phase == 'train':
                scheduler.step()
            epoch_loss = running_loss / dataset_sizes[phase]
            epoch_acc = running_corrects.double() / dataset_sizes[phase]
            print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss,
                epoch_acc))
            if phase == 'train':
                writer.add_scalar('Train/Loss', epoch_loss, epoch)
                writer.add_scalar('Train/Accuracy', epoch_acc, epoch)
                writer.flush()
            else:
                writer.add_scalar('Valid/Loss', epoch_loss, epoch)
                writer.add_scalar('Valid/Accuracy', epoch_acc, epoch)
                writer.flush()
            if phase == 'valid' and epoch_acc > best_acc:
                best_acc = epoch_acc
                best_model_wts = copy.deepcopy(model.state_dict())
        print()
    time_elapsed = time.time() - since
    print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60,
        time_elapsed % 60))
    print('Best val Acc: {:4f}'.format(best_acc))
    model.load_state_dict(best_model_wts)
    return model


model_ft = train_model(model_ft, criterion, optimizer_ft, exp_lr_scheduler,
    num_epochs=10)
print("""
Saving the model...""")
torch.save(model_ft, saving_path['SpinalNet_ResNet'])

import os
import pandas as pd
import torch
import torch.nn as nn
from torchvision import datasets,models,transforms
from torch.utils.tensorboard import SummaryWriter
import time, os, copy

image_transforms = { 
    'train': transforms.Compose([
        transforms.RandomResizedCrop(size=256, scale=(0.8, 1.0)),
        transforms.RandomRotation(degrees=15),
        transforms.CenterCrop(size=224),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406],
                             [0.229, 0.224, 0.225])
    ]),
    'valid': transforms.Compose([
        transforms.Resize(size=256),
        transforms.CenterCrop(size=224),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406],
                             [0.229, 0.224, 0.225])
    ])
}

train_path='/home/abhinav/kaggle/intel_image_classification/input/train/'
val_path='/home/abhinav/kaggle/intel_image_classification/input/val/'

saving_path={
    'mobile_net':"/home/abhinav/kaggle/intel_image_classification/models/mobilenetv2.pth",
    'resnet50':"/home/abhinav/kaggle/intel_image_classification/models/resnet50.pth",
    'SpinalNet_ResNet':"/home/abhinav/kaggle/intel_image_classification/models/SpinalNet_ResNet.pth"
}



dataset = {
    'train': datasets.ImageFolder(root=train_path, transform=image_transforms['train']),
    'valid': datasets.ImageFolder(root=val_path, transform=image_transforms['valid'])
}
 
dataset_sizes = {
    'train':len(dataset['train']),
    'valid':len(dataset['valid'])
}

dataloaders = {
    'train':torch.utils.data.DataLoader(dataset['train'], batch_size=32, shuffle=True),
    'valid':torch.utils.data.DataLoader(dataset['valid'], batch_size=32, shuffle=True,)
}

class_names = dataset['train'].classes
print("Classes:", class_names)
 
print("Training-set size:",dataset_sizes['train'],
      "\nValidation-set size:", dataset_sizes['valid'])

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

print(device," is available")

# print("\nLoading mobilenetv2 ...\n")
# model_ft = models.mobilenet_v2(pretrained=True)    
# for params in list(model_ft.parameters())[0:-5]:
#     params.requires_grad = False

# model_ft = models.resnet50(pretrained=True)    
# for params in list(model_ft.parameters())[0:-5]:
#     params.requires_grad = False

# # num_ftrs=model_ft.classifier[-1].in_features
# num_ftrs = model_ft.fc.in_features
# model_ft.fc=nn.Sequential(
#     nn.Dropout(p=0.2, inplace=False),
#     nn.Linear(in_features=num_ftrs, out_features=6, bias=True)
#     )    
# model_ft = model_ft.to(device)


model_ft = models.resnet50(pretrained=True)
num_ftrs = model_ft.fc.in_features

half_in_size = round(num_ftrs/2)
layer_width = 12 #Small for Resnet, large for VGG
Num_class=6

class SpinalNet_ResNet(nn.Module):
    def __init__(self):
        super(SpinalNet_ResNet, self).__init__()
        
        self.fc_spinal_layer1 = nn.Sequential(
            #nn.Dropout(p = 0.5), 
            nn.Linear(half_in_size, layer_width),
            #nn.BatchNorm1d(layer_width), 
            nn.ReLU(inplace=True),)
        self.fc_spinal_layer2 = nn.Sequential(
            #nn.Dropout(p = 0.5), 
            nn.Linear(half_in_size+layer_width, layer_width),
            #nn.BatchNorm1d(layer_width), 
            nn.ReLU(inplace=True),)
        self.fc_spinal_layer3 = nn.Sequential(
            #nn.Dropout(p = 0.5), 
            nn.Linear(half_in_size+layer_width, layer_width),
            #nn.BatchNorm1d(layer_width), 
            nn.ReLU(inplace=True),)
        self.fc_spinal_layer4 = nn.Sequential(
            #nn.Dropout(p = 0.5), 
            nn.Linear(half_in_size+layer_width, layer_width),
            #nn.BatchNorm1d(layer_width), 
            nn.ReLU(inplace=True),)
        self.fc_out = nn.Sequential(
            #nn.Dropout(p = 0.5), 
            nn.Linear(layer_width*4, Num_class),)
        
    def forward(self, x):
        x1 = self.fc_spinal_layer1(x[:, 0:half_in_size])
        x2 = self.fc_spinal_layer2(torch.cat([ x[:,half_in_size:2*half_in_size], x1], dim=1))
        x3 = self.fc_spinal_layer3(torch.cat([ x[:,0:half_in_size], x2], dim=1))
        x4 = self.fc_spinal_layer4(torch.cat([ x[:,half_in_size:2*half_in_size], x3], dim=1))
        
        
        x = torch.cat([x1, x2], dim=1)
        x = torch.cat([x, x3], dim=1)
        x = torch.cat([x, x4], dim=1)

        
        x = self.fc_out(x)
        return x

model_ft.fc=SpinalNet_ResNet()
model_ft.to(device)


# print('Model Summary:-\n')
for num, (name, param) in enumerate(model_ft.named_parameters()):
    print(num, name, param.requires_grad )
# summary(model_ft, input_size=(3, 224, 224))
print(model_ft)

criterion = nn.CrossEntropyLoss()

optimizer_ft = torch.optim.SGD(model_ft.parameters(), lr=0.001, momentum=0.9)

exp_lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer_ft, step_size=7, gamma=0.1)

print("\nTraining:-\n")
def train_model(model, criterion, optimizer, scheduler, num_epochs=30):
    since = time.time()

    best_model_wts = copy.deepcopy(model.state_dict())
    best_acc = 0.0

    # Tensorboard summary
    writer = SummaryWriter()
    
    for epoch in range(num_epochs):
        print('Epoch {}/{}'.format(epoch, num_epochs - 1))
        print('-' * 10)

        # Each epoch has a training and validation phase
        for phase in ['train', 'valid']:
            if phase == 'train':
                model.train()  # Set model to training mode
            else:
                model.eval()   # Set model to evaluate mode

            running_loss = 0.0
            running_corrects = 0

            # Iterate over data.
            for inputs, labels in dataloaders[phase]:
                inputs = inputs.to(device, non_blocking=True)
                labels = labels.to(device, non_blocking=True)

                # zero the parameter gradients
                optimizer.zero_grad()

                # forward
                # track history if only in train
                with torch.set_grad_enabled(phase == 'train'):
                    outputs = model(inputs)
                    _, preds = torch.max(outputs, 1)
                    loss = criterion(outputs, labels)

                    # backward + optimize only if in training phase
                    if phase == 'train':
                        loss.backward()
                        optimizer.step()

                # statistics
                running_loss += loss.item() * inputs.size(0)
                running_corrects += torch.sum(preds == labels.data)
            if phase == 'train':
                scheduler.step()

            epoch_loss = running_loss / dataset_sizes[phase]
            epoch_acc = running_corrects.double() / dataset_sizes[phase]

            print('{} Loss: {:.4f} Acc: {:.4f}'.format(
                phase, epoch_loss, epoch_acc))

            # Record training loss and accuracy for each phase
            if phase == 'train':
                writer.add_scalar('Train/Loss', epoch_loss, epoch)
                writer.add_scalar('Train/Accuracy', epoch_acc, epoch)
                writer.flush()
            else:
                writer.add_scalar('Valid/Loss', epoch_loss, epoch)
                writer.add_scalar('Valid/Accuracy', epoch_acc, epoch)
                writer.flush()

            # deep copy the model
            if phase == 'valid' and epoch_acc > best_acc:
                best_acc = epoch_acc
                best_model_wts = copy.deepcopy(model.state_dict())

        print()

    time_elapsed = time.time() - since
    print('Training complete in {:.0f}m {:.0f}s'.format(
        time_elapsed // 60, time_elapsed % 60))
    print('Best val Acc: {:4f}'.format(best_acc))

    # load best model weights
    model.load_state_dict(best_model_wts)
    return model


model_ft = train_model(model_ft, criterion, optimizer_ft, exp_lr_scheduler,
                       num_epochs=10)
# Save the entire model
print("\nSaving the model...")
torch.save(model_ft, saving_path['SpinalNet_ResNet'])
