######################################################################################
#LEDNet: A Lightweight Encoder-Decoder Network for Real-Time Semantic Segmentation
#Paper-Link: https://arxiv.org/abs/1905.02423
######################################################################################


import torch
import torch.nn as nn
import torch.nn.functional as F
from torchsummary import summary  



__all__ = ["LEDNetG3"]

def Split(x):
    c = int(x.size()[1])
    c1 = round(c * 0.5)
    x1 = x[:, :c1, :, :].contiguous() #把tensor变成在内存中连续分布的形式
    x2 = x[:, c1:, :, :].contiguous()
    return x1, x2 


def Merge(x1,x2):
    return torch.cat((x1,x2),1) 
    

def Channel_shuffle(x,groups):
    batchsize, num_channels, height, width = x.data.size()
    
    channels_per_group = num_channels // groups
    
    #reshape
    x = x.view(batchsize,groups,
        channels_per_group,height,width)
    
    x = torch.transpose(x,1,2).contiguous()
    
    #flatten
    x = x.view(batchsize,-1,height,width)
    
    return x


class PermutationBlock(nn.Module):
    def __init__(self, groups):
        super(PermutationBlock, self).__init__()
        self.groups = groups

    def forward(self, input):
        n, c, h, w = input.size()
        G = self.groups
        output = input.view(n, G, c // G, h, w).permute(0, 2, 1, 3, 4).contiguous().view(n, c, h, w)
        return output



class Conv2dBnRelu(nn.Module):
    def __init__(self,in_ch,out_ch,kernel_size=3,stride=1,padding=0,dilation=1,bias=True):
        super(Conv2dBnRelu,self).__init__()
        self.conv = nn.Sequential(
            nn.Conv2d(in_ch,out_ch,kernel_size,stride,padding,dilation=dilation,bias=bias),
            nn.BatchNorm2d(out_ch, eps=1e-3),
            nn.ReLU(inplace=True)
        )

    def forward(self, x):
        return self.conv(x)


class DownsamplerBlock(nn.Module):
    def __init__(self, ninput, noutput):
        super().__init__()

        self.conv = nn.Conv2d(ninput, noutput-ninput, (2, 2), stride=2, padding=0, bias=True)
        self.pool = nn.MaxPool2d(2, stride=2)
        self.bn = nn.BatchNorm2d(noutput, eps=1e-3)
        self.relu = nn.ReLU(inplace=True)

    def forward(self, input):
        x1 = self.pool(input)
        x2 = self.conv(input)

        diffY = x2.size()[2] - x1.size()[2]
        diffX = x2.size()[3] - x1.size()[3]

        x1 = F.pad(x1, [diffX // 2, diffX - diffX // 2,
                        diffY // 2, diffY - diffY // 2])

        output = torch.cat([x2, x1], 1)
        output = self.bn(output)
        output = self.relu(output)
        return output



# class Interpolate(nn.Module):
#     def __init__(self,size,mode):
#         super(Interpolate,self).__init__()
#         self.size = size
#         self.mode = mode
#     def forward(self,x):
#         x = F.interpolate(x,size=self.size,mode=self.mode,align_corners=True)
#         return x

        

class SS_nbt_module_paper(nn.Module):
    def __init__(self, chann, dropprob, dilated):        
        super().__init__()

        oup_inc = chann//2
        
        #dw
        self.conv3x1_1_l = nn.Conv2d(oup_inc, oup_inc, (3,1), stride=1, padding=(1,0), bias=True)

        self.conv1x3_1_l = nn.Conv2d(oup_inc, oup_inc, (1,3), stride=1, padding=(0,1), bias=True)

        self.bn1_l = nn.BatchNorm2d(oup_inc, eps=1e-03)

        self.conv3x1_2_l = nn.Conv2d(oup_inc, oup_inc, (3,1), stride=1, padding=(1*dilated,0), bias=True, dilation = (dilated,1))

        self.conv1x3_2_l = nn.Conv2d(oup_inc, oup_inc, (1,3), stride=1, padding=(0,1*dilated), bias=True, dilation = (1,dilated))

        self.bn2_l = nn.BatchNorm2d(oup_inc, eps=1e-03)
        
        #dw
        self.conv3x1_1_r = nn.Conv2d(oup_inc, oup_inc, (3,1), stride=1, padding=(1,0), bias=True)

        self.conv1x3_1_r = nn.Conv2d(oup_inc, oup_inc, (1,3), stride=1, padding=(0,1), bias=True)

        self.bn1_r = nn.BatchNorm2d(oup_inc, eps=1e-03)

        self.conv3x1_2_r = nn.Conv2d(oup_inc, oup_inc, (3,1), stride=1, padding=(1*dilated,0), bias=True, dilation = (dilated,1))

        self.conv1x3_2_r = nn.Conv2d(oup_inc, oup_inc, (1,3), stride=1, padding=(0,1*dilated), bias=True, dilation = (1,dilated))

        self.bn2_r = nn.BatchNorm2d(oup_inc, eps=1e-03)       
        
        self.relu = nn.ReLU(inplace=True)
        self.dropout = nn.Dropout2d(dropprob)

        # self.channel_shuffle = PermutationBlock(2)
       
    
    def forward(self, x):
    
        residual = x
    
        x1, x2 = Split(x)
    
        output1 = self.conv3x1_1_l(x1)
        output1 = self.relu(output1)
        output1 = self.conv1x3_1_l(output1)
        output1 = self.bn1_l(output1)
        output1_mid = self.relu(output1)

        output2 = self.conv1x3_1_r(x2)
        output2 = self.relu(output2)
        output2 = self.conv3x1_1_r(output2)
        output2 = self.bn1_r(output2)
        output2_mid = self.relu(output2)

        output1 = self.conv3x1_2_l(output1_mid)
        output1 = self.relu(output1)
        output1 = self.conv1x3_2_l(output1)
        output1 = self.bn2_l(output1)
      
        output2 = self.conv1x3_2_r(output2_mid)
        output2 = self.relu(output2)
        output2 = self.conv3x1_2_r(output2)
        output2 = self.bn2_r(output2)

        if (self.dropout.p != 0):
            output1 = self.dropout(output1)
            output2 = self.dropout(output2)

        out = Merge(output1, output2)
        
        out = F.relu(residual + out)

        # out = self.channel_shuffle(out)   ### channel shuffle
        out = Channel_shuffle(out,2)   ### channel shuffle

        return out

        # return    ### channel shuffle


class APNModule(nn.Module):
    def __init__(self, in_ch, out_ch):
        super(APNModule, self).__init__()
        # global pooling branch
        self.branch1 = nn.Sequential(
                nn.AdaptiveAvgPool2d(1),
                Conv2dBnRelu(in_ch, out_ch, kernel_size=1, stride=1, padding=0)
        )

        # midddle branch
        self.mid = nn.Sequential(
            Conv2dBnRelu(in_ch, out_ch, kernel_size=1, stride=1, padding=0)
        )


        self.down1 = nn.Sequential(
            # nn.Conv2d(in_ch,1,kernel_size=(7,1),stride=(2,1),padding=(3,0),bias=True),
            # nn.Conv2d(1,1,kernel_size=(1,7),stride=(1,2),padding=(0,3),bias=True),
            nn.Conv2d(in_ch, 1, kernel_size=(2, 1), stride=(2, 1), padding=(0, 0), bias=True),
            nn.Conv2d(1, 1, kernel_size=(1, 2), stride=(1, 2), padding=(0, 0), bias=True),
            nn.BatchNorm2d(1, eps=1e-03),
            nn.ReLU(inplace=True),
            nn.Conv2d(1, 1, kernel_size=(7, 1), stride=1, padding=(3, 0), bias=True),
            nn.Conv2d(1, 1, kernel_size=(1, 7), stride=1, padding=(0, 3), bias=True),
            nn.BatchNorm2d(1, eps=1e-03),
            nn.ReLU(inplace=True)
        )


        self.down2 = nn.Sequential(
            # nn.Conv2d(1,1,kernel_size=(5,1),stride=(2,1),padding=(2,0),bias=True),
            # nn.Conv2d(1,1,kernel_size=(1,5),stride=(1,2),padding=(0,2),bias=True),
            nn.Conv2d(1, 1, kernel_size=(2, 1), stride=(2, 1), padding=(0, 0), bias=True),
            nn.Conv2d(1, 1, kernel_size=(1, 2), stride=(1, 2), padding=(0, 0), bias=True),
            nn.BatchNorm2d(1, eps=1e-03),
            nn.ReLU(inplace=True),
            nn.Conv2d(1,1,kernel_size=(5,1),stride=1,padding=(2,0),bias=True),
            nn.Conv2d(1,1,kernel_size=(1,5),stride=1,padding=(0,2),bias=True),
            nn.BatchNorm2d(1, eps=1e-03),
            nn.ReLU(inplace=True)
        )

        self.down3 = nn.Sequential(
            # nn.Conv2d(1,1,kernel_size=(3,1),stride=(2,1),padding=(1,0),bias=True),
            # nn.Conv2d(1,1,kernel_size=(1,3),stride=(1,2),padding=(0,1),bias=True),
            nn.Conv2d(1, 1, kernel_size=(2, 1), stride=(2, 1), padding=(0, 0), bias=True),
            nn.Conv2d(1, 1, kernel_size=(1, 2), stride=(1, 2), padding=(0, 0), bias=True),
            nn.BatchNorm2d(1, eps=1e-03),
            nn.ReLU(inplace=True),
            #
            nn.Conv2d(1,1,kernel_size=(3,1),stride=1,padding=(1,0),bias=True),
            nn.Conv2d(1,1,kernel_size=(1,3),stride=1,padding=(0,1),bias=True),
            nn.BatchNorm2d(1, eps=1e-03),
            nn.ReLU(inplace=True)
        )
        self.conv2 = nn.Sequential(
            nn.Conv2d(1,1,kernel_size=(5,1),stride=1,padding=(2,0),bias=True),
            nn.Conv2d(1,1,kernel_size=(1,5),stride=1,padding=(0,2),bias=True),
            nn.BatchNorm2d(1, eps=1e-03),
            nn.ReLU(inplace=True)
        )
        self.conv1 = nn.Sequential(
            nn.Conv2d(1,1,kernel_size=(7,1),stride=1,padding=(3,0),bias=True),
            nn.Conv2d(1,1,kernel_size=(1,7),stride=1,padding=(0,3),bias=True),
            nn.BatchNorm2d(1, eps=1e-03),
            nn.ReLU(inplace=True)
        )
    def forward(self, x):

        h,w = x.size()[2:]

        b1 = self.branch1(x)
        b1= F.interpolate(b1, size=(h, w), mode="bilinear", align_corners=True)

        mid = self.mid(x)

        x1 = self.down1(x)
        x2 = self.down2(x1)
        x3 = self.down3(x2)
        x3= F.interpolate(x3, size=((h+3) // 4, (w+3) // 4), mode="bilinear", align_corners=True)

        x2 = self.conv2(x2)
        x = x2 + x3
        x= F.interpolate(x, size=((h+1) // 2, (w+1) // 2), mode="bilinear", align_corners=True)


        x1 = self.conv1(x1)
        x = x + x1
        x= F.interpolate(x, size=(h, w), mode="bilinear", align_corners=True)

        x = torch.mul(x, mid)

        x = x + b1

        return x


class LEDNetG3(nn.Module):
    def __init__(self, classes):
        super().__init__()
        self.initial_block = DownsamplerBlock(3,32)
        
        self.layers = nn.ModuleList()

        for x in range(0, 3):   
           self.layers.append(SS_nbt_module_paper(32, 0.03, 1)) 
        

        self.layers.append(DownsamplerBlock(32,64))
        

        for x in range(0, 2):   
           self.layers.append(SS_nbt_module_paper(64, 0.03, 1)) 
  
        self.layers.append(DownsamplerBlock(64,128))

        for x in range(0, 1):    
            self.layers.append(SS_nbt_module_paper(128, 0.3, 1))
            self.layers.append(SS_nbt_module_paper(128, 0.3, 2))
            self.layers.append(SS_nbt_module_paper(128, 0.3, 5))
            self.layers.append(SS_nbt_module_paper(128, 0.3, 9))
            
        for x in range(0, 1):    
            self.layers.append(SS_nbt_module_paper(128, 0.3, 2))
            self.layers.append(SS_nbt_module_paper(128, 0.3, 5))
            self.layers.append(SS_nbt_module_paper(128, 0.3, 9))
            self.layers.append(SS_nbt_module_paper(128, 0.3, 17))
                    
        self.apn = APNModule(in_ch=128,out_ch=classes)

        #self.output_conv = nn.ConvTranspose2d(128, num_classes, kernel_size=4, stride=2, padding=1, output_padding=0, bias=True)
        #self.output_conv = nn.ConvTranspose2d(128, num_classes, kernel_size=3, stride=2, padding=1, output_padding=1, bias=True)
        #self.output_conv = nn.ConvTranspose2d(128, num_classes, kernel_size=2, stride=2, padding=0, output_padding=0, bias=True)

        # self.output_conv = nn.Conv2d(128, num_classes, 1, stride=1, padding=0, bias=True)

    def forward(self, input):
        
        output = self.initial_block(input)

        for layer in self.layers:
            output = layer(output)
            
        output = self.apn(output)
        out = F.interpolate(output, input.size()[2:], mode="bilinear", align_corners=True)
        # print(out.shape)

        return out

         

"""print layers and params of network"""
if __name__ == '__main__':
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    model = LEDNetG3(classes=19).to(device)
    summary(model,(3,512,1024))

    from fvcore.nn.flop_count import flop_count  # https://github.com/facebookresearch/fvcore
    from tools.flops_counter.ptflops import get_model_complexity_info
    from thop import profile  # https://github.com/Lyken17/pytorch-OpCounter

    x = torch.randn(2, 3, 512, 1024).to(device)

    from fvcore.nn.jit_handles import batchnorm_flop_jit
    from fvcore.nn.jit_handles import generic_activation_jit
    supported_ops = {
                        "aten::batch_norm": batchnorm_flop_jit,
                    }
    flop_dict, _ = flop_count(model, (x,), supported_ops)


    flops_count, params_count = get_model_complexity_info(model, (3, 512, 1024),
                                              as_strings=False,
                                              print_per_layer_stat=True)
    input = x
    macs, params = profile(model, inputs=(input,))
    print(flop_dict)
    print(flops_count, params_count)
    print(macs, params)

'''
/home/ethan/anaconda3/envs/py36_cuda101/bin/python /home/ethan/codes/Efficient-Segmentation-Networks/model/LEDNetG3.py
----------------------------------------------------------------
        Layer (type)               Output Shape         Param #
================================================================
         MaxPool2d-1          [-1, 3, 256, 512]               0
            Conv2d-2         [-1, 29, 256, 512]             377
       BatchNorm2d-3         [-1, 32, 256, 512]              64
              ReLU-4         [-1, 32, 256, 512]               0
  DownsamplerBlock-5         [-1, 32, 256, 512]               0
            Conv2d-6         [-1, 16, 256, 512]             784
              ReLU-7         [-1, 16, 256, 512]               0
            Conv2d-8         [-1, 16, 256, 512]             784
       BatchNorm2d-9         [-1, 16, 256, 512]              32
             ReLU-10         [-1, 16, 256, 512]               0
           Conv2d-11         [-1, 16, 256, 512]             784
             ReLU-12         [-1, 16, 256, 512]               0
           Conv2d-13         [-1, 16, 256, 512]             784
      BatchNorm2d-14         [-1, 16, 256, 512]              32
             ReLU-15         [-1, 16, 256, 512]               0
           Conv2d-16         [-1, 16, 256, 512]             784
             ReLU-17         [-1, 16, 256, 512]               0
           Conv2d-18         [-1, 16, 256, 512]             784
      BatchNorm2d-19         [-1, 16, 256, 512]              32
           Conv2d-20         [-1, 16, 256, 512]             784
             ReLU-21         [-1, 16, 256, 512]               0
           Conv2d-22         [-1, 16, 256, 512]             784
      BatchNorm2d-23         [-1, 16, 256, 512]              32
        Dropout2d-24         [-1, 16, 256, 512]               0
        Dropout2d-25         [-1, 16, 256, 512]               0
SS_nbt_module_paper-26         [-1, 32, 256, 512]               0
           Conv2d-27         [-1, 16, 256, 512]             784
             ReLU-28         [-1, 16, 256, 512]               0
           Conv2d-29         [-1, 16, 256, 512]             784
      BatchNorm2d-30         [-1, 16, 256, 512]              32
             ReLU-31         [-1, 16, 256, 512]               0
           Conv2d-32         [-1, 16, 256, 512]             784
             ReLU-33         [-1, 16, 256, 512]               0
           Conv2d-34         [-1, 16, 256, 512]             784
      BatchNorm2d-35         [-1, 16, 256, 512]              32
             ReLU-36         [-1, 16, 256, 512]               0
           Conv2d-37         [-1, 16, 256, 512]             784
             ReLU-38         [-1, 16, 256, 512]               0
           Conv2d-39         [-1, 16, 256, 512]             784
      BatchNorm2d-40         [-1, 16, 256, 512]              32
           Conv2d-41         [-1, 16, 256, 512]             784
             ReLU-42         [-1, 16, 256, 512]               0
           Conv2d-43         [-1, 16, 256, 512]             784
      BatchNorm2d-44         [-1, 16, 256, 512]              32
        Dropout2d-45         [-1, 16, 256, 512]               0
        Dropout2d-46         [-1, 16, 256, 512]               0
SS_nbt_module_paper-47         [-1, 32, 256, 512]               0
           Conv2d-48         [-1, 16, 256, 512]             784
             ReLU-49         [-1, 16, 256, 512]               0
           Conv2d-50         [-1, 16, 256, 512]             784
      BatchNorm2d-51         [-1, 16, 256, 512]              32
             ReLU-52         [-1, 16, 256, 512]               0
           Conv2d-53         [-1, 16, 256, 512]             784
             ReLU-54         [-1, 16, 256, 512]               0
           Conv2d-55         [-1, 16, 256, 512]             784
      BatchNorm2d-56         [-1, 16, 256, 512]              32
             ReLU-57         [-1, 16, 256, 512]               0
           Conv2d-58         [-1, 16, 256, 512]             784
             ReLU-59         [-1, 16, 256, 512]               0
           Conv2d-60         [-1, 16, 256, 512]             784
      BatchNorm2d-61         [-1, 16, 256, 512]              32
           Conv2d-62         [-1, 16, 256, 512]             784
             ReLU-63         [-1, 16, 256, 512]               0
           Conv2d-64         [-1, 16, 256, 512]             784
      BatchNorm2d-65         [-1, 16, 256, 512]              32
        Dropout2d-66         [-1, 16, 256, 512]               0
        Dropout2d-67         [-1, 16, 256, 512]               0
SS_nbt_module_paper-68         [-1, 32, 256, 512]               0
        MaxPool2d-69         [-1, 32, 128, 256]               0
           Conv2d-70         [-1, 32, 128, 256]           4,128
      BatchNorm2d-71         [-1, 64, 128, 256]             128
             ReLU-72         [-1, 64, 128, 256]               0
 DownsamplerBlock-73         [-1, 64, 128, 256]               0
           Conv2d-74         [-1, 32, 128, 256]           3,104
             ReLU-75         [-1, 32, 128, 256]               0
           Conv2d-76         [-1, 32, 128, 256]           3,104
      BatchNorm2d-77         [-1, 32, 128, 256]              64
             ReLU-78         [-1, 32, 128, 256]               0
           Conv2d-79         [-1, 32, 128, 256]           3,104
             ReLU-80         [-1, 32, 128, 256]               0
           Conv2d-81         [-1, 32, 128, 256]           3,104
      BatchNorm2d-82         [-1, 32, 128, 256]              64
             ReLU-83         [-1, 32, 128, 256]               0
           Conv2d-84         [-1, 32, 128, 256]           3,104
             ReLU-85         [-1, 32, 128, 256]               0
           Conv2d-86         [-1, 32, 128, 256]           3,104
      BatchNorm2d-87         [-1, 32, 128, 256]              64
           Conv2d-88         [-1, 32, 128, 256]           3,104
             ReLU-89         [-1, 32, 128, 256]               0
           Conv2d-90         [-1, 32, 128, 256]           3,104
      BatchNorm2d-91         [-1, 32, 128, 256]              64
        Dropout2d-92         [-1, 32, 128, 256]               0
        Dropout2d-93         [-1, 32, 128, 256]               0
SS_nbt_module_paper-94         [-1, 64, 128, 256]               0
           Conv2d-95         [-1, 32, 128, 256]           3,104
             ReLU-96         [-1, 32, 128, 256]               0
           Conv2d-97         [-1, 32, 128, 256]           3,104
      BatchNorm2d-98         [-1, 32, 128, 256]              64
             ReLU-99         [-1, 32, 128, 256]               0
          Conv2d-100         [-1, 32, 128, 256]           3,104
            ReLU-101         [-1, 32, 128, 256]               0
          Conv2d-102         [-1, 32, 128, 256]           3,104
     BatchNorm2d-103         [-1, 32, 128, 256]              64
            ReLU-104         [-1, 32, 128, 256]               0
          Conv2d-105         [-1, 32, 128, 256]           3,104
            ReLU-106         [-1, 32, 128, 256]               0
          Conv2d-107         [-1, 32, 128, 256]           3,104
     BatchNorm2d-108         [-1, 32, 128, 256]              64
          Conv2d-109         [-1, 32, 128, 256]           3,104
            ReLU-110         [-1, 32, 128, 256]               0
          Conv2d-111         [-1, 32, 128, 256]           3,104
     BatchNorm2d-112         [-1, 32, 128, 256]              64
       Dropout2d-113         [-1, 32, 128, 256]               0
       Dropout2d-114         [-1, 32, 128, 256]               0
SS_nbt_module_paper-115         [-1, 64, 128, 256]               0
       MaxPool2d-116          [-1, 64, 64, 128]               0
          Conv2d-117          [-1, 64, 64, 128]          16,448
     BatchNorm2d-118         [-1, 128, 64, 128]             256
            ReLU-119         [-1, 128, 64, 128]               0
DownsamplerBlock-120         [-1, 128, 64, 128]               0
          Conv2d-121          [-1, 64, 64, 128]          12,352
            ReLU-122          [-1, 64, 64, 128]               0
          Conv2d-123          [-1, 64, 64, 128]          12,352
     BatchNorm2d-124          [-1, 64, 64, 128]             128
            ReLU-125          [-1, 64, 64, 128]               0
          Conv2d-126          [-1, 64, 64, 128]          12,352
            ReLU-127          [-1, 64, 64, 128]               0
          Conv2d-128          [-1, 64, 64, 128]          12,352
     BatchNorm2d-129          [-1, 64, 64, 128]             128
            ReLU-130          [-1, 64, 64, 128]               0
          Conv2d-131          [-1, 64, 64, 128]          12,352
            ReLU-132          [-1, 64, 64, 128]               0
          Conv2d-133          [-1, 64, 64, 128]          12,352
     BatchNorm2d-134          [-1, 64, 64, 128]             128
          Conv2d-135          [-1, 64, 64, 128]          12,352
            ReLU-136          [-1, 64, 64, 128]               0
          Conv2d-137          [-1, 64, 64, 128]          12,352
     BatchNorm2d-138          [-1, 64, 64, 128]             128
       Dropout2d-139          [-1, 64, 64, 128]               0
       Dropout2d-140          [-1, 64, 64, 128]               0
SS_nbt_module_paper-141         [-1, 128, 64, 128]               0
          Conv2d-142          [-1, 64, 64, 128]          12,352
            ReLU-143          [-1, 64, 64, 128]               0
          Conv2d-144          [-1, 64, 64, 128]          12,352
     BatchNorm2d-145          [-1, 64, 64, 128]             128
            ReLU-146          [-1, 64, 64, 128]               0
          Conv2d-147          [-1, 64, 64, 128]          12,352
            ReLU-148          [-1, 64, 64, 128]               0
          Conv2d-149          [-1, 64, 64, 128]          12,352
     BatchNorm2d-150          [-1, 64, 64, 128]             128
            ReLU-151          [-1, 64, 64, 128]               0
          Conv2d-152          [-1, 64, 64, 128]          12,352
            ReLU-153          [-1, 64, 64, 128]               0
          Conv2d-154          [-1, 64, 64, 128]          12,352
     BatchNorm2d-155          [-1, 64, 64, 128]             128
          Conv2d-156          [-1, 64, 64, 128]          12,352
            ReLU-157          [-1, 64, 64, 128]               0
          Conv2d-158          [-1, 64, 64, 128]          12,352
     BatchNorm2d-159          [-1, 64, 64, 128]             128
       Dropout2d-160          [-1, 64, 64, 128]               0
       Dropout2d-161          [-1, 64, 64, 128]               0
SS_nbt_module_paper-162         [-1, 128, 64, 128]               0
          Conv2d-163          [-1, 64, 64, 128]          12,352
            ReLU-164          [-1, 64, 64, 128]               0
          Conv2d-165          [-1, 64, 64, 128]          12,352
     BatchNorm2d-166          [-1, 64, 64, 128]             128
            ReLU-167          [-1, 64, 64, 128]               0
          Conv2d-168          [-1, 64, 64, 128]          12,352
            ReLU-169          [-1, 64, 64, 128]               0
          Conv2d-170          [-1, 64, 64, 128]          12,352
     BatchNorm2d-171          [-1, 64, 64, 128]             128
            ReLU-172          [-1, 64, 64, 128]               0
          Conv2d-173          [-1, 64, 64, 128]          12,352
            ReLU-174          [-1, 64, 64, 128]               0
          Conv2d-175          [-1, 64, 64, 128]          12,352
     BatchNorm2d-176          [-1, 64, 64, 128]             128
          Conv2d-177          [-1, 64, 64, 128]          12,352
            ReLU-178          [-1, 64, 64, 128]               0
          Conv2d-179          [-1, 64, 64, 128]          12,352
     BatchNorm2d-180          [-1, 64, 64, 128]             128
       Dropout2d-181          [-1, 64, 64, 128]               0
       Dropout2d-182          [-1, 64, 64, 128]               0
SS_nbt_module_paper-183         [-1, 128, 64, 128]               0
          Conv2d-184          [-1, 64, 64, 128]          12,352
            ReLU-185          [-1, 64, 64, 128]               0
          Conv2d-186          [-1, 64, 64, 128]          12,352
     BatchNorm2d-187          [-1, 64, 64, 128]             128
            ReLU-188          [-1, 64, 64, 128]               0
          Conv2d-189          [-1, 64, 64, 128]          12,352
            ReLU-190          [-1, 64, 64, 128]               0
          Conv2d-191          [-1, 64, 64, 128]          12,352
     BatchNorm2d-192          [-1, 64, 64, 128]             128
            ReLU-193          [-1, 64, 64, 128]               0
          Conv2d-194          [-1, 64, 64, 128]          12,352
            ReLU-195          [-1, 64, 64, 128]               0
          Conv2d-196          [-1, 64, 64, 128]          12,352
     BatchNorm2d-197          [-1, 64, 64, 128]             128
          Conv2d-198          [-1, 64, 64, 128]          12,352
            ReLU-199          [-1, 64, 64, 128]               0
          Conv2d-200          [-1, 64, 64, 128]          12,352
     BatchNorm2d-201          [-1, 64, 64, 128]             128
       Dropout2d-202          [-1, 64, 64, 128]               0
       Dropout2d-203          [-1, 64, 64, 128]               0
SS_nbt_module_paper-204         [-1, 128, 64, 128]               0
          Conv2d-205          [-1, 64, 64, 128]          12,352
            ReLU-206          [-1, 64, 64, 128]               0
          Conv2d-207          [-1, 64, 64, 128]          12,352
     BatchNorm2d-208          [-1, 64, 64, 128]             128
            ReLU-209          [-1, 64, 64, 128]               0
          Conv2d-210          [-1, 64, 64, 128]          12,352
            ReLU-211          [-1, 64, 64, 128]               0
          Conv2d-212          [-1, 64, 64, 128]          12,352
     BatchNorm2d-213          [-1, 64, 64, 128]             128
            ReLU-214          [-1, 64, 64, 128]               0
          Conv2d-215          [-1, 64, 64, 128]          12,352
            ReLU-216          [-1, 64, 64, 128]               0
          Conv2d-217          [-1, 64, 64, 128]          12,352
     BatchNorm2d-218          [-1, 64, 64, 128]             128
          Conv2d-219          [-1, 64, 64, 128]          12,352
            ReLU-220          [-1, 64, 64, 128]               0
          Conv2d-221          [-1, 64, 64, 128]          12,352
     BatchNorm2d-222          [-1, 64, 64, 128]             128
       Dropout2d-223          [-1, 64, 64, 128]               0
       Dropout2d-224          [-1, 64, 64, 128]               0
SS_nbt_module_paper-225         [-1, 128, 64, 128]               0
          Conv2d-226          [-1, 64, 64, 128]          12,352
            ReLU-227          [-1, 64, 64, 128]               0
          Conv2d-228          [-1, 64, 64, 128]          12,352
     BatchNorm2d-229          [-1, 64, 64, 128]             128
            ReLU-230          [-1, 64, 64, 128]               0
          Conv2d-231          [-1, 64, 64, 128]          12,352
            ReLU-232          [-1, 64, 64, 128]               0
          Conv2d-233          [-1, 64, 64, 128]          12,352
     BatchNorm2d-234          [-1, 64, 64, 128]             128
            ReLU-235          [-1, 64, 64, 128]               0
          Conv2d-236          [-1, 64, 64, 128]          12,352
            ReLU-237          [-1, 64, 64, 128]               0
          Conv2d-238          [-1, 64, 64, 128]          12,352
     BatchNorm2d-239          [-1, 64, 64, 128]             128
          Conv2d-240          [-1, 64, 64, 128]          12,352
            ReLU-241          [-1, 64, 64, 128]               0
          Conv2d-242          [-1, 64, 64, 128]          12,352
     BatchNorm2d-243          [-1, 64, 64, 128]             128
       Dropout2d-244          [-1, 64, 64, 128]               0
       Dropout2d-245          [-1, 64, 64, 128]               0
SS_nbt_module_paper-246         [-1, 128, 64, 128]               0
          Conv2d-247          [-1, 64, 64, 128]          12,352
            ReLU-248          [-1, 64, 64, 128]               0
          Conv2d-249          [-1, 64, 64, 128]          12,352
     BatchNorm2d-250          [-1, 64, 64, 128]             128
            ReLU-251          [-1, 64, 64, 128]               0
          Conv2d-252          [-1, 64, 64, 128]          12,352
            ReLU-253          [-1, 64, 64, 128]               0
          Conv2d-254          [-1, 64, 64, 128]          12,352
     BatchNorm2d-255          [-1, 64, 64, 128]             128
            ReLU-256          [-1, 64, 64, 128]               0
          Conv2d-257          [-1, 64, 64, 128]          12,352
            ReLU-258          [-1, 64, 64, 128]               0
          Conv2d-259          [-1, 64, 64, 128]          12,352
     BatchNorm2d-260          [-1, 64, 64, 128]             128
          Conv2d-261          [-1, 64, 64, 128]          12,352
            ReLU-262          [-1, 64, 64, 128]               0
          Conv2d-263          [-1, 64, 64, 128]          12,352
     BatchNorm2d-264          [-1, 64, 64, 128]             128
       Dropout2d-265          [-1, 64, 64, 128]               0
       Dropout2d-266          [-1, 64, 64, 128]               0
SS_nbt_module_paper-267         [-1, 128, 64, 128]               0
          Conv2d-268          [-1, 64, 64, 128]          12,352
            ReLU-269          [-1, 64, 64, 128]               0
          Conv2d-270          [-1, 64, 64, 128]          12,352
     BatchNorm2d-271          [-1, 64, 64, 128]             128
            ReLU-272          [-1, 64, 64, 128]               0
          Conv2d-273          [-1, 64, 64, 128]          12,352
            ReLU-274          [-1, 64, 64, 128]               0
          Conv2d-275          [-1, 64, 64, 128]          12,352
     BatchNorm2d-276          [-1, 64, 64, 128]             128
            ReLU-277          [-1, 64, 64, 128]               0
          Conv2d-278          [-1, 64, 64, 128]          12,352
            ReLU-279          [-1, 64, 64, 128]               0
          Conv2d-280          [-1, 64, 64, 128]          12,352
     BatchNorm2d-281          [-1, 64, 64, 128]             128
          Conv2d-282          [-1, 64, 64, 128]          12,352
            ReLU-283          [-1, 64, 64, 128]               0
          Conv2d-284          [-1, 64, 64, 128]          12,352
     BatchNorm2d-285          [-1, 64, 64, 128]             128
       Dropout2d-286          [-1, 64, 64, 128]               0
       Dropout2d-287          [-1, 64, 64, 128]               0
SS_nbt_module_paper-288         [-1, 128, 64, 128]               0
AdaptiveAvgPool2d-289            [-1, 128, 1, 1]               0
          Conv2d-290             [-1, 19, 1, 1]           2,451
     BatchNorm2d-291             [-1, 19, 1, 1]              38
            ReLU-292             [-1, 19, 1, 1]               0
    Conv2dBnRelu-293             [-1, 19, 1, 1]               0
          Conv2d-294          [-1, 19, 64, 128]           2,451
     BatchNorm2d-295          [-1, 19, 64, 128]              38
            ReLU-296          [-1, 19, 64, 128]               0
    Conv2dBnRelu-297          [-1, 19, 64, 128]               0
          Conv2d-298           [-1, 1, 32, 128]             257
          Conv2d-299            [-1, 1, 32, 64]               3
     BatchNorm2d-300            [-1, 1, 32, 64]               2
            ReLU-301            [-1, 1, 32, 64]               0
          Conv2d-302            [-1, 1, 32, 64]               8
          Conv2d-303            [-1, 1, 32, 64]               8
     BatchNorm2d-304            [-1, 1, 32, 64]               2
            ReLU-305            [-1, 1, 32, 64]               0
          Conv2d-306            [-1, 1, 16, 64]               3
          Conv2d-307            [-1, 1, 16, 32]               3
     BatchNorm2d-308            [-1, 1, 16, 32]               2
            ReLU-309            [-1, 1, 16, 32]               0
          Conv2d-310            [-1, 1, 16, 32]               6
          Conv2d-311            [-1, 1, 16, 32]               6
     BatchNorm2d-312            [-1, 1, 16, 32]               2
            ReLU-313            [-1, 1, 16, 32]               0
          Conv2d-314             [-1, 1, 8, 32]               3
          Conv2d-315             [-1, 1, 8, 16]               3
     BatchNorm2d-316             [-1, 1, 8, 16]               2
            ReLU-317             [-1, 1, 8, 16]               0
          Conv2d-318             [-1, 1, 8, 16]               4
          Conv2d-319             [-1, 1, 8, 16]               4
     BatchNorm2d-320             [-1, 1, 8, 16]               2
            ReLU-321             [-1, 1, 8, 16]               0
          Conv2d-322            [-1, 1, 16, 32]               6
          Conv2d-323            [-1, 1, 16, 32]               6
     BatchNorm2d-324            [-1, 1, 16, 32]               2
            ReLU-325            [-1, 1, 16, 32]               0
          Conv2d-326            [-1, 1, 32, 64]               8
          Conv2d-327            [-1, 1, 32, 64]               8
     BatchNorm2d-328            [-1, 1, 32, 64]               2
            ReLU-329            [-1, 1, 32, 64]               0
       APNModule-330          [-1, 19, 64, 128]               0
================================================================
Total params: 890,731
Trainable params: 890,731
Non-trainable params: 0
----------------------------------------------------------------
Input size (MB): 6.00
Forward/backward pass size (MB): 2342.20
Params size (MB): 3.40
Estimated Total Size (MB): 2351.60
----------------------------------------------------------------
/home/ethan/codes/Efficient-Segmentation-Networks/model/LEDNetG3.py:17: TracerWarning: Converting a tensor to a Python integer might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
  c = int(x.size()[1])
Skipped operation aten::max_pool2d 3 time(s)
Skipped operation aten::sub 12 time(s)
Skipped operation aten::relu_ 91 time(s)
Skipped operation aten::feature_dropout 26 time(s)
Skipped operation aten::add 20 time(s)
Skipped operation aten::relu 13 time(s)
Skipped operation aten::adaptive_avg_pool2d 1 time(s)
Skipped operation aten::upsample_bilinear2d 5 time(s)
Skipped operation aten::mul 1 time(s)
LEDNetG3(
  11.118 GMac, 100.000% MACs, 
  (initial_block): DownsamplerBlock(
    0.064 GMac, 0.572% MACs, 
    (conv): Conv2d(0.049 GMac, 0.444% MACs, 3, 29, kernel_size=(2, 2), stride=(2, 2))
    (pool): MaxPool2d(0.002 GMac, 0.014% MACs, kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
    (bn): BatchNorm2d(0.008 GMac, 0.075% MACs, 32, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
    (relu): ReLU(0.004 GMac, 0.038% MACs, inplace=True)
  )
  (layers): ModuleList(
    11.032 GMac, 99.223% MACs, 
    (0): SS_nbt_module_paper(
      0.851 GMac, 7.658% MACs, 
      (conv3x1_1_l): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_l): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_l): BatchNorm2d(0.004 GMac, 0.038% MACs, 16, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_l): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_2_l): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn2_l): BatchNorm2d(0.004 GMac, 0.038% MACs, 16, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_1_r): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_r): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_r): BatchNorm2d(0.004 GMac, 0.038% MACs, 16, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_r): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_2_r): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn2_r): BatchNorm2d(0.004 GMac, 0.038% MACs, 16, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(0.013 GMac, 0.113% MACs, inplace=True)
      (dropout): Dropout2d(0.0 GMac, 0.000% MACs, p=0.03, inplace=False)
    )
    (1): SS_nbt_module_paper(
      0.851 GMac, 7.658% MACs, 
      (conv3x1_1_l): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_l): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_l): BatchNorm2d(0.004 GMac, 0.038% MACs, 16, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_l): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_2_l): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn2_l): BatchNorm2d(0.004 GMac, 0.038% MACs, 16, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_1_r): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_r): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_r): BatchNorm2d(0.004 GMac, 0.038% MACs, 16, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_r): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_2_r): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn2_r): BatchNorm2d(0.004 GMac, 0.038% MACs, 16, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(0.013 GMac, 0.113% MACs, inplace=True)
      (dropout): Dropout2d(0.0 GMac, 0.000% MACs, p=0.03, inplace=False)
    )
    (2): SS_nbt_module_paper(
      0.851 GMac, 7.658% MACs, 
      (conv3x1_1_l): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_l): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_l): BatchNorm2d(0.004 GMac, 0.038% MACs, 16, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_l): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_2_l): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn2_l): BatchNorm2d(0.004 GMac, 0.038% MACs, 16, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_1_r): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_r): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_r): BatchNorm2d(0.004 GMac, 0.038% MACs, 16, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_r): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_2_r): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn2_r): BatchNorm2d(0.004 GMac, 0.038% MACs, 16, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(0.013 GMac, 0.113% MACs, inplace=True)
      (dropout): Dropout2d(0.0 GMac, 0.000% MACs, p=0.03, inplace=False)
    )
    (3): DownsamplerBlock(
      0.146 GMac, 1.311% MACs, 
      (conv): Conv2d(0.135 GMac, 1.217% MACs, 32, 32, kernel_size=(2, 2), stride=(2, 2))
      (pool): MaxPool2d(0.004 GMac, 0.038% MACs, kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
      (bn): BatchNorm2d(0.004 GMac, 0.038% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(0.002 GMac, 0.019% MACs, inplace=True)
    )
    (4): SS_nbt_module_paper(
      0.828 GMac, 7.451% MACs, 
      (conv3x1_1_l): Conv2d(0.102 GMac, 0.915% MACs, 32, 32, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_l): Conv2d(0.102 GMac, 0.915% MACs, 32, 32, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_l): BatchNorm2d(0.002 GMac, 0.019% MACs, 32, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_l): Conv2d(0.102 GMac, 0.915% MACs, 32, 32, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_2_l): Conv2d(0.102 GMac, 0.915% MACs, 32, 32, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn2_l): BatchNorm2d(0.002 GMac, 0.019% MACs, 32, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_1_r): Conv2d(0.102 GMac, 0.915% MACs, 32, 32, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_r): Conv2d(0.102 GMac, 0.915% MACs, 32, 32, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_r): BatchNorm2d(0.002 GMac, 0.019% MACs, 32, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_r): Conv2d(0.102 GMac, 0.915% MACs, 32, 32, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_2_r): Conv2d(0.102 GMac, 0.915% MACs, 32, 32, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn2_r): BatchNorm2d(0.002 GMac, 0.019% MACs, 32, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(0.006 GMac, 0.057% MACs, inplace=True)
      (dropout): Dropout2d(0.0 GMac, 0.000% MACs, p=0.03, inplace=False)
    )
    (5): SS_nbt_module_paper(
      0.828 GMac, 7.451% MACs, 
      (conv3x1_1_l): Conv2d(0.102 GMac, 0.915% MACs, 32, 32, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_l): Conv2d(0.102 GMac, 0.915% MACs, 32, 32, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_l): BatchNorm2d(0.002 GMac, 0.019% MACs, 32, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_l): Conv2d(0.102 GMac, 0.915% MACs, 32, 32, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_2_l): Conv2d(0.102 GMac, 0.915% MACs, 32, 32, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn2_l): BatchNorm2d(0.002 GMac, 0.019% MACs, 32, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_1_r): Conv2d(0.102 GMac, 0.915% MACs, 32, 32, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_r): Conv2d(0.102 GMac, 0.915% MACs, 32, 32, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_r): BatchNorm2d(0.002 GMac, 0.019% MACs, 32, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_r): Conv2d(0.102 GMac, 0.915% MACs, 32, 32, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_2_r): Conv2d(0.102 GMac, 0.915% MACs, 32, 32, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn2_r): BatchNorm2d(0.002 GMac, 0.019% MACs, 32, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(0.006 GMac, 0.057% MACs, inplace=True)
      (dropout): Dropout2d(0.0 GMac, 0.000% MACs, p=0.03, inplace=False)
    )
    (6): DownsamplerBlock(
      0.14 GMac, 1.259% MACs, 
      (conv): Conv2d(0.135 GMac, 1.212% MACs, 64, 64, kernel_size=(2, 2), stride=(2, 2))
      (pool): MaxPool2d(0.002 GMac, 0.019% MACs, kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
      (bn): BatchNorm2d(0.002 GMac, 0.019% MACs, 128, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(0.001 GMac, 0.009% MACs, inplace=True)
    )
    (7): SS_nbt_module_paper(
      0.817 GMac, 7.347% MACs, 
      (conv3x1_1_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_l): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_2_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn2_l): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_1_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_r): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_2_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn2_r): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(0.003 GMac, 0.028% MACs, inplace=True)
      (dropout): Dropout2d(0.0 GMac, 0.000% MACs, p=0.3, inplace=False)
    )
    (8): SS_nbt_module_paper(
      0.817 GMac, 7.347% MACs, 
      (conv3x1_1_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_l): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(2, 0), dilation=(2, 1))
      (conv1x3_2_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 2), dilation=(1, 2))
      (bn2_l): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_1_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_r): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(2, 0), dilation=(2, 1))
      (conv1x3_2_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 2), dilation=(1, 2))
      (bn2_r): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(0.003 GMac, 0.028% MACs, inplace=True)
      (dropout): Dropout2d(0.0 GMac, 0.000% MACs, p=0.3, inplace=False)
    )
    (9): SS_nbt_module_paper(
      0.817 GMac, 7.347% MACs, 
      (conv3x1_1_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_l): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(5, 0), dilation=(5, 1))
      (conv1x3_2_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 5), dilation=(1, 5))
      (bn2_l): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_1_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_r): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(5, 0), dilation=(5, 1))
      (conv1x3_2_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 5), dilation=(1, 5))
      (bn2_r): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(0.003 GMac, 0.028% MACs, inplace=True)
      (dropout): Dropout2d(0.0 GMac, 0.000% MACs, p=0.3, inplace=False)
    )
    (10): SS_nbt_module_paper(
      0.817 GMac, 7.347% MACs, 
      (conv3x1_1_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_l): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(9, 0), dilation=(9, 1))
      (conv1x3_2_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 9), dilation=(1, 9))
      (bn2_l): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_1_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_r): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(9, 0), dilation=(9, 1))
      (conv1x3_2_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 9), dilation=(1, 9))
      (bn2_r): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(0.003 GMac, 0.028% MACs, inplace=True)
      (dropout): Dropout2d(0.0 GMac, 0.000% MACs, p=0.3, inplace=False)
    )
    (11): SS_nbt_module_paper(
      0.817 GMac, 7.347% MACs, 
      (conv3x1_1_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_l): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(2, 0), dilation=(2, 1))
      (conv1x3_2_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 2), dilation=(1, 2))
      (bn2_l): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_1_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_r): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(2, 0), dilation=(2, 1))
      (conv1x3_2_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 2), dilation=(1, 2))
      (bn2_r): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(0.003 GMac, 0.028% MACs, inplace=True)
      (dropout): Dropout2d(0.0 GMac, 0.000% MACs, p=0.3, inplace=False)
    )
    (12): SS_nbt_module_paper(
      0.817 GMac, 7.347% MACs, 
      (conv3x1_1_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_l): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(5, 0), dilation=(5, 1))
      (conv1x3_2_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 5), dilation=(1, 5))
      (bn2_l): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_1_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_r): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(5, 0), dilation=(5, 1))
      (conv1x3_2_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 5), dilation=(1, 5))
      (bn2_r): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(0.003 GMac, 0.028% MACs, inplace=True)
      (dropout): Dropout2d(0.0 GMac, 0.000% MACs, p=0.3, inplace=False)
    )
    (13): SS_nbt_module_paper(
      0.817 GMac, 7.347% MACs, 
      (conv3x1_1_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_l): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(9, 0), dilation=(9, 1))
      (conv1x3_2_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 9), dilation=(1, 9))
      (bn2_l): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_1_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_r): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(9, 0), dilation=(9, 1))
      (conv1x3_2_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 9), dilation=(1, 9))
      (bn2_r): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(0.003 GMac, 0.028% MACs, inplace=True)
      (dropout): Dropout2d(0.0 GMac, 0.000% MACs, p=0.3, inplace=False)
    )
    (14): SS_nbt_module_paper(
      0.817 GMac, 7.347% MACs, 
      (conv3x1_1_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_l): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(17, 0), dilation=(17, 1))
      (conv1x3_2_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 17), dilation=(1, 17))
      (bn2_l): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_1_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_r): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(17, 0), dilation=(17, 1))
      (conv1x3_2_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 17), dilation=(1, 17))
      (bn2_r): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(0.003 GMac, 0.028% MACs, inplace=True)
      (dropout): Dropout2d(0.0 GMac, 0.000% MACs, p=0.3, inplace=False)
    )
  )
  (apn): APNModule(
    0.023 GMac, 0.205% MACs, 
    (branch1): Sequential(
      0.001 GMac, 0.009% MACs, 
      (0): AdaptiveAvgPool2d(0.001 GMac, 0.009% MACs, output_size=1)
      (1): Conv2dBnRelu(
        0.0 GMac, 0.000% MACs, 
        (conv): Sequential(
          0.0 GMac, 0.000% MACs, 
          (0): Conv2d(0.0 GMac, 0.000% MACs, 128, 19, kernel_size=(1, 1), stride=(1, 1))
          (1): BatchNorm2d(0.0 GMac, 0.000% MACs, 19, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
          (2): ReLU(0.0 GMac, 0.000% MACs, inplace=True)
        )
      )
    )
    (mid): Sequential(
      0.021 GMac, 0.185% MACs, 
      (0): Conv2dBnRelu(
        0.021 GMac, 0.185% MACs, 
        (conv): Sequential(
          0.021 GMac, 0.185% MACs, 
          (0): Conv2d(0.02 GMac, 0.181% MACs, 128, 19, kernel_size=(1, 1), stride=(1, 1))
          (1): BatchNorm2d(0.0 GMac, 0.003% MACs, 19, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
          (2): ReLU(0.0 GMac, 0.001% MACs, inplace=True)
        )
      )
    )
    (down1): Sequential(
      0.001 GMac, 0.010% MACs, 
      (0): Conv2d(0.001 GMac, 0.009% MACs, 128, 1, kernel_size=(2, 1), stride=(2, 1))
      (1): Conv2d(0.0 GMac, 0.000% MACs, 1, 1, kernel_size=(1, 2), stride=(1, 2))
      (2): BatchNorm2d(0.0 GMac, 0.000% MACs, 1, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (3): ReLU(0.0 GMac, 0.000% MACs, inplace=True)
      (4): Conv2d(0.0 GMac, 0.000% MACs, 1, 1, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0))
      (5): Conv2d(0.0 GMac, 0.000% MACs, 1, 1, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3))
      (6): BatchNorm2d(0.0 GMac, 0.000% MACs, 1, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (7): ReLU(0.0 GMac, 0.000% MACs, inplace=True)
    )
    (down2): Sequential(
      0.0 GMac, 0.000% MACs, 
      (0): Conv2d(0.0 GMac, 0.000% MACs, 1, 1, kernel_size=(2, 1), stride=(2, 1))
      (1): Conv2d(0.0 GMac, 0.000% MACs, 1, 1, kernel_size=(1, 2), stride=(1, 2))
      (2): BatchNorm2d(0.0 GMac, 0.000% MACs, 1, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (3): ReLU(0.0 GMac, 0.000% MACs, inplace=True)
      (4): Conv2d(0.0 GMac, 0.000% MACs, 1, 1, kernel_size=(5, 1), stride=(1, 1), padding=(2, 0))
      (5): Conv2d(0.0 GMac, 0.000% MACs, 1, 1, kernel_size=(1, 5), stride=(1, 1), padding=(0, 2))
      (6): BatchNorm2d(0.0 GMac, 0.000% MACs, 1, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (7): ReLU(0.0 GMac, 0.000% MACs, inplace=True)
    )
    (down3): Sequential(
      0.0 GMac, 0.000% MACs, 
      (0): Conv2d(0.0 GMac, 0.000% MACs, 1, 1, kernel_size=(2, 1), stride=(2, 1))
      (1): Conv2d(0.0 GMac, 0.000% MACs, 1, 1, kernel_size=(1, 2), stride=(1, 2))
      (2): BatchNorm2d(0.0 GMac, 0.000% MACs, 1, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (3): ReLU(0.0 GMac, 0.000% MACs, inplace=True)
      (4): Conv2d(0.0 GMac, 0.000% MACs, 1, 1, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (5): Conv2d(0.0 GMac, 0.000% MACs, 1, 1, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (6): BatchNorm2d(0.0 GMac, 0.000% MACs, 1, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (7): ReLU(0.0 GMac, 0.000% MACs, inplace=True)
    )
    (conv2): Sequential(
      0.0 GMac, 0.000% MACs, 
      (0): Conv2d(0.0 GMac, 0.000% MACs, 1, 1, kernel_size=(5, 1), stride=(1, 1), padding=(2, 0))
      (1): Conv2d(0.0 GMac, 0.000% MACs, 1, 1, kernel_size=(1, 5), stride=(1, 1), padding=(0, 2))
      (2): BatchNorm2d(0.0 GMac, 0.000% MACs, 1, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (3): ReLU(0.0 GMac, 0.000% MACs, inplace=True)
    )
    (conv1): Sequential(
      0.0 GMac, 0.000% MACs, 
      (0): Conv2d(0.0 GMac, 0.000% MACs, 1, 1, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0))
      (1): Conv2d(0.0 GMac, 0.000% MACs, 1, 1, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3))
      (2): BatchNorm2d(0.0 GMac, 0.000% MACs, 1, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (3): ReLU(0.0 GMac, 0.000% MACs, inplace=True)
    )
  )
)
[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.
[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.
[INFO] Register count_bn() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.
[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.
[WARN] Cannot find rule for <class '__main__.DownsamplerBlock'>. Treat it as zero Macs and zero Params.
[WARN] Cannot find rule for <class 'torch.nn.modules.dropout.Dropout2d'>. Treat it as zero Macs and zero Params.
[WARN] Cannot find rule for <class '__main__.SS_nbt_module_paper'>. Treat it as zero Macs and zero Params.
[WARN] Cannot find rule for <class 'torch.nn.modules.container.ModuleList'>. Treat it as zero Macs and zero Params.
[INFO] Register count_adap_avgpool() for <class 'torch.nn.modules.pooling.AdaptiveAvgPool2d'>.
[WARN] Cannot find rule for <class 'torch.nn.modules.container.Sequential'>. Treat it as zero Macs and zero Params.
[WARN] Cannot find rule for <class '__main__.Conv2dBnRelu'>. Treat it as zero Macs and zero Params.
[WARN] Cannot find rule for <class '__main__.APNModule'>. Treat it as zero Macs and zero Params.
[WARN] Cannot find rule for <class '__main__.LEDNetG3'>. Treat it as zero Macs and zero Params.
defaultdict(<class 'float'>, {'conv': 21.608163072, 'batchnorm': 0.462682264})
11117877580.0 890731
22054024562.0 890731.0

Process finished with exit code 0


'''
7210ea2f1304321e9eae5c97bc268d39ad917f70
<mask token>


class Conv2dBnRelu(nn.Module):
    <mask token>
    <mask token>


class DownsamplerBlock(nn.Module):

    def __init__(self, ninput, noutput):
        super().__init__()
        self.conv = nn.Conv2d(ninput, noutput - ninput, (2, 2), stride=2,
            padding=0, bias=True)
        self.pool = nn.MaxPool2d(2, stride=2)
        self.bn = nn.BatchNorm2d(noutput, eps=0.001)
        self.relu = nn.ReLU(inplace=True)

    def forward(self, input):
        x1 = self.pool(input)
        x2 = self.conv(input)
        diffY = x2.size()[2] - x1.size()[2]
        diffX = x2.size()[3] - x1.size()[3]
        x1 = F.pad(x1, [diffX // 2, diffX - diffX // 2, diffY // 2, diffY -
            diffY // 2])
        output = torch.cat([x2, x1], 1)
        output = self.bn(output)
        output = self.relu(output)
        return output


class SS_nbt_module_paper(nn.Module):

    def __init__(self, chann, dropprob, dilated):
        super().__init__()
        oup_inc = chann // 2
        self.conv3x1_1_l = nn.Conv2d(oup_inc, oup_inc, (3, 1), stride=1,
            padding=(1, 0), bias=True)
        self.conv1x3_1_l = nn.Conv2d(oup_inc, oup_inc, (1, 3), stride=1,
            padding=(0, 1), bias=True)
        self.bn1_l = nn.BatchNorm2d(oup_inc, eps=0.001)
        self.conv3x1_2_l = nn.Conv2d(oup_inc, oup_inc, (3, 1), stride=1,
            padding=(1 * dilated, 0), bias=True, dilation=(dilated, 1))
        self.conv1x3_2_l = nn.Conv2d(oup_inc, oup_inc, (1, 3), stride=1,
            padding=(0, 1 * dilated), bias=True, dilation=(1, dilated))
        self.bn2_l = nn.BatchNorm2d(oup_inc, eps=0.001)
        self.conv3x1_1_r = nn.Conv2d(oup_inc, oup_inc, (3, 1), stride=1,
            padding=(1, 0), bias=True)
        self.conv1x3_1_r = nn.Conv2d(oup_inc, oup_inc, (1, 3), stride=1,
            padding=(0, 1), bias=True)
        self.bn1_r = nn.BatchNorm2d(oup_inc, eps=0.001)
        self.conv3x1_2_r = nn.Conv2d(oup_inc, oup_inc, (3, 1), stride=1,
            padding=(1 * dilated, 0), bias=True, dilation=(dilated, 1))
        self.conv1x3_2_r = nn.Conv2d(oup_inc, oup_inc, (1, 3), stride=1,
            padding=(0, 1 * dilated), bias=True, dilation=(1, dilated))
        self.bn2_r = nn.BatchNorm2d(oup_inc, eps=0.001)
        self.relu = nn.ReLU(inplace=True)
        self.dropout = nn.Dropout2d(dropprob)

    def forward(self, x):
        residual = x
        x1, x2 = Split(x)
        output1 = self.conv3x1_1_l(x1)
        output1 = self.relu(output1)
        output1 = self.conv1x3_1_l(output1)
        output1 = self.bn1_l(output1)
        output1_mid = self.relu(output1)
        output2 = self.conv1x3_1_r(x2)
        output2 = self.relu(output2)
        output2 = self.conv3x1_1_r(output2)
        output2 = self.bn1_r(output2)
        output2_mid = self.relu(output2)
        output1 = self.conv3x1_2_l(output1_mid)
        output1 = self.relu(output1)
        output1 = self.conv1x3_2_l(output1)
        output1 = self.bn2_l(output1)
        output2 = self.conv1x3_2_r(output2_mid)
        output2 = self.relu(output2)
        output2 = self.conv3x1_2_r(output2)
        output2 = self.bn2_r(output2)
        if self.dropout.p != 0:
            output1 = self.dropout(output1)
            output2 = self.dropout(output2)
        out = Merge(output1, output2)
        out = F.relu(residual + out)
        out = Channel_shuffle(out, 2)
        return out


class APNModule(nn.Module):

    def __init__(self, in_ch, out_ch):
        super(APNModule, self).__init__()
        self.branch1 = nn.Sequential(nn.AdaptiveAvgPool2d(1), Conv2dBnRelu(
            in_ch, out_ch, kernel_size=1, stride=1, padding=0))
        self.mid = nn.Sequential(Conv2dBnRelu(in_ch, out_ch, kernel_size=1,
            stride=1, padding=0))
        self.down1 = nn.Sequential(nn.Conv2d(in_ch, 1, kernel_size=(2, 1),
            stride=(2, 1), padding=(0, 0), bias=True), nn.Conv2d(1, 1,
            kernel_size=(1, 2), stride=(1, 2), padding=(0, 0), bias=True),
            nn.BatchNorm2d(1, eps=0.001), nn.ReLU(inplace=True), nn.Conv2d(
            1, 1, kernel_size=(7, 1), stride=1, padding=(3, 0), bias=True),
            nn.Conv2d(1, 1, kernel_size=(1, 7), stride=1, padding=(0, 3),
            bias=True), nn.BatchNorm2d(1, eps=0.001), nn.ReLU(inplace=True))
        self.down2 = nn.Sequential(nn.Conv2d(1, 1, kernel_size=(2, 1),
            stride=(2, 1), padding=(0, 0), bias=True), nn.Conv2d(1, 1,
            kernel_size=(1, 2), stride=(1, 2), padding=(0, 0), bias=True),
            nn.BatchNorm2d(1, eps=0.001), nn.ReLU(inplace=True), nn.Conv2d(
            1, 1, kernel_size=(5, 1), stride=1, padding=(2, 0), bias=True),
            nn.Conv2d(1, 1, kernel_size=(1, 5), stride=1, padding=(0, 2),
            bias=True), nn.BatchNorm2d(1, eps=0.001), nn.ReLU(inplace=True))
        self.down3 = nn.Sequential(nn.Conv2d(1, 1, kernel_size=(2, 1),
            stride=(2, 1), padding=(0, 0), bias=True), nn.Conv2d(1, 1,
            kernel_size=(1, 2), stride=(1, 2), padding=(0, 0), bias=True),
            nn.BatchNorm2d(1, eps=0.001), nn.ReLU(inplace=True), nn.Conv2d(
            1, 1, kernel_size=(3, 1), stride=1, padding=(1, 0), bias=True),
            nn.Conv2d(1, 1, kernel_size=(1, 3), stride=1, padding=(0, 1),
            bias=True), nn.BatchNorm2d(1, eps=0.001), nn.ReLU(inplace=True))
        self.conv2 = nn.Sequential(nn.Conv2d(1, 1, kernel_size=(5, 1),
            stride=1, padding=(2, 0), bias=True), nn.Conv2d(1, 1,
            kernel_size=(1, 5), stride=1, padding=(0, 2), bias=True), nn.
            BatchNorm2d(1, eps=0.001), nn.ReLU(inplace=True))
        self.conv1 = nn.Sequential(nn.Conv2d(1, 1, kernel_size=(7, 1),
            stride=1, padding=(3, 0), bias=True), nn.Conv2d(1, 1,
            kernel_size=(1, 7), stride=1, padding=(0, 3), bias=True), nn.
            BatchNorm2d(1, eps=0.001), nn.ReLU(inplace=True))

    def forward(self, x):
        h, w = x.size()[2:]
        b1 = self.branch1(x)
        b1 = F.interpolate(b1, size=(h, w), mode='bilinear', align_corners=True
            )
        mid = self.mid(x)
        x1 = self.down1(x)
        x2 = self.down2(x1)
        x3 = self.down3(x2)
        x3 = F.interpolate(x3, size=((h + 3) // 4, (w + 3) // 4), mode=
            'bilinear', align_corners=True)
        x2 = self.conv2(x2)
        x = x2 + x3
        x = F.interpolate(x, size=((h + 1) // 2, (w + 1) // 2), mode=
            'bilinear', align_corners=True)
        x1 = self.conv1(x1)
        x = x + x1
        x = F.interpolate(x, size=(h, w), mode='bilinear', align_corners=True)
        x = torch.mul(x, mid)
        x = x + b1
        return x


class LEDNetG3(nn.Module):

    def __init__(self, classes):
        super().__init__()
        self.initial_block = DownsamplerBlock(3, 32)
        self.layers = nn.ModuleList()
        for x in range(0, 3):
            self.layers.append(SS_nbt_module_paper(32, 0.03, 1))
        self.layers.append(DownsamplerBlock(32, 64))
        for x in range(0, 2):
            self.layers.append(SS_nbt_module_paper(64, 0.03, 1))
        self.layers.append(DownsamplerBlock(64, 128))
        for x in range(0, 1):
            self.layers.append(SS_nbt_module_paper(128, 0.3, 1))
            self.layers.append(SS_nbt_module_paper(128, 0.3, 2))
            self.layers.append(SS_nbt_module_paper(128, 0.3, 5))
            self.layers.append(SS_nbt_module_paper(128, 0.3, 9))
        for x in range(0, 1):
            self.layers.append(SS_nbt_module_paper(128, 0.3, 2))
            self.layers.append(SS_nbt_module_paper(128, 0.3, 5))
            self.layers.append(SS_nbt_module_paper(128, 0.3, 9))
            self.layers.append(SS_nbt_module_paper(128, 0.3, 17))
        self.apn = APNModule(in_ch=128, out_ch=classes)

    def forward(self, input):
        output = self.initial_block(input)
        for layer in self.layers:
            output = layer(output)
        output = self.apn(output)
        out = F.interpolate(output, input.size()[2:], mode='bilinear',
            align_corners=True)
        return out


<mask token>

<mask token>


class PermutationBlock(nn.Module):

    def __init__(self, groups):
        super(PermutationBlock, self).__init__()
        self.groups = groups

    def forward(self, input):
        n, c, h, w = input.size()
        G = self.groups
        output = input.view(n, G, c // G, h, w).permute(0, 2, 1, 3, 4
            ).contiguous().view(n, c, h, w)
        return output


class Conv2dBnRelu(nn.Module):

    def __init__(self, in_ch, out_ch, kernel_size=3, stride=1, padding=0,
        dilation=1, bias=True):
        super(Conv2dBnRelu, self).__init__()
        self.conv = nn.Sequential(nn.Conv2d(in_ch, out_ch, kernel_size,
            stride, padding, dilation=dilation, bias=bias), nn.BatchNorm2d(
            out_ch, eps=0.001), nn.ReLU(inplace=True))

    def forward(self, x):
        return self.conv(x)


class DownsamplerBlock(nn.Module):

    def __init__(self, ninput, noutput):
        super().__init__()
        self.conv = nn.Conv2d(ninput, noutput - ninput, (2, 2), stride=2,
            padding=0, bias=True)
        self.pool = nn.MaxPool2d(2, stride=2)
        self.bn = nn.BatchNorm2d(noutput, eps=0.001)
        self.relu = nn.ReLU(inplace=True)

    def forward(self, input):
        x1 = self.pool(input)
        x2 = self.conv(input)
        diffY = x2.size()[2] - x1.size()[2]
        diffX = x2.size()[3] - x1.size()[3]
        x1 = F.pad(x1, [diffX // 2, diffX - diffX // 2, diffY // 2, diffY -
            diffY // 2])
        output = torch.cat([x2, x1], 1)
        output = self.bn(output)
        output = self.relu(output)
        return output


class SS_nbt_module_paper(nn.Module):

    def __init__(self, chann, dropprob, dilated):
        super().__init__()
        oup_inc = chann // 2
        self.conv3x1_1_l = nn.Conv2d(oup_inc, oup_inc, (3, 1), stride=1,
            padding=(1, 0), bias=True)
        self.conv1x3_1_l = nn.Conv2d(oup_inc, oup_inc, (1, 3), stride=1,
            padding=(0, 1), bias=True)
        self.bn1_l = nn.BatchNorm2d(oup_inc, eps=0.001)
        self.conv3x1_2_l = nn.Conv2d(oup_inc, oup_inc, (3, 1), stride=1,
            padding=(1 * dilated, 0), bias=True, dilation=(dilated, 1))
        self.conv1x3_2_l = nn.Conv2d(oup_inc, oup_inc, (1, 3), stride=1,
            padding=(0, 1 * dilated), bias=True, dilation=(1, dilated))
        self.bn2_l = nn.BatchNorm2d(oup_inc, eps=0.001)
        self.conv3x1_1_r = nn.Conv2d(oup_inc, oup_inc, (3, 1), stride=1,
            padding=(1, 0), bias=True)
        self.conv1x3_1_r = nn.Conv2d(oup_inc, oup_inc, (1, 3), stride=1,
            padding=(0, 1), bias=True)
        self.bn1_r = nn.BatchNorm2d(oup_inc, eps=0.001)
        self.conv3x1_2_r = nn.Conv2d(oup_inc, oup_inc, (3, 1), stride=1,
            padding=(1 * dilated, 0), bias=True, dilation=(dilated, 1))
        self.conv1x3_2_r = nn.Conv2d(oup_inc, oup_inc, (1, 3), stride=1,
            padding=(0, 1 * dilated), bias=True, dilation=(1, dilated))
        self.bn2_r = nn.BatchNorm2d(oup_inc, eps=0.001)
        self.relu = nn.ReLU(inplace=True)
        self.dropout = nn.Dropout2d(dropprob)

    def forward(self, x):
        residual = x
        x1, x2 = Split(x)
        output1 = self.conv3x1_1_l(x1)
        output1 = self.relu(output1)
        output1 = self.conv1x3_1_l(output1)
        output1 = self.bn1_l(output1)
        output1_mid = self.relu(output1)
        output2 = self.conv1x3_1_r(x2)
        output2 = self.relu(output2)
        output2 = self.conv3x1_1_r(output2)
        output2 = self.bn1_r(output2)
        output2_mid = self.relu(output2)
        output1 = self.conv3x1_2_l(output1_mid)
        output1 = self.relu(output1)
        output1 = self.conv1x3_2_l(output1)
        output1 = self.bn2_l(output1)
        output2 = self.conv1x3_2_r(output2_mid)
        output2 = self.relu(output2)
        output2 = self.conv3x1_2_r(output2)
        output2 = self.bn2_r(output2)
        if self.dropout.p != 0:
            output1 = self.dropout(output1)
            output2 = self.dropout(output2)
        out = Merge(output1, output2)
        out = F.relu(residual + out)
        out = Channel_shuffle(out, 2)
        return out


class APNModule(nn.Module):

    def __init__(self, in_ch, out_ch):
        super(APNModule, self).__init__()
        self.branch1 = nn.Sequential(nn.AdaptiveAvgPool2d(1), Conv2dBnRelu(
            in_ch, out_ch, kernel_size=1, stride=1, padding=0))
        self.mid = nn.Sequential(Conv2dBnRelu(in_ch, out_ch, kernel_size=1,
            stride=1, padding=0))
        self.down1 = nn.Sequential(nn.Conv2d(in_ch, 1, kernel_size=(2, 1),
            stride=(2, 1), padding=(0, 0), bias=True), nn.Conv2d(1, 1,
            kernel_size=(1, 2), stride=(1, 2), padding=(0, 0), bias=True),
            nn.BatchNorm2d(1, eps=0.001), nn.ReLU(inplace=True), nn.Conv2d(
            1, 1, kernel_size=(7, 1), stride=1, padding=(3, 0), bias=True),
            nn.Conv2d(1, 1, kernel_size=(1, 7), stride=1, padding=(0, 3),
            bias=True), nn.BatchNorm2d(1, eps=0.001), nn.ReLU(inplace=True))
        self.down2 = nn.Sequential(nn.Conv2d(1, 1, kernel_size=(2, 1),
            stride=(2, 1), padding=(0, 0), bias=True), nn.Conv2d(1, 1,
            kernel_size=(1, 2), stride=(1, 2), padding=(0, 0), bias=True),
            nn.BatchNorm2d(1, eps=0.001), nn.ReLU(inplace=True), nn.Conv2d(
            1, 1, kernel_size=(5, 1), stride=1, padding=(2, 0), bias=True),
            nn.Conv2d(1, 1, kernel_size=(1, 5), stride=1, padding=(0, 2),
            bias=True), nn.BatchNorm2d(1, eps=0.001), nn.ReLU(inplace=True))
        self.down3 = nn.Sequential(nn.Conv2d(1, 1, kernel_size=(2, 1),
            stride=(2, 1), padding=(0, 0), bias=True), nn.Conv2d(1, 1,
            kernel_size=(1, 2), stride=(1, 2), padding=(0, 0), bias=True),
            nn.BatchNorm2d(1, eps=0.001), nn.ReLU(inplace=True), nn.Conv2d(
            1, 1, kernel_size=(3, 1), stride=1, padding=(1, 0), bias=True),
            nn.Conv2d(1, 1, kernel_size=(1, 3), stride=1, padding=(0, 1),
            bias=True), nn.BatchNorm2d(1, eps=0.001), nn.ReLU(inplace=True))
        self.conv2 = nn.Sequential(nn.Conv2d(1, 1, kernel_size=(5, 1),
            stride=1, padding=(2, 0), bias=True), nn.Conv2d(1, 1,
            kernel_size=(1, 5), stride=1, padding=(0, 2), bias=True), nn.
            BatchNorm2d(1, eps=0.001), nn.ReLU(inplace=True))
        self.conv1 = nn.Sequential(nn.Conv2d(1, 1, kernel_size=(7, 1),
            stride=1, padding=(3, 0), bias=True), nn.Conv2d(1, 1,
            kernel_size=(1, 7), stride=1, padding=(0, 3), bias=True), nn.
            BatchNorm2d(1, eps=0.001), nn.ReLU(inplace=True))

    def forward(self, x):
        h, w = x.size()[2:]
        b1 = self.branch1(x)
        b1 = F.interpolate(b1, size=(h, w), mode='bilinear', align_corners=True
            )
        mid = self.mid(x)
        x1 = self.down1(x)
        x2 = self.down2(x1)
        x3 = self.down3(x2)
        x3 = F.interpolate(x3, size=((h + 3) // 4, (w + 3) // 4), mode=
            'bilinear', align_corners=True)
        x2 = self.conv2(x2)
        x = x2 + x3
        x = F.interpolate(x, size=((h + 1) // 2, (w + 1) // 2), mode=
            'bilinear', align_corners=True)
        x1 = self.conv1(x1)
        x = x + x1
        x = F.interpolate(x, size=(h, w), mode='bilinear', align_corners=True)
        x = torch.mul(x, mid)
        x = x + b1
        return x


class LEDNetG3(nn.Module):

    def __init__(self, classes):
        super().__init__()
        self.initial_block = DownsamplerBlock(3, 32)
        self.layers = nn.ModuleList()
        for x in range(0, 3):
            self.layers.append(SS_nbt_module_paper(32, 0.03, 1))
        self.layers.append(DownsamplerBlock(32, 64))
        for x in range(0, 2):
            self.layers.append(SS_nbt_module_paper(64, 0.03, 1))
        self.layers.append(DownsamplerBlock(64, 128))
        for x in range(0, 1):
            self.layers.append(SS_nbt_module_paper(128, 0.3, 1))
            self.layers.append(SS_nbt_module_paper(128, 0.3, 2))
            self.layers.append(SS_nbt_module_paper(128, 0.3, 5))
            self.layers.append(SS_nbt_module_paper(128, 0.3, 9))
        for x in range(0, 1):
            self.layers.append(SS_nbt_module_paper(128, 0.3, 2))
            self.layers.append(SS_nbt_module_paper(128, 0.3, 5))
            self.layers.append(SS_nbt_module_paper(128, 0.3, 9))
            self.layers.append(SS_nbt_module_paper(128, 0.3, 17))
        self.apn = APNModule(in_ch=128, out_ch=classes)

    def forward(self, input):
        output = self.initial_block(input)
        for layer in self.layers:
            output = layer(output)
        output = self.apn(output)
        out = F.interpolate(output, input.size()[2:], mode='bilinear',
            align_corners=True)
        return out


<mask token>

<mask token>


def Channel_shuffle(x, groups):
    batchsize, num_channels, height, width = x.data.size()
    channels_per_group = num_channels // groups
    x = x.view(batchsize, groups, channels_per_group, height, width)
    x = torch.transpose(x, 1, 2).contiguous()
    x = x.view(batchsize, -1, height, width)
    return x


class PermutationBlock(nn.Module):

    def __init__(self, groups):
        super(PermutationBlock, self).__init__()
        self.groups = groups

    def forward(self, input):
        n, c, h, w = input.size()
        G = self.groups
        output = input.view(n, G, c // G, h, w).permute(0, 2, 1, 3, 4
            ).contiguous().view(n, c, h, w)
        return output


class Conv2dBnRelu(nn.Module):

    def __init__(self, in_ch, out_ch, kernel_size=3, stride=1, padding=0,
        dilation=1, bias=True):
        super(Conv2dBnRelu, self).__init__()
        self.conv = nn.Sequential(nn.Conv2d(in_ch, out_ch, kernel_size,
            stride, padding, dilation=dilation, bias=bias), nn.BatchNorm2d(
            out_ch, eps=0.001), nn.ReLU(inplace=True))

    def forward(self, x):
        return self.conv(x)


class DownsamplerBlock(nn.Module):

    def __init__(self, ninput, noutput):
        super().__init__()
        self.conv = nn.Conv2d(ninput, noutput - ninput, (2, 2), stride=2,
            padding=0, bias=True)
        self.pool = nn.MaxPool2d(2, stride=2)
        self.bn = nn.BatchNorm2d(noutput, eps=0.001)
        self.relu = nn.ReLU(inplace=True)

    def forward(self, input):
        x1 = self.pool(input)
        x2 = self.conv(input)
        diffY = x2.size()[2] - x1.size()[2]
        diffX = x2.size()[3] - x1.size()[3]
        x1 = F.pad(x1, [diffX // 2, diffX - diffX // 2, diffY // 2, diffY -
            diffY // 2])
        output = torch.cat([x2, x1], 1)
        output = self.bn(output)
        output = self.relu(output)
        return output


class SS_nbt_module_paper(nn.Module):

    def __init__(self, chann, dropprob, dilated):
        super().__init__()
        oup_inc = chann // 2
        self.conv3x1_1_l = nn.Conv2d(oup_inc, oup_inc, (3, 1), stride=1,
            padding=(1, 0), bias=True)
        self.conv1x3_1_l = nn.Conv2d(oup_inc, oup_inc, (1, 3), stride=1,
            padding=(0, 1), bias=True)
        self.bn1_l = nn.BatchNorm2d(oup_inc, eps=0.001)
        self.conv3x1_2_l = nn.Conv2d(oup_inc, oup_inc, (3, 1), stride=1,
            padding=(1 * dilated, 0), bias=True, dilation=(dilated, 1))
        self.conv1x3_2_l = nn.Conv2d(oup_inc, oup_inc, (1, 3), stride=1,
            padding=(0, 1 * dilated), bias=True, dilation=(1, dilated))
        self.bn2_l = nn.BatchNorm2d(oup_inc, eps=0.001)
        self.conv3x1_1_r = nn.Conv2d(oup_inc, oup_inc, (3, 1), stride=1,
            padding=(1, 0), bias=True)
        self.conv1x3_1_r = nn.Conv2d(oup_inc, oup_inc, (1, 3), stride=1,
            padding=(0, 1), bias=True)
        self.bn1_r = nn.BatchNorm2d(oup_inc, eps=0.001)
        self.conv3x1_2_r = nn.Conv2d(oup_inc, oup_inc, (3, 1), stride=1,
            padding=(1 * dilated, 0), bias=True, dilation=(dilated, 1))
        self.conv1x3_2_r = nn.Conv2d(oup_inc, oup_inc, (1, 3), stride=1,
            padding=(0, 1 * dilated), bias=True, dilation=(1, dilated))
        self.bn2_r = nn.BatchNorm2d(oup_inc, eps=0.001)
        self.relu = nn.ReLU(inplace=True)
        self.dropout = nn.Dropout2d(dropprob)

    def forward(self, x):
        residual = x
        x1, x2 = Split(x)
        output1 = self.conv3x1_1_l(x1)
        output1 = self.relu(output1)
        output1 = self.conv1x3_1_l(output1)
        output1 = self.bn1_l(output1)
        output1_mid = self.relu(output1)
        output2 = self.conv1x3_1_r(x2)
        output2 = self.relu(output2)
        output2 = self.conv3x1_1_r(output2)
        output2 = self.bn1_r(output2)
        output2_mid = self.relu(output2)
        output1 = self.conv3x1_2_l(output1_mid)
        output1 = self.relu(output1)
        output1 = self.conv1x3_2_l(output1)
        output1 = self.bn2_l(output1)
        output2 = self.conv1x3_2_r(output2_mid)
        output2 = self.relu(output2)
        output2 = self.conv3x1_2_r(output2)
        output2 = self.bn2_r(output2)
        if self.dropout.p != 0:
            output1 = self.dropout(output1)
            output2 = self.dropout(output2)
        out = Merge(output1, output2)
        out = F.relu(residual + out)
        out = Channel_shuffle(out, 2)
        return out


class APNModule(nn.Module):

    def __init__(self, in_ch, out_ch):
        super(APNModule, self).__init__()
        self.branch1 = nn.Sequential(nn.AdaptiveAvgPool2d(1), Conv2dBnRelu(
            in_ch, out_ch, kernel_size=1, stride=1, padding=0))
        self.mid = nn.Sequential(Conv2dBnRelu(in_ch, out_ch, kernel_size=1,
            stride=1, padding=0))
        self.down1 = nn.Sequential(nn.Conv2d(in_ch, 1, kernel_size=(2, 1),
            stride=(2, 1), padding=(0, 0), bias=True), nn.Conv2d(1, 1,
            kernel_size=(1, 2), stride=(1, 2), padding=(0, 0), bias=True),
            nn.BatchNorm2d(1, eps=0.001), nn.ReLU(inplace=True), nn.Conv2d(
            1, 1, kernel_size=(7, 1), stride=1, padding=(3, 0), bias=True),
            nn.Conv2d(1, 1, kernel_size=(1, 7), stride=1, padding=(0, 3),
            bias=True), nn.BatchNorm2d(1, eps=0.001), nn.ReLU(inplace=True))
        self.down2 = nn.Sequential(nn.Conv2d(1, 1, kernel_size=(2, 1),
            stride=(2, 1), padding=(0, 0), bias=True), nn.Conv2d(1, 1,
            kernel_size=(1, 2), stride=(1, 2), padding=(0, 0), bias=True),
            nn.BatchNorm2d(1, eps=0.001), nn.ReLU(inplace=True), nn.Conv2d(
            1, 1, kernel_size=(5, 1), stride=1, padding=(2, 0), bias=True),
            nn.Conv2d(1, 1, kernel_size=(1, 5), stride=1, padding=(0, 2),
            bias=True), nn.BatchNorm2d(1, eps=0.001), nn.ReLU(inplace=True))
        self.down3 = nn.Sequential(nn.Conv2d(1, 1, kernel_size=(2, 1),
            stride=(2, 1), padding=(0, 0), bias=True), nn.Conv2d(1, 1,
            kernel_size=(1, 2), stride=(1, 2), padding=(0, 0), bias=True),
            nn.BatchNorm2d(1, eps=0.001), nn.ReLU(inplace=True), nn.Conv2d(
            1, 1, kernel_size=(3, 1), stride=1, padding=(1, 0), bias=True),
            nn.Conv2d(1, 1, kernel_size=(1, 3), stride=1, padding=(0, 1),
            bias=True), nn.BatchNorm2d(1, eps=0.001), nn.ReLU(inplace=True))
        self.conv2 = nn.Sequential(nn.Conv2d(1, 1, kernel_size=(5, 1),
            stride=1, padding=(2, 0), bias=True), nn.Conv2d(1, 1,
            kernel_size=(1, 5), stride=1, padding=(0, 2), bias=True), nn.
            BatchNorm2d(1, eps=0.001), nn.ReLU(inplace=True))
        self.conv1 = nn.Sequential(nn.Conv2d(1, 1, kernel_size=(7, 1),
            stride=1, padding=(3, 0), bias=True), nn.Conv2d(1, 1,
            kernel_size=(1, 7), stride=1, padding=(0, 3), bias=True), nn.
            BatchNorm2d(1, eps=0.001), nn.ReLU(inplace=True))

    def forward(self, x):
        h, w = x.size()[2:]
        b1 = self.branch1(x)
        b1 = F.interpolate(b1, size=(h, w), mode='bilinear', align_corners=True
            )
        mid = self.mid(x)
        x1 = self.down1(x)
        x2 = self.down2(x1)
        x3 = self.down3(x2)
        x3 = F.interpolate(x3, size=((h + 3) // 4, (w + 3) // 4), mode=
            'bilinear', align_corners=True)
        x2 = self.conv2(x2)
        x = x2 + x3
        x = F.interpolate(x, size=((h + 1) // 2, (w + 1) // 2), mode=
            'bilinear', align_corners=True)
        x1 = self.conv1(x1)
        x = x + x1
        x = F.interpolate(x, size=(h, w), mode='bilinear', align_corners=True)
        x = torch.mul(x, mid)
        x = x + b1
        return x


class LEDNetG3(nn.Module):

    def __init__(self, classes):
        super().__init__()
        self.initial_block = DownsamplerBlock(3, 32)
        self.layers = nn.ModuleList()
        for x in range(0, 3):
            self.layers.append(SS_nbt_module_paper(32, 0.03, 1))
        self.layers.append(DownsamplerBlock(32, 64))
        for x in range(0, 2):
            self.layers.append(SS_nbt_module_paper(64, 0.03, 1))
        self.layers.append(DownsamplerBlock(64, 128))
        for x in range(0, 1):
            self.layers.append(SS_nbt_module_paper(128, 0.3, 1))
            self.layers.append(SS_nbt_module_paper(128, 0.3, 2))
            self.layers.append(SS_nbt_module_paper(128, 0.3, 5))
            self.layers.append(SS_nbt_module_paper(128, 0.3, 9))
        for x in range(0, 1):
            self.layers.append(SS_nbt_module_paper(128, 0.3, 2))
            self.layers.append(SS_nbt_module_paper(128, 0.3, 5))
            self.layers.append(SS_nbt_module_paper(128, 0.3, 9))
            self.layers.append(SS_nbt_module_paper(128, 0.3, 17))
        self.apn = APNModule(in_ch=128, out_ch=classes)

    def forward(self, input):
        output = self.initial_block(input)
        for layer in self.layers:
            output = layer(output)
        output = self.apn(output)
        out = F.interpolate(output, input.size()[2:], mode='bilinear',
            align_corners=True)
        return out


<mask token>

<mask token>


def Split(x):
    c = int(x.size()[1])
    c1 = round(c * 0.5)
    x1 = x[:, :c1, :, :].contiguous()
    x2 = x[:, c1:, :, :].contiguous()
    return x1, x2


def Merge(x1, x2):
    return torch.cat((x1, x2), 1)


def Channel_shuffle(x, groups):
    batchsize, num_channels, height, width = x.data.size()
    channels_per_group = num_channels // groups
    x = x.view(batchsize, groups, channels_per_group, height, width)
    x = torch.transpose(x, 1, 2).contiguous()
    x = x.view(batchsize, -1, height, width)
    return x


class PermutationBlock(nn.Module):

    def __init__(self, groups):
        super(PermutationBlock, self).__init__()
        self.groups = groups

    def forward(self, input):
        n, c, h, w = input.size()
        G = self.groups
        output = input.view(n, G, c // G, h, w).permute(0, 2, 1, 3, 4
            ).contiguous().view(n, c, h, w)
        return output


class Conv2dBnRelu(nn.Module):

    def __init__(self, in_ch, out_ch, kernel_size=3, stride=1, padding=0,
        dilation=1, bias=True):
        super(Conv2dBnRelu, self).__init__()
        self.conv = nn.Sequential(nn.Conv2d(in_ch, out_ch, kernel_size,
            stride, padding, dilation=dilation, bias=bias), nn.BatchNorm2d(
            out_ch, eps=0.001), nn.ReLU(inplace=True))

    def forward(self, x):
        return self.conv(x)


class DownsamplerBlock(nn.Module):

    def __init__(self, ninput, noutput):
        super().__init__()
        self.conv = nn.Conv2d(ninput, noutput - ninput, (2, 2), stride=2,
            padding=0, bias=True)
        self.pool = nn.MaxPool2d(2, stride=2)
        self.bn = nn.BatchNorm2d(noutput, eps=0.001)
        self.relu = nn.ReLU(inplace=True)

    def forward(self, input):
        x1 = self.pool(input)
        x2 = self.conv(input)
        diffY = x2.size()[2] - x1.size()[2]
        diffX = x2.size()[3] - x1.size()[3]
        x1 = F.pad(x1, [diffX // 2, diffX - diffX // 2, diffY // 2, diffY -
            diffY // 2])
        output = torch.cat([x2, x1], 1)
        output = self.bn(output)
        output = self.relu(output)
        return output


class SS_nbt_module_paper(nn.Module):

    def __init__(self, chann, dropprob, dilated):
        super().__init__()
        oup_inc = chann // 2
        self.conv3x1_1_l = nn.Conv2d(oup_inc, oup_inc, (3, 1), stride=1,
            padding=(1, 0), bias=True)
        self.conv1x3_1_l = nn.Conv2d(oup_inc, oup_inc, (1, 3), stride=1,
            padding=(0, 1), bias=True)
        self.bn1_l = nn.BatchNorm2d(oup_inc, eps=0.001)
        self.conv3x1_2_l = nn.Conv2d(oup_inc, oup_inc, (3, 1), stride=1,
            padding=(1 * dilated, 0), bias=True, dilation=(dilated, 1))
        self.conv1x3_2_l = nn.Conv2d(oup_inc, oup_inc, (1, 3), stride=1,
            padding=(0, 1 * dilated), bias=True, dilation=(1, dilated))
        self.bn2_l = nn.BatchNorm2d(oup_inc, eps=0.001)
        self.conv3x1_1_r = nn.Conv2d(oup_inc, oup_inc, (3, 1), stride=1,
            padding=(1, 0), bias=True)
        self.conv1x3_1_r = nn.Conv2d(oup_inc, oup_inc, (1, 3), stride=1,
            padding=(0, 1), bias=True)
        self.bn1_r = nn.BatchNorm2d(oup_inc, eps=0.001)
        self.conv3x1_2_r = nn.Conv2d(oup_inc, oup_inc, (3, 1), stride=1,
            padding=(1 * dilated, 0), bias=True, dilation=(dilated, 1))
        self.conv1x3_2_r = nn.Conv2d(oup_inc, oup_inc, (1, 3), stride=1,
            padding=(0, 1 * dilated), bias=True, dilation=(1, dilated))
        self.bn2_r = nn.BatchNorm2d(oup_inc, eps=0.001)
        self.relu = nn.ReLU(inplace=True)
        self.dropout = nn.Dropout2d(dropprob)

    def forward(self, x):
        residual = x
        x1, x2 = Split(x)
        output1 = self.conv3x1_1_l(x1)
        output1 = self.relu(output1)
        output1 = self.conv1x3_1_l(output1)
        output1 = self.bn1_l(output1)
        output1_mid = self.relu(output1)
        output2 = self.conv1x3_1_r(x2)
        output2 = self.relu(output2)
        output2 = self.conv3x1_1_r(output2)
        output2 = self.bn1_r(output2)
        output2_mid = self.relu(output2)
        output1 = self.conv3x1_2_l(output1_mid)
        output1 = self.relu(output1)
        output1 = self.conv1x3_2_l(output1)
        output1 = self.bn2_l(output1)
        output2 = self.conv1x3_2_r(output2_mid)
        output2 = self.relu(output2)
        output2 = self.conv3x1_2_r(output2)
        output2 = self.bn2_r(output2)
        if self.dropout.p != 0:
            output1 = self.dropout(output1)
            output2 = self.dropout(output2)
        out = Merge(output1, output2)
        out = F.relu(residual + out)
        out = Channel_shuffle(out, 2)
        return out


class APNModule(nn.Module):

    def __init__(self, in_ch, out_ch):
        super(APNModule, self).__init__()
        self.branch1 = nn.Sequential(nn.AdaptiveAvgPool2d(1), Conv2dBnRelu(
            in_ch, out_ch, kernel_size=1, stride=1, padding=0))
        self.mid = nn.Sequential(Conv2dBnRelu(in_ch, out_ch, kernel_size=1,
            stride=1, padding=0))
        self.down1 = nn.Sequential(nn.Conv2d(in_ch, 1, kernel_size=(2, 1),
            stride=(2, 1), padding=(0, 0), bias=True), nn.Conv2d(1, 1,
            kernel_size=(1, 2), stride=(1, 2), padding=(0, 0), bias=True),
            nn.BatchNorm2d(1, eps=0.001), nn.ReLU(inplace=True), nn.Conv2d(
            1, 1, kernel_size=(7, 1), stride=1, padding=(3, 0), bias=True),
            nn.Conv2d(1, 1, kernel_size=(1, 7), stride=1, padding=(0, 3),
            bias=True), nn.BatchNorm2d(1, eps=0.001), nn.ReLU(inplace=True))
        self.down2 = nn.Sequential(nn.Conv2d(1, 1, kernel_size=(2, 1),
            stride=(2, 1), padding=(0, 0), bias=True), nn.Conv2d(1, 1,
            kernel_size=(1, 2), stride=(1, 2), padding=(0, 0), bias=True),
            nn.BatchNorm2d(1, eps=0.001), nn.ReLU(inplace=True), nn.Conv2d(
            1, 1, kernel_size=(5, 1), stride=1, padding=(2, 0), bias=True),
            nn.Conv2d(1, 1, kernel_size=(1, 5), stride=1, padding=(0, 2),
            bias=True), nn.BatchNorm2d(1, eps=0.001), nn.ReLU(inplace=True))
        self.down3 = nn.Sequential(nn.Conv2d(1, 1, kernel_size=(2, 1),
            stride=(2, 1), padding=(0, 0), bias=True), nn.Conv2d(1, 1,
            kernel_size=(1, 2), stride=(1, 2), padding=(0, 0), bias=True),
            nn.BatchNorm2d(1, eps=0.001), nn.ReLU(inplace=True), nn.Conv2d(
            1, 1, kernel_size=(3, 1), stride=1, padding=(1, 0), bias=True),
            nn.Conv2d(1, 1, kernel_size=(1, 3), stride=1, padding=(0, 1),
            bias=True), nn.BatchNorm2d(1, eps=0.001), nn.ReLU(inplace=True))
        self.conv2 = nn.Sequential(nn.Conv2d(1, 1, kernel_size=(5, 1),
            stride=1, padding=(2, 0), bias=True), nn.Conv2d(1, 1,
            kernel_size=(1, 5), stride=1, padding=(0, 2), bias=True), nn.
            BatchNorm2d(1, eps=0.001), nn.ReLU(inplace=True))
        self.conv1 = nn.Sequential(nn.Conv2d(1, 1, kernel_size=(7, 1),
            stride=1, padding=(3, 0), bias=True), nn.Conv2d(1, 1,
            kernel_size=(1, 7), stride=1, padding=(0, 3), bias=True), nn.
            BatchNorm2d(1, eps=0.001), nn.ReLU(inplace=True))

    def forward(self, x):
        h, w = x.size()[2:]
        b1 = self.branch1(x)
        b1 = F.interpolate(b1, size=(h, w), mode='bilinear', align_corners=True
            )
        mid = self.mid(x)
        x1 = self.down1(x)
        x2 = self.down2(x1)
        x3 = self.down3(x2)
        x3 = F.interpolate(x3, size=((h + 3) // 4, (w + 3) // 4), mode=
            'bilinear', align_corners=True)
        x2 = self.conv2(x2)
        x = x2 + x3
        x = F.interpolate(x, size=((h + 1) // 2, (w + 1) // 2), mode=
            'bilinear', align_corners=True)
        x1 = self.conv1(x1)
        x = x + x1
        x = F.interpolate(x, size=(h, w), mode='bilinear', align_corners=True)
        x = torch.mul(x, mid)
        x = x + b1
        return x


class LEDNetG3(nn.Module):

    def __init__(self, classes):
        super().__init__()
        self.initial_block = DownsamplerBlock(3, 32)
        self.layers = nn.ModuleList()
        for x in range(0, 3):
            self.layers.append(SS_nbt_module_paper(32, 0.03, 1))
        self.layers.append(DownsamplerBlock(32, 64))
        for x in range(0, 2):
            self.layers.append(SS_nbt_module_paper(64, 0.03, 1))
        self.layers.append(DownsamplerBlock(64, 128))
        for x in range(0, 1):
            self.layers.append(SS_nbt_module_paper(128, 0.3, 1))
            self.layers.append(SS_nbt_module_paper(128, 0.3, 2))
            self.layers.append(SS_nbt_module_paper(128, 0.3, 5))
            self.layers.append(SS_nbt_module_paper(128, 0.3, 9))
        for x in range(0, 1):
            self.layers.append(SS_nbt_module_paper(128, 0.3, 2))
            self.layers.append(SS_nbt_module_paper(128, 0.3, 5))
            self.layers.append(SS_nbt_module_paper(128, 0.3, 9))
            self.layers.append(SS_nbt_module_paper(128, 0.3, 17))
        self.apn = APNModule(in_ch=128, out_ch=classes)

    def forward(self, input):
        output = self.initial_block(input)
        for layer in self.layers:
            output = layer(output)
        output = self.apn(output)
        out = F.interpolate(output, input.size()[2:], mode='bilinear',
            align_corners=True)
        return out


<mask token>

######################################################################################
#LEDNet: A Lightweight Encoder-Decoder Network for Real-Time Semantic Segmentation
#Paper-Link: https://arxiv.org/abs/1905.02423
######################################################################################


import torch
import torch.nn as nn
import torch.nn.functional as F
from torchsummary import summary  



__all__ = ["LEDNetG3"]

def Split(x):
    c = int(x.size()[1])
    c1 = round(c * 0.5)
    x1 = x[:, :c1, :, :].contiguous() #把tensor变成在内存中连续分布的形式
    x2 = x[:, c1:, :, :].contiguous()
    return x1, x2 


def Merge(x1,x2):
    return torch.cat((x1,x2),1) 
    

def Channel_shuffle(x,groups):
    batchsize, num_channels, height, width = x.data.size()
    
    channels_per_group = num_channels // groups
    
    #reshape
    x = x.view(batchsize,groups,
        channels_per_group,height,width)
    
    x = torch.transpose(x,1,2).contiguous()
    
    #flatten
    x = x.view(batchsize,-1,height,width)
    
    return x


class PermutationBlock(nn.Module):
    def __init__(self, groups):
        super(PermutationBlock, self).__init__()
        self.groups = groups

    def forward(self, input):
        n, c, h, w = input.size()
        G = self.groups
        output = input.view(n, G, c // G, h, w).permute(0, 2, 1, 3, 4).contiguous().view(n, c, h, w)
        return output



class Conv2dBnRelu(nn.Module):
    def __init__(self,in_ch,out_ch,kernel_size=3,stride=1,padding=0,dilation=1,bias=True):
        super(Conv2dBnRelu,self).__init__()
        self.conv = nn.Sequential(
            nn.Conv2d(in_ch,out_ch,kernel_size,stride,padding,dilation=dilation,bias=bias),
            nn.BatchNorm2d(out_ch, eps=1e-3),
            nn.ReLU(inplace=True)
        )

    def forward(self, x):
        return self.conv(x)


class DownsamplerBlock(nn.Module):
    def __init__(self, ninput, noutput):
        super().__init__()

        self.conv = nn.Conv2d(ninput, noutput-ninput, (2, 2), stride=2, padding=0, bias=True)
        self.pool = nn.MaxPool2d(2, stride=2)
        self.bn = nn.BatchNorm2d(noutput, eps=1e-3)
        self.relu = nn.ReLU(inplace=True)

    def forward(self, input):
        x1 = self.pool(input)
        x2 = self.conv(input)

        diffY = x2.size()[2] - x1.size()[2]
        diffX = x2.size()[3] - x1.size()[3]

        x1 = F.pad(x1, [diffX // 2, diffX - diffX // 2,
                        diffY // 2, diffY - diffY // 2])

        output = torch.cat([x2, x1], 1)
        output = self.bn(output)
        output = self.relu(output)
        return output



# class Interpolate(nn.Module):
#     def __init__(self,size,mode):
#         super(Interpolate,self).__init__()
#         self.size = size
#         self.mode = mode
#     def forward(self,x):
#         x = F.interpolate(x,size=self.size,mode=self.mode,align_corners=True)
#         return x

        

class SS_nbt_module_paper(nn.Module):
    def __init__(self, chann, dropprob, dilated):        
        super().__init__()

        oup_inc = chann//2
        
        #dw
        self.conv3x1_1_l = nn.Conv2d(oup_inc, oup_inc, (3,1), stride=1, padding=(1,0), bias=True)

        self.conv1x3_1_l = nn.Conv2d(oup_inc, oup_inc, (1,3), stride=1, padding=(0,1), bias=True)

        self.bn1_l = nn.BatchNorm2d(oup_inc, eps=1e-03)

        self.conv3x1_2_l = nn.Conv2d(oup_inc, oup_inc, (3,1), stride=1, padding=(1*dilated,0), bias=True, dilation = (dilated,1))

        self.conv1x3_2_l = nn.Conv2d(oup_inc, oup_inc, (1,3), stride=1, padding=(0,1*dilated), bias=True, dilation = (1,dilated))

        self.bn2_l = nn.BatchNorm2d(oup_inc, eps=1e-03)
        
        #dw
        self.conv3x1_1_r = nn.Conv2d(oup_inc, oup_inc, (3,1), stride=1, padding=(1,0), bias=True)

        self.conv1x3_1_r = nn.Conv2d(oup_inc, oup_inc, (1,3), stride=1, padding=(0,1), bias=True)

        self.bn1_r = nn.BatchNorm2d(oup_inc, eps=1e-03)

        self.conv3x1_2_r = nn.Conv2d(oup_inc, oup_inc, (3,1), stride=1, padding=(1*dilated,0), bias=True, dilation = (dilated,1))

        self.conv1x3_2_r = nn.Conv2d(oup_inc, oup_inc, (1,3), stride=1, padding=(0,1*dilated), bias=True, dilation = (1,dilated))

        self.bn2_r = nn.BatchNorm2d(oup_inc, eps=1e-03)       
        
        self.relu = nn.ReLU(inplace=True)
        self.dropout = nn.Dropout2d(dropprob)

        # self.channel_shuffle = PermutationBlock(2)
       
    
    def forward(self, x):
    
        residual = x
    
        x1, x2 = Split(x)
    
        output1 = self.conv3x1_1_l(x1)
        output1 = self.relu(output1)
        output1 = self.conv1x3_1_l(output1)
        output1 = self.bn1_l(output1)
        output1_mid = self.relu(output1)

        output2 = self.conv1x3_1_r(x2)
        output2 = self.relu(output2)
        output2 = self.conv3x1_1_r(output2)
        output2 = self.bn1_r(output2)
        output2_mid = self.relu(output2)

        output1 = self.conv3x1_2_l(output1_mid)
        output1 = self.relu(output1)
        output1 = self.conv1x3_2_l(output1)
        output1 = self.bn2_l(output1)
      
        output2 = self.conv1x3_2_r(output2_mid)
        output2 = self.relu(output2)
        output2 = self.conv3x1_2_r(output2)
        output2 = self.bn2_r(output2)

        if (self.dropout.p != 0):
            output1 = self.dropout(output1)
            output2 = self.dropout(output2)

        out = Merge(output1, output2)
        
        out = F.relu(residual + out)

        # out = self.channel_shuffle(out)   ### channel shuffle
        out = Channel_shuffle(out,2)   ### channel shuffle

        return out

        # return    ### channel shuffle


class APNModule(nn.Module):
    def __init__(self, in_ch, out_ch):
        super(APNModule, self).__init__()
        # global pooling branch
        self.branch1 = nn.Sequential(
                nn.AdaptiveAvgPool2d(1),
                Conv2dBnRelu(in_ch, out_ch, kernel_size=1, stride=1, padding=0)
        )

        # midddle branch
        self.mid = nn.Sequential(
            Conv2dBnRelu(in_ch, out_ch, kernel_size=1, stride=1, padding=0)
        )


        self.down1 = nn.Sequential(
            # nn.Conv2d(in_ch,1,kernel_size=(7,1),stride=(2,1),padding=(3,0),bias=True),
            # nn.Conv2d(1,1,kernel_size=(1,7),stride=(1,2),padding=(0,3),bias=True),
            nn.Conv2d(in_ch, 1, kernel_size=(2, 1), stride=(2, 1), padding=(0, 0), bias=True),
            nn.Conv2d(1, 1, kernel_size=(1, 2), stride=(1, 2), padding=(0, 0), bias=True),
            nn.BatchNorm2d(1, eps=1e-03),
            nn.ReLU(inplace=True),
            nn.Conv2d(1, 1, kernel_size=(7, 1), stride=1, padding=(3, 0), bias=True),
            nn.Conv2d(1, 1, kernel_size=(1, 7), stride=1, padding=(0, 3), bias=True),
            nn.BatchNorm2d(1, eps=1e-03),
            nn.ReLU(inplace=True)
        )


        self.down2 = nn.Sequential(
            # nn.Conv2d(1,1,kernel_size=(5,1),stride=(2,1),padding=(2,0),bias=True),
            # nn.Conv2d(1,1,kernel_size=(1,5),stride=(1,2),padding=(0,2),bias=True),
            nn.Conv2d(1, 1, kernel_size=(2, 1), stride=(2, 1), padding=(0, 0), bias=True),
            nn.Conv2d(1, 1, kernel_size=(1, 2), stride=(1, 2), padding=(0, 0), bias=True),
            nn.BatchNorm2d(1, eps=1e-03),
            nn.ReLU(inplace=True),
            nn.Conv2d(1,1,kernel_size=(5,1),stride=1,padding=(2,0),bias=True),
            nn.Conv2d(1,1,kernel_size=(1,5),stride=1,padding=(0,2),bias=True),
            nn.BatchNorm2d(1, eps=1e-03),
            nn.ReLU(inplace=True)
        )

        self.down3 = nn.Sequential(
            # nn.Conv2d(1,1,kernel_size=(3,1),stride=(2,1),padding=(1,0),bias=True),
            # nn.Conv2d(1,1,kernel_size=(1,3),stride=(1,2),padding=(0,1),bias=True),
            nn.Conv2d(1, 1, kernel_size=(2, 1), stride=(2, 1), padding=(0, 0), bias=True),
            nn.Conv2d(1, 1, kernel_size=(1, 2), stride=(1, 2), padding=(0, 0), bias=True),
            nn.BatchNorm2d(1, eps=1e-03),
            nn.ReLU(inplace=True),
            #
            nn.Conv2d(1,1,kernel_size=(3,1),stride=1,padding=(1,0),bias=True),
            nn.Conv2d(1,1,kernel_size=(1,3),stride=1,padding=(0,1),bias=True),
            nn.BatchNorm2d(1, eps=1e-03),
            nn.ReLU(inplace=True)
        )
        self.conv2 = nn.Sequential(
            nn.Conv2d(1,1,kernel_size=(5,1),stride=1,padding=(2,0),bias=True),
            nn.Conv2d(1,1,kernel_size=(1,5),stride=1,padding=(0,2),bias=True),
            nn.BatchNorm2d(1, eps=1e-03),
            nn.ReLU(inplace=True)
        )
        self.conv1 = nn.Sequential(
            nn.Conv2d(1,1,kernel_size=(7,1),stride=1,padding=(3,0),bias=True),
            nn.Conv2d(1,1,kernel_size=(1,7),stride=1,padding=(0,3),bias=True),
            nn.BatchNorm2d(1, eps=1e-03),
            nn.ReLU(inplace=True)
        )
    def forward(self, x):

        h,w = x.size()[2:]

        b1 = self.branch1(x)
        b1= F.interpolate(b1, size=(h, w), mode="bilinear", align_corners=True)

        mid = self.mid(x)

        x1 = self.down1(x)
        x2 = self.down2(x1)
        x3 = self.down3(x2)
        x3= F.interpolate(x3, size=((h+3) // 4, (w+3) // 4), mode="bilinear", align_corners=True)

        x2 = self.conv2(x2)
        x = x2 + x3
        x= F.interpolate(x, size=((h+1) // 2, (w+1) // 2), mode="bilinear", align_corners=True)


        x1 = self.conv1(x1)
        x = x + x1
        x= F.interpolate(x, size=(h, w), mode="bilinear", align_corners=True)

        x = torch.mul(x, mid)

        x = x + b1

        return x


class LEDNetG3(nn.Module):
    def __init__(self, classes):
        super().__init__()
        self.initial_block = DownsamplerBlock(3,32)
        
        self.layers = nn.ModuleList()

        for x in range(0, 3):   
           self.layers.append(SS_nbt_module_paper(32, 0.03, 1)) 
        

        self.layers.append(DownsamplerBlock(32,64))
        

        for x in range(0, 2):   
           self.layers.append(SS_nbt_module_paper(64, 0.03, 1)) 
  
        self.layers.append(DownsamplerBlock(64,128))

        for x in range(0, 1):    
            self.layers.append(SS_nbt_module_paper(128, 0.3, 1))
            self.layers.append(SS_nbt_module_paper(128, 0.3, 2))
            self.layers.append(SS_nbt_module_paper(128, 0.3, 5))
            self.layers.append(SS_nbt_module_paper(128, 0.3, 9))
            
        for x in range(0, 1):    
            self.layers.append(SS_nbt_module_paper(128, 0.3, 2))
            self.layers.append(SS_nbt_module_paper(128, 0.3, 5))
            self.layers.append(SS_nbt_module_paper(128, 0.3, 9))
            self.layers.append(SS_nbt_module_paper(128, 0.3, 17))
                    
        self.apn = APNModule(in_ch=128,out_ch=classes)

        #self.output_conv = nn.ConvTranspose2d(128, num_classes, kernel_size=4, stride=2, padding=1, output_padding=0, bias=True)
        #self.output_conv = nn.ConvTranspose2d(128, num_classes, kernel_size=3, stride=2, padding=1, output_padding=1, bias=True)
        #self.output_conv = nn.ConvTranspose2d(128, num_classes, kernel_size=2, stride=2, padding=0, output_padding=0, bias=True)

        # self.output_conv = nn.Conv2d(128, num_classes, 1, stride=1, padding=0, bias=True)

    def forward(self, input):
        
        output = self.initial_block(input)

        for layer in self.layers:
            output = layer(output)
            
        output = self.apn(output)
        out = F.interpolate(output, input.size()[2:], mode="bilinear", align_corners=True)
        # print(out.shape)

        return out

         

"""print layers and params of network"""
if __name__ == '__main__':
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    model = LEDNetG3(classes=19).to(device)
    summary(model,(3,512,1024))

    from fvcore.nn.flop_count import flop_count  # https://github.com/facebookresearch/fvcore
    from tools.flops_counter.ptflops import get_model_complexity_info
    from thop import profile  # https://github.com/Lyken17/pytorch-OpCounter

    x = torch.randn(2, 3, 512, 1024).to(device)

    from fvcore.nn.jit_handles import batchnorm_flop_jit
    from fvcore.nn.jit_handles import generic_activation_jit
    supported_ops = {
                        "aten::batch_norm": batchnorm_flop_jit,
                    }
    flop_dict, _ = flop_count(model, (x,), supported_ops)


    flops_count, params_count = get_model_complexity_info(model, (3, 512, 1024),
                                              as_strings=False,
                                              print_per_layer_stat=True)
    input = x
    macs, params = profile(model, inputs=(input,))
    print(flop_dict)
    print(flops_count, params_count)
    print(macs, params)

'''
/home/ethan/anaconda3/envs/py36_cuda101/bin/python /home/ethan/codes/Efficient-Segmentation-Networks/model/LEDNetG3.py
----------------------------------------------------------------
        Layer (type)               Output Shape         Param #
================================================================
         MaxPool2d-1          [-1, 3, 256, 512]               0
            Conv2d-2         [-1, 29, 256, 512]             377
       BatchNorm2d-3         [-1, 32, 256, 512]              64
              ReLU-4         [-1, 32, 256, 512]               0
  DownsamplerBlock-5         [-1, 32, 256, 512]               0
            Conv2d-6         [-1, 16, 256, 512]             784
              ReLU-7         [-1, 16, 256, 512]               0
            Conv2d-8         [-1, 16, 256, 512]             784
       BatchNorm2d-9         [-1, 16, 256, 512]              32
             ReLU-10         [-1, 16, 256, 512]               0
           Conv2d-11         [-1, 16, 256, 512]             784
             ReLU-12         [-1, 16, 256, 512]               0
           Conv2d-13         [-1, 16, 256, 512]             784
      BatchNorm2d-14         [-1, 16, 256, 512]              32
             ReLU-15         [-1, 16, 256, 512]               0
           Conv2d-16         [-1, 16, 256, 512]             784
             ReLU-17         [-1, 16, 256, 512]               0
           Conv2d-18         [-1, 16, 256, 512]             784
      BatchNorm2d-19         [-1, 16, 256, 512]              32
           Conv2d-20         [-1, 16, 256, 512]             784
             ReLU-21         [-1, 16, 256, 512]               0
           Conv2d-22         [-1, 16, 256, 512]             784
      BatchNorm2d-23         [-1, 16, 256, 512]              32
        Dropout2d-24         [-1, 16, 256, 512]               0
        Dropout2d-25         [-1, 16, 256, 512]               0
SS_nbt_module_paper-26         [-1, 32, 256, 512]               0
           Conv2d-27         [-1, 16, 256, 512]             784
             ReLU-28         [-1, 16, 256, 512]               0
           Conv2d-29         [-1, 16, 256, 512]             784
      BatchNorm2d-30         [-1, 16, 256, 512]              32
             ReLU-31         [-1, 16, 256, 512]               0
           Conv2d-32         [-1, 16, 256, 512]             784
             ReLU-33         [-1, 16, 256, 512]               0
           Conv2d-34         [-1, 16, 256, 512]             784
      BatchNorm2d-35         [-1, 16, 256, 512]              32
             ReLU-36         [-1, 16, 256, 512]               0
           Conv2d-37         [-1, 16, 256, 512]             784
             ReLU-38         [-1, 16, 256, 512]               0
           Conv2d-39         [-1, 16, 256, 512]             784
      BatchNorm2d-40         [-1, 16, 256, 512]              32
           Conv2d-41         [-1, 16, 256, 512]             784
             ReLU-42         [-1, 16, 256, 512]               0
           Conv2d-43         [-1, 16, 256, 512]             784
      BatchNorm2d-44         [-1, 16, 256, 512]              32
        Dropout2d-45         [-1, 16, 256, 512]               0
        Dropout2d-46         [-1, 16, 256, 512]               0
SS_nbt_module_paper-47         [-1, 32, 256, 512]               0
           Conv2d-48         [-1, 16, 256, 512]             784
             ReLU-49         [-1, 16, 256, 512]               0
           Conv2d-50         [-1, 16, 256, 512]             784
      BatchNorm2d-51         [-1, 16, 256, 512]              32
             ReLU-52         [-1, 16, 256, 512]               0
           Conv2d-53         [-1, 16, 256, 512]             784
             ReLU-54         [-1, 16, 256, 512]               0
           Conv2d-55         [-1, 16, 256, 512]             784
      BatchNorm2d-56         [-1, 16, 256, 512]              32
             ReLU-57         [-1, 16, 256, 512]               0
           Conv2d-58         [-1, 16, 256, 512]             784
             ReLU-59         [-1, 16, 256, 512]               0
           Conv2d-60         [-1, 16, 256, 512]             784
      BatchNorm2d-61         [-1, 16, 256, 512]              32
           Conv2d-62         [-1, 16, 256, 512]             784
             ReLU-63         [-1, 16, 256, 512]               0
           Conv2d-64         [-1, 16, 256, 512]             784
      BatchNorm2d-65         [-1, 16, 256, 512]              32
        Dropout2d-66         [-1, 16, 256, 512]               0
        Dropout2d-67         [-1, 16, 256, 512]               0
SS_nbt_module_paper-68         [-1, 32, 256, 512]               0
        MaxPool2d-69         [-1, 32, 128, 256]               0
           Conv2d-70         [-1, 32, 128, 256]           4,128
      BatchNorm2d-71         [-1, 64, 128, 256]             128
             ReLU-72         [-1, 64, 128, 256]               0
 DownsamplerBlock-73         [-1, 64, 128, 256]               0
           Conv2d-74         [-1, 32, 128, 256]           3,104
             ReLU-75         [-1, 32, 128, 256]               0
           Conv2d-76         [-1, 32, 128, 256]           3,104
      BatchNorm2d-77         [-1, 32, 128, 256]              64
             ReLU-78         [-1, 32, 128, 256]               0
           Conv2d-79         [-1, 32, 128, 256]           3,104
             ReLU-80         [-1, 32, 128, 256]               0
           Conv2d-81         [-1, 32, 128, 256]           3,104
      BatchNorm2d-82         [-1, 32, 128, 256]              64
             ReLU-83         [-1, 32, 128, 256]               0
           Conv2d-84         [-1, 32, 128, 256]           3,104
             ReLU-85         [-1, 32, 128, 256]               0
           Conv2d-86         [-1, 32, 128, 256]           3,104
      BatchNorm2d-87         [-1, 32, 128, 256]              64
           Conv2d-88         [-1, 32, 128, 256]           3,104
             ReLU-89         [-1, 32, 128, 256]               0
           Conv2d-90         [-1, 32, 128, 256]           3,104
      BatchNorm2d-91         [-1, 32, 128, 256]              64
        Dropout2d-92         [-1, 32, 128, 256]               0
        Dropout2d-93         [-1, 32, 128, 256]               0
SS_nbt_module_paper-94         [-1, 64, 128, 256]               0
           Conv2d-95         [-1, 32, 128, 256]           3,104
             ReLU-96         [-1, 32, 128, 256]               0
           Conv2d-97         [-1, 32, 128, 256]           3,104
      BatchNorm2d-98         [-1, 32, 128, 256]              64
             ReLU-99         [-1, 32, 128, 256]               0
          Conv2d-100         [-1, 32, 128, 256]           3,104
            ReLU-101         [-1, 32, 128, 256]               0
          Conv2d-102         [-1, 32, 128, 256]           3,104
     BatchNorm2d-103         [-1, 32, 128, 256]              64
            ReLU-104         [-1, 32, 128, 256]               0
          Conv2d-105         [-1, 32, 128, 256]           3,104
            ReLU-106         [-1, 32, 128, 256]               0
          Conv2d-107         [-1, 32, 128, 256]           3,104
     BatchNorm2d-108         [-1, 32, 128, 256]              64
          Conv2d-109         [-1, 32, 128, 256]           3,104
            ReLU-110         [-1, 32, 128, 256]               0
          Conv2d-111         [-1, 32, 128, 256]           3,104
     BatchNorm2d-112         [-1, 32, 128, 256]              64
       Dropout2d-113         [-1, 32, 128, 256]               0
       Dropout2d-114         [-1, 32, 128, 256]               0
SS_nbt_module_paper-115         [-1, 64, 128, 256]               0
       MaxPool2d-116          [-1, 64, 64, 128]               0
          Conv2d-117          [-1, 64, 64, 128]          16,448
     BatchNorm2d-118         [-1, 128, 64, 128]             256
            ReLU-119         [-1, 128, 64, 128]               0
DownsamplerBlock-120         [-1, 128, 64, 128]               0
          Conv2d-121          [-1, 64, 64, 128]          12,352
            ReLU-122          [-1, 64, 64, 128]               0
          Conv2d-123          [-1, 64, 64, 128]          12,352
     BatchNorm2d-124          [-1, 64, 64, 128]             128
            ReLU-125          [-1, 64, 64, 128]               0
          Conv2d-126          [-1, 64, 64, 128]          12,352
            ReLU-127          [-1, 64, 64, 128]               0
          Conv2d-128          [-1, 64, 64, 128]          12,352
     BatchNorm2d-129          [-1, 64, 64, 128]             128
            ReLU-130          [-1, 64, 64, 128]               0
          Conv2d-131          [-1, 64, 64, 128]          12,352
            ReLU-132          [-1, 64, 64, 128]               0
          Conv2d-133          [-1, 64, 64, 128]          12,352
     BatchNorm2d-134          [-1, 64, 64, 128]             128
          Conv2d-135          [-1, 64, 64, 128]          12,352
            ReLU-136          [-1, 64, 64, 128]               0
          Conv2d-137          [-1, 64, 64, 128]          12,352
     BatchNorm2d-138          [-1, 64, 64, 128]             128
       Dropout2d-139          [-1, 64, 64, 128]               0
       Dropout2d-140          [-1, 64, 64, 128]               0
SS_nbt_module_paper-141         [-1, 128, 64, 128]               0
          Conv2d-142          [-1, 64, 64, 128]          12,352
            ReLU-143          [-1, 64, 64, 128]               0
          Conv2d-144          [-1, 64, 64, 128]          12,352
     BatchNorm2d-145          [-1, 64, 64, 128]             128
            ReLU-146          [-1, 64, 64, 128]               0
          Conv2d-147          [-1, 64, 64, 128]          12,352
            ReLU-148          [-1, 64, 64, 128]               0
          Conv2d-149          [-1, 64, 64, 128]          12,352
     BatchNorm2d-150          [-1, 64, 64, 128]             128
            ReLU-151          [-1, 64, 64, 128]               0
          Conv2d-152          [-1, 64, 64, 128]          12,352
            ReLU-153          [-1, 64, 64, 128]               0
          Conv2d-154          [-1, 64, 64, 128]          12,352
     BatchNorm2d-155          [-1, 64, 64, 128]             128
          Conv2d-156          [-1, 64, 64, 128]          12,352
            ReLU-157          [-1, 64, 64, 128]               0
          Conv2d-158          [-1, 64, 64, 128]          12,352
     BatchNorm2d-159          [-1, 64, 64, 128]             128
       Dropout2d-160          [-1, 64, 64, 128]               0
       Dropout2d-161          [-1, 64, 64, 128]               0
SS_nbt_module_paper-162         [-1, 128, 64, 128]               0
          Conv2d-163          [-1, 64, 64, 128]          12,352
            ReLU-164          [-1, 64, 64, 128]               0
          Conv2d-165          [-1, 64, 64, 128]          12,352
     BatchNorm2d-166          [-1, 64, 64, 128]             128
            ReLU-167          [-1, 64, 64, 128]               0
          Conv2d-168          [-1, 64, 64, 128]          12,352
            ReLU-169          [-1, 64, 64, 128]               0
          Conv2d-170          [-1, 64, 64, 128]          12,352
     BatchNorm2d-171          [-1, 64, 64, 128]             128
            ReLU-172          [-1, 64, 64, 128]               0
          Conv2d-173          [-1, 64, 64, 128]          12,352
            ReLU-174          [-1, 64, 64, 128]               0
          Conv2d-175          [-1, 64, 64, 128]          12,352
     BatchNorm2d-176          [-1, 64, 64, 128]             128
          Conv2d-177          [-1, 64, 64, 128]          12,352
            ReLU-178          [-1, 64, 64, 128]               0
          Conv2d-179          [-1, 64, 64, 128]          12,352
     BatchNorm2d-180          [-1, 64, 64, 128]             128
       Dropout2d-181          [-1, 64, 64, 128]               0
       Dropout2d-182          [-1, 64, 64, 128]               0
SS_nbt_module_paper-183         [-1, 128, 64, 128]               0
          Conv2d-184          [-1, 64, 64, 128]          12,352
            ReLU-185          [-1, 64, 64, 128]               0
          Conv2d-186          [-1, 64, 64, 128]          12,352
     BatchNorm2d-187          [-1, 64, 64, 128]             128
            ReLU-188          [-1, 64, 64, 128]               0
          Conv2d-189          [-1, 64, 64, 128]          12,352
            ReLU-190          [-1, 64, 64, 128]               0
          Conv2d-191          [-1, 64, 64, 128]          12,352
     BatchNorm2d-192          [-1, 64, 64, 128]             128
            ReLU-193          [-1, 64, 64, 128]               0
          Conv2d-194          [-1, 64, 64, 128]          12,352
            ReLU-195          [-1, 64, 64, 128]               0
          Conv2d-196          [-1, 64, 64, 128]          12,352
     BatchNorm2d-197          [-1, 64, 64, 128]             128
          Conv2d-198          [-1, 64, 64, 128]          12,352
            ReLU-199          [-1, 64, 64, 128]               0
          Conv2d-200          [-1, 64, 64, 128]          12,352
     BatchNorm2d-201          [-1, 64, 64, 128]             128
       Dropout2d-202          [-1, 64, 64, 128]               0
       Dropout2d-203          [-1, 64, 64, 128]               0
SS_nbt_module_paper-204         [-1, 128, 64, 128]               0
          Conv2d-205          [-1, 64, 64, 128]          12,352
            ReLU-206          [-1, 64, 64, 128]               0
          Conv2d-207          [-1, 64, 64, 128]          12,352
     BatchNorm2d-208          [-1, 64, 64, 128]             128
            ReLU-209          [-1, 64, 64, 128]               0
          Conv2d-210          [-1, 64, 64, 128]          12,352
            ReLU-211          [-1, 64, 64, 128]               0
          Conv2d-212          [-1, 64, 64, 128]          12,352
     BatchNorm2d-213          [-1, 64, 64, 128]             128
            ReLU-214          [-1, 64, 64, 128]               0
          Conv2d-215          [-1, 64, 64, 128]          12,352
            ReLU-216          [-1, 64, 64, 128]               0
          Conv2d-217          [-1, 64, 64, 128]          12,352
     BatchNorm2d-218          [-1, 64, 64, 128]             128
          Conv2d-219          [-1, 64, 64, 128]          12,352
            ReLU-220          [-1, 64, 64, 128]               0
          Conv2d-221          [-1, 64, 64, 128]          12,352
     BatchNorm2d-222          [-1, 64, 64, 128]             128
       Dropout2d-223          [-1, 64, 64, 128]               0
       Dropout2d-224          [-1, 64, 64, 128]               0
SS_nbt_module_paper-225         [-1, 128, 64, 128]               0
          Conv2d-226          [-1, 64, 64, 128]          12,352
            ReLU-227          [-1, 64, 64, 128]               0
          Conv2d-228          [-1, 64, 64, 128]          12,352
     BatchNorm2d-229          [-1, 64, 64, 128]             128
            ReLU-230          [-1, 64, 64, 128]               0
          Conv2d-231          [-1, 64, 64, 128]          12,352
            ReLU-232          [-1, 64, 64, 128]               0
          Conv2d-233          [-1, 64, 64, 128]          12,352
     BatchNorm2d-234          [-1, 64, 64, 128]             128
            ReLU-235          [-1, 64, 64, 128]               0
          Conv2d-236          [-1, 64, 64, 128]          12,352
            ReLU-237          [-1, 64, 64, 128]               0
          Conv2d-238          [-1, 64, 64, 128]          12,352
     BatchNorm2d-239          [-1, 64, 64, 128]             128
          Conv2d-240          [-1, 64, 64, 128]          12,352
            ReLU-241          [-1, 64, 64, 128]               0
          Conv2d-242          [-1, 64, 64, 128]          12,352
     BatchNorm2d-243          [-1, 64, 64, 128]             128
       Dropout2d-244          [-1, 64, 64, 128]               0
       Dropout2d-245          [-1, 64, 64, 128]               0
SS_nbt_module_paper-246         [-1, 128, 64, 128]               0
          Conv2d-247          [-1, 64, 64, 128]          12,352
            ReLU-248          [-1, 64, 64, 128]               0
          Conv2d-249          [-1, 64, 64, 128]          12,352
     BatchNorm2d-250          [-1, 64, 64, 128]             128
            ReLU-251          [-1, 64, 64, 128]               0
          Conv2d-252          [-1, 64, 64, 128]          12,352
            ReLU-253          [-1, 64, 64, 128]               0
          Conv2d-254          [-1, 64, 64, 128]          12,352
     BatchNorm2d-255          [-1, 64, 64, 128]             128
            ReLU-256          [-1, 64, 64, 128]               0
          Conv2d-257          [-1, 64, 64, 128]          12,352
            ReLU-258          [-1, 64, 64, 128]               0
          Conv2d-259          [-1, 64, 64, 128]          12,352
     BatchNorm2d-260          [-1, 64, 64, 128]             128
          Conv2d-261          [-1, 64, 64, 128]          12,352
            ReLU-262          [-1, 64, 64, 128]               0
          Conv2d-263          [-1, 64, 64, 128]          12,352
     BatchNorm2d-264          [-1, 64, 64, 128]             128
       Dropout2d-265          [-1, 64, 64, 128]               0
       Dropout2d-266          [-1, 64, 64, 128]               0
SS_nbt_module_paper-267         [-1, 128, 64, 128]               0
          Conv2d-268          [-1, 64, 64, 128]          12,352
            ReLU-269          [-1, 64, 64, 128]               0
          Conv2d-270          [-1, 64, 64, 128]          12,352
     BatchNorm2d-271          [-1, 64, 64, 128]             128
            ReLU-272          [-1, 64, 64, 128]               0
          Conv2d-273          [-1, 64, 64, 128]          12,352
            ReLU-274          [-1, 64, 64, 128]               0
          Conv2d-275          [-1, 64, 64, 128]          12,352
     BatchNorm2d-276          [-1, 64, 64, 128]             128
            ReLU-277          [-1, 64, 64, 128]               0
          Conv2d-278          [-1, 64, 64, 128]          12,352
            ReLU-279          [-1, 64, 64, 128]               0
          Conv2d-280          [-1, 64, 64, 128]          12,352
     BatchNorm2d-281          [-1, 64, 64, 128]             128
          Conv2d-282          [-1, 64, 64, 128]          12,352
            ReLU-283          [-1, 64, 64, 128]               0
          Conv2d-284          [-1, 64, 64, 128]          12,352
     BatchNorm2d-285          [-1, 64, 64, 128]             128
       Dropout2d-286          [-1, 64, 64, 128]               0
       Dropout2d-287          [-1, 64, 64, 128]               0
SS_nbt_module_paper-288         [-1, 128, 64, 128]               0
AdaptiveAvgPool2d-289            [-1, 128, 1, 1]               0
          Conv2d-290             [-1, 19, 1, 1]           2,451
     BatchNorm2d-291             [-1, 19, 1, 1]              38
            ReLU-292             [-1, 19, 1, 1]               0
    Conv2dBnRelu-293             [-1, 19, 1, 1]               0
          Conv2d-294          [-1, 19, 64, 128]           2,451
     BatchNorm2d-295          [-1, 19, 64, 128]              38
            ReLU-296          [-1, 19, 64, 128]               0
    Conv2dBnRelu-297          [-1, 19, 64, 128]               0
          Conv2d-298           [-1, 1, 32, 128]             257
          Conv2d-299            [-1, 1, 32, 64]               3
     BatchNorm2d-300            [-1, 1, 32, 64]               2
            ReLU-301            [-1, 1, 32, 64]               0
          Conv2d-302            [-1, 1, 32, 64]               8
          Conv2d-303            [-1, 1, 32, 64]               8
     BatchNorm2d-304            [-1, 1, 32, 64]               2
            ReLU-305            [-1, 1, 32, 64]               0
          Conv2d-306            [-1, 1, 16, 64]               3
          Conv2d-307            [-1, 1, 16, 32]               3
     BatchNorm2d-308            [-1, 1, 16, 32]               2
            ReLU-309            [-1, 1, 16, 32]               0
          Conv2d-310            [-1, 1, 16, 32]               6
          Conv2d-311            [-1, 1, 16, 32]               6
     BatchNorm2d-312            [-1, 1, 16, 32]               2
            ReLU-313            [-1, 1, 16, 32]               0
          Conv2d-314             [-1, 1, 8, 32]               3
          Conv2d-315             [-1, 1, 8, 16]               3
     BatchNorm2d-316             [-1, 1, 8, 16]               2
            ReLU-317             [-1, 1, 8, 16]               0
          Conv2d-318             [-1, 1, 8, 16]               4
          Conv2d-319             [-1, 1, 8, 16]               4
     BatchNorm2d-320             [-1, 1, 8, 16]               2
            ReLU-321             [-1, 1, 8, 16]               0
          Conv2d-322            [-1, 1, 16, 32]               6
          Conv2d-323            [-1, 1, 16, 32]               6
     BatchNorm2d-324            [-1, 1, 16, 32]               2
            ReLU-325            [-1, 1, 16, 32]               0
          Conv2d-326            [-1, 1, 32, 64]               8
          Conv2d-327            [-1, 1, 32, 64]               8
     BatchNorm2d-328            [-1, 1, 32, 64]               2
            ReLU-329            [-1, 1, 32, 64]               0
       APNModule-330          [-1, 19, 64, 128]               0
================================================================
Total params: 890,731
Trainable params: 890,731
Non-trainable params: 0
----------------------------------------------------------------
Input size (MB): 6.00
Forward/backward pass size (MB): 2342.20
Params size (MB): 3.40
Estimated Total Size (MB): 2351.60
----------------------------------------------------------------
/home/ethan/codes/Efficient-Segmentation-Networks/model/LEDNetG3.py:17: TracerWarning: Converting a tensor to a Python integer might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
  c = int(x.size()[1])
Skipped operation aten::max_pool2d 3 time(s)
Skipped operation aten::sub 12 time(s)
Skipped operation aten::relu_ 91 time(s)
Skipped operation aten::feature_dropout 26 time(s)
Skipped operation aten::add 20 time(s)
Skipped operation aten::relu 13 time(s)
Skipped operation aten::adaptive_avg_pool2d 1 time(s)
Skipped operation aten::upsample_bilinear2d 5 time(s)
Skipped operation aten::mul 1 time(s)
LEDNetG3(
  11.118 GMac, 100.000% MACs, 
  (initial_block): DownsamplerBlock(
    0.064 GMac, 0.572% MACs, 
    (conv): Conv2d(0.049 GMac, 0.444% MACs, 3, 29, kernel_size=(2, 2), stride=(2, 2))
    (pool): MaxPool2d(0.002 GMac, 0.014% MACs, kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
    (bn): BatchNorm2d(0.008 GMac, 0.075% MACs, 32, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
    (relu): ReLU(0.004 GMac, 0.038% MACs, inplace=True)
  )
  (layers): ModuleList(
    11.032 GMac, 99.223% MACs, 
    (0): SS_nbt_module_paper(
      0.851 GMac, 7.658% MACs, 
      (conv3x1_1_l): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_l): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_l): BatchNorm2d(0.004 GMac, 0.038% MACs, 16, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_l): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_2_l): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn2_l): BatchNorm2d(0.004 GMac, 0.038% MACs, 16, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_1_r): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_r): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_r): BatchNorm2d(0.004 GMac, 0.038% MACs, 16, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_r): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_2_r): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn2_r): BatchNorm2d(0.004 GMac, 0.038% MACs, 16, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(0.013 GMac, 0.113% MACs, inplace=True)
      (dropout): Dropout2d(0.0 GMac, 0.000% MACs, p=0.03, inplace=False)
    )
    (1): SS_nbt_module_paper(
      0.851 GMac, 7.658% MACs, 
      (conv3x1_1_l): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_l): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_l): BatchNorm2d(0.004 GMac, 0.038% MACs, 16, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_l): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_2_l): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn2_l): BatchNorm2d(0.004 GMac, 0.038% MACs, 16, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_1_r): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_r): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_r): BatchNorm2d(0.004 GMac, 0.038% MACs, 16, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_r): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_2_r): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn2_r): BatchNorm2d(0.004 GMac, 0.038% MACs, 16, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(0.013 GMac, 0.113% MACs, inplace=True)
      (dropout): Dropout2d(0.0 GMac, 0.000% MACs, p=0.03, inplace=False)
    )
    (2): SS_nbt_module_paper(
      0.851 GMac, 7.658% MACs, 
      (conv3x1_1_l): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_l): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_l): BatchNorm2d(0.004 GMac, 0.038% MACs, 16, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_l): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_2_l): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn2_l): BatchNorm2d(0.004 GMac, 0.038% MACs, 16, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_1_r): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_r): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_r): BatchNorm2d(0.004 GMac, 0.038% MACs, 16, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_r): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_2_r): Conv2d(0.103 GMac, 0.924% MACs, 16, 16, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn2_r): BatchNorm2d(0.004 GMac, 0.038% MACs, 16, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(0.013 GMac, 0.113% MACs, inplace=True)
      (dropout): Dropout2d(0.0 GMac, 0.000% MACs, p=0.03, inplace=False)
    )
    (3): DownsamplerBlock(
      0.146 GMac, 1.311% MACs, 
      (conv): Conv2d(0.135 GMac, 1.217% MACs, 32, 32, kernel_size=(2, 2), stride=(2, 2))
      (pool): MaxPool2d(0.004 GMac, 0.038% MACs, kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
      (bn): BatchNorm2d(0.004 GMac, 0.038% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(0.002 GMac, 0.019% MACs, inplace=True)
    )
    (4): SS_nbt_module_paper(
      0.828 GMac, 7.451% MACs, 
      (conv3x1_1_l): Conv2d(0.102 GMac, 0.915% MACs, 32, 32, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_l): Conv2d(0.102 GMac, 0.915% MACs, 32, 32, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_l): BatchNorm2d(0.002 GMac, 0.019% MACs, 32, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_l): Conv2d(0.102 GMac, 0.915% MACs, 32, 32, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_2_l): Conv2d(0.102 GMac, 0.915% MACs, 32, 32, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn2_l): BatchNorm2d(0.002 GMac, 0.019% MACs, 32, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_1_r): Conv2d(0.102 GMac, 0.915% MACs, 32, 32, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_r): Conv2d(0.102 GMac, 0.915% MACs, 32, 32, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_r): BatchNorm2d(0.002 GMac, 0.019% MACs, 32, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_r): Conv2d(0.102 GMac, 0.915% MACs, 32, 32, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_2_r): Conv2d(0.102 GMac, 0.915% MACs, 32, 32, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn2_r): BatchNorm2d(0.002 GMac, 0.019% MACs, 32, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(0.006 GMac, 0.057% MACs, inplace=True)
      (dropout): Dropout2d(0.0 GMac, 0.000% MACs, p=0.03, inplace=False)
    )
    (5): SS_nbt_module_paper(
      0.828 GMac, 7.451% MACs, 
      (conv3x1_1_l): Conv2d(0.102 GMac, 0.915% MACs, 32, 32, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_l): Conv2d(0.102 GMac, 0.915% MACs, 32, 32, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_l): BatchNorm2d(0.002 GMac, 0.019% MACs, 32, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_l): Conv2d(0.102 GMac, 0.915% MACs, 32, 32, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_2_l): Conv2d(0.102 GMac, 0.915% MACs, 32, 32, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn2_l): BatchNorm2d(0.002 GMac, 0.019% MACs, 32, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_1_r): Conv2d(0.102 GMac, 0.915% MACs, 32, 32, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_r): Conv2d(0.102 GMac, 0.915% MACs, 32, 32, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_r): BatchNorm2d(0.002 GMac, 0.019% MACs, 32, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_r): Conv2d(0.102 GMac, 0.915% MACs, 32, 32, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_2_r): Conv2d(0.102 GMac, 0.915% MACs, 32, 32, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn2_r): BatchNorm2d(0.002 GMac, 0.019% MACs, 32, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(0.006 GMac, 0.057% MACs, inplace=True)
      (dropout): Dropout2d(0.0 GMac, 0.000% MACs, p=0.03, inplace=False)
    )
    (6): DownsamplerBlock(
      0.14 GMac, 1.259% MACs, 
      (conv): Conv2d(0.135 GMac, 1.212% MACs, 64, 64, kernel_size=(2, 2), stride=(2, 2))
      (pool): MaxPool2d(0.002 GMac, 0.019% MACs, kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
      (bn): BatchNorm2d(0.002 GMac, 0.019% MACs, 128, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(0.001 GMac, 0.009% MACs, inplace=True)
    )
    (7): SS_nbt_module_paper(
      0.817 GMac, 7.347% MACs, 
      (conv3x1_1_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_l): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_2_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn2_l): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_1_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_r): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_2_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn2_r): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(0.003 GMac, 0.028% MACs, inplace=True)
      (dropout): Dropout2d(0.0 GMac, 0.000% MACs, p=0.3, inplace=False)
    )
    (8): SS_nbt_module_paper(
      0.817 GMac, 7.347% MACs, 
      (conv3x1_1_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_l): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(2, 0), dilation=(2, 1))
      (conv1x3_2_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 2), dilation=(1, 2))
      (bn2_l): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_1_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_r): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(2, 0), dilation=(2, 1))
      (conv1x3_2_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 2), dilation=(1, 2))
      (bn2_r): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(0.003 GMac, 0.028% MACs, inplace=True)
      (dropout): Dropout2d(0.0 GMac, 0.000% MACs, p=0.3, inplace=False)
    )
    (9): SS_nbt_module_paper(
      0.817 GMac, 7.347% MACs, 
      (conv3x1_1_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_l): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(5, 0), dilation=(5, 1))
      (conv1x3_2_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 5), dilation=(1, 5))
      (bn2_l): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_1_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_r): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(5, 0), dilation=(5, 1))
      (conv1x3_2_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 5), dilation=(1, 5))
      (bn2_r): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(0.003 GMac, 0.028% MACs, inplace=True)
      (dropout): Dropout2d(0.0 GMac, 0.000% MACs, p=0.3, inplace=False)
    )
    (10): SS_nbt_module_paper(
      0.817 GMac, 7.347% MACs, 
      (conv3x1_1_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_l): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(9, 0), dilation=(9, 1))
      (conv1x3_2_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 9), dilation=(1, 9))
      (bn2_l): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_1_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_r): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(9, 0), dilation=(9, 1))
      (conv1x3_2_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 9), dilation=(1, 9))
      (bn2_r): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(0.003 GMac, 0.028% MACs, inplace=True)
      (dropout): Dropout2d(0.0 GMac, 0.000% MACs, p=0.3, inplace=False)
    )
    (11): SS_nbt_module_paper(
      0.817 GMac, 7.347% MACs, 
      (conv3x1_1_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_l): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(2, 0), dilation=(2, 1))
      (conv1x3_2_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 2), dilation=(1, 2))
      (bn2_l): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_1_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_r): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(2, 0), dilation=(2, 1))
      (conv1x3_2_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 2), dilation=(1, 2))
      (bn2_r): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(0.003 GMac, 0.028% MACs, inplace=True)
      (dropout): Dropout2d(0.0 GMac, 0.000% MACs, p=0.3, inplace=False)
    )
    (12): SS_nbt_module_paper(
      0.817 GMac, 7.347% MACs, 
      (conv3x1_1_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_l): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(5, 0), dilation=(5, 1))
      (conv1x3_2_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 5), dilation=(1, 5))
      (bn2_l): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_1_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_r): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(5, 0), dilation=(5, 1))
      (conv1x3_2_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 5), dilation=(1, 5))
      (bn2_r): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(0.003 GMac, 0.028% MACs, inplace=True)
      (dropout): Dropout2d(0.0 GMac, 0.000% MACs, p=0.3, inplace=False)
    )
    (13): SS_nbt_module_paper(
      0.817 GMac, 7.347% MACs, 
      (conv3x1_1_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_l): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(9, 0), dilation=(9, 1))
      (conv1x3_2_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 9), dilation=(1, 9))
      (bn2_l): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_1_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_r): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(9, 0), dilation=(9, 1))
      (conv1x3_2_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 9), dilation=(1, 9))
      (bn2_r): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(0.003 GMac, 0.028% MACs, inplace=True)
      (dropout): Dropout2d(0.0 GMac, 0.000% MACs, p=0.3, inplace=False)
    )
    (14): SS_nbt_module_paper(
      0.817 GMac, 7.347% MACs, 
      (conv3x1_1_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_l): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(17, 0), dilation=(17, 1))
      (conv1x3_2_l): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 17), dilation=(1, 17))
      (bn2_l): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_1_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (conv1x3_1_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (bn1_r): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (conv3x1_2_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(3, 1), stride=(1, 1), padding=(17, 0), dilation=(17, 1))
      (conv1x3_2_r): Conv2d(0.101 GMac, 0.910% MACs, 64, 64, kernel_size=(1, 3), stride=(1, 1), padding=(0, 17), dilation=(1, 17))
      (bn2_r): BatchNorm2d(0.001 GMac, 0.009% MACs, 64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(0.003 GMac, 0.028% MACs, inplace=True)
      (dropout): Dropout2d(0.0 GMac, 0.000% MACs, p=0.3, inplace=False)
    )
  )
  (apn): APNModule(
    0.023 GMac, 0.205% MACs, 
    (branch1): Sequential(
      0.001 GMac, 0.009% MACs, 
      (0): AdaptiveAvgPool2d(0.001 GMac, 0.009% MACs, output_size=1)
      (1): Conv2dBnRelu(
        0.0 GMac, 0.000% MACs, 
        (conv): Sequential(
          0.0 GMac, 0.000% MACs, 
          (0): Conv2d(0.0 GMac, 0.000% MACs, 128, 19, kernel_size=(1, 1), stride=(1, 1))
          (1): BatchNorm2d(0.0 GMac, 0.000% MACs, 19, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
          (2): ReLU(0.0 GMac, 0.000% MACs, inplace=True)
        )
      )
    )
    (mid): Sequential(
      0.021 GMac, 0.185% MACs, 
      (0): Conv2dBnRelu(
        0.021 GMac, 0.185% MACs, 
        (conv): Sequential(
          0.021 GMac, 0.185% MACs, 
          (0): Conv2d(0.02 GMac, 0.181% MACs, 128, 19, kernel_size=(1, 1), stride=(1, 1))
          (1): BatchNorm2d(0.0 GMac, 0.003% MACs, 19, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
          (2): ReLU(0.0 GMac, 0.001% MACs, inplace=True)
        )
      )
    )
    (down1): Sequential(
      0.001 GMac, 0.010% MACs, 
      (0): Conv2d(0.001 GMac, 0.009% MACs, 128, 1, kernel_size=(2, 1), stride=(2, 1))
      (1): Conv2d(0.0 GMac, 0.000% MACs, 1, 1, kernel_size=(1, 2), stride=(1, 2))
      (2): BatchNorm2d(0.0 GMac, 0.000% MACs, 1, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (3): ReLU(0.0 GMac, 0.000% MACs, inplace=True)
      (4): Conv2d(0.0 GMac, 0.000% MACs, 1, 1, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0))
      (5): Conv2d(0.0 GMac, 0.000% MACs, 1, 1, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3))
      (6): BatchNorm2d(0.0 GMac, 0.000% MACs, 1, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (7): ReLU(0.0 GMac, 0.000% MACs, inplace=True)
    )
    (down2): Sequential(
      0.0 GMac, 0.000% MACs, 
      (0): Conv2d(0.0 GMac, 0.000% MACs, 1, 1, kernel_size=(2, 1), stride=(2, 1))
      (1): Conv2d(0.0 GMac, 0.000% MACs, 1, 1, kernel_size=(1, 2), stride=(1, 2))
      (2): BatchNorm2d(0.0 GMac, 0.000% MACs, 1, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (3): ReLU(0.0 GMac, 0.000% MACs, inplace=True)
      (4): Conv2d(0.0 GMac, 0.000% MACs, 1, 1, kernel_size=(5, 1), stride=(1, 1), padding=(2, 0))
      (5): Conv2d(0.0 GMac, 0.000% MACs, 1, 1, kernel_size=(1, 5), stride=(1, 1), padding=(0, 2))
      (6): BatchNorm2d(0.0 GMac, 0.000% MACs, 1, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (7): ReLU(0.0 GMac, 0.000% MACs, inplace=True)
    )
    (down3): Sequential(
      0.0 GMac, 0.000% MACs, 
      (0): Conv2d(0.0 GMac, 0.000% MACs, 1, 1, kernel_size=(2, 1), stride=(2, 1))
      (1): Conv2d(0.0 GMac, 0.000% MACs, 1, 1, kernel_size=(1, 2), stride=(1, 2))
      (2): BatchNorm2d(0.0 GMac, 0.000% MACs, 1, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (3): ReLU(0.0 GMac, 0.000% MACs, inplace=True)
      (4): Conv2d(0.0 GMac, 0.000% MACs, 1, 1, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))
      (5): Conv2d(0.0 GMac, 0.000% MACs, 1, 1, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1))
      (6): BatchNorm2d(0.0 GMac, 0.000% MACs, 1, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (7): ReLU(0.0 GMac, 0.000% MACs, inplace=True)
    )
    (conv2): Sequential(
      0.0 GMac, 0.000% MACs, 
      (0): Conv2d(0.0 GMac, 0.000% MACs, 1, 1, kernel_size=(5, 1), stride=(1, 1), padding=(2, 0))
      (1): Conv2d(0.0 GMac, 0.000% MACs, 1, 1, kernel_size=(1, 5), stride=(1, 1), padding=(0, 2))
      (2): BatchNorm2d(0.0 GMac, 0.000% MACs, 1, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (3): ReLU(0.0 GMac, 0.000% MACs, inplace=True)
    )
    (conv1): Sequential(
      0.0 GMac, 0.000% MACs, 
      (0): Conv2d(0.0 GMac, 0.000% MACs, 1, 1, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0))
      (1): Conv2d(0.0 GMac, 0.000% MACs, 1, 1, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3))
      (2): BatchNorm2d(0.0 GMac, 0.000% MACs, 1, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (3): ReLU(0.0 GMac, 0.000% MACs, inplace=True)
    )
  )
)
[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.
[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.
[INFO] Register count_bn() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.
[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.
[WARN] Cannot find rule for <class '__main__.DownsamplerBlock'>. Treat it as zero Macs and zero Params.
[WARN] Cannot find rule for <class 'torch.nn.modules.dropout.Dropout2d'>. Treat it as zero Macs and zero Params.
[WARN] Cannot find rule for <class '__main__.SS_nbt_module_paper'>. Treat it as zero Macs and zero Params.
[WARN] Cannot find rule for <class 'torch.nn.modules.container.ModuleList'>. Treat it as zero Macs and zero Params.
[INFO] Register count_adap_avgpool() for <class 'torch.nn.modules.pooling.AdaptiveAvgPool2d'>.
[WARN] Cannot find rule for <class 'torch.nn.modules.container.Sequential'>. Treat it as zero Macs and zero Params.
[WARN] Cannot find rule for <class '__main__.Conv2dBnRelu'>. Treat it as zero Macs and zero Params.
[WARN] Cannot find rule for <class '__main__.APNModule'>. Treat it as zero Macs and zero Params.
[WARN] Cannot find rule for <class '__main__.LEDNetG3'>. Treat it as zero Macs and zero Params.
defaultdict(<class 'float'>, {'conv': 21.608163072, 'batchnorm': 0.462682264})
11117877580.0 890731
22054024562.0 890731.0

Process finished with exit code 0


'''