import io
import requests
from PIL import Image
import torch
from torchvision import models, transforms
from torch.autograd import Variable
from torch.nn import functional as F
from models.resnetall import resnet34
import numpy as np
import cv2
import os
import pdb


# input image
# LABELS_URL = 'https://s3.amazonaws.com/outcome-blog/imagenet/labels.json'
# IMG_URL = 'http://media.mlive.com/news_impact/photo/9933031-large.jpg'

# networks such as googlenet, resnet, densenet already use global average pooling at the end, so CAM could be used directly.
def returnCAM(feature_conv, weight_softmax, class_idx):
    # generate the class activation maps upsample to 256x256
    size_upsample = (100, 100)
    bz, nc, h, w = feature_conv.shape
    output_cam = []
    for idx in class_idx:
        cam = weight_softmax[idx].dot(feature_conv.reshape((nc, h*w)))
        cam = cam.reshape(h, w)
        cam = cam - np.min(cam)
        cam_img = cam / np.max(cam)
        cam_img = np.uint8(255 * cam_img)
        output_cam.append(cv2.resize(cam_img, size_upsample))
    return output_cam

model_id = 4
if model_id == 1:
    net = models.squeezenet1_1(pretrained=True)
    finalconv_name = 'features' # this is the last conv layer of the network
elif model_id == 2:
    net = models.resnet18(pretrained=True)
    finalconv_name = 'layer4'
elif model_id == 3:
    net = models.densenet161(pretrained=True)
    finalconv_name = 'features'
elif model_id == 4:
    net = resnet34(num_classes=7)
    print(net)
    finalconv_name = 'conv5_x'

checkpoint = torch.load(os.path.join('../RAF_Resnet34-100', 'PublicTest_model.t7'),map_location='cpu')
net.load_state_dict(checkpoint['net'])
net.eval()

# hook the feature extractor
features_blobs = []
def hook_feature(module, input, output):
    features_blobs.append(output.data.cpu().numpy())

net._modules.get(finalconv_name).register_forward_hook(hook_feature)

# get the softmax weight
params = list(net.parameters())
weight_softmax = np.squeeze(params[-2].data.numpy())


# normalize = transforms.Normalize(
#    mean=[0.485, 0.456, 0.406],
#    std=[0.229, 0.224, 0.225]
# )
# preprocess = transforms.Compose([
#    transforms.Resize((224,224)),
#    transforms.ToTensor(),
#    normalize
# ])
cut_size = 96
transform_test = transforms.Compose([
    transforms.CenterCrop(cut_size),
    transforms.ToTensor(),
])
path = '/media/jinbo/DATA1/data_all/Face/RAF-basic/thirdTrain/aligned100/'
imagelist = os.listdir(path)

if __name__ == '__main__':
    # response = requests.get(IMG_URL)
    # img_pil = Image.open(io.BytesIO(response.content))
    num = 0
    savepath = 'GAP_CAM'
    for image in imagelist:
        imgpath = path + image
        img_pil = Image.open(imgpath)
        cv_img_pil = cv2.cvtColor(np.asarray(img_pil), cv2.COLOR_RGB2BGR)
        savesourceimg = os.path.join(savepath,'GAP_HEATMAP_'+ str(num) + '.jpg')
        cv2.imshow("cv_img_pil",cv_img_pil)
        # cv2.imwrite(savesourceimg,cv_img_pil)
        img_tensor = transform_test(img_pil)
        img_variable = Variable(img_tensor.unsqueeze(0))
        logit = net(img_variable)

        # download the imagenet category list
        # classes = {int(key):value for (key, value)
        #           in requests.get(LABELS_URL).json().items()}
        classes = {0:'Surprise',1:'Fear',2:'Disgust',3:'Happy',4:'Sadness',5:'Angry',6:'Neutral'}

        h_x = F.softmax(logit, dim=1).data.squeeze()
        probs, idx = h_x.sort(0, True)
        probs = probs.numpy()
        idx = idx.numpy()

        # output the prediction
        for i in range(0, 5):
            print('{:.3f} -> {}'.format(probs[i], classes[idx[i]]))

        # generate class activation mapping for the top1 prediction
        CAMs = returnCAM(features_blobs[0], weight_softmax, [idx[0]])

        # render the CAM and output
        print('output CAM.jpg for the top1 prediction: %s'%classes[idx[0]])
        # img = cv2.imread(savesourceimg)
        # cv2.imshow("img",img)
        height, width, _ = cv_img_pil.shape
        heatmap = cv2.applyColorMap(cv2.resize(CAMs[0],(width, height)), cv2.COLORMAP_JET)
        result = heatmap * 0.3 + cv_img_pil * 0.5
        cv2.imshow("result",result.astype('uint8'))
        saveheatimg = os.path.join(savepath,'heatmap_' + str(num) + '.jpg')
        # cv2.imwrite(saveheatimg, result)
        num = num + 1
        cv2.waitKey(0)

81dd5d997e13013202dab23e6c7ebff576d8d1cf
<mask token>


def returnCAM(feature_conv, weight_softmax, class_idx):
    size_upsample = 100, 100
    bz, nc, h, w = feature_conv.shape
    output_cam = []
    for idx in class_idx:
        cam = weight_softmax[idx].dot(feature_conv.reshape((nc, h * w)))
        cam = cam.reshape(h, w)
        cam = cam - np.min(cam)
        cam_img = cam / np.max(cam)
        cam_img = np.uint8(255 * cam_img)
        output_cam.append(cv2.resize(cam_img, size_upsample))
    return output_cam


<mask token>


def hook_feature(module, input, output):
    features_blobs.append(output.data.cpu().numpy())


<mask token>

<mask token>


def returnCAM(feature_conv, weight_softmax, class_idx):
    size_upsample = 100, 100
    bz, nc, h, w = feature_conv.shape
    output_cam = []
    for idx in class_idx:
        cam = weight_softmax[idx].dot(feature_conv.reshape((nc, h * w)))
        cam = cam.reshape(h, w)
        cam = cam - np.min(cam)
        cam_img = cam / np.max(cam)
        cam_img = np.uint8(255 * cam_img)
        output_cam.append(cv2.resize(cam_img, size_upsample))
    return output_cam


<mask token>
if model_id == 1:
    net = models.squeezenet1_1(pretrained=True)
    finalconv_name = 'features'
elif model_id == 2:
    net = models.resnet18(pretrained=True)
    finalconv_name = 'layer4'
elif model_id == 3:
    net = models.densenet161(pretrained=True)
    finalconv_name = 'features'
elif model_id == 4:
    net = resnet34(num_classes=7)
    print(net)
    finalconv_name = 'conv5_x'
<mask token>
net.load_state_dict(checkpoint['net'])
net.eval()
<mask token>


def hook_feature(module, input, output):
    features_blobs.append(output.data.cpu().numpy())


net._modules.get(finalconv_name).register_forward_hook(hook_feature)
<mask token>
if __name__ == '__main__':
    num = 0
    savepath = 'GAP_CAM'
    for image in imagelist:
        imgpath = path + image
        img_pil = Image.open(imgpath)
        cv_img_pil = cv2.cvtColor(np.asarray(img_pil), cv2.COLOR_RGB2BGR)
        savesourceimg = os.path.join(savepath, 'GAP_HEATMAP_' + str(num) +
            '.jpg')
        cv2.imshow('cv_img_pil', cv_img_pil)
        img_tensor = transform_test(img_pil)
        img_variable = Variable(img_tensor.unsqueeze(0))
        logit = net(img_variable)
        classes = {(0): 'Surprise', (1): 'Fear', (2): 'Disgust', (3):
            'Happy', (4): 'Sadness', (5): 'Angry', (6): 'Neutral'}
        h_x = F.softmax(logit, dim=1).data.squeeze()
        probs, idx = h_x.sort(0, True)
        probs = probs.numpy()
        idx = idx.numpy()
        for i in range(0, 5):
            print('{:.3f} -> {}'.format(probs[i], classes[idx[i]]))
        CAMs = returnCAM(features_blobs[0], weight_softmax, [idx[0]])
        print('output CAM.jpg for the top1 prediction: %s' % classes[idx[0]])
        height, width, _ = cv_img_pil.shape
        heatmap = cv2.applyColorMap(cv2.resize(CAMs[0], (width, height)),
            cv2.COLORMAP_JET)
        result = heatmap * 0.3 + cv_img_pil * 0.5
        cv2.imshow('result', result.astype('uint8'))
        saveheatimg = os.path.join(savepath, 'heatmap_' + str(num) + '.jpg')
        num = num + 1
        cv2.waitKey(0)

<mask token>


def returnCAM(feature_conv, weight_softmax, class_idx):
    size_upsample = 100, 100
    bz, nc, h, w = feature_conv.shape
    output_cam = []
    for idx in class_idx:
        cam = weight_softmax[idx].dot(feature_conv.reshape((nc, h * w)))
        cam = cam.reshape(h, w)
        cam = cam - np.min(cam)
        cam_img = cam / np.max(cam)
        cam_img = np.uint8(255 * cam_img)
        output_cam.append(cv2.resize(cam_img, size_upsample))
    return output_cam


model_id = 4
if model_id == 1:
    net = models.squeezenet1_1(pretrained=True)
    finalconv_name = 'features'
elif model_id == 2:
    net = models.resnet18(pretrained=True)
    finalconv_name = 'layer4'
elif model_id == 3:
    net = models.densenet161(pretrained=True)
    finalconv_name = 'features'
elif model_id == 4:
    net = resnet34(num_classes=7)
    print(net)
    finalconv_name = 'conv5_x'
checkpoint = torch.load(os.path.join('../RAF_Resnet34-100',
    'PublicTest_model.t7'), map_location='cpu')
net.load_state_dict(checkpoint['net'])
net.eval()
features_blobs = []


def hook_feature(module, input, output):
    features_blobs.append(output.data.cpu().numpy())


net._modules.get(finalconv_name).register_forward_hook(hook_feature)
params = list(net.parameters())
weight_softmax = np.squeeze(params[-2].data.numpy())
cut_size = 96
transform_test = transforms.Compose([transforms.CenterCrop(cut_size),
    transforms.ToTensor()])
path = '/media/jinbo/DATA1/data_all/Face/RAF-basic/thirdTrain/aligned100/'
imagelist = os.listdir(path)
if __name__ == '__main__':
    num = 0
    savepath = 'GAP_CAM'
    for image in imagelist:
        imgpath = path + image
        img_pil = Image.open(imgpath)
        cv_img_pil = cv2.cvtColor(np.asarray(img_pil), cv2.COLOR_RGB2BGR)
        savesourceimg = os.path.join(savepath, 'GAP_HEATMAP_' + str(num) +
            '.jpg')
        cv2.imshow('cv_img_pil', cv_img_pil)
        img_tensor = transform_test(img_pil)
        img_variable = Variable(img_tensor.unsqueeze(0))
        logit = net(img_variable)
        classes = {(0): 'Surprise', (1): 'Fear', (2): 'Disgust', (3):
            'Happy', (4): 'Sadness', (5): 'Angry', (6): 'Neutral'}
        h_x = F.softmax(logit, dim=1).data.squeeze()
        probs, idx = h_x.sort(0, True)
        probs = probs.numpy()
        idx = idx.numpy()
        for i in range(0, 5):
            print('{:.3f} -> {}'.format(probs[i], classes[idx[i]]))
        CAMs = returnCAM(features_blobs[0], weight_softmax, [idx[0]])
        print('output CAM.jpg for the top1 prediction: %s' % classes[idx[0]])
        height, width, _ = cv_img_pil.shape
        heatmap = cv2.applyColorMap(cv2.resize(CAMs[0], (width, height)),
            cv2.COLORMAP_JET)
        result = heatmap * 0.3 + cv_img_pil * 0.5
        cv2.imshow('result', result.astype('uint8'))
        saveheatimg = os.path.join(savepath, 'heatmap_' + str(num) + '.jpg')
        num = num + 1
        cv2.waitKey(0)

import io
import requests
from PIL import Image
import torch
from torchvision import models, transforms
from torch.autograd import Variable
from torch.nn import functional as F
from models.resnetall import resnet34
import numpy as np
import cv2
import os
import pdb


def returnCAM(feature_conv, weight_softmax, class_idx):
    size_upsample = 100, 100
    bz, nc, h, w = feature_conv.shape
    output_cam = []
    for idx in class_idx:
        cam = weight_softmax[idx].dot(feature_conv.reshape((nc, h * w)))
        cam = cam.reshape(h, w)
        cam = cam - np.min(cam)
        cam_img = cam / np.max(cam)
        cam_img = np.uint8(255 * cam_img)
        output_cam.append(cv2.resize(cam_img, size_upsample))
    return output_cam


model_id = 4
if model_id == 1:
    net = models.squeezenet1_1(pretrained=True)
    finalconv_name = 'features'
elif model_id == 2:
    net = models.resnet18(pretrained=True)
    finalconv_name = 'layer4'
elif model_id == 3:
    net = models.densenet161(pretrained=True)
    finalconv_name = 'features'
elif model_id == 4:
    net = resnet34(num_classes=7)
    print(net)
    finalconv_name = 'conv5_x'
checkpoint = torch.load(os.path.join('../RAF_Resnet34-100',
    'PublicTest_model.t7'), map_location='cpu')
net.load_state_dict(checkpoint['net'])
net.eval()
features_blobs = []


def hook_feature(module, input, output):
    features_blobs.append(output.data.cpu().numpy())


net._modules.get(finalconv_name).register_forward_hook(hook_feature)
params = list(net.parameters())
weight_softmax = np.squeeze(params[-2].data.numpy())
cut_size = 96
transform_test = transforms.Compose([transforms.CenterCrop(cut_size),
    transforms.ToTensor()])
path = '/media/jinbo/DATA1/data_all/Face/RAF-basic/thirdTrain/aligned100/'
imagelist = os.listdir(path)
if __name__ == '__main__':
    num = 0
    savepath = 'GAP_CAM'
    for image in imagelist:
        imgpath = path + image
        img_pil = Image.open(imgpath)
        cv_img_pil = cv2.cvtColor(np.asarray(img_pil), cv2.COLOR_RGB2BGR)
        savesourceimg = os.path.join(savepath, 'GAP_HEATMAP_' + str(num) +
            '.jpg')
        cv2.imshow('cv_img_pil', cv_img_pil)
        img_tensor = transform_test(img_pil)
        img_variable = Variable(img_tensor.unsqueeze(0))
        logit = net(img_variable)
        classes = {(0): 'Surprise', (1): 'Fear', (2): 'Disgust', (3):
            'Happy', (4): 'Sadness', (5): 'Angry', (6): 'Neutral'}
        h_x = F.softmax(logit, dim=1).data.squeeze()
        probs, idx = h_x.sort(0, True)
        probs = probs.numpy()
        idx = idx.numpy()
        for i in range(0, 5):
            print('{:.3f} -> {}'.format(probs[i], classes[idx[i]]))
        CAMs = returnCAM(features_blobs[0], weight_softmax, [idx[0]])
        print('output CAM.jpg for the top1 prediction: %s' % classes[idx[0]])
        height, width, _ = cv_img_pil.shape
        heatmap = cv2.applyColorMap(cv2.resize(CAMs[0], (width, height)),
            cv2.COLORMAP_JET)
        result = heatmap * 0.3 + cv_img_pil * 0.5
        cv2.imshow('result', result.astype('uint8'))
        saveheatimg = os.path.join(savepath, 'heatmap_' + str(num) + '.jpg')
        num = num + 1
        cv2.waitKey(0)

import io
import requests
from PIL import Image
import torch
from torchvision import models, transforms
from torch.autograd import Variable
from torch.nn import functional as F
from models.resnetall import resnet34
import numpy as np
import cv2
import os
import pdb


# input image
# LABELS_URL = 'https://s3.amazonaws.com/outcome-blog/imagenet/labels.json'
# IMG_URL = 'http://media.mlive.com/news_impact/photo/9933031-large.jpg'

# networks such as googlenet, resnet, densenet already use global average pooling at the end, so CAM could be used directly.
def returnCAM(feature_conv, weight_softmax, class_idx):
    # generate the class activation maps upsample to 256x256
    size_upsample = (100, 100)
    bz, nc, h, w = feature_conv.shape
    output_cam = []
    for idx in class_idx:
        cam = weight_softmax[idx].dot(feature_conv.reshape((nc, h*w)))
        cam = cam.reshape(h, w)
        cam = cam - np.min(cam)
        cam_img = cam / np.max(cam)
        cam_img = np.uint8(255 * cam_img)
        output_cam.append(cv2.resize(cam_img, size_upsample))
    return output_cam

model_id = 4
if model_id == 1:
    net = models.squeezenet1_1(pretrained=True)
    finalconv_name = 'features' # this is the last conv layer of the network
elif model_id == 2:
    net = models.resnet18(pretrained=True)
    finalconv_name = 'layer4'
elif model_id == 3:
    net = models.densenet161(pretrained=True)
    finalconv_name = 'features'
elif model_id == 4:
    net = resnet34(num_classes=7)
    print(net)
    finalconv_name = 'conv5_x'

checkpoint = torch.load(os.path.join('../RAF_Resnet34-100', 'PublicTest_model.t7'),map_location='cpu')
net.load_state_dict(checkpoint['net'])
net.eval()

# hook the feature extractor
features_blobs = []
def hook_feature(module, input, output):
    features_blobs.append(output.data.cpu().numpy())

net._modules.get(finalconv_name).register_forward_hook(hook_feature)

# get the softmax weight
params = list(net.parameters())
weight_softmax = np.squeeze(params[-2].data.numpy())


# normalize = transforms.Normalize(
#    mean=[0.485, 0.456, 0.406],
#    std=[0.229, 0.224, 0.225]
# )
# preprocess = transforms.Compose([
#    transforms.Resize((224,224)),
#    transforms.ToTensor(),
#    normalize
# ])
cut_size = 96
transform_test = transforms.Compose([
    transforms.CenterCrop(cut_size),
    transforms.ToTensor(),
])
path = '/media/jinbo/DATA1/data_all/Face/RAF-basic/thirdTrain/aligned100/'
imagelist = os.listdir(path)

if __name__ == '__main__':
    # response = requests.get(IMG_URL)
    # img_pil = Image.open(io.BytesIO(response.content))
    num = 0
    savepath = 'GAP_CAM'
    for image in imagelist:
        imgpath = path + image
        img_pil = Image.open(imgpath)
        cv_img_pil = cv2.cvtColor(np.asarray(img_pil), cv2.COLOR_RGB2BGR)
        savesourceimg = os.path.join(savepath,'GAP_HEATMAP_'+ str(num) + '.jpg')
        cv2.imshow("cv_img_pil",cv_img_pil)
        # cv2.imwrite(savesourceimg,cv_img_pil)
        img_tensor = transform_test(img_pil)
        img_variable = Variable(img_tensor.unsqueeze(0))
        logit = net(img_variable)

        # download the imagenet category list
        # classes = {int(key):value for (key, value)
        #           in requests.get(LABELS_URL).json().items()}
        classes = {0:'Surprise',1:'Fear',2:'Disgust',3:'Happy',4:'Sadness',5:'Angry',6:'Neutral'}

        h_x = F.softmax(logit, dim=1).data.squeeze()
        probs, idx = h_x.sort(0, True)
        probs = probs.numpy()
        idx = idx.numpy()

        # output the prediction
        for i in range(0, 5):
            print('{:.3f} -> {}'.format(probs[i], classes[idx[i]]))

        # generate class activation mapping for the top1 prediction
        CAMs = returnCAM(features_blobs[0], weight_softmax, [idx[0]])

        # render the CAM and output
        print('output CAM.jpg for the top1 prediction: %s'%classes[idx[0]])
        # img = cv2.imread(savesourceimg)
        # cv2.imshow("img",img)
        height, width, _ = cv_img_pil.shape
        heatmap = cv2.applyColorMap(cv2.resize(CAMs[0],(width, height)), cv2.COLORMAP_JET)
        result = heatmap * 0.3 + cv_img_pil * 0.5
        cv2.imshow("result",result.astype('uint8'))
        saveheatimg = os.path.join(savepath,'heatmap_' + str(num) + '.jpg')
        # cv2.imwrite(saveheatimg, result)
        num = num + 1
        cv2.waitKey(0)
