# -*- coding: utf-8 -*-
"""
Created on Mon Feb 09 16:50:14 2015

@author: VishnuC
@email: vrajs5@gmail.com
Beating the benchmark for Microsoft Malware Classification Challenge (BIG 2015)
"""
from joblib import Parallel, delayed
import os
import gzip
from csv import writer
import six
import numpy as np

read_mode, write_mode = ('r','w') if six.PY2 else ('rt','wt')

path = '/home/zak/kaggle/malware/' #Path to project 
os.chdir(path)

if six.PY2:
    from itertools import izip
    zp = izip
else:
    zp = zip

# Give path to gzip of asm files
paths = ['train','test']

 
def consolidate(path):
    ''' A consolidation of given train or test files

        This function reads each asm files (stored in gzip format)
        and prepare summary. asm gzip files are stored in train_gz 
        and test_gz locating.
    '''
    
    s_path = path + '/'
    Files = os.listdir(s_path)
    byteFiles = [i for i in Files if '.bytes.gz' in i]
    consolidatedFile = path + '_consolidation.gz'
    
    with gzip.open(consolidatedFile, write_mode) as f:
        # Preparing header part
        fw = writer(f)
        colnames = ['filename', 'no_que_mark']
        colnames += ['file_size', 'two_byte_sum']
        colnames += ['TB_'+hex(i)[2:] for i in range(16**2)]
        colnames += ['FB_'+hex(i)[2:] for i in range(16**4)]
        fw.writerow(colnames)
        
        # Creating row set
        consolidation = []
        for t, fname in enumerate(byteFiles):
            file_size = os.path.getsize(s_path+fname)
            f = gzip.open(s_path+fname, read_mode)
            twoByte = np.zeros(16**2, dtype=np.int)
            #fourByte = np.zeros(16**4, dtype=np.int)
            no_que_mark = 0
            for row in f:
                codes = row[:-2].split()[1:]
                # Finding number of times ?? appears
                no_que_mark += codes.count('??')
                
                # Conversion of code to to two byte
                
                twoByteCode = np.fromiter((int(i,16) for i in codes if i != '??'), np.int)
                #fourByteCode = np.fromiter((int(codes[i]+codes[i+1],16) for i in range(len(codes)-1) if ((codes[i] != '??') and (codes[i+1] != '??'))), np.int)
                                                    
                # Frequency calculation of two byte codes
                for i in twoByteCode:
                    twoByte[i] += 1
                #for i in fourByteCode:
                #    fourByte[i] += 1
                two_byte_sum = np.sum(twoByte)
                if two_byte_sum == 0:
                    print codes
                #four_byte_sum = np.sum(fourByte)
                if two_byte_sum == 0:
                    two_byte = twoByte
                else:
                    two_byte = np.fromiter((float(i)/two_byte_sum for i in twoByte), np.float)
                #if four_byte_sum == 0:
                #    four_byte = fourByte
                #else:
                    #four_byte = np.fromiter((float(i)/four_byte_sum for i in fourByte), np.float)
            # Row added
            consolidation.append([fname[:fname.find('.bytes.gz')], no_que_mark] + \
                                 [file_size] + [two_byte_sum] + \
                                 np.ndarray.tolist(two_byte))# + np.ndarray.tolist(four_byte))
                                    
            # Writing rows after every 100 files processed
            if (t+1)%100==0:
                print(t+1, 'files loaded for ', path)
                fw.writerows(consolidation)
                consolidation = []
                
        # Writing remaining files
        if len(consolidation)>0:
            fw.writerows(consolidation)
            consolidation = []
    
    del Files, byteFiles, colnames, s_path, consolidation, f, fw, \
        twoByte, twoByteCode, consolidatedFile

if __name__ == '__main__':
    #for path in paths:
    #    consolidate(path)
    Parallel(n_jobs=2)(delayed(consolidate)(subpath) for subpath in paths)

8bcf943657aa93117c951a2e31ac62e71cdb3f5f
# -*- coding: utf-8 -*-
"""
Created on Mon Feb 09 16:50:14 2015

@author: VishnuC
@email: vrajs5@gmail.com
Beating the benchmark for Microsoft Malware Classification Challenge (BIG 2015)
"""
from joblib import Parallel, delayed
import os
import gzip
from csv import writer
import six
import numpy as np

read_mode, write_mode = ('r','w') if six.PY2 else ('rt','wt')

path = '/home/zak/kaggle/malware/' #Path to project 
os.chdir(path)

if six.PY2:
    from itertools import izip
    zp = izip
else:
    zp = zip

# Give path to gzip of asm files
paths = ['train','test']

 
def consolidate(path):
    ''' A consolidation of given train or test files

        This function reads each asm files (stored in gzip format)
        and prepare summary. asm gzip files are stored in train_gz 
        and test_gz locating.
    '''
    
    s_path = path + '/'
    Files = os.listdir(s_path)
    byteFiles = [i for i in Files if '.bytes.gz' in i]
    consolidatedFile = path + '_consolidation.gz'
    
    with gzip.open(consolidatedFile, write_mode) as f:
        # Preparing header part
        fw = writer(f)
        colnames = ['filename', 'no_que_mark']
        colnames += ['file_size', 'two_byte_sum']
        colnames += ['TB_'+hex(i)[2:] for i in range(16**2)]
        colnames += ['FB_'+hex(i)[2:] for i in range(16**4)]
        fw.writerow(colnames)
        
        # Creating row set
        consolidation = []
        for t, fname in enumerate(byteFiles):
            file_size = os.path.getsize(s_path+fname)
            f = gzip.open(s_path+fname, read_mode)
            twoByte = np.zeros(16**2, dtype=np.int)
            #fourByte = np.zeros(16**4, dtype=np.int)
            no_que_mark = 0
            for row in f:
                codes = row[:-2].split()[1:]
                # Finding number of times ?? appears
                no_que_mark += codes.count('??')
                
                # Conversion of code to to two byte
                
                twoByteCode = np.fromiter((int(i,16) for i in codes if i != '??'), np.int)
                #fourByteCode = np.fromiter((int(codes[i]+codes[i+1],16) for i in range(len(codes)-1) if ((codes[i] != '??') and (codes[i+1] != '??'))), np.int)
                                                    
                # Frequency calculation of two byte codes
                for i in twoByteCode:
                    twoByte[i] += 1
                #for i in fourByteCode:
                #    fourByte[i] += 1
                two_byte_sum = np.sum(twoByte)
                if two_byte_sum == 0:
                    print codes
                #four_byte_sum = np.sum(fourByte)
                if two_byte_sum == 0:
                    two_byte = twoByte
                else:
                    two_byte = np.fromiter((float(i)/two_byte_sum for i in twoByte), np.float)
                #if four_byte_sum == 0:
                #    four_byte = fourByte
                #else:
                    #four_byte = np.fromiter((float(i)/four_byte_sum for i in fourByte), np.float)
            # Row added
            consolidation.append([fname[:fname.find('.bytes.gz')], no_que_mark] + \
                                 [file_size] + [two_byte_sum] + \
                                 np.ndarray.tolist(two_byte))# + np.ndarray.tolist(four_byte))
                                    
            # Writing rows after every 100 files processed
            if (t+1)%100==0:
                print(t+1, 'files loaded for ', path)
                fw.writerows(consolidation)
                consolidation = []
                
        # Writing remaining files
        if len(consolidation)>0:
            fw.writerows(consolidation)
            consolidation = []
    
    del Files, byteFiles, colnames, s_path, consolidation, f, fw, \
        twoByte, twoByteCode, consolidatedFile

if __name__ == '__main__':
    #for path in paths:
    #    consolidate(path)
    Parallel(n_jobs=2)(delayed(consolidate)(subpath) for subpath in paths)
