
# coding: utf-8

# In[1]:


import os
import gc
import pandas as pd
import numpy as np

data_path = './data'


# In[2]:


train = pd.read_csv(os.path.join(data_path, 'security_test.csv'))
train_data = train[['file_id']].drop_duplicates()


# # Uni-Gram Features

# In[3]:


### File_id (Api): count,nunique
agg_func = ['count','nunique'] 
for func in agg_func:
    tmp = train.groupby(['file_id'])['api'].agg({'fileid_api_' + func: func}).reset_index() 
    train_data = pd.merge(train_data,tmp,how='left', on='file_id')  

### File_id (Tid): count,nunique,max,min,quantile(20,40,50,60,80),std,range
agg_func = ['count','nunique','max','min','median','std'] 
for func in agg_func:
    tmp = train.groupby(['file_id'])['tid'].agg({'fileid_tid_' + func: func}).reset_index() 
    train_data = pd.merge(train_data,tmp,how='left', on='file_id') 

secs = [0.2,0.4,0.6,0.8]
for sec in secs: 
    train_data['fileid_tid_quantile_' + str(sec * 100)] = train.groupby(['file_id'])['tid'].quantile(sec).values
 
train_data['fileid_tid_range'] = train.groupby(['file_id'])['tid'].quantile(0.975).values - train.groupby(['file_id'])['tid'].quantile(0.0125).values

### File_id (Index): count,nunique,max,min,quantile(20,40,50,60,80),std,range
agg_func = ['count','nunique','max','min','median','std'] 
for func in agg_func:
    tmp = train.groupby(['file_id'])['index'].agg({'fileid_index_' + func: func}).reset_index() 
    train_data = pd.merge(train_data,tmp,how='left', on='file_id') 
    
secs = [0.2,0.4,0.6,0.8]
for sec in secs: 
    train_data['fileid_index_quantile_' + str(sec * 100)] = train.groupby(['file_id'])['index'].quantile(sec).values
 
train_data['fileid_index_range'] = train.groupby(['file_id'])['index'].quantile(0.975).values - train.groupby(['file_id'])['index'].quantile(0.0125).values


# In[6]:


def groupby_pivot_features(data_merge, data_orig , groupby_features,
                           col1 = None, col2 = None, agg_func = None):
    for func in agg_func:
        train_split = data_orig.groupby(['file_id',col1])[col2].agg({'fileid_' + col1 + '_'+col2+'_'+ str(func):func}).reset_index() 
        
        train_split_ =  pd.pivot_table(train_split, values = 'fileid_' + col1 + '_'+col2+'_'+ str(func), index=['file_id'],columns=[col1])
        new_cols = [ 'fileid_' + col1 + '_'+col2+  '_' + func + '_' + str(col) for col in train_split_.columns]
        
        groupby_features.append(new_cols)
        train_split_.columns = new_cols 

        train_split_.reset_index(inplace = True)
        
        data_merge = pd.merge(data_merge,train_split_,how='left', on='file_id') 
    return data_merge,groupby_features 


# In[ ]:


# File_id + Api (tid): count,nunique
groupby_features = []
agg_func = ['count', 'nunique']
train_data_,groupby_features = groupby_pivot_features(train_data, train, groupby_features, 
                                                      col1 = 'api', col2 = 'tid', agg_func = agg_func)

# File_id + Api(index): nunique, max, min, median, std
agg_func = ['nunique','max','min','median','std']
train_data_,groupby_features = groupby_pivot_features(train_data_, train, groupby_features, 
                                                      col1 = 'api', col2 = 'index', agg_func = agg_func) 

train_diff = train.groupby(['file_id','tid'])['index'].diff().fillna(-999).values
train['index_diff'] = train_diff
train_diff = train.loc[train.index_diff!=-999] 

agg_func = ['nunique','max','min','median','std']
train_data_,groupby_features = groupby_pivot_features(train_data_, train_diff, groupby_features, 
                                                      col1 = 'api', col2 = 'index_diff', agg_func = agg_func) 


# # Bi-Gram Features

# In[ ]:


train['api_shift'] = train['api'].shift(-1)
train['api_2'] = train['api'] +'_' + train['api_shift']
train.drop(['api_shift'],axis=1,inplace=True)
api_count = train['api_2'].value_counts()

agg_func = ['count','nunique'] 
for func in agg_func:
    tmp = train.groupby(['file_id'])['api_2'].agg({'fileid_api_2_' + func: func}).reset_index() 
    train_data_ = pd.merge(train_data_,tmp,how='left', on='file_id')
    
    
api_value_counts = pd.DataFrame(api_count).reset_index()
api_value_counts.columns = ['api_2','api_2_count']
train = pd.merge(train, api_value_counts, on ='api_2' , how='left')

agg_func = ['count']
groupby_features =  []
train_data_,groupby_features = groupby_pivot_features(train_data_, train.loc[train.api_2_count>=20], groupby_features, 
                                                      col1 = 'api_2', col2 = 'tid', agg_func = agg_func)


# In[ ]:


train_data_.to_csv('./feats/test_feats_stats.csv',index = None)

