<repo_name>Zorqe/DataMiningProjectCTU-13<file_sep>/KnnModel.py
import pandas as pd
import numpy as np
from sklearn.neighbors import KNeighborsClassifier
from sklearn.metrics import accuracy_score


dataFrameTrain = pd.read_csv('trainDataCombinedCaptureNewFeatureGenerated.csv')
dataFrameTest = pd.read_csv('testDataCombinedCaptureNewFeatureGenerated.csv')


#Mapping taken from tutorial https://www.kaggle.com/parasjindal96/how-to-normalize-dataframe-pandas
#Function to map strings and discretize for discretization
def mapping(data,feature):
    featureMap=dict()
    count=0
    for i in sorted(data[feature].unique(),reverse=True):
        featureMap[i]=count
        count=count+1
    data[feature]=data[feature].map(featureMap)
    return data


#Function to normalize all data for similarity calculations is KNN
def normalizeData(dfTrain):

    #Normalizing all values between 0 to 15
    dfNormalized = ((dfTrain-dfTrain.min())/(dfTrain.max()-dfTrain.min()))*15
    
    dfNormalized = dfNormalized.fillna(0)
    return dfNormalized




#Function to predict the class of all entries using the KnnClassifier from Sikit-learn
#Returns the list of predictions
def knnPredict(k, dataFrameTest,dataFrameTrain):

    knn = KNeighborsClassifier(n_neighbors=k)
    knn.fit(dataFrameTrain.drop(['LabelDisc'],axis=1), dataFrameTrain['LabelDisc'])

    predictions = knn.predict(dataFrameTest.drop(['LabelDisc'],axis=1))
    return predictions
    




#Drop all columns which we won't use for training data, instead use their discretized values

print("Dropping cols to not train on")

dataFrameTrain = dataFrameTrain.drop(['StartTime','SrcAddr','DstAddr','SrcAddr_App','SrcDst_Sport_unique'], axis=1)
dataFrameTest = dataFrameTest.drop(['StartTime','SrcAddr','DstAddr','SrcAddr_App','SrcDst_Sport_unique'], axis=1)

print("Normalizing data")

dataFrameTrain = normalizeData(dataFrameTrain)
dataFrameTest = normalizeData(dataFrameTest)

#Predicting using the KNN model
k = 5
print("\nPredicting values with k value: " + str(k))
predictions = knnPredict(k, dataFrameTest, dataFrameTrain)

countMaliciousPredicted = np.count_nonzero(predictions == 15.0)
countBackgroundPredicted = np.count_nonzero(predictions == 0.0)

totalMalicious = np.count_nonzero(dataFrameTest['LabelDisc'] == 1)

print("The count of malicious predictions are: "  + str( countMaliciousPredicted )  )
print("The count of background predictions are: "  + str( countBackgroundPredicted )  )

#Calculating the accuracy
accuracy =  accuracy_score(dataFrameTest['LabelDisc'], predictions) *100
print ("The accuracy is: " + str(accuracy)+"%")

correctlyClassified = 0
correctlyClassifiedBackground = 0
index = 0
for classification in list(dataFrameTest.LabelDisc):
    if predictions[index] == classification and classification == 15.0:
        correctlyClassified += 1
    elif predictions[index] == classification and classification == 0.0:
        correctlyClassifiedBackground += 1
    index += 1

print("The number of correctly classified malicious predictions are: " + str(correctlyClassified))


print("The precision on malicious data classification is: {:0.2f}%\n".format( (correctlyClassified/countMaliciousPredicted)*100))
print("The recall on malicious data classification is: {:0.2f}%\n".format( (correctlyClassified/totalMalicious)*100))

print("The precision on benign data classification is: {:0.2f}%\n".format( (correctlyClassifiedBackground/countBackgroundPredicted)*100))<file_sep>/processData.py
import pandas as pd
import numpy as np
import os
import time
from pathlib import Path
from sklearn import preprocessing
from sklearn.preprocessing import LabelEncoder


"""
Input: All 13 binetflow CSV files for the CTU-13 Dataset (Local directory)
Output: Two files, the processed training data, and the processed testing data (Output directory)

Script to perform preprocessing on all 13 CSV files, then discretization, followed by feature generation
The final feature generated files are then merged as follows, training (scenario 1-8 and 11-13) and testing (scenario 9-10)
The merged files are the two final outputs provided by this script
"""

#------------------------------- Functions -----------------------------------------
#Deletes row's where the column values are null,nan,nat, or blank
def deleteNullRow(dataFrame, column):
    newDataFrame = dataFrame
    
    #dataframe dropna won't replace empty values only NaN and NaT so convert blank space to NaN then drop
    newDataFrame[column].replace('', np.nan, inplace=True)
    newDataFrame = newDataFrame.dropna(subset=[column])
    return newDataFrame


# Partially inspired from: https://github.com/mgarzon/cybersec/blob/master/MalwareDetection.ipynb
def preprocessData(dataFrame):

    '''
    This function is used to perform
    the necessary operations to 
    convert the raw data into a
    clean data set.
    '''
    
    #Outputting number of rows and column names before preprocessing
    print("----------Before pre-processing-----------")
    print("Number of rows: " + str(len(dataFrame.index)))
    print("The columns are: " + str(list(dataFrame)))
    
    
    
    #dropping columns specified
    listOfFeaturesToDrop = [
    'Dir',
    'sTos',
    'dTos']
    # Ignore columns that might not exist (ex. UNB dataset)
    dataFrame = dataFrame.drop(listOfFeaturesToDrop, axis=1, errors="ignore")

    #Dropping all null value rows from specified columns
    dataFrame = deleteNullRow(dataFrame,'Sport')
    dataFrame = deleteNullRow(dataFrame,'SrcAddr')
    dataFrame = deleteNullRow(dataFrame,'Dport')
    dataFrame = deleteNullRow(dataFrame,'DstAddr')
    
    
    # TODO
    #dp.convertColumnToTimeStamp(dataFrame,'StartTime') # ?? already a timestamp
    
    
    #Outputting number of rows and column names after preprocessing
    print("\n----------After pre-processing-----------")
    print("Number of rows: " + str(len(dataFrame.index)))
    print("The columns are: " + str(list(dataFrame)))
    return dataFrame

    #Function to perform discretization on the data
def discretizeData(dataFrame):
    
    '''
    This function is used discretize the data
    '''

    dfNew = dataFrame
    
    # Binning technique from
    # https://towardsdatascience.com/understanding-feature-engineering-part-1-continuous-numeric-data-da4e47099a7b
    quantile_list = [0, .25, .5, .75, 1.] # Change the quantile_list for more or less accuracy
    
    dfNew['TotBytesDisc'] = ""
    dfNew['SrcBytesDisc'] = ""
    dfNew['TotBytesDisc'] = pd.qcut(dataFrame['TotBytes'], quantile_list, duplicates='drop')
    dfNew['SrcBytesDisc'] = pd.qcut(dataFrame['SrcBytes'], quantile_list, duplicates='drop')
    
    # Bin Src/Dest port
    # According to 0-1023(WELLKNOWN_PORTNUMBER)
    #              1024-49151(REGISTERED_PORTNUMBER)
    #              49152-65535(DYNAMIC_PORTNUMBER)
    Sport = dataFrame['Sport']#[0x0303].astype('int64')
    if(Sport.dtype != 'int64'):
        Sport = Sport.apply(lambda x: int(x, 0)) # 0 as base means 10 will be converted to 10 and 0x10 will be converted to 16

    dfNew['SportDisc'] = ""
    dfNew['SportDisc'] = pd.cut(Sport, [0, 1023, 49151, 65535], duplicates='drop')
    
    Dport = dataFrame['Dport']#[0x0303].astype('int64')
    if(Dport.dtype != 'int64'):
        Dport = Dport.apply(lambda x: int(x, 0))

    dfNew['DportDisc'] = ""
    dfNew['DportDisc'] = pd.cut(Dport, [0, 1023, 49151, 65535], duplicates='drop')

    
    #LabelEncoder for unique values for Proto column and stored as column ProtoDisc
    le = preprocessing.LabelEncoder()
    le.fit(dfNew.Proto.unique())
    dfNew["ProtoDisc"] = ""
    dfNew.ProtoDisc = le.transform(dfNew.Proto)
    
    
    #Encoding "label" column to "labelDisc"
    #0 = Background/Normal             1=Botnet
    dfNew["LabelDisc"] = ""
    dfNew['LabelDisc'] = dfNew['Label']
    dfNew['LabelDisc'] = dfNew.LabelDisc.replace(r'(^.*Background.*$)', '0')
    dfNew['LabelDisc'] = dfNew.LabelDisc.replace(r'(^.*Normal.*$)', '0')
    dfNew['LabelDisc'] = dfNew.LabelDisc.replace(r'(^.*Botnet.*$)', '1')
    
    
    return dfNew


#helper function to count the distinct values of second column
#where SRCaddr's match in rolling window of size windowSize
def countDistinctMatchingForSrcAddr(sliceDF):
    
    '''
    This function is a helper function to perform the custom rolling window tasks
    '''
    
    SrcAddr = sliceDF["SrcAddr"].iloc[-1]     #SrcAddr of the rolling window to calculate for
    DstAddr = sliceDF["DstAddr"].iloc[-1]
    
    returnData = pd.DataFrame()
    
    srcAddrRows = sliceDF[sliceDF.SrcAddr == SrcAddr]
    destAddrRows = sliceDF[sliceDF.DstAddr == DstAddr]
    srcAndDestRows = srcAddrRows[srcAddrRows.DstAddr == DstAddr]
    
    # SrcAddr statistics
    returnData["SrcAddr_App"] = [srcAddrRows.shape[0]]   #counting total SrcAddr matches
    returnData["Src_Dport_unique"] =  srcAddrRows.Dport.nunique() #only counting distinct dports by using set
    returnData["Src_DstAddr_unique"] =  srcAddrRows.DstAddr.nunique()
    returnData["Src_Sport_unique"] =  srcAddrRows.Sport.nunique()
    returnData["Src_TotPkts_mean"] = srcAddrRows.TotPkts.mean()
    returnData["Src_TotBytesDisc_mode"] = srcAddrRows.TotBytesDisc.mode() # not quite mean but close enough
    
    # DstAddr statistics
    returnData["DstAddr_App"] = [destAddrRows.shape[0]]   #counting total DstAddr matches
    returnData["Dst_Dport_unique"] =  destAddrRows.Dport.nunique()
    returnData["Dst_SrcAddr_unique"] =  destAddrRows.SrcAddr.nunique()
    returnData["Dst_Sport_unique"] =  destAddrRows.Sport.nunique()
    returnData["Dst_TotPkts_mean"] = destAddrRows.TotPkts.mean()
    returnData["Dst_TotBytesDisc_mode"] = destAddrRows.TotBytesDisc.mode() # not quite mean but close enough
    
    # Src+Dstaddr statistics
    returnData["SrcDst_Sport_unique"] =  srcAndDestRows.Sport.nunique()
    returnData["SrcDst_Dport_unique"] =  srcAndDestRows.Dport.nunique()
    
    return returnData



#Function to generate connection based features for the source address
def generateSrcAddrFeaturesConnectionBased(dataFrame, windowSize):
    
    dfNew = dataFrame
    
    #How many times the SRCADDRESS has appeared within the last X netflows (SrcAddr_Dis)
    #For any of the flow records that SRCADDRESS has appeared within the last X netflows, count the distinct destination ports (Src_Dist_Des_Port) 
    #For any of the flow records that SRCADDRESS has appeared within the last X netflows, count the distinct destination addresses (Src_Dist_Des_Addr)
    #For any of the flow records that SRCADDRESS has appeared within the last X netflows, count the distinct source ports (Src_Dist_Src_Port)
    #For any of the flow records that SRCADDRESS AND DSTADDRESS has appeared within the last X netflows, count the distinct source ports   
    #For any of the flow records that SRCADDRESS AND DSTADDRESS has appeared within the last X netflows, count the distinct destinations ports
    
    #For any of the flow records that SRCADDRESS has appeared within the last X netflows, average the packets
    #For any of the flow records that SRCADDRESS has appeared within the last X netflows, average the bytes
    
    additionalCol = []
    for i in range(windowSize - 1, len(dfNew.index) + 1):
        #Feature generation feedback every 10000 generated rows
        if (i%10000 == 0):
            print(i)

        window = dfNew[i - (windowSize-1):i+1]
        
        slice_df = countDistinctMatchingForSrcAddr(window)

        additionalCol.append(slice_df)
    
    # Set the right index
    newCol = pd.concat(additionalCol, axis=0)
    del additionalCol
    newCol.index = np.arange(windowSize - 1, windowSize + len(newCol) - 1)
    dfNew = dfNew.join(newCol)
    
    #Dropping beginning rows of size: windowsize since all the generated features are null
    dfNew = deleteNullRow(dfNew, 'SrcAddr_App')

    return dfNew


#Adjustment Function to further adjust generated features and label encode them to ensure consistency and good data
def adjustFeatures(df):
    #Ensuring valid entries in Sport and Dport
    df['Sport']=pd.to_numeric(df['Sport'], errors='coerce')
    df['Dport']=pd.to_numeric(df['Dport'], errors='coerce')

    #Removing invalid entries that've been converted to NaN
    df = deleteNullRow(df,'Sport')
    df = deleteNullRow(df,'Sport')
    df = df.dropna()

    stringColsToMap = ['TotBytesDisc','SrcBytesDisc','SportDisc','DportDisc','Src_TotBytesDisc_mode','Dst_TotBytesDisc_mode']
    for col in stringColsToMap:
        LE = LabelEncoder()
        df[col] = LE.fit_transform(df[col])


    #Drop the original cols which have already been label encoded
    colsAlreadyLabelEncoded = ['Label','Proto']
    df = df.drop(colsAlreadyLabelEncoded, axis=1)

    return df

#Adjust merged data to maintain common quantile list ranges
def modifyMergedData(dataf):
    dataFrame = dataf
    #Drop all previous incorrectly computed columns
    #Note: State is dropped and not recomputed --> Is not used for training
    colsToDrop = ["State","TotBytesDisc","SrcBytesDisc","SportDisc","SportDisc","DportDisc","Src_TotBytesDisc_mode","Dst_TotBytesDisc_mode"]

    dataFrame.drop(colsToDrop, axis=1, inplace=True, errors="ignore")

    quantile_list = [0, .25, .5, .75, 1.] # Change the quantile_list for more or less accuracy

    dataFrame['TotBytesDisc'] = ""
    dataFrame['SrcBytesDisc'] = ""
    dataFrame['TotBytesDisc'] = pd.qcut(dataFrame['TotBytes'], quantile_list, duplicates='drop')
    dataFrame['SrcBytesDisc'] = pd.qcut(dataFrame['SrcBytes'], quantile_list, duplicates='drop')

    #Label encode discretized byte vals
    le = preprocessing.LabelEncoder()
    le.fit(dataFrame.TotBytesDisc.unique())
    dataFrame.TotBytesDisc = le.transform(dataFrame.TotBytesDisc)

    le = preprocessing.LabelEncoder()
    le.fit(dataFrame.SrcBytesDisc.unique())
    dataFrame.SrcBytesDisc = le.transform(dataFrame.SrcBytesDisc)

    dataFrame['SportDisc'] = ""
    dataFrame['DportDisc'] = ""

    dataFrame['SportDisc'] = pd.cut(dataFrame['Sport'],[0,1023,49151,65535],labels=[0,1,2])
    dataFrame['DportDisc'] = pd.cut(dataFrame['Dport'],[0,1023,49151,65535],labels=[0,1,2])

    return dataFrame

#Have all files to clean,discretize,featuregenerate in local directory
#Getting all .csv files(local directory) to be feature generated
localFiles = [file for file in os.listdir('.') if file.endswith(".csv")]

if len(localFiles) == 0:
    print("No files to process. Exiting.")
    exit()

testingIndexes = []
try:
    # If these files are found, put them in testing data
    testingIndex1 = localFiles.index("capture20110817.csv")        #index of testing scenario 9
    testingIndex2 = localFiles.index("capture20110818.csv")        #index of testing scenario 10
    testingIndexes = [testingIndex1, testingIndex2]
except ValueError:
    # Only create training data
    pass


# Perform preprocessing, discretizing, and feature generation on each file seperately
processedDataTraining = []
processedDataTesting = []
for scenarioNum,file in enumerate(localFiles):
    print("Reading file: "+file)
    dataFrame = pd.read_csv(file)

    print("Preprocessing file: "+file)
    dataFrame = preprocessData(dataFrame)

    print("Discretizing file: "+file)
    dataFrame = discretizeData(dataFrame)

    print("Feature generating file: "+file)
    #Window size 10,000
    now = time.time()
    dataFrame = generateSrcAddrFeaturesConnectionBased(dataFrame,10000)
    dataFrame = adjustFeatures(dataFrame)

    if scenarioNum in testingIndexes:
        #Testing set
        processedDataTesting.append(dataFrame)
    else:
        #Training set
        processedDataTraining.append(dataFrame)

def writeToDisk(filename, data):
    print("Merging scenario", filename)
    dfOut = pd.concat(data)
    del data

    print("Adjusting merged data for", filename)
    modifyMergedData(dfOut)

    dfOut.to_csv(filename, encoding='utf-8', index=False)
    print(filename + " Created")


#--------------------Outputting files--------------------------
dataFrameOutDir = Path("Output")
dataFrameOutDir.mkdir(parents=True, exist_ok=True)

writeToDisk(dataFrameOutDir / "finalTrainingData.csv", processedDataTraining)
if len(processedDataTesting) > 0:
    writeToDisk(dataFrameOutDir / "finalTestingData.csv", processedDataTesting)<file_sep>/runModel.py
import pandas as pd
import os
import pickle
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier, AdaBoostClassifier
from sklearn.svm import LinearSVC
from sklearn.metrics import classification_report
from sklearn.metrics import accuracy_score

"""
Input: Processed testing data (local directory) and the trained models
Output Display: Predicted classes with classification reports

Script to test all the models: AdaBoost, Gradient Boost, SVM, and Decision Tree
After predicting the classes for the test data the classification reports are printed
"""

#Load the testing dataframe
print("Loading testing data...")

test_dataFrame = pd.read_csv('Output/finalTestingData.csv')

#Dropping values that weren't trained on
test_dataFrame = test_dataFrame.drop(['StartTime','SrcAddr','DstAddr'], axis=1)

#Splitting up the test dataframe into one with only features, other with classifications
test_dataFrame_Classification = test_dataFrame[['LabelDisc']].copy()
test_dataFrame = test_dataFrame.drop(['LabelDisc'], axis=1)

#Getting vals to use for testing
test_data = test_dataFrame.values
test_data_class = test_dataFrame_Classification.values

test_features = test_data[0::]
test_results = test_data_class[0::,0]

#Testing
modelsFolder = "Models"
modelFileNames = [modelsFolder + "/" + file for file in os.listdir(modelsFolder) if file.endswith(".pkl")]

model = None
for model_filename in modelFileNames:
    print("\n\nTesting using model "+model_filename+"...")
    with open(model_filename, 'rb') as file:  
        model = pickle.load(file)

    predictionClassification = model.predict(test_features)

    print("Scikit-learn classification report: ")
    print(classification_report(test_dataFrame_Classification.LabelDisc, predictionClassification))
    print ('Accuracy = ' + str(accuracy_score(test_results, predictionClassification)) )<file_sep>/trainModel.py
import pandas as pd
import os
import pickle
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier, AdaBoostClassifier
from sklearn.svm import LinearSVC
from sklearn import tree
from sklearn.ensemble import GradientBoostingClassifier

"""
Input: Processed training data (local directory)
Output: All trained machine learning models

Script to train all the models: AdaBoost, Gradient Boost, SVM, and Decision Tree
Outputs the saved models in a pickle formate to be used by runModel.py
"""

#Load the training dataframe
print("Loading training data...")
train_dataFrame = pd.read_csv('finalTrainingData.csv')

#Drop columns to not train on
print("Dropping columns not required for training...")
train_dataFrame = train_dataFrame.drop(['StartTime','SrcAddr','DstAddr'], axis=1)

#Training classes and data seperated
train_dataframe_Classification = train_dataFrame[['LabelDisc']].copy()
train_dataFrame = train_dataFrame.drop(['LabelDisc'], axis=1)

#Getting the values for the classes and the training data
train_data = train_dataFrame.values
train_data_labels = train_dataframe_Classification.values

#Storing the features to use in model as well as the classifications
train_features = train_data[0::]
train_labels = train_data_labels[0::,0]


#Creating all models to be trained on (except KNN)
models = [
        ["adaBoostModel", AdaBoostClassifier(RandomForestClassifier(n_estimators = 1000), algorithm="SAMME", n_estimators=500)],
        ["LinearSVCModel", LinearSVC(max_iter=4000)],
        ["CART", tree.DecisionTreeClassifier()],
        ["GradientBoost", GradientBoostingClassifier()]
        ]

#Training on all models and saving them locally
for modelToTrain in models:
    modelName = modelToTrain[0]
    model = modelToTrain[1]
    print("\n\nModel training for model " + modelName + "...")
    
    #Fit the training data to the model
    model = model.fit(train_features, train_labels)

    print("Saving model "+ modelName + "...")
    model_filename = modelName + ".pkl"
    with open(model_filename, 'wb') as file:  
        pickle.dump(model, file)<file_sep>/README.md
 <NAME>
<NAME>

Google Drive File Download Link:
https://drive.google.com/drive/folders/114W78sGZ_F3vhY6IK1YwFu8GnXDMX5ty?usp=sharing

Includes:

    - finalTestingData.csv        (csv with the feature generated test data)
    - finalTrainingData.csv       (csv with the feature generated train data)
    - adaBoostModel.pkl           (Trained adaBoostModel)
    - testingDataUNB.csv          (UNB malware dataset converted into the CTU dataset format)
    - testingDataUNB-filtered.csv          (UNB malware dataset with only the Neris and Rbot botnets)

CTU-13 training set has bots:

    - Neris
    - Rbot
    - e4f816462c4fc84bb250e2b1d295bf23_85f9a5247afbe51e64794193f1dd72eb_unpacked (unknown name)
    - svchosta
    - sogou_explorer_silent_1.4.0.418_2136
    - QvodSetuPuls23
    - 3d3d%3F%3F%3F%3F.xls
    
CTU-13 testing set has bots:

    - Neris
    - Rbot

UNB testing set has bots: (https://www.unb.ca/cic/datasets/botnet.html)

    - Neris | IRC | 25967 (5.67%)
    - Rbot | IRC | 83 (0.018%)
    - Menti | IRC | 2878(0.62%)
    - Sogou | HTTP | 89 (0.019%)
    - Murlo | IRC | 4881 (1.06%)
    - Virut | HTTP | 58576 (12.80%)
    - NSIS | P2P | 757 (0.165%)
    - Zeus | P2P | 502 (0.109%)
    - SMTP Spam | P2P | 21633 (4.72%)
    - UDP Storm | P2P | 44062 (9.63%)
    - Tbot | IRC | 1296 (0.283%)
    - Zero Access | P2P | 1011 (0.221%)
    - Weasel | P2P | 42313 (9.25%)
    - Smoke Bot | P2P | 78 (0.017%)
    - Zeus Control (C&C) | P2P | 31 (0.006%)
    - ISCX IRC bot | P2P | 1816 (0.387%)

They have neris and rbot in common.
