text_plain_output_1.png
from copy import deepcopy

from sklearn.preprocessing import MinMaxScaler

import math

import numpy as np

import pandas as pd

import warnings

import pandas as pd
import warnings
import os
import numpy as np
import folium
from folium import plugins
from sklearn.preprocessing import MinMaxScaler
from copy import deepcopy
import math
import time
from sklearn.cluster import AgglomerativeClustering
from sklearn import metrics
import matplotlib.pyplot as plt
from sklearn.cluster import KMeans
from sklearn.cluster import DBSCAN
from sklearn import mixture
warnings.filterwarnings('ignore')
pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', None)

all_data = pd.read_csv('../input/globalterrorismdb_0718dist.csv', encoding='ISO-8859-1')

all_data = all_data[['eventid', 'iyear', 'imonth', 'iday', 'extended', 'country', 'region', 'vicinity', 'latitude', 'longitude', 'specificity', 'crit1', 'crit2', 'crit3', 'doubtterr', 'multiple', 'attacktype1', 'success', 'suicide', 'weaptype1', 'targtype1', 'nkill', 'nkillter', 'nwound', 'nwoundte', 'property', 'propextent', 'ishostkid', 'ransom', 'INT_LOG', 'INT_IDEO', 'INT_MISC', 'INT_ANY']]

all_data = all_data.dropna(subset=['latitude'])
all_data = all_data.dropna(subset=['longitude'])

all_data.shape

all_data['specificity'] = pd.to_numeric(all_data['specificity'], downcast='integer')
all_data['doubtterr'] = pd.to_numeric(all_data['doubtterr'], downcast='integer')
all_data['multiple'] = pd.to_numeric(all_data['multiple'], downcast='integer')
all_data['nkill'] = pd.to_numeric(all_data['nkill'], downcast='integer')
all_data['nkillter'] = pd.to_numeric(all_data['nkillter'], downcast='integer')
all_data['nwound'] = pd.to_numeric(all_data['nwound'], downcast='integer')
all_data['nwoundte'] = pd.to_numeric(all_data['nwoundte'], downcast='integer')
all_data['ishostkid'] = pd.to_numeric(all_data['ishostkid'], downcast='integer')
all_data['ransom'] = pd.to_numeric(all_data['ransom'], downcast='integer')

id_ = list(all_data['eventid'])
all_data.drop('eventid', axis=1, inplace=True)

all_data = all_data[['crit1', 'crit2', 'crit3', 'doubtterr', 'multiple', 'attacktype1', 'success', 'suicide', 'weaptype1', 'targtype1', 'nkill', 'nkillter', 'nwound', 'nwoundte', 'property', 'propextent', 'ishostkid', 'ransom']]

"""
author: zhenyu wu
time: 2019/04/07 21:48
function: 对数据样本进行归一化处理，将区间放缩到某个范围内
   在MinMaxScaler中是给定了一个明确的最大值与最小值。它的计算公式如下: X_std = (X - X.min(axis=0)) / (X.max(axis=0) - X.min(axis=0)) | X_scaled = X_std / (max - min) + min 每个特征中的最小值变成了MIN，最大值变成了MAX。这样做是为了在熵权法取对数过程中避免向下溢出。
params: 
    data: 数据集
    MIN: 下限
    MAX: 上限
return:
    data_norm: 归一化后的数据集
"""
def normalization(data, MIN=0.002, MAX=0.998):
    min_max_scaler = MinMaxScaler(feature_range=(MIN, MAX))
    data_norm = min_max_scaler.fit_transform(data)
    data_norm = pd.DataFrame(data_norm)
    data_norm.columns = data.columns
    return data_norm

all_data_norm = normalization(all_data, MIN=0.002, MAX=0.998)

"""
author: zhenyu wu
time: 2019/04/07 21:47
function: 熵权法确定特征权重
params: 
    data_norm: 数据集
    threshold: 阈值
return:
    Entropy: 特征的熵值
    difference_coefficient: 特征的差异系数
    important_features: 重要特征名称序列
    entropy_weight: 特征熵权
    overall: 样本的重要性分数
"""
def entropy(data_norm, data_id, threshold=0.8):
    feature_weight = pd.DataFrame({'temp': list(np.zeros(len(data_norm)))})
    for i in data_norm.columns:
        Sum = data_norm[i].sum()
        temp = data_norm[i] / Sum
        feature_weight[i] = temp
    feature_weight.drop('temp', axis=1, inplace=True)
    Entropy = {}
    for i in feature_weight.columns:
        Sum = 0
        column = list(deepcopy(feature_weight[i]))
        for j in range(len(feature_weight)):
            Sum += column[j] * math.log(column[j])
        Entropy[i] = -1 / math.log(len(feature_weight)) * Sum
    important_features = []
    for key, value in Entropy.items():
        important_features.append(key)
    difference_coefficient = {}
    for i in important_features:
        difference_coefficient[i] = 1 - Entropy[i]
    Diff_sum = sum(list(difference_coefficient.values()))
    entropy_weight = {}
    for i in important_features:
        entropy_weight[i] = difference_coefficient[i] / Diff_sum
    feature_weight = feature_weight[important_features]
    feature_weight = np.mat(feature_weight)
    weight = np.array(list(entropy_weight.values()))
    overall_merit = weight * feature_weight.T
    overall_merit = overall_merit.T
    overall_merit = np.array(overall_merit)
    overall_list = []
    for i in range(len(feature_weight)):
        overall_list.append(overall_merit[i][0])
    overall = pd.DataFrame({'eventid': data_id, 'overall': overall_list})
    overall = overall.sort_values(by=['overall'], ascending=False)
    overall.index = list(np.arange(len(data_norm)))
    data_norm = data_norm[important_features]
    overall = overall.sort_values(by=['eventid'], ascending=True)
    overall.index = list(np.arange(len(data_norm)))
    feature_names = list(entropy_weight.keys())
    entropy_weight = list(entropy_weight.values())
    norm_entropy_weight = []
    for sub_weight in entropy_weight:
        norm_entropy_weight.append(sub_weight / sum(entropy_weight))
    entropy_weight = dict(zip(feature_names, norm_entropy_weight))
    return (entropy_weight, overall)

"""
author: zhenyu wu
time: 2019/04/09 16:51
function: 利用模糊层次分析法确定指标的权重
params: 
    r_1_n: r11~r1n的模糊关系
return:
    B: FAHP的权重计算结果
"""
def FAHP(r_1_n):
    R = np.zeros((len(r_1_n), len(r_1_n)))
    E = np.zeros((len(r_1_n), len(r_1_n)))
    B = np.zeros(len(r_1_n))
    R[0] = r_1_n
    col_1 = 1 - R[0]
    for i in range(len(r_1_n)):
        R[i] = R[0] - (R[0][0] - col_1[i])
    e = R.sum(axis=1)
    for i in range(len(e)):
        for j in range(len(e)):
            E[i][j] = (e[i] - e[j]) / (2 * len(e)) + 0.5
    e = E.sum(axis=0)
    for i in range(len(e)):
        B[i] = (2 * e[i] - 1) / (len(e) * (len(e) - 1))
    return B

"""
author: zhenyu wu
time: 2019/04/09 17:39
function: 利用模糊层次分析法确定所有指标权重
params: 
    feature_names: 特征名称
return:
    all_fAHP_weight: 所有指标的fAHP权重
"""
def fAHP_weight(feature_names):
    r_1_n = [0.5, 0.8, 0.8, 0.8]
    level_1_fAHP_weight = FAHP(r_1_n)
    r_1_n = [0.5, 0.6, 0.7, 0.8, 0.8]
    level_2_std_fAHP_weight = FAHP(r_1_n)
    level_2_std_fAHP_weight = list(level_2_std_fAHP_weight / sum(level_2_std_fAHP_weight) * level_1_fAHP_weight[0])
    r_1_n = [0.5, 0.7, 0.8, 0.8]
    level_2_att_fAHP_weight = FAHP(r_1_n)
    level_2_att_fAHP_weight = list(level_2_att_fAHP_weight / sum(level_2_att_fAHP_weight) * level_1_fAHP_weight[1])
    level_2_targ_fAHP_weight = [level_1_fAHP_weight[2]]
    r_1_n = [0.5, 0.5, 0.4, 0.4, 0.3, 0.4, 0.6, 0.6]
    level_2_kill_fAHP_weight = FAHP(r_1_n)
    level_2_kill_fAHP_weight = list(level_2_kill_fAHP_weight / sum(level_2_kill_fAHP_weight) * level_1_fAHP_weight[3])
    all_fAHP_weight = level_2_std_fAHP_weight + level_2_att_fAHP_weight + level_2_targ_fAHP_weight + level_2_kill_fAHP_weight
    all_fAHP_weight = all_fAHP_weight / sum(all_fAHP_weight)
    AHP_weight = dict(zip(feature_names, all_fAHP_weight))
    return AHP_weight

fAHP_weight = fAHP_weight(all_data_norm.columns)

print(fAHP_weight)