第七色在线视频,2021少妇久久久久久久久久,亚洲欧洲精品成人久久av18,亚洲国产精品特色大片观看完整版,孙宇晨将参加特朗普的晚宴

統(tǒng)計指標--缺失值

特殊描述值--頻數(shù)

特殊描述值-分位點

最大/小值

統(tǒng)計描述-眾數(shù)

均值中位數(shù)

均值/中位數(shù)

基本描述統(tǒng)計

數(shù)據的讀取

數(shù)據診斷1

數(shù)據獲取Kaggle

numpy:

pandas:把數(shù)據讀成dataframe形式，讀寫

scipy:計算，函數(shù)，眾數(shù)等計算

1、編碼

2、導入division返回值是一個浮點型的數(shù)

3、導入pd，np，scipy中的stats函數(shù)

#?-*-?coding:utf-8
from?__future__?import?division?#除的結果是小數(shù)
import?pandas?as?pd
import?numpy?as?np
from?scipy?import?stats

def?fill_fre_top_5(x):?#頻數(shù)前五位的函數(shù)：若不足5，則nan補齊
????if?(len(x))?<=?5:
????????new_array?=?np.full(5,?np.nan)
????????new_array[0:len(x)]?=?x
????????return?new_array

def?eda_analysis(missSet?=?[np.nan,?9999999999,?-999999],?df=None):
????#種類計數(shù)
????count_un?=?df.apply(lambda?x:?len(x.unique()))
????count_un?=?count_un.to_frame('count')?#轉成dataframe
????
????#零的計數(shù)
????count_zero?=?df.apply(lambda?x:?np.sum(x==0))
????count_zero?=?count_zero.to_frame('count_zero')
????
????#均值
????df_mean?=?df.apply(lambda?x:?np.mean(x[~np.isin(x,?missSet)]))
????df_mean?=?df_mean.to_frame('mean')
????
????#中位數(shù)
????df_median?=?df.apply(lambda?x:?np.median(x[~np.isin(x,?missSet)]))
????df_median?=?df_median.to_frame('median')
????
????#眾數(shù)
????df_mode?=?df.apply(lambda?x:?stats.mode(x[~np.isin(x,?missSet)])[0][0])
????df_mode?=?df_mode.to_frame('mode')
????
????#眾數(shù)的頻數(shù)和比例
????df_mode_count?=?df.apply(lambda?x:?stats.mode(x[~np.isin(x,?missSet)])[1][0])
????df_mode_count?=?df_mode_count.to_frame('mode_count')
????df_mode_perct?=?df_mode_count?/?df.shape[0]
????df_mode_perct.columns?=?['mode_perct']
????
????#最小值
????df_min?=?df.apply(lambda?x:?np.min(x[~np.isin(x,?missSet)]))
????df_min?=?df_min.to_frame('min')
????
????#最大值
????df_max?=?df.apply(lambda?x:?np.min(x[~np.isin(x,?missSet)]))
????df_max?=?df_max.to_frame('max')
????
????#分位數(shù)
????json_quantile?=?{}
????for?i,?name?in?enumerate(df.columns):
????????json_quantile[name]?=?np.percentile(df[name][~np.isin(df[name],?missSet)],?(1,?5,?25,?50,?75,?95,?99))
????df_quantile?=?pd.DataFrame(json_quantile)[df.columns].T
????df_quantile.columns?=?['quan01',?'quan05',?'quan25',?'quan50',?'quan75',?'quan95',?'quan99']
????
????#頻數(shù)（產出為數(shù)組，不能使用apply）
????json_fre_name?=?{}
????json_fre_count?=?{}
????
????for?i,?name?in?enumerate(df.columns):
????????#頻數(shù)前五的數(shù)
????????index_name?=?df[name][~np.isin(df[name],?missSet)].value_counts().iloc[0:5,?].index.values
????????index_name?=?fill_fre_top_5(index_name)
????????
????????json_fre_name[name]?=?index_name
????????
????????#頻數(shù)前五的數(shù)的頻數(shù)
????????value_count?=?df[name][~np.isin(df[name],?missSet)].value_counts().iloc[0:5,?].values
????????value_count?=?fill_fre_top_5(value_count)
????????
????????json_fre_count[name]?=?value_count?
????????
????df_fre_name?=?pd.DataFrame(json_fre_name)[df.columns].T
????df_fre_count?=?pd.DataFrame(json_fre_count)[df.columns].T
????
????df_fre?=?pd.concat([df_fre_name,?df_fre_count],?axis=1)?#合并
????df_fre.columns?=?['Value1',?'Value2',?'Value3',?'Value4',?'Value5',?'freq1',?'freq2',?'freq3',?'freq4',?'freq5']
????
????#缺失值
????df_miss?=?df.apply(lambda?x:?np.sum(np.isin(x,?missSet)))
????df_miss?=?df_miss.to_frame('freq_miss')
????
????#合并所有統(tǒng)計指標
????df_eda_summary?=?pd.concat([count_un,?count_zero,?df_mean,?df_median,?df_mode,?df_mode_count,?df_mode_perct,?df_min,?df.max,?df_fre,?df_miss],?axis=1)
????
????return?df_eda_summary

調用以上模塊：

import?wrap_un?import?*

df?=?pd.read_csv(".csv")
label?=?df['TARGET']
df?=?df.drop(['ID',?'TARGET'],?axis=1)

df_eda_summary?=?eda_analysis(missSet=[np.nan,?9999999999,?-999999],?df=df.iloc[:,?0:3])

測試：

start?=?timeit.default_timer()
df_eda_summary?=?eda_analysis(missSet=[np.nan,?9999999999,?-999999],?df=df.iloc[:,?0:3])
print?"EDA?Running?Time:?{0:.2f}?seconds".format(timeit.default_timer()-start)

'''

Created on 2018年7月16日

@author: Administrator

'''

#-*-coding:utf-8-*-

from __future__ import division

import pandas as pd

import numpy as np

from scipy import stats

from datetime import datetime

def fill_fre_top_5(x):

? ? if(len(x))<=5:

? ? ? ? new_array =np.full(5, np.nan)

? ? ? ? new_array[0:len(x)] =x

? ? ? ? return new_array

? ? ? ?

def eda_analysis(missSet=[np.nan, 9999999999, -999999], df=None):

? ? ##1.Count

? ? count_un = df.apply(lambda x:len(x.unique()))

? ? count_un = count_un.to_frame('count')

? ??

? ? ##2.Count Zero

? ? count_zero = df.apply(lambda x:np.sum(x==0))

? ? count_un = count_zero.to_frame('count_zero')

? ??

? ? ##3.Mean

? ? df_mean = df.apply(lambda x:np.mean(x[~np.isin(x, missSet)]))

? ? df_mean = df_mean.to_frame('mean')

? ??

? ? ##4.Median

? ? df_median = df.apply(lambda x:np.median(x[~np.isin(x, missSet)])) ?

? ? df_median = df_median.to_frame('median')

? ??

? ? ##5.Mode

? ? df_mode = df.apply(lambda x:stats.mode(x[~np.isin(x, missSet)])[0][0])

? ? df_mode = df_mode.to_frame('mode')

? ??

? ? ##6.Mode Percentage

? ? df_mode_count = df.apply(lambda x:stats.mode(x[~np.isin(x, missSet)])[1][0])

? ? df_mode_count = df_mode_count.to_frame('mode_count')

? ??

? ? df_mode_perct = df_mode_count/df.shape[0]

? ? df_mode_perct.columns =['mode_perct']

? ??

? ? ##7.Min

? ? df_min = df.apply(lambda x:np.min(x[~np.isin(x, missSet)])) ?

? ? df_min = df_min.to_frame('min')

? ??

? ? ##8.Max

? ? df_max = df.apply(lambda x:np.max(x[~np.isin(x, missSet)]))

? ? df_max = df_max.to_frame('max')

? ??

? ? ##9.Quantile

? ? json_quantile ={}

? ? for i, name in enumerate(df.columns):

? ? ? ? json_quantile[name] = np.percentile(df[name][~np.isin(df[name],missSet)],(1,5,25,50,75,95,99))

? ? df_quantile = pd.DataFrame(json_quantile)[df.columns].T

? ? df_quantile.columns=['quan01','quan05','quan25','quan50','quan75','quan95','quan99']

? ??

? ? ##10.Frequence

? ? json_fre_name ={}

? ? json_fre_count ={}

? ??

? ? for i,name in enumerate(df.columns):

? ? ? ? ##1.Index Name

? ? ? ? index_name = df[name][~np.isin(df[name],missSet)].value_counts().iloc[0:5, ].index.values

? ? ? ? ##1.1 If the length of array is less than 5

? ? ? ? index_name=fill_fre_top_5(index_name)

? ? ? ??

? ? ? ? ##2.Value Count

? ? ? ? values_count=df[name][~np.isin(df[name],missSet)].value_counts().iloc[0:5, ].values

? ? ? ? ##2.1 If the length of array is less than 5

? ? ? ? values_count = fill_fre_top_5(values_count)

? ? ? ??

? ? ? ? json_fre_count[name]=values_count

? ??

? ? df_fre_name =pd.DataFrame(json_fre_name)[df.columns].T

? ? df_fre_count =pd.DataFrame(json_fre_count)[df.columns].T

? ? df_fre = pd.concat([df_fre_name, df_fre_count],axis=1)

? ? df_fre.columns=['value1','value2','value3','value4','value5','freq1','freq2','freq3','freq4','freq5']

? ? ##11.Miss Value Count

? ? df_miss = df.apply(lambda x:np.sum(np.isin(x,missSet)))

? ? df_miss = df_miss.to_frame('freq_miss')

? ??

? ? #####12.Combine All Informations#####

? ? df_eda_summary = pd.concat(

? ? ? ? [count_un, count_zero, df_mean, df_median,?

? ? ? ? df_mode, df_mode_count,df_mode_perct,

? ? ? ? df_min,df_max,df_fre,df_miss], axis=1)

? ??

? ? return df_eda_summary