导入库
import pandas as pd
import numpy as np
pd.set_option("display.max_columns",33)
import seaborn as sns
import matplotlib.pyplot as plt
import plotly.express as px
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split,GridSearchCV
from sklearn.metrics import accuracy_score,confusion_matrix
import warnings
warnings.filterwarnings("ignore")
数据基本信息
df = pd.read_excel("Data_Train.xlsx")
df.head()
image.png
df.shape
image.png
df.isnull().sum()
image.png
df.dtypes
image.png
columns = df.columns.tolist()
columns
image.png
具体字段的中文含义：
Airline：不同类型的航空公司
Date_of_Journey：旅客的旅行开始日期
Source：旅客出发地
Destination：旅客目的地
Route：航班路线
Dep_Time：出发时间
Arrival_Time：抵达时间
Duration：持续时间；指的是航班完成从出发到目的地的旅程的整个时间
Total_Stops：总共停留地
Additional_Info：其他信息，比如：食物、设备信息等
Price：整个旅程的航班票价
df.info()
image.png
df.describe()
image.png
缺失值处理
import missingno as mso
mso.bar(df,color="blue")
plt.show()
image.png
# 缺失值删除
df.dropna(inplace=True)
df.isnull().sum()
image.png
时间相关字段处理
# 时间处理
# 通过pd.to_datetime()直接将字符型的数据转成时间类型的数据
# 通过dt.day或者df.month 直接获取天或者月的信息
def change_to_datetime(col):
df[col] = pd.to_datetime(df[col])
for col in ["Date_of_Journey","Dep_Time","Arrival_Time"]:
change_to_datetime(col)
df.dtypes
image.png
# 提取天和月
df["day"] = df["Date_of_Journey"].dt.day
df["month"] = df["Date_of_Journey"].dt.month
df.head()
image.png
df.drop("Date_of_Journey",axis=1,inplace=True)
# 起飞时间和抵达时间处理
def extract_hour(data,col):
data[col+ "_hour"] = data[col].dt.hour
def extract_minute(data,col):
data[col+ "_minute"] = data[col].dt.minute
def drop_col(data,col):
data.drop(col,axis=1,inplace=True)
extract_hour(df,"Dep_Time")
extract_minute(df,"Dep_Time")
drop_col(df,"Dep_Time")
extract_hour(df,"Arrival_Time")
extract_minute(df,"Arrival_Time")
drop_col(df,"Arrival_Time")
df.head()
image.png
# 航班持续时间
# 1、将持续时间规范化处理，统一变成0h 1m
# duration = list(df["Duration"])
# for i in range(len(duration)):
# if len(duration[i].split(' ')) == 2:
# pass
# else:
# if 'h' in duration[i]:
# duration[i] = duration[i] + ' 0m'
# else:
# duration[i] = '0h ' + duration[i]
def change_duration(x):
if "h" in x and "m" in x:
return x
else:
if "h" in x:
return x + " 0m"
else:
return "0h " + x
df["Duration"] = df["Duration"].apply(change_duration)
df.head()
image.png
# 2、从Duration字段中提取小时和分钟
df1 = df["Duration"].str.extract(r'(?P<dur_hour>\d+)h (?P<dur_minute>\d+)m')
df1.head()
image.png
df = df.join(df1)
df.head()
image.png
df.drop("Duration",inplace=True,axis=1)
# 3、字段类型转化：查看dur_hour和dur_minute的字段类型变化
df.dtypes
image.png
df["dur_hour"] = df["dur_hour"].astype(int)
df["dur_minute"] = df["dur_minute"].astype(int)
df.dtypes
image.png
字段编码
# 1、针对字符型的字段
column = [column for column in df.columns if df[column].dtype == "object"]
column
image.png
# 2、数值型（连续型）字段
continuous_col = [column for column in df.columns if df[column].dtype != "object"]
continuous_col
image.png
2种编码技术
标称数据：没有任何顺序，使用独热编码oneot encoding
有序数据：存在一定的顺序，使用类型编码labelEncoder
# 生成标称型字段组成的数据
categorical = df[column]
categorical.head()
image.png
不同字段编码处理
# 航空公司-Airline
# 1、不同航空公司的数量统计：
airline = categorical["Airline"].value_counts().reset_index()
airline
image.png
# 2、查看航空公司与价格关系
plt.figure(figsize=(15,8))
sns.boxplot(x="Airline",y="Price",data=df.sort_values("Price",ascending=False))
plt.show()
image.png
Jet Airways Business公司的机票价格是最高的
其他公司的价格中位数是比较接近的
# 3、实现独热编码
Airline = pd.get_dummies(categorical["Airline"],drop_first=True)
Airline.head()
image.png
# 停留地-Total_Stops
# 1、和价格的关系
plt.figure(figsize=(15,8))
sns.boxplot(x="Total_Stops",y="Price",data=df.sort_values("Price",ascending=False))
plt.show()
image.png
# 2、实施硬编码；区别于航空公司的独热编码
dict_stops = {"non-stop":0, "1 stop":1, "2 stops":2, "3 stops":3, "4 stops":4}
categorical["Total_Stops"] = categorical["Total_Stops"].map(dict_stops)
categorical.head()
image.png
# 出发地source
# 出发地和价格的关系：
plt.figure(figsize=(18,12))
sns.catplot(x="Source",y="Price",data=df.sort_values("Price",ascending=False),kind="boxen")
plt.show()
image.png
# 独热编码的过程：
source = pd.get_dummies(categorical["Source"],drop_first=True)
source.head()
image.png
# 目的地-destination
# 目的地和价格的关系
plt.figure(figsize=(18, 12))
sns.boxplot(x="Destination",
y="Price",
data=df.sort_values("Price", ascending=False))
plt.show()
image.png
# 独热编码的实现
destination = pd.get_dummies(categorical["Destination"], drop_first=True)
destination.head()
image.png
# 路线Route
# 1.不同路线的数量统计
categorical["Route"].value_counts()
image.png
# 2.路线名称提取
# 从上面结果看出来最长的路线中有5个地名，我们一次性提取
# 没有出现的数据则用NaN来表示:
categorical["Route1"] = categorical["Route"].str.split("→").str[0]
categorical["Route2"] = categorical["Route"].str.split("→").str[1]
categorical["Route3"] = categorical["Route"].str.split("→").str[2]
categorical["Route4"] = categorical["Route"].str.split("→").str[3]
categorical["Route5"] = categorical["Route"].str.split("→").str[4]
categorical.head()
image.png
# 3.缺失值字段
categorical.drop("Route", axis=1, inplace=True)
categorical.isnull().sum()
image.png
for i in ["Route3", "Route4", "Route5"]:
categorical[i].fillna("None", inplace=True)
# 4.类型编码LabelEncoder
from sklearn import preprocessing
le =preprocessing.LabelEncoder()
for i in ["Route1", "Route2", "Route3", "Route4", "Route5"]:
categorical[i] = le.fit_transform(categorical[i])
categorical.head()
image.png
# 抵达时间/小时-Arrival_Time_hour
# 抵达目的地时间和价格的关系
df.plot.hexbin(x="Arrival_Time_hour", y="Price", gridsize=15)
plt.show()
image.png
建模数据
# 删除无效字段
# 生成的全部字段信息
categorical.columns
image.png
# 将原始的无效字段直接删除
drop_col(categorical, "Airline")
drop_col(categorical, "Source")
drop_col(categorical, "Destination")
drop_col(categorical, "Additional_Info")
# 最终数据
final_df = pd.concat([categorical, Airline, source, destination, df[continuous_col]], axis=1)
final_df.head()
image.png
# 离群点检测
# 对上面生成的最终数据进行离群点检测
def plot(data, col):
fig, (ax1, ax2) = plt.subplots(2, 1)
sns.distplot(data[col], ax=ax1)
sns.boxplot(data[col], ax=ax2)
plot(final_df, "Price")
image.png
# 对离群点填充均值，查看填充后的效果
final_df["Price"] = np.where(final_df["Price"]>=40000,
final_df["Price"].median(),
final_df["Price"])
plot(final_df, "Price")
image.png
# 数据切分
X = final_df.drop("Price", axis=1)
y = final_df["Price"]
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=123)
特征选择
from sklearn.feature_selection import mutual_info_classif
imp = pd.DataFrame(mutual_info_classif(X ,y), index=X.columns)
imp.columns = ["importance"]
imp.sort_values(by="importance", ascending=False)
image.png
评价指标
# r2_score(重点关注), mean_absolute_error, mean_squared_error
from sklearn.metrics import r2_score, mean_absolute_error, mean_squared_error
def predict(ml_model):
print("Model is: ", ml_model)
model = ml_model.fit(X_train, y_train)
print("Training score: ", model.score(X_train, y_train))
predictions = model.predict(X_test)
print("Predictions: ", predictions)
print("----------")
r2score = r2_score(y_test, predictions)
print("r2 score is: ", r2score)
print("MAE:{}", mean_absolute_error(y_test, predictions))
print("MSE:{}", mean_squared_error(y_test, predictions))
print("RMSE:{}", np.sqrt(mean_squared_error(y_test, predictions)))
sns.distplot(y_test - predictions)
建模
# 导入多种模型
# 逻辑回归
from sklearn.linear_model import LogisticRegression
# K近邻回归
from sklearn.neighbors import KNeighborsRegressor
# 决策树回归
from sklearn.tree import DecisionTreeRegressor
# 支持向量机回归
from sklearn.svm import SVR
# 梯度提升回归，随机森林回归
from sklearn.ensemble import GradientBoostingRegressor, RandomForestRegressor
# 随机森林回归树
predict(RandomForestRegressor())
image.png
# 逻辑回归
predict(LogisticRegression())
image.png
# K近邻回归
predict(KNeighborsRegressor())
image.png
# 决策树回归
predict(DecisionTreeRegressor())
image.png
# 支持向量机回归
predict(SVR())
image.png
# 梯度提升回归
predict(GradientBoostingRegressor())
image.png
模型调优
# 调优寻参
# 采用随机搜索调优
from sklearn.model_selection import RandomizedSearchCV
random_grid = {
"n_estimators":[100, 120, 150, 180, 200, 220],
"max_features":["auto", "sqrt"],
"max_depth":[5, 10, 15, 20]
}
rf = RandomForestRegressor()
rf_random = RandomizedSearchCV(estimator=rf, param_distributions=random_grid, cv=3, verbose=2, n_jobs=-1)
rf_random.fit(X_train, y_train)
image.png
rf_random.best_params_
image.png
# 调优后结果
prediction = rf_random.predict(X_test)
sns.distplot(y_test - prediction)
image.png
r2_score(y_test, prediction)
image.png
两种常见求解r2方式
# 利用python间接求解
from sklearn.metrics import mean_squared_error
y_test = [1, 2, 3]
y_pred = [1.3, 2.1, 3.5]
1 - mean_squared_error(y_test, y_pred)/np.var(y_test)
image.png
# sklearn直接求解
from sklearn.metrics import r2_score
y_test = [1, 2, 3]
y_pred = [1.3, 2.1, 3.5]
r2_score(y_test, y_pred)
image.png
来源：尤而小屋
