1.模型的评估与参数调优
#引入工具库
import warnings
warings.filterwarnings('ignore')
%matplotlib.inline
import matplotlib as mpl
import numpy as np
from matplotlib import pyplot as plt
mpl.rcParams['legend.numpoints'] = 1
#基本建模流程
from sklearn.dataset import make_blobs
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
#构建数据集
X , y = make_blobs(random_state = 0)
#切分train 和test data
X_train , X_test , y_train , y_test = train_test_split(X , y ,random_state = 0)
#初始化模型对象并拟合
logreg = LogisticRegression().fit(X_train , y_train)
#模型评估
logreg.score(X_test , y_test)
1.1交叉验证/Cross - validation
from tools import *
plots.plot_cross_validation()
1
1.1.1 K折交叉验证
from sklearn.datasets import load_iris
iris = load_iris()
print(iris.target)
2
plots.plot_stratified_cross_validation()
3
1.1.2 sklearn中的交叉验证
from sklearn.model_selection import cross_val_score
from skelarn.datasets import load_iris
from skelarn.linear_model import LogisticRegression
iris = load_iris()
logreg = LogisticRegression()
scores = cross_val_score(logreg, iris.data, iris.target)
print("cross-validation scores: ", scores)
4
scores = cross_val_score(logreg, iris.data, iris.target, cv=5)
scores
5
scores.mean()
6
1.1.3不同的数据切分方式
1.1.3.1 k折交叉验证（手动指定k折切分）
from sklearn.model_selection import KFold
kfold = KFold(n_split = 5)
cross_val_score(logreg , iris.data , iris.target , cv = kfold)
7
kfold = KFold(n_splits = 3)
cross_val_score(logreg , iris.data , iris.target , cv = kfold)
8
kfold = KFold(n_splits=3, shuffle=True, random_state=0)
cross_val_score(logreg, iris.data, iris.target, cv=kfold)
9
1.1.3.2 留一交叉验证
from sklearn.model_selection import LeaveOneOut
loo = LeaveOneOut()
scores = cross_val_score(logreg , iris.data , iris.target , cv = loo)
print("number of cv iterations: ", len(scores))
print("mean accuracy: ", scores.mean())
10
1.1.3.3 乱序分割交叉验证
from sklearn.model_selection import ShuffleSplit
shuffle_split = ShuffleSplit(test_size = 5 , train_size = 5 , n_split = 10)
cross_val_scroe(logreg,iris.data , iris.target , cv=shuffle_split)
11
1.1.3.4 分层抽样交叉验证
print("根据标签类别分层抽样的kfold")
plots.plot_label_kfold()
12
from sklearn.model_selection import StratifiedKFold
from tools.datasets import make_blobs
#构建数据集
X , y = make_blobs(n_samples = 12 , random_state = 0 )
#敲定一组label , 做分层抽样交叉验证
labels = [0,0,0,1,1,1,1,2,2,3,3,3]
cross_val_score(logerg , X , y , labels , cv = StratifiedKFlod(n_splits = 3))
13
1,2网格搜索/Grid Search
1.2.1 手动遍历超参数进行超参数选择（训练集+验证集）
我们用训练集进行模型拟合，用验证集做效果评估和参数选择
#naive grid search implementation
from sklearn.svm import svc
from sklearn.model_selection import train_test_split
X_train , X_test , y_train , y_test = train_test_split(iris.data , iris.target , random_state = 0)
print("Size of training set: %d size of test set: %d" % (X_train.shape[0], X_test.shape[0]))
best_score = 0
#for 循环遍历参数列表
for gamma in [0.001 , 0.01 , 0.1 , 1 , 10 , 100 ]:
for C in [0.001 , 0.01 , 0.1 , 1 , 10 , 100 ]:
#使用对应的参数组初始化模型对象
svm = SVC(gamma = gamma , C = C)
svm.fit(X_train , y_train)
#评估svm
score = svm.score(X_test , y_test)
#保留最高分和对应参数
if score > best_score:
best_score = score
best_parameters = {'C‘ ：C ，'gamma' :gamma}
print("最高得分: ", best_score)
print("最好的参数: ", best_parameters)
14
best_score
15
1.2.2 训练集+测试集+验证集
- 训练集：拟合和训练模型
- 验证集：使用不同参数组在验证集上试验，用户参数调优
- 测试集：模型评估
plots.plot_threefold.split()
16
from sklearn.svm import SVC
#训练集+测试集
X_trainval , X_test , y_trainval , y_test = train_test_split(iris.data , iris.target , random_state = 0)
#真正的训练集+验证集
X_train , X_valid , y_train , y_vaild = train_test_split(X_trainval , y_trainval , random_state = 1)
print("训练集数据量: %d，验证集数据量: %d，测试集数据量: %d" % (X_train.shape[0], X_valid.shape[0], X_test.shape[0]))
best_score = 0
for gamma in [0.001, 0.01, 0.1, 1, 10, 100]:
for C in [0.001, 0.01, 0.1, 1, 10, 100]:
svm = SVC(gamma=gamma, C=C)
svm.fit(X_train, y_train)
# 评估
score = svm.score(X_valid, y_valid)
# 保留最高得分
if score > best_score:
best_score = score
best_parameters = {'C': C, 'gamma': gamma}
# 在测试数据上评估
svm = SVC(**best_parameters)
svm.fit(X_trainval, y_trainval)
test_score = svm.score(X_test, y_test)
print("验证集上最高得分: ", best_score)
print("最佳参数: ", best_parameters)
print("验证集选出最好的参数上测试集的得分为: ", test_score)
17
1.2.3 GridSearchCV = grid_search(产出候选超参数) + cross_validation(评估方式)
1.2.4 RandomizedSearchCV
param_grid = {'C' : [0.001 , 0.01 , 0.1, 1 , 10, 100],
'gamma' :[0.001 , 0.01 , 0.1 ,1 , 10,100]}
param_grid
#超参数1:5种取值
#超参数2:5种取值
#超参数3:6种取值
#5-fold 交叉验证 ， 要建多少次模型用于评估？
#5*3*6*5 +1
#有加速方法吗？（并行化，加资源。。）
#depth:[3,5,7,10]
#min_child :[20,50,100]
#lr :[0.01 , 0.1 , 1 , 10]
# 7 50 0.1
# 周边搜索
# [10,20,10]
18
#param_grid是参数列表
#GridSearchCV是网格搜索交叉验证对象，fit之后可以对参数列表中的参数组进行拟合和交叉验证评估
from sklearn.model_selection import GridSearchCV
from sklearn.svm import SVC
grid_search = GridSearchCV(SVC() , param_grid , cv = 5)
X_train , X_test , y_train , y_test = train_test_split(iris.data , iris.target , random_state = 0)
import warnings
warings.fitlerwarnings("ignore")
grid_search.fit(X_train , y_train)
19
grid_search.score(X_test , y_test)
20
#最好的参数 和最高得分
print(grid_search.best_params_)
print(grid_search.best_score_)
21
grid_search.best_estimator_
22
1.2.5 附：模型候选超参数参考表
23
1.2.6 检视交叉验证的结果
grid_search.cv_result_
24
import numpy as np
scores = grid_search.cv_result_['mean_test_score']
scores = np.array(scores).reshape(6,6)
#plot the mean cross-validation scores
tools.heatmap(scores , xlabel = 'gamma' , ylabel = 'C' , xticklabels = param_grid['gamma'] , yticklabels = param_grid['C'] , cmap = 'viridis')
25
fig , axes = plt.subplots(1 , 3 , figsize = (13 , 5))
param_grid_linear = {'C' : np.linspace(1,2,6) ,'gamma' : np.linspace(1,2,6)}
param_grid_one_log = {'C' :np.linspace(1,2,6) ,'gamma' :np.linspace(-3 , 2, 6)}
param_grid_range = {'C' : np.linspace(-3 , 2 , 6) , 'gamma' : np.logspace(-7 ,-2 ,6)}
for param_grid , ax in zip([param_grid_linear , param_grid_one_log , param_grid_range] , axes):
grid_search = GridSearchCV(SVC(),param_grid , cv = 5)
grid_search.fit(X_train , y_train)
scores = grid_search.cv_results_['mean_test_score']
scores = np.array(scores).reshape(6,6)
#plot the mean cross-validation scores
scores_image = tools.heatmap(scores, xlabel = 'gamma' , ylabel = 'C' , xticklabels = param_grid['gamma'] , yticklabels = param_grid['C'] , cmap = 'viridis' , ax = ax)
plt.colorbar(score_image , ax = axes.tolist())
26
1.2.7 手动切分数据交叉验证
1.2.7 手动切分数据交叉验证
在有一些问题中，我们不能直接对数据进行随机切分，比如分类问题中，如果类别是不均衡的(非1:1)，我们不能直接随机切分，更多的情况下，我们会手动切分，并且保证每个fold中的样本比例一致。
scores= cross_val_score(GridSearch(SVC() , param_grid , cv = 5) , iris.data , iris.target , cv = 5)
print("交叉验证得分: ", scores)
print("平均交叉验证得分: ", scores.mean())
27
def nested_cv(X, y, inner_cv, outer_cv, Classifier, parameter_grid):
outer_scores = []
# 手动切分训练集和测试集(几折交叉)
for training_samples, test_samples in outer_cv.split(X, y):
# 初始化最好的参数存储的字典:
best_parms = {}
best_score = -np.inf
# 遍历参数
for parameters in parameter_grid:
# 记录不同参数交叉验证实验得分
cv_scores = []
# 再把训练集做几折切分内部切分为 真正的训练集 和 验证集
for inner_train, inner_test in inner_cv.split(X[training_samples], y[training_samples]):
clf = Classifier(**parameters)
clf.fit(X[inner_train], y[inner_train])
score = clf.score(X[inner_test], y[inner_test])
cv_scores.append(score)
# 交叉验证的平均结果
mean_score = np.mean(cv_scores)
if mean_score > best_score:
best_score = mean_score
best_params = parameters
clf = Classifier(**best_params)
clf.fit(X[training_samples], y[training_samples])
outer_scores.append(clf.score(X[test_samples], y[test_samples]))
return outer_scores
from sklearn.model_selection import ParameterGrid , StratifieKFold
nested_cv(iris.data , iris.target , StratifieKFold(5) , StratifieKFold(5) , SVC , ParameterGrid(param_grid))
28
