4、Iris数据集上的非嵌套和嵌套交叉验证

from sklearn.datasets import load_iris

from matplotlib import pyplot as plt

from sklearn.svm import SVC

from sklearn.model_selection import GridSearchCV, cross_val_score, KFold

import numpy as np

plt.rcParams['font.sans-serif'] = ['SimHei']

plt.rcParams['axes.unicode_minus'] = False

# 随机试验次数

NUM_TRIALS = 30

# 导入数据集

iris = load_iris()

X_iris = iris.data

y_iris = iris.target

# 设置参数的可能值以优化

p_grid = {"C": [1, 10, 100],

          "gamma": [.01, .1]}

# 我们将使用带有“ rbf”内核的支持向量分类器

svm = SVC(kernel="rbf")

# 存储分数的数组

non_nested_scores = np.zeros(NUM_TRIALS)

nested_scores = np.zeros(NUM_TRIALS)

# 每次试用循环

for i in range(NUM_TRIALS):

    # 独立于数据集，为内部和外部循环选择交叉验证技术。

    # 例如“ GroupKFold”，“ LeaveOneOut”，“ LeaveOneGroupOut”等。

    inner_cv = KFold(n_splits=4, shuffle=True, random_state=i)

    outer_cv = KFold(n_splits=4, shuffle=True, random_state=i)

    # 非嵌套参数搜索和评分

    clf = GridSearchCV(estimator=svm, param_grid=p_grid, cv=inner_cv)

    clf.fit(X_iris, y_iris)

    non_nested_scores[i] = clf.best_score_

    # 带有参数优化的嵌套简历

    nested_score = cross_val_score(clf, X=X_iris, y=y_iris, cv=outer_cv)

    nested_scores[i] = nested_score.mean()

score_difference = non_nested_scores - nested_scores

print("Average difference of {:6f} with std. dev. of {:6f}."

      .format(score_difference.mean(), score_difference.std()))

# 嵌套和非嵌套CV在每个试验中的得分

plt.figure()

plt.subplot(211)

non_nested_scores_line, = plt.plot(non_nested_scores, color='r')

nested_line, = plt.plot(nested_scores, color='b')

plt.ylabel("score", fontsize="14")

plt.legend([non_nested_scores_line, nested_line],

          ["Non-Nested CV", "Nested CV"],

          bbox_to_anchor=(0, .4, .5, 0))

plt.title("Iris数据集上的非嵌套和嵌套交叉验证",

          x=.5, y=1.1, fontsize="15")

# 绘制差异柱状图

plt.subplot(212)

difference_plot = plt.bar(range(NUM_TRIALS), score_difference)

plt.xlabel("Individual Trial #")

plt.legend([difference_plot],

          ["Non-Nested CV - Nested CV Score"],

          bbox_to_anchor=(0, 1, .8, 0))

plt.ylabel("score difference", fontsize="14")

plt.show()