机器学习 (五): 模型评估

约定不等于承诺〃 2022-02-22 19:22 331阅读 0赞

### 机器学习 模型评估 ###

*  交叉验证
 *  Baseline 模型
 *   *  数值型baseline
     *  分类型baseline
 *  ROC 曲线
 *  Confusion Matrix

# 交叉验证 #

from sklearn.preprocessing import StandardScaler
    from sklearn.pipeline import make_pipeline
    from sklearn.linear_model import LogisticRegression
    from sklearn.model_selection import KFold, cross_val_score
    from sklearn.datasets import load_digits
    
    # 加载数据 (手写数字图像)
    digits = load_digits()
    features = digits.data
    target = digits.target
    
    # 创建一个流水线, 流水线由
    # 将输入特征变换为0均值,1方差的缩放器
    # 逻辑回归模型
    # 组成
    pipeline = make_pipeline(StandardScaler(), LogisticRegression())
    
    # 交叉验证, Fold=10
    cv_res = cross_val_score(pipeline, features, target, cv=KFold(10, shuffle=True, random_state=1), scoring='accuracy', n_jobs=-1)
    print(cv_res.mean())

# Baseline 模型 #

## 数值型baseline ##

from sklearn.datasets import load_boston
    from sklearn.linear_model import LinearRegression
    from sklearn.preprocessing import train_test_split
    from sklearn.dummy import DummyRegressor
    from sklearn.model_selection import train_test_split
    
    # 加载数据
    boston = load_boston()
    features, target = boston.data, boston.target
    x_train, x_test, y_train, y_test = train_test_split(features, target, test_size = 0.2)
    
    # 预处理, 缩放数据
    std_scaler = StandardScaler()
    std_scaler.fit(x_train)
    x_train = std_scaler.transform(x_train)
    x_test = std_scaler.transform(x_test)
    
    # baselien model
    baseline = DummyRegressor(strategy='mean')
    baseline.fit(x_train, y_train)
    baseline.score(x_test, y_test)
    # 得分 -0.05
    
    # my model
    clf = LinearRegression()
    clf.fit(x_train, y_train)
    clf.score(x_test, y_test)
    # 得分 0.74

## 分类型baseline ##

from sklearn.datasets import load_iris
    from sklearn.preprocessing import StandardScaler
    from sklearn.model_selection import train_test_split
    from sklearn.dummy import DummyClassifier
    from sklearn.linear_model import LogisticRegression
    
    # 加载数据
    iris = load_iris()
    features, target = iris.data, iris.target
    x_train, x_test, y_train, y_test = train_test_split(features, target, test_size = 0.2)
    
    # 预处理
    std_scaler = StandardScaler()
    std_scaler.fit(x_train)
    x_train = std_scaler.transform(x_train)
    x_test = std_scaler.transform(x_test)
    
    # baseline
    baseline = DummyClassifier(strategy='stratified', random_state=1)
    baseline.fit(x_train, y_train)
    print(baseline.score(x_test, y_test))
    # 得分 0.4
    
    # my model
    clf = LogisticRegression(solver='lbfgs', multi_class='auto')
    clf.fit(x_train, y_train)
    clf.score(x_test, y_test)
    # 得分 0.97

# ROC 曲线 #

from sklearn.datasets import make_classification
    from sklearn.linear_model import LogisticRegression
    from sklearn.metrics import roc_curve, roc_auc_score
    from sklearn.model_selection import train_test_split
    from sklearn.preprocessing import StandardScaler
    from sklearn.dummy import DummyClassifier
    import matplotlib.pyplot as plt
    %matplotlib inline
    
    features, target = make_classification(n_samples = 1000, n_features = 10, n_informative = 2, n_classes = 2)
    x_train, x_test, y_train, y_test = train_test_split(features, target, test_size = 0.2)
    
    std_scaler = StandardScaler()
    std_scaler.fit(x_train)
    x_train = std_scaler.transform(x_train)
    x_test = std_scaler.transform(x_test)
    
    baseline = DummyClassifier(strategy='stratified')
    baseline.fit(x_train, y_train)
    y_pred = baseline.predict_proba(x_test)[:,1]
    fp, tp, threshold = roc_curve(y_test, y_pred)
    plt.plot(fp, tp, label='baseline (%f)' % roc_auc_score(y_test, y_pred))
    
    clf = LogisticRegression(solver='lbfgs')
    clf.fit(x_train, y_train)
    y_pred = clf.predict_proba(x_test)[:,1]
    fp, tp, threshold = roc_curve(y_test, y_pred)
    
    plt.plot(fp, tp, label='mine (%f)' % roc_auc_score(y_test, y_pred))
    
    plt.legend()

![0069McTXly1g1wk5qyzs5j30af070t8r.jpg][]

# Confusion Matrix #

from sklearn.datasets import load_iris
    from sklearn.preprocessing import StandardScaler
    from sklearn.model_selection import train_test_split
    from sklearn.metrics import confusion_matrix
    from sklearn.linear_model import LogisticRegression
    import seaborn as sns
    
    # 加载数据
    iris = load_iris()
    features, target = iris.data, iris.target
    x_train, x_test, y_train, y_test = train_test_split(features, target, test_size = 0.5)
    
    # 预处理
    std_scaler = StandardScaler()
    std_scaler.fit(x_train)
    x_train = std_scaler.transform(x_train)
    x_test = std_scaler.transform(x_test)
    
    # my model
    clf = LogisticRegression(solver='lbfgs', multi_class='auto')
    clf.fit(x_train, y_train)
    y_pred = clf.predict(x_test)
    sns.heatmap(confusion_matrix(y_test, y_pred), annot=True, cmap='Blues')

![0069McTXly1g1wki3awx9j309m070745.jpg][]

[0069McTXly1g1wk5qyzs5j30af070t8r.jpg]: /images/20220222/5a269d3c57c6492f9d2e4807e5b64ce2.png
[0069McTXly1g1wki3awx9j309m070745.jpg]: /images/20220222/8bbabbdea30f4578aad493b4ba08765f.png