sklearn 绘制多分类ROC曲线

妖狐艹你老母 2022-01-14 12:03 471阅读 0赞

from rasa_nlu.model import Interpreter
    from sklearn.metrics import precision_score
    from sklearn.metrics import recall_score
    from sklearn import metrics
    from sklearn.preprocessing import OneHotEncoder
    import json
    import time
    import numpy as np
    import pandas as pd
    import matplotlib
    matplotlib.use('Agg')
    import os
    from matplotlib import pyplot as plt 
    def get_path(root):
        paths = []
        if os.path.isdir(root):
            lists = os.listdir(root)
            for i in lists:
                paths.append(os.path.join(root,i))
        return paths
    root = '/app/split_data/test'
    paths = get_path(root)
    examples = []
    for i in paths:
        print(i)
        f = open(i,'r')
        data = json.load(f)
        examples.append(data['rasa_nlu_data']['common_examples'])
        f.close()
    interpreter2 = Interpreter.load("./models/nlu/ECS-nlu/models_shuffle/")
    
    result = []
    intent = []
    ranking = []
    for i in examples:
        for item in i:
            rasa_result= interpreter2.parse(item['text'])
            result.append(rasa_result['intent']['name'])
            intent.append(item['intent'])
            ranking.append(rasa_result['intent_ranking'])
    '''
    for k,v in ecs_test_data.items():
        rasa_res = interpreter2.parse(k)
        result.append(rasa_res['intent']['name'])
        intent.append(v)
        ranking.append(rasa_result['intent_ranking'])
    '''
    acc_score = precision_score(intent,result,average='weighted')
    print('accuracy score:{}'.format(acc_score))
    rec_score = recall_score(intent,result,average='weighted')
    print('recall score:{}'.format(rec_score))
    intent_lists = set(intent)
    raw = {}
    arrays = []
    for i in intent_lists:
        raw[i] = 0
    for j in ranking:
        tmp = raw.copy()
        for k in j:
             tmp[k['name']] = k['confidence']
        confs = list(tmp.values())
        arrays.append(confs)
    arrays = np.array(arrays)
    intent = np.array(intent).T
    label = pd.DataFrame(intent,columns = ['label'])
    unique = label.ix[:,'label'].unique()
    for j in range(len(unique)):
        label.ix[:,'label'] = label.ix[:,'label'].apply(lambda x:j if x == unique[j] else x)
    enc = OneHotEncoder()
    enc.fit(label)
    label_one_hot = enc.transform(label).toarray()
    print(metrics.roc_auc_score(label_one_hot,arrays,average='weighted'))