sklearn 绘制多分类ROC曲线-蒲公英云

sklearn 绘制多分类ROC曲线

from rasa_nlu.model import Interpreter
from sklearn.metrics import precision_score
from sklearn.metrics import recall_score
from sklearn import metrics
from sklearn.preprocessing import OneHotEncoder
import json
import time
import numpy as np
import pandas as pd
import matplotlib
matplotlib.use('Agg')
import os
from matplotlib import pyplot as plt 
def get_path(root):
    paths = []
    if os.path.isdir(root):
        lists = os.listdir(root)
        for i in lists:
            paths.append(os.path.join(root,i))
    return paths
root = '/app/split_data/test'
paths = get_path(root)
examples = []
for i in paths:
    print(i)
    f = open(i,'r')
    data = json.load(f)
    examples.append(data['rasa_nlu_data']['common_examples'])
    f.close()
interpreter2 = Interpreter.load("./models/nlu/ECS-nlu/models_shuffle/")
result = []
intent = []
ranking = []
for i in examples:
    for item in i:
        rasa_result= interpreter2.parse(item['text'])
        result.append(rasa_result['intent']['name'])
        intent.append(item['intent'])
        ranking.append(rasa_result['intent_ranking'])
'''
for k,v in ecs_test_data.items():
    rasa_res = interpreter2.parse(k)
    result.append(rasa_res['intent']['name'])
    intent.append(v)
    ranking.append(rasa_result['intent_ranking'])
'''
acc_score = precision_score(intent,result,average='weighted')
print('accuracy score:{}'.format(acc_score))
rec_score = recall_score(intent,result,average='weighted')
print('recall score:{}'.format(rec_score))
intent_lists = set(intent)
raw = {}
arrays = []
for i in intent_lists:
    raw[i] = 0
for j in ranking:
    tmp = raw.copy()
    for k in j:
         tmp[k['name']] = k['confidence']
    confs = list(tmp.values())
    arrays.append(confs)
arrays = np.array(arrays)
intent = np.array(intent).T
label = pd.DataFrame(intent,columns = ['label'])
unique = label.ix[:,'label'].unique()
for j in range(len(unique)):
    label.ix[:,'label'] = label.ix[:,'label'].apply(lambda x:j if x == unique[j] else x)
enc = OneHotEncoder()
enc.fit(label)
label_one_hot = enc.transform(label).toarray()
print(metrics.roc_auc_score(label_one_hot,arrays,average='weighted'))