输出多标签分类模型每class指标OP,OR,OF1,CP,CR,CF1

曾经终败给现在 2022-01-31 01:04 187阅读 0赞

需要逐类对预测结果进行分析。

相关代码地址：[https://github.com/Xingxiangrui/multi\_label\_analyse][https_github.com_Xingxiangrui_multi_label_analyse]

**目录**

一、结果读出

1.1 写出结果

1.2 结果读出

二、运算指标

2.1 指标与print

2.2 指标的运算

2.3 输出

三、precision，recall，F1

3.1 定义函数进行运算

3.2 函数的调用

3.3 运行结果

--------------------

# 一、结果读出 #

## 1.1 写出结果 ##

见 [PyTorch实操(五)加载模型验证并将所有结果写入文件 ][PyTorch_]

## 1.2 结果读出 ##

一定要确定路径之中为rb，不然会将文件重写写入wb，文件会被清零。

with open('checkpoint/coco/resnet101_on_coco/model_results.pkl', 'rb') as f:
        print("loading checkpoint/coco/resnet101_on_coco/model_results.pkl")
        model_results = pickle.load(f)
    with open('checkpoint/coco/resnet101_on_coco/coco_labels_in_np.pkl','rb') as f:
        print("loading checkpoint/coco/resnet101_on_coco/model_results.pkl")
        labels=pickle.load(f)
    
    print('model_results[3]',model_results[3])
    print('labels[3]',labels[3])

# 二、运算指标 #

## 2.1 指标与print ##

需要算出OP，OR，OF1等各个指标。直接运用self.state\['ap\_meter'\]得出相应的指标

def on_end_epoch(self, training, model, criterion, data_loader, optimizer=None, display=True):
            map = 100 * self.state['ap_meter'].value().mean()
            loss = self.state['meter_loss'].value()[0]
            OP, OR, OF1, CP, CR, CF1 = self.state['ap_meter'].overall()
            OP_k, OR_k, OF1_k, CP_k, CR_k, CF1_k = self.state['ap_meter'].overall_topk(3)
            if display:
                if training:
                    print('Epoch: [{0}]\t'
                          'Loss {loss:.4f}\t'
                          'mAP {map:.3f}'.format(self.state['epoch'], loss=loss, map=map))
                    print('OP: {OP:.4f}\t'
                          'OR: {OR:.4f}\t'
                          'OF1: {OF1:.4f}\t'
                          'CP: {CP:.4f}\t'
                          'CR: {CR:.4f}\t'
                          'CF1: {CF1:.4f}'.format(OP=OP, OR=OR, OF1=OF1, CP=CP, CR=CR, CF1=CF1))
                else:
                    print('Test: \t Loss {loss:.4f}\t mAP {map:.3f}'.format(loss=loss, map=map))
                    print('OP: {OP:.4f}\t'
                          'OR: {OR:.4f}\t'
                          'OF1: {OF1:.4f}\t'
                          'CP: {CP:.4f}\t'
                          'CR: {CR:.4f}\t'
                          'CF1: {CF1:.4f}'.format(OP=OP, OR=OR, OF1=OF1, CP=CP, CR=CR, CF1=CF1))
                    print('OP_3: {OP:.4f}\t'
                          'OR_3: {OR:.4f}\t'
                          'OF1_3: {OF1:.4f}\t'
                          'CP_3: {CP:.4f}\t'
                          'CR_3: {CR:.4f}\t'
                          'CF1_3: {CF1:.4f}'.format(OP=OP_k, OR=OR_k, OF1=OF1_k, CP=CP_k, CR=CR_k, CF1=CF1_k))
    
            return map

相应的函数为.overall()

## 2.2 指标的运算 ##

def overall(self):
            if self.scores.numel() == 0:
                return 0
            scores = self.scores.cpu().numpy()
            targets = self.targets.cpu().numpy()
            targets[targets == -1] = 0
            return self.evaluation(scores, targets)
    
        def overall_topk(self, k):
            targets = self.targets.cpu().numpy()
            targets[targets == -1] = 0
            n, c = self.scores.size()
            scores = np.zeros((n, c)) - 1
            index = self.scores.topk(k, 1, True, True)[1].cpu().numpy()
            tmp = self.scores.cpu().numpy()
            for i in range(n):
                for ind in index[i]:
                    scores[i, ind] = 1 if tmp[i, ind] >= 0 else -1
            return self.evaluation(scores, targets)
    
    
        def evaluation(self, scores_, targets_):
            n, n_class = scores_.shape
            Nc, Np, Ng = np.zeros(n_class), np.zeros(n_class), np.zeros(n_class)
            for k in range(n_class):
                scores = scores_[:, k]
                targets = targets_[:, k]
                targets[targets == -1] = 0
                Ng[k] = np.sum(targets == 1)
                Np[k] = np.sum(scores >= 0)
                Nc[k] = np.sum(targets * (scores >= 0))
            Np[Np == 0] = 1
            OP = np.sum(Nc) / np.sum(Np)
            OR = np.sum(Nc) / np.sum(Ng)
            OF1 = (2 * OP * OR) / (OP + OR)
    
            CP = np.sum(Nc / Np) / n_class
            CR = np.sum(Nc / Ng) / n_class
            CF1 = (2 * CP * CR) / (CP + CR)
            return OP, OR, OF1, CP, CR, CF1

## 2.3 输出 ##

输入为  self.state\['ap\_meter'\]

其中包括了 self.scores和self.targets,且最终转为了numpy数组

scores = self.scores.cpu().numpy() 
    targets = self.targets.cpu().numpy()

在往上，此变量是通过add加入了网络的预测输出，与label

def on_end_batch(self, training, model, criterion, data_loader, optimizer=None, display=True):
    
            Engine.on_end_batch(self, training, model, criterion, data_loader, optimizer, display=False)
    
            # measure mAP
            self.state['ap_meter'].add(self.state['output'].data, self.state['target_gt'])

# 三、precision，recall，F1 #

## 3.1 定义函数进行运算 ##

我们自己定义函数运算相应的指标

def scores_evaluation(scores_, targets_):
        print('evaluation start...')
        n, n_class = scores_.shape
        print('img_numbers=',n,'n_class=',n_class)
        Nc, Np, Ng = np.zeros(n_class), np.zeros(n_class), np.zeros(n_class)
        cls_P,cls_R,cls_F1=np.zeros(n_class),np.zeros(n_class),np.zeros(n_class)
        for k in range(n_class):
            scores = scores_[:, k]         # all img scores on class_k
            targets = targets_[:, k]       # all img labels on class_k
            targets[targets == -1] = 0     # set img labels from -1 to 0
            Ng[k] = np.sum(targets == 1)   # ture:     all ture labels sum number
            Np[k] = np.sum(scores >= 0)    # positive: all predict positive sum number
            Nc[k] = np.sum(targets * (scores >= 0)) # true_positive: true_positive sum number
            cls_P[k]=Nc[k]/Np[k]
            cls_R[k]=Nc[k]/Ng[k]
            cls_F1[k]=(2 * cls_P[k] * cls_R[k]) / (cls_P[k] + cls_R[k])
        Np[Np == 0] = 1
        print('np.sum(Nc),true_positive=',np.sum(Nc))
        print('np.sum(Np),positive=',np.sum(Np))
        print('np.sum(Ng),ture=',np.sum(Ng))
    
        # for all labels num_imgs*n_classes
        OP = np.sum(Nc) / np.sum(Np)        # precision: true_positive/positive
        OR = np.sum(Nc) / np.sum(Ng)        # recall:    true_positive/true
        OF1 = (2 * OP * OR) / (OP + OR)     # F1_score: harmonic mean of precision and recall
        # average by class
        CP = np.sum(Nc / Np) / n_class      # precision: true_positive/positive
        CR = np.sum(Nc / Ng) / n_class      # recall:    true_positive/true
        CF1 = (2 * CP * CR) / (CP + CR)     # F1_score: harmonic mean of precision and recall
    
        return OP, OR, OF1, CP, CR, CF1,cls_P,cls_R,cls_F1

## 3.2 函数的调用 ##

直接调用函数进行运算，加载出模型，需要为numpy格式，dim0为img\_idx,  dim1为classes，直接讲numpy送入函数即可returen出需要的值。

# load results on pkl files
    # file in format { 0: [[ -4.905565    -7.9314375  ... -6.8639855   -8.622047    -8.28002]] numpy 1*80
    #                  1: [[ -8.905565    -8.9314375  ... -5.8639855   -6.622047    -4.28002]] numpy 1*80
    #                   ...
    #                  1355: ..........    numpy 1*80 }
    # label in format (0: [[1. 0. 0. 0. 0. 0. 0. 0. ..... 1 ]]  numpy 1*80
    #                  1: [[0. 1. .....                  ...]]}
    with open('checkpoint/coco/resnet101_on_coco/model_results.pkl', 'rb') as f:
        print("loading checkpoint/coco/resnet101_on_coco/model_results.pkl")
        model_results = pickle.load(f)
    with open('checkpoint/coco/resnet101_on_coco/coco_labels_in_np.pkl','rb') as f:
        print("loading checkpoint/coco/resnet101_on_coco/model_results.pkl")
        labels=pickle.load(f)
    # print('model_results[3]',model_results[3])
    # print('labels[3]',labels[3])
    
    # concat all numpy
    total_results=model_results[0]
    total_labels=labels[0]
    for img_idx in range(len(model_results)-1):
        if img_idx%1000==0:
            print(img_idx,'/',len(model_results))
        total_results=np.append(total_results,model_results[img_idx+1],axis=0)
        total_labels=np.append(total_labels,labels[img_idx+1],axis=0)
    
    print('np.shape(total_results)',np.shape(total_results))
    print('np.shape(total_labels)',np.shape(total_labels))
    
    with open('checkpoint/coco/resnet101_on_coco/model_results_numpy.pkl', 'wb') as f:
        print("writing checkpoint/coco/resnet101_on_coco/model_results_numpy.pkl")
        pickle.dump(total_results, f)
    with open('checkpoint/coco/resnet101_on_coco/coco_labels_numpy.pkl','wb') as f:
        print("writing checkpoint/coco/resnet101_on_coco/oco_labels_numpy.pkl")
        pickle.dump(total_labels, f)
    
    OP,OR,OF1,CP,CR,CF1,cls_P,cls_R,cls_F1=scores_evaluation(scores_=total_results,targets_=total_labels)
    all_evaluate_results={'OP':OP,'OR':OR,'OF1':OF1,'CP':CP,'CR':CR,'CF1':CF1,'cls_P':cls_P,'cls_R':cls_R,'cls_F1':cls_F1}
    with open('checkpoint/coco/resnet101_on_coco/all_evaluate_results.pkl','wb') as f:
        print('writing checkpoint/coco/resnet101_on_coco/all_evaluate_results.pkl')
        pickle.dump(all_evaluate_results, f)

最终将模型写入文件。

## 3.3 运行结果 ##

针对结果

np.shape(total_results) (40137, 80)
    np.shape(total_labels) (40137, 80)
    writing checkpoint/coco/resnet101_on_coco/model_results_numpy.pkl
    writing checkpoint/coco/resnet101_on_coco/oco_labels_numpy.pkl
    evaluation start...
    img_numbers= 40137 n_class= 80
    np.sum(Nc),true_positive= 93107.0
    np.sum(Np),positive= 122286.0
    np.sum(Ng),ture= 116592.0
    class_Precison [0.96273115 0.7880597  0.76769912 0.86618005 0.93787336 0.83281005
     0.91071429 0.6272687  0.7920434  0.74237508 0.83457249 0.826
     0.76213592 0.5688509  0.82555781 0.94503171 0.88401697 0.89315353
     0.9        0.80672269 0.96170213 0.93786982 0.99096386 0.98452381
     0.45626478 0.83036406 0.45948866 0.80294358 0.70716889 0.83989835
     0.86316776 0.7015625  0.71230398 0.91428571 0.82326112 0.8512931
     0.89946619 0.90539484 0.96890756 0.6308514  0.71782763 0.63008487
     0.59097222 0.53621561 0.51172414 0.61934082 0.79247911 0.62579281
     0.66046512 0.73846154 0.77116402 0.71869328 0.66125761 0.8952734
     0.72968198 0.68949343 0.65346734 0.66802089 0.61627201 0.79196217
     0.64725882 0.88643791 0.77614679 0.82966102 0.6955414  0.66727773
     0.73536585 0.59127907 0.57725948 0.69874477 0.2293578  0.74602304
     0.58472086 0.57300144 0.83785617 0.69811321 0.58131488 0.8566879
     0.42105263 0.60055096]
    class_recall [0.9385227  0.71095153 0.83014354 0.87612797 0.93452381 0.78592593
     0.91569087 0.68920233 0.83587786 0.79610299 0.75844595 0.70118846
     0.60153257 0.6109128  0.72613738 0.90608108 0.82182774 0.86013986
     0.90184049 0.86486486 0.94957983 0.92961877 0.97193501 0.97408716
     0.52674672 0.7695621  0.56161972 0.76242236 0.66438356 0.87318362
     0.92749245 0.8424015  0.81730104 0.92434663 0.90362954 0.93491124
     0.92582418 0.89628483 0.96647108 0.68956044 0.69724771 0.70336491
     0.7254902  0.67730496 0.62405383 0.71339174 0.78159341 0.60285132
     0.69437653 0.76056338 0.87014925 0.68512111 0.72123894 0.86481647
     0.78967495 0.75851393 0.73819255 0.7948895  0.65909091 0.77786378
     0.76919192 0.91561181 0.80469245 0.79464286 0.81008902 0.69932757
     0.804      0.6        0.7734375  0.84428716 0.33783784 0.86404066
     0.7556962  0.654814   0.72476526 0.70916667 0.55629139 0.74309392
     0.22857143 0.63929619]
    class F1 score [0.9504728  0.74752242 0.79770115 0.87112561 0.93619559 0.80868902
     0.9131958  0.65677868 0.81337047 0.76830087 0.79469027 0.75849403
     0.67237687 0.58913204 0.77266255 0.9251466  0.85178876 0.87633588
     0.90091931 0.83478261 0.95560254 0.93372607 0.9813572  0.97927768
     0.48897897 0.79880775 0.50544662 0.7821585  0.68510889 0.85621762
     0.89417476 0.7655584  0.76119884 0.91928865 0.86157518 0.89114495
     0.91245487 0.9008168  0.96768779 0.65890074 0.70738802 0.66471133
     0.65135859 0.59855845 0.56233422 0.66304769 0.78699862 0.61410788
     0.67699642 0.74934952 0.81767181 0.70150576 0.68994709 0.87978142
     0.75849403 0.72235872 0.69325088 0.72595396 0.63696266 0.78484967
     0.70297715 0.90078871 0.79016189 0.81177446 0.74845785 0.68292683
     0.76815287 0.59560761 0.66110184 0.76465201 0.27322404 0.80070651
     0.65930425 0.61118203 0.77721838 0.70359653 0.56852792 0.79585799
     0.2962963  0.61931818]
    All evaluate results:
    OP: 0.7614	OR: 0.7986	OF1: 0.7795	CP: 0.7446	CR: 0.7693	CF1: 0.7568

[https_github.com_Xingxiangrui_multi_label_analyse]: https://github.com/Xingxiangrui/multi_label_analyse
[PyTorch_]: https://blog.csdn.net/weixin_36474809/article/details/90207499