【darknet源码】network.c

绝地灬酷狼 2022-05-16 22:56 242阅读 0赞

network.c

#include <stdio.h> 
         #include <time.h> 
         #include <assert.h> 
         #include "network.h" 
         #include "image.h" 
         #include "data.h" 
         #include "utils.h" 
         #include "blas.h" 
         
         #include "crop_layer.h" 
         #include "connected_layer.h" 
         #include "gru_layer.h" 
         #include "rnn_layer.h" 
         #include "crnn_layer.h" 
         #include "local_layer.h" 
         #include "convolutional_layer.h" 
         #include "activation_layer.h" 
         #include "detection_layer.h" 
         #include "region_layer.h" 
         #include "yolo_layer.h" 
         #include "normalization_layer.h" 
         #include "batchnorm_layer.h" 
         #include "maxpool_layer.h" 
         #include "reorg_layer.h" 
         #include "avgpool_layer.h" 
         #include "cost_layer.h" 
         #include "softmax_layer.h" 
         #include "dropout_layer.h" 
         #include "route_layer.h" 
         #include "upsample_layer.h" 
         #include "shortcut_layer.h" 
         #include "parser.h" 
         #include "data.h"

load_args get_base_args(network net) 
         { 
         load_args args = { 0}; 
         args.w = net->w; 
         args.h = net->h; 
         args.size = net->w; 
         
         args.min = net->min_crop; 
         args.max = net->max_crop; 
         args.angle = net->angle; 
         args.aspect = net->aspect; 
         args.exposure = net->exposure; 
         args.center = net->center; 
         args.saturation = net->saturation; 
         args.hue = net->hue; 
         return args; 
         } 
         
         network load_network(char cfg, char *weights, int clear) 
         { 
         network *net = parse_network_cfg(cfg); 
         if(weights && weights[ 0] != 0){ 
         load_weights(net, weights); 
         } 
         if(clear) (*net->seen) = 0; 
         return net; 
         } 
         
         // 获取当前的迭代次数 
         size_t get_current_batch(network *net) 
         { 
         size_t batch_num = (*net->seen)/(net->batch*net->subdivisions); 
         return batch_num; 
         }

void reset_network_state(network *net, int b) 
         { 
         int i; 
         for (i = 0; i < net->n; ++i) { 
         #ifdef GPU 
         layer l = net->layers[i]; 
         if(l.state_gpu){ 
         fill_gpu(l.outputs, 0, l.state_gpu + l.outputs*b, 1); 
         } 
         if(l.h_gpu){ 
         fill_gpu(l.outputs, 0, l.h_gpu + l.outputs*b, 1); 
         } 
         #endif 
         } 
         } 
         
         void reset_rnn(network *net) 
         { 
         reset_network_state(net, 0); 
         }

// 得到当前迭代的学习率 
         float get_current_rate(network *net) 
         { 
         size_t batch_num = get_current_batch(net); 
         int i; 
         float rate; 
         
         // 当迭代次数小于 burn_in 时，learning_rate = learning_rate ((batch_num / burn_in) ^ power) 
         if (batch_num < net->burn_in) return net->learning_rate * pow(( float)batch_num / net->burn_in, net->power); 
         
         // 根据策略计算学习率 
         switch (net->policy) { 
         case CONSTANT: 
         return net->learning_rate; 
         case STEP: 
         return net->learning_rate * pow(net->scale, batch_num/net->step); 
         case STEPS: 
         rate = net->learning_rate; 
         for(i = 0; i < net->num_steps; ++i){ 
         if(net->steps[i] > batch_num) return rate; 
         rate = net->scales[i]; 
         } 
         return rate; 
         case EXP: 
         return net->learning_rate pow(net->gamma, batch_num); 
         case POLY: 
         return net->learning_rate * pow( 1 - ( float)batch_num / net->max_batches, net->power); 
         case RANDOM: 
         return net->learning_rate * pow(rand_uniform( 0, 1), net->power); 
         case SIG: 
         return net->learning_rate * ( 1./( 1.+ exp(net->gamma*(batch_num - net->step)))); 
         default: 
         fprintf( stderr, "Policy is weird!\n"); 
         return net->learning_rate; 
         } 
         }

// 根据 layer 的枚举类型返回对应的 layer 名称字符串 
         char get_layer_string(LAYER_TYPE a) 
         { 
         switch(a){ 
         case CONVOLUTIONAL: 
         return "convolutional"; 
         case ACTIVE: 
         return "activation"; 
         case LOCAL: 
         return "local"; 
         case DECONVOLUTIONAL: 
         return "deconvolutional"; 
         case CONNECTED: 
         return "connected"; 
         case RNN: 
         return "rnn"; 
         case GRU: 
         return "gru"; 
         case LSTM: 
         return "lstm"; 
         case CRNN: 
         return "crnn"; 
         case MAXPOOL: 
         return "maxpool"; 
         case REORG: 
         return "reorg"; 
         case AVGPOOL: 
         return "avgpool"; 
         case SOFTMAX: 
         return "softmax"; 
         case DETECTION: 
         return "detection"; 
         case REGION: 
         return "region"; 
         case YOLO: 
         return "yolo"; 
         case DROPOUT: 
         return "dropout"; 
         case CROP: 
         return "crop"; 
         case COST: 
         return "cost"; 
         case ROUTE: 
         return "route"; 
         case SHORTCUT: 
         return "shortcut"; 
         case NORMALIZATION: 
         return "normalization"; 
         case BATCHNORM: 
         return "batchnorm"; 
         default: 
         break; 
         } 
         return "none"; 
         }

// 创建并初始化一个 net 对象 
         network make_network(int n) 
         { 
         network net = calloc(1, sizeof(network)); 
         net->n = n; 
         net->layers = calloc(net->n, sizeof(layer)); 
         net->seen = calloc( 1, sizeof( size_t)); 
         net->t = calloc( 1, sizeof( int)); 
         net->cost = calloc( 1, sizeof( float)); 
         return net; 
         }

// 前向传播 
         void forward_network(network *netp) 
         { 
         #ifdef GPU 
         if(netp->gpu_index >= 0){ 
         forward_network_gpu(netp); 
         return; 
         } 
         #endif 
         network net = *netp; 
         int i; 
         for(i = 0; i < net.n; ++i){ 
         // 网络当前活跃层 (index) 为第 i 层 
         net.index = i; 
         
         layer l = net.layers[i]; 
         
         if(l.delta){ 
         fill_cpu(l.outputs l.batch, 0, l.delta, 1); 
         } 
         
         // 前向传播 
         l.forward(l, net); 
         
         // 上一层的输出为下一层的输入 
         net.input = l.output; 
         
         if(l.truth) { 
         net.truth = l.output; 
         } 
         } 
         
         // 计算网络的 loss 
         calc_network_cost(netp); 
         } 
         
         // 更新网络参数 
         void update_network(network netp) 
         { 
         #ifdef GPU 
         if(netp->gpu_index >= 0){ 
         update_network_gpu(netp); 
         return; 
         } 
         #endif 
         network net = *netp; 
         int i; 
         update_args a = { 0}; 
         a.batch = net.batch*net.subdivisions; 
         a.learning_rate = get_current_rate(netp); 
         a.momentum = net.momentum; 
         a.decay = net.decay; 
         a.adam = net.adam; 
         a.B1 = net.B1; 
         a.B2 = net.B2; 
         a.eps = net.eps; 
         ++*net.t; 
         a.t = *net.t; 
         
         for(i = 0; i < net.n; ++i){ 
         layer l = net.layers[i]; 
         if(l.update){ 
         l.update(l, a); 
         } 
         } 
         } 
         
         // 计算网络的 loss 
         void calc_network_cost(network *netp) 
         { 
         network net = *netp; 
         int i; 
         float sum = 0; 
         int count = 0; 
         for(i = 0; i < net.n; ++i){ 
         if(net.layers[i].cost){ 
         sum += net.layers[i].cost[ 0]; 
         ++count; 
         } 
         } 
         *net.cost = sum/count; 
         } 
         
         // 获取网络预测的类别 
         int get_predicted_class_network(network *net) 
         { 
         return max_index(net->output, net->outputs); 
         } 
         
         // 反向传播 
         void backward_network(network *netp) 
         { 
         #ifdef GPU 
         if(netp->gpu_index >= 0){ 
         backward_network_gpu(netp); 
         return; 
         } 
         #endif 
         network net = *netp; 
         int i; 
         network orig = net; 
         for(i = net.n -1; i >= 0; --i){ 
         layer l = net.layers[i]; 
         
         if(l.stopbackward) break; 
         
         // i == 0 表示第1个隐含层，第1个隐含层的前一层就是输入层 
         if(i == 0){ 
         // orig.input 就是第1层的输入 
         // 第1层无需计算上一层的delta 
         net = orig; 
         } else{ 
         layer prev = net.layers[i -1]; 
         net.input = prev.output; 
         net.delta = prev.delta; 
         } 
         
         // 网络当前活跃层 (index) 为第 i 层 
         net.index = i; 
         
         // 反向传播 
         l.backward(l, net); 
         } 
         }

// 批量梯度下降 
         float train_network_datum(network *net) 
         { 
         *net->seen += net->batch; 
         net->train = 1; 
         forward_network(net); 
         backward_network(net); 
         float error = *net->cost; 
         if(((*net->seen)/net->batch)%net->subdivisions == 0) update_network(net); 
         return error; 
         } 
         
         // 随机梯度下降 
         float train_network_sgd(network *net, data d, int n) 
         { 
         int batch = net->batch; 
         
         int i; 
         float sum = 0; 
         for(i = 0; i < n; ++i){ 
         // 随机构造1个batch 
         get_random_batch(d, batch, net->input, net->truth); 
         float err = train_network_datum(net); 
         sum += err; 
         } 
         return ( float)sum/(n*batch); 
         } 
         
         // 训练网络 
         float train_network(network *net, data d) 
         { 
         assert(d.X.rows % net->batch == 0); 
         
         // 计算1个epoch需要多少个batch 
         int batch = net->batch; 
         int n = d.X.rows / batch; 
         
         int i; 
         float sum = 0; 
         for(i = 0; i < n; ++i){ 
         // 获取下一个batch 
         get_next_batch(d, batch, i*batch, net->input, net->truth); 
         float err = train_network_datum(net); 
         sum += err; 
         } 
         return ( float)sum/(n*batch); 
         }

void set_temp_network(network *net, float t) 
         { 
         int i; 
         for(i = 0; i < net->n; ++i){ 
         net->layers[i].temperature = t; 
         } 
         } 
         
         // 设置1个batch的输入图像个数 
         void set_batch_network(network *net, int b) 
         { 
         net->batch = b; 
         int i; 
         for(i = 0; i < net->n; ++i){ 
         net->layers[i].batch = b; 
         #ifdef CUDNN 
         if(net->layers[i].type == CONVOLUTIONAL){ 
         cudnn_convolutional_setup(net->layers + i); 
         } 
         if(net->layers[i].type == DECONVOLUTIONAL){ 
         layer *l = net->layers + i; 
         cudnnSetTensor4dDescriptor(l->dstTensorDesc, CUDNN_TENSOR_NCHW, CUDNN_DATA_FLOAT, 1, l->out_c, l->out_h, l->out_w); 
         cudnnSetTensor4dDescriptor(l->normTensorDesc, CUDNN_TENSOR_NCHW, CUDNN_DATA_FLOAT, 1, l->out_c, 1, 1); 
         } 
         #endif 
         } 
         }

// resize 网络每一层的 w,h 
         int resize_network(network *net, int w, int h) 
         { 
         #ifdef GPU 
         cuda_set_device(net->gpu_index); 
         cuda_free(net->workspace); 
         #endif 
         int i; 
         //if(w == net->w && h == net->h) return 0; 
         net->w = w; 
         net->h = h; 
         int inputs = 0; 
         size_t workspace_size = 0; 
         //fprintf(stderr, "Resizing to %d x %d...\n", w, h); 
         //fflush(stderr); 
         
         // 根据网络类型，对每一个层执行 resize 操作 
         for (i = 0; i < net->n; ++i){ 
         layer l = net->layers[i]; 
         if(l.type == CONVOLUTIONAL){ 
         resize_convolutional_layer(&l, w, h); 
         } else if(l.type == CROP){ 
         resize_crop_layer(&l, w, h); 
         } else if(l.type == MAXPOOL){ 
         resize_maxpool_layer(&l, w, h); 
         } else if(l.type == REGION){ 
         resize_region_layer(&l, w, h); 
         } else if(l.type == YOLO){ 
         resize_yolo_layer(&l, w, h); 
         } else if(l.type == ROUTE){ 
         resize_route_layer(&l, net); 
         } else if(l.type == SHORTCUT){ 
         resize_shortcut_layer(&l, w, h); 
         } else if(l.type == UPSAMPLE){ 
         resize_upsample_layer(&l, w, h); 
         } else if(l.type == REORG){ 
         resize_reorg_layer(&l, w, h); 
         } else if(l.type == AVGPOOL){ 
         resize_avgpool_layer(&l, w, h); 
         } else if(l.type == NORMALIZATION){ 
         resize_normalization_layer(&l, w, h); 
         } else if(l.type == COST){ 
         resize_cost_layer(&l, inputs); 
         } else{ 
         error( "Cannot resize this type of layer"); 
         } 
         
         if(l.workspace_size > workspace_size) workspace_size = l.workspace_size; 
         if(l.workspace_size > 2000000000) assert( 0); 
         
         inputs = l.outputs; 
         
         net->layers[i] = l; 
         w = l.out_w; 
         h = l.out_h; 
         
         if(l.type == AVGPOOL) break; 
         } 
         
         // 获取网络最后一层(除去loss层) 
         layer out = get_network_output_layer(net); 
         net->inputs = net->layers[ 0].inputs; 
         net->outputs = out.outputs; 
         net->truths = out.outputs; 
         if(net->layers[net->n -1].truths) net->truths = net->layers[net->n -1].truths; 
         net->output = out.output; 
         
         free(net->input); 
         free(net->truth); 
         net->input = calloc(net->inputs*net->batch, sizeof( float)); 
         net->truth = calloc(net->truths*net->batch, sizeof( float)); 
         #ifdef GPU 
         if(gpu_index >= 0){ 
         cuda_free(net->input_gpu); 
         cuda_free(net->truth_gpu); 
         net->input_gpu = cuda_make_array(net->input, net->inputs*net->batch); 
         net->truth_gpu = cuda_make_array(net->truth, net->truths*net->batch); 
         if(workspace_size){ 
         net->workspace = cuda_make_array( 0, (workspace_size -1)/ sizeof( float)+ 1); 
         } 
         } else { 
         free(net->workspace); 
         net->workspace = calloc( 1, workspace_size); 
         } 
         #else 
         free(net->workspace); 
         net->workspace = calloc( 1, workspace_size); 
         #endif 
         //fprintf(stderr, " Done!\n"); 
         return 0; 
         }

// 返回网络中的 DETECTION 层 
         layer get_network_detection_layer(network *net) 
         { 
         int i; 
         for(i = 0; i < net->n; ++i){ 
         if(net->layers[i].type == DETECTION){ 
         return net->layers[i]; 
         } 
         } 
         fprintf( stderr, "Detection layer not found!!\n"); 
         layer l = { 0}; 
         return l; 
         } 
         
         
         // 将网络第 i 层转换为 image 
         image get_network_image_layer(network *net, int i) 
         { 
         layer l = net->layers[i]; 
         #ifdef GPU 
         //cuda_pull_array(l.output_gpu, l.output, l.outputs); 
         #endif 
         if (l.out_w && l.out_h && l.out_c){ 
         return float_to_image(l.out_w, l.out_h, l.out_c, l.output); 
         } 
         image def = { 0}; 
         return def; 
         } 
         
         image get_network_image(network *net) 
         { 
         int i; 
         for(i = net->n -1; i >= 0; --i){ 
         image m = get_network_image_layer(net, i); 
         if(m.h != 0) return m; 
         } 
         image def = { 0}; 
         return def; 
         } 
         
         // 可视化所有卷基层 
         void visualize_network(network *net) 
         { 
         image *prev = 0; 
         int i; 
         char buff[ 256]; 
         for(i = 0; i < net->n; ++i){ 
         sprintf(buff, "Layer %d", i); 
         layer l = net->layers[i]; 
         if(l.type == CONVOLUTIONAL){ 
         prev = visualize_convolutional_layer(l, buff, prev); 
         } 
         } 
         }

// 返回网络 output 的 top-k 
         void top_predictions(network *net, int k, int *index) 
         { 
         top_k(net->output, net->outputs, k, index); 
         } 
         
         // 给定输入，网络预测输出，并返回 
         float network_predict(network net, float *input) 
         { 
         network orig = *net; 
         net->input = input; 
         net->truth = 0; 
         net->train = 0; 
         net->delta = 0; 
         forward_network(net); 
         float *out = net->output; 
         *net = orig; 
         return out; 
         }

// 返回网络中 bbox 的总数 
         int num_detections(network *net, float thresh) 
         { 
         int i; 
         int s = 0; 
         for(i = 0; i < net->n; ++i){ 
         layer l = net->layers[i]; 
         if(l.type == YOLO){ 
         s += yolo_num_detections(l, thresh); 
         } 
         if(l.type == DETECTION || l.type == REGION){ 
         s += l.w*l.h*l.n; 
         } 
         } 
         return s; 
         } 
         
         // 根据最后一层 bbox 的数目，构造 detection 数组 dets 
         detection make_network_boxes(network net, float thresh, int *num) 
         { 
         layer l = net->layers[net->n - 1]; 
         int i; 
         int nboxes = num_detections(net, thresh); 
         if(num) *num = nboxes; 
         detection *dets = calloc(nboxes, sizeof(detection)); 
         for(i = 0; i < nboxes; ++i){ 
         dets[i].prob = calloc(l.classes, sizeof( float)); 
         if(l.coords > 4){ 
         dets[i].mask = calloc(l.coords -4, sizeof( float)); 
         } 
         } 
         return dets; 
         } 
         
         // 填充 dets 中的信息 
         void fill_network_boxes(network *net, int w, int h, float thresh, float hier, int map, int relative, detection dets) 
         { 
         int j; 
         for(j = 0; j < net->n; ++j){ 
         layer l = net->layers[j]; 
         if(l.type == YOLO){ 
         int count = get_yolo_detections(l, w, h, net->w, net->h, thresh, map, relative, dets); 
         dets += count; 
         } 
         if(l.type == REGION){ 
         get_region_detections(l, w, h, net->w, net->h, thresh, map, hier, relative, dets); 
         dets += l.w*l.h*l.n; 
         } 
         if(l.type == DETECTION){ 
         get_detection_detections(l, w, h, thresh, dets); 
         dets += l.w*l.h*l.n; 
         } 
         } 
         } 
         
         // 获取网络的所有检测的 bbox (detection 数组 dets) 
         detection get_network_boxes(network net, int w, int h, float thresh, float hier, int map, int relative, int num) 
         { 
         detection *dets = make_network_boxes(net, thresh, num); 
         fill_network_boxes(net, w, h, thresh, hier, map, relative, dets); 
         return dets; 
         } 
         
         // 释放 detection 数组 dets 的内存 
         void free_detections(detection *dets, int n) 
         { 
         int i; 
         for(i = 0; i < n; ++i){ 
         free(dets[i].prob); 
         if(dets[i].mask) free(dets[i].mask); 
         } 
         free(dets); 
         }

// 输入一张图像，网络预测输出 
         float network_predict_image(network net, image im) 
         { 
         // 调整输入图像大小，图像的 上下 或者 左右 多余的值为 0.5 
         image imr = letterbox_image(im, net->w, net->h); 
         
         // 设置1个batch的输入图像个数为1 
         set_batch_network(net, 1); 
         
         // 网络预测 
         float *p = network_predict(net, imr.data); 
         
         free_image(imr); 
         return p; 
         } 
         
         int network_width(network *net){ return net->w;} 
         int network_height(network *net){ return net->h;} 
         
         // 输入多张图像，网络预测输出，预测 n 次，取平均值 
         matrix network_predict_data_multi(network *net, data test, int n) 
         { 
         int i,j,b,m; 
         int k = net->outputs; 
         
         // 构造矩阵存储预测结果，行为 test.X.rows, 列为 k 
         matrix pred = make_matrix(test.X.rows, k); 
         
         float *X = calloc(net->batch test.X.cols, sizeof( float)); 
         
         for(i = 0; i < test.X.rows; i += net->batch){ 
         for(b = 0; b < net->batch; ++b){ 
         // 最后一个batch 
         if(i + b == test.X.rows) break; 
         
         memcpy(X + b*test.X.cols, test.X.vals[i+b], test.X.cols* sizeof( float)); 
         } 
         
         // 预测 n 次，取平均值 
         for(m = 0; m < n; ++m){ 
         float out = network_predict(net, X); 
         for(b = 0; b < net->batch; ++b){ 
         if(i+b == test.X.rows) break; 
         
         for(j = 0; j < k; ++j){ 
         pred.vals[i+b][j] += out[j+b*k]/n; 
         } 
         } 
         } 
         } 
         free(X); 
         return pred; 
         } 
         
         // 输入多张图像，网络预测输出 
         matrix network_predict_data(network *net, data test) 
         { 
         int i,j,b; 
         int k = net->outputs; 
         matrix pred = make_matrix(test.X.rows, k); 
         float *X = calloc(net->batch*test.X.cols, sizeof( float)); 
         for(i = 0; i < test.X.rows; i += net->batch){ 
         for(b = 0; b < net->batch; ++b){ 
         if(i+b == test.X.rows) break; 
         memcpy(X+b*test.X.cols, test.X.vals[i+b], test.X.cols sizeof( float)); 
         } 
         float out = network_predict(net, X); 
         for(b = 0; b < net->batch; ++b){ 
         if(i+b == test.X.rows) break; 
         for(j = 0; j < k; ++j){ 
         pred.vals[i+b][j] = out[j+b*k]; 
         } 
         } 
         } 
         free(X); 
         return pred; 
         }

// 打印网络每一层的均值，方差和前100个输出 
         void print_network(network *net) 
         { 
         int i,j; 
         for(i = 0; i < net->n; ++i){ 
         layer l = net->layers[i]; 
         float *output = l.output; 
         int n = l.outputs; 
         // 均值 
         float mean = mean_array(output, n); 
         // 方差 
         float vari = variance_array(output, n); 
         fprintf( stderr, "Layer %d - Mean: %f, Variance: %f\n",i,mean, vari); 
         // 前100个输出 
         if(n > 100) n = 100; 
         for(j = 0; j < n; ++j) fprintf( stderr, "%f, ", output[j]); 
         if(n == 100) fprintf( stderr, ".....\n"); 
         fprintf( stderr, "\n"); 
         } 
         } 
         
         // 比较 n1 n2 预测结果的差距 
         void compare_networks(network *n1, network *n2, data test) 
         { 
         // 获取 n1 n2 的预测结果 
         matrix g1 = network_predict_data(n1, test); 
         matrix g2 = network_predict_data(n2, test); 
         
         int i; 
         int a,b,c,d; 
         a = b = c = d = 0; 
         for(i = 0; i < g1.rows; ++i){ 
         // ground truth 以及 n1, n2 的预测结果 
         int truth = max_index(test.y.vals[i], test.y.cols); 
         int p1 = max_index(g1.vals[i], g1.cols); 
         int p2 = max_index(g2.vals[i], g2.cols); 
         
         / 
         a: n1 n2 均预测错误 
         b: n1 预测错误，n2 预测正确 
         c: n1 预测正确，n2 预测错误 
         d: n1 n2 均预测正确 
         / 
         if(p1 == truth){ 
         if(p2 == truth) ++d; 
         else ++c; 
         } else{ 
         if(p2 == truth) ++b; 
         else ++a; 
         } 
         } 
         printf( "%5d %5d\n%5d %5d\n", a, b, c, d); 
         float num = pow(( abs(b - c) - 1.), 2.); 
         float den = b + c; 
         printf( "%f\n", num/den); 
         } 
         
         // 计算网络 top-1 准确度 
         float network_accuracy(network *net, data d) 
         { 
         matrix guess = network_predict_data(net, d); 
         float acc = matrix_topk_accuracy(d.y, guess, 1); 
         free_matrix(guess); 
         return acc; 
         } 
         
         // 计算网络 top-1 准确度 和 top-n 准确度 
         float network_accuracies(network net, data d, int n) 
         { 
         static float acc[ 2]; 
         matrix guess = network_predict_data(net, d); 
         acc[ 0] = matrix_topk_accuracy(d.y, guess, 1); 
         acc[ 1] = matrix_topk_accuracy(d.y, guess, n); 
         free_matrix(guess); 
         return acc; 
         } 
         
         // 获取网络最后一层(除去loss层) 
         layer get_network_output_layer(network *net) 
         { 
         int i; 
         for(i = net->n - 1; i >= 0; --i){ 
         if(net->layers[i].type != COST) break; 
         } 
         return net->layers[i]; 
         } 
         
         // 网络预测 n 次，对结果取平均，计算网络 top-1 准确度 
         float network_accuracy_multi(network *net, data d, int n) 
         { 
         matrix guess = network_predict_data_multi(net, d, n); 
         float acc = matrix_topk_accuracy(d.y, guess, 1); 
         free_matrix(guess); 
         return acc; 
         } 
         
         // 释放网络内存 
         void free_network(network *net) 
         { 
         int i; 
         for(i = 0; i < net->n; ++i){ 
         free_layer(net->layers[i]); 
         } 
         free(net->layers); 
         if(net->input) free(net->input); 
         if(net->truth) free(net->truth); 
         #ifdef GPU 
         if(net->input_gpu) cuda_free(net->input_gpu); 
         if(net->truth_gpu) cuda_free(net->truth_gpu); 
         #endif 
         free(net); 
         }

// 返回网络输出层 
         layer network_output_layer(network *net) 
         { 
         int i; 
         for(i = net->n - 1; i >= 0; --i){ 
         if(net->layers[i].type != COST) break; 
         } 
         return net->layers[i]; 
         } 
         
         // 返回网络输入个数 
         int network_inputs(network *net) 
         { 
         return net->layers[ 0].inputs; 
         } 
         
         // 返回网络输出个数 
         int network_outputs(network *net) 
         { 
         return network_output_layer(net).outputs; 
         } 
         
         // 返回网络输出 
         float network_output(network net) 
         { 
         return network_output_layer(net).output; 
         }

#ifdef GPU 
         
         // GPU版前向传播 
         void forward_network_gpu(network *netp) 
         { 
         network net = *netp; 
         cuda_set_device(net.gpu_index); 
         
         // 将主机内存拷贝到GPU内存 
         cuda_push_array(net.input_gpu, net.input, net.inputs*net.batch); 
         if(net.truth){ 
         cuda_push_array(net.truth_gpu, net.truth, net.truths*net.batch); 
         } 
         
         int i; 
         for(i = 0; i < net.n; ++i){ 
         net.index = i; 
         layer l = net.layers[i]; 
         
         if(l.delta_gpu){ 
         fill_gpu(l.outputs l.batch, 0, l.delta_gpu, 1); 
         } 
         
         // GPU版前向传播 
         l.forward_gpu(l, net); 
         net.input_gpu = l.output_gpu; 
         net.input = l.output; 
         
         if(l.truth) { 
         net.truth_gpu = l.output_gpu; 
         net.truth = l.output; 
         } 
         } 
         
         // 将网络输出从GPU内存拷贝到主机内存 
         pull_network_output(netp); 
         
         // 计算网络的 loss 
         calc_network_cost(netp); 
         } 
         
         // GPU版反向传播 
         void backward_network_gpu(network netp) 
         { 
         int i; 
         network net = *netp; 
         network orig = net; 
         cuda_set_device(net.gpu_index); 
         
         for(i = net.n -1; i >= 0; --i){ 
         layer l = net.layers[i]; 
         if(l.stopbackward) break; 
         if(i == 0){ 
         net = orig; 
         } else{ 
         layer prev = net.layers[i -1]; 
         net.input = prev.output; 
         net.delta = prev.delta; 
         net.input_gpu = prev.output_gpu; 
         net.delta_gpu = prev.delta_gpu; 
         } 
         net.index = i; 
         
         // GPU版反向传播 
         l.backward_gpu(l, net); 
         } 
         } 
         
         // GPU版更新网络参数 
         void update_network_gpu(network *netp) 
         { 
         network net = *netp; 
         cuda_set_device(net.gpu_index); 
         
         update_args a = { 0}; 
         a.batch = net.batch*net.subdivisions; 
         a.learning_rate = get_current_rate(netp); 
         a.momentum = net.momentum; 
         a.decay = net.decay; 
         a.adam = net.adam; 
         a.B1 = net.B1; 
         a.B2 = net.B2; 
         a.eps = net.eps; 
         ++*net.t; 
         a.t = (*net.t); 
         
         int i; 
         for(i = 0; i < net.n; ++i){ 
         layer l = net.layers[i]; 
         if(l.update_gpu){ 
         // GPU版更新网络参数 
         l.update_gpu(l, a); 
         } 
         } 
         } 
         
         // 将网络每一层的 weight bias scale 的 updates 全部置为0 
         void harmless_update_network_gpu(network *netp) 
         { 
         network net = *netp; 
         cuda_set_device(net.gpu_index); 
         
         int i; 
         for(i = 0; i < net.n; ++i){ 
         layer l = net.layers[i]; 
         
         // 将 weight bias scale 的 updates 全部置为0 
         if(l.weight_updates_gpu) fill_gpu(l.nweights, 0, l.weight_updates_gpu, 1); 
         if(l.bias_updates_gpu) fill_gpu(l.nbiases, 0, l.bias_updates_gpu, 1); 
         if(l.scale_updates_gpu) fill_gpu(l.nbiases, 0, l.scale_updates_gpu, 1); 
         } 
         } 
         
         typedef struct { 
         network *net; 
         data d; 
         float *err; 
         } train_args; 
         
         // 训练网络的线程 
         void train_thread(void ptr) 
         { 
         train_args args = (train_args*)ptr; 
         free(ptr); 
         cuda_set_device(args.net->gpu_index); 
         args.err = train_network(args.net, args.d); 
         return 0; 
         } 
         
         // 创建并返回训练网络的线程 
         pthread_t train_network_in_thread(network *net, data d, float *err) 
         { 
         pthread_t thread; 
         train_args *ptr = (train_args ) calloc( 1, sizeof(train_args)); 
         ptr->net = net; 
         ptr->d = d; 
         ptr->err = err; 
         if(pthread_create(&thread, 0, train_thread, ptr)) error( "Thread creation failed"); 
         return thread; 
         } 
         
         // 更新 bias 和 weight 
         void merge_weights(layer l, layer base) 
         { 
         if (l.type == CONVOLUTIONAL) { 
         axpy_cpu(l.n, 1, l.bias_updates, 1, base.biases, 1); 
         axpy_cpu(l.nweights, 1, l.weight_updates, 1, base.weights, 1); 
         if (l.scales) { 
         axpy_cpu(l.n, 1, l.scale_updates, 1, base.scales, 1); 
         } 
         } else if(l.type == CONNECTED) { 
         axpy_cpu(l.outputs, 1, l.bias_updates, 1, base.biases, 1); 
         axpy_cpu(l.outputs*l.inputs, 1, l.weight_updates, 1, base.weights, 1); 
         } 
         } 
         
         // 缩放 bias 和 weight 
         void scale_weights(layer l, float s) 
         { 
         if (l.type == CONVOLUTIONAL) { 
         scal_cpu(l.n, s, l.biases, 1); 
         scal_cpu(l.nweights, s, l.weights, 1); 
         if (l.scales) { 
         scal_cpu(l.n, s, l.scales, 1); 
         } 
         } else if(l.type == CONNECTED) { 
         scal_cpu(l.outputs, s, l.biases, 1); 
         scal_cpu(l.outputs*l.inputs, s, l.weights, 1); 
         } 
         } 
         
         // 将 bias 和 weight 从GPU内存拷贝到主机内存 
         void pull_weights(layer l) 
         { 
         if(l.type == CONVOLUTIONAL || l.type == DECONVOLUTIONAL){ 
         cuda_pull_array(l.biases_gpu, l.bias_updates, l.n); 
         cuda_pull_array(l.weights_gpu, l.weight_updates, l.nweights); 
         if(l.scales) cuda_pull_array(l.scales_gpu, l.scale_updates, l.n); 
         } else if(l.type == CONNECTED){ 
         cuda_pull_array(l.biases_gpu, l.bias_updates, l.outputs); 
         cuda_pull_array(l.weights_gpu, l.weight_updates, l.outputs*l.inputs); 
         } 
         } 
         
         // 将 bias 和 weight 从主机内存拷贝到GPU内存 
         void push_weights(layer l) 
         { 
         if(l.type == CONVOLUTIONAL || l.type == DECONVOLUTIONAL){ 
         cuda_push_array(l.biases_gpu, l.biases, l.n); 
         cuda_push_array(l.weights_gpu, l.weights, l.nweights); 
         if(l.scales) cuda_push_array(l.scales_gpu, l.scales, l.n); 
         } else if(l.type == CONNECTED){ 
         cuda_push_array(l.biases_gpu, l.biases, l.outputs); 
         cuda_push_array(l.weights_gpu, l.weights, l.outputs*l.inputs); 
         } 
         } 
         
         // 将 layer base 的 bias 和 weight 从主机内存拷贝到 layer l 的GPU内存 
         void distribute_weights(layer l, layer base) 
         { 
         if (l.type == CONVOLUTIONAL || l.type == DECONVOLUTIONAL) { 
         cuda_push_array(l.biases_gpu, base.biases, l.n); 
         cuda_push_array(l.weights_gpu, base.weights, l.nweights); 
         if (base.scales) cuda_push_array(l.scales_gpu, base.scales, l.n); 
         } else if (l.type == CONNECTED) { 
         cuda_push_array(l.biases_gpu, base.biases, l.outputs); 
         cuda_push_array(l.weights_gpu, base.weights, l.outputs*l.inputs); 
         } 
         } 
         
         // 同步所有网络第 j 层 layer 
         void sync_layer(network *nets, int n, int j) 
         { 
         int i; 
         network *net = nets[ 0]; 
         
         // 将网络 第 j 层 layer 拷贝到 base，并将 bias 和 weight 初始化为 0 
         layer base = net->layers[j]; 
         scale_weights(base, 0); 
         
         // 将所有网络第 j 层的 bias 和 weight 累加到 base，并求平均 
         for (i = 0; i < n; ++i) { 
         cuda_set_device(nets[i]->gpu_index); 
         layer l = nets[i]->layers[j]; 
         pull_weights(l); 
         merge_weights(l, base); 
         } 
         scale_weights(base, 1./n); 
         
         // 将 base 的 bias 和 weight 拷贝到所有网络第 j 层的GPU内存 
         for (i = 0; i < n; ++i) { 
         cuda_set_device(nets[i]->gpu_index); 
         layer l = nets[i]->layers[j]; 
         distribute_weights(l, base); 
         } 
         } 
         
         typedef struct{ 
         network **nets; 
         int n; 
         int j; 
         } sync_args; 
         
         // 同步网络的线程 
         void sync_layer_thread(void ptr) 
         { 
         sync_args args = (sync_args*)ptr; 
         sync_layer(args.nets, args.n, args.j); 
         free(ptr); 
         return 0; 
         } 
         
         // 创建并返回同步网络的线程 
         pthread_t sync_layer_in_thread(network *nets, int n, int j) 
         { 
         pthread_t thread; 
         sync_args *ptr = (sync_args ) calloc( 1, sizeof(sync_args)); 
         ptr->nets = nets; 
         ptr->n = n; 
         ptr->j = j; 
         if(pthread_create(&thread, 0, sync_layer_thread, ptr)) error( "Thread creation failed"); 
         return thread; 
         } 
         
         // interval个batch后，同步网络 
         void sync_nets(network *nets, int n, int interval) 
         { 
         int j; 
         int layers = nets[ 0]->n; 
         pthread_t *threads = ( pthread_t ) calloc(layers, sizeof( pthread_t)); 
         
         // 更新网络已经看到的图片数量，并同步到所有网络 
         (nets[0]->seen) += interval (n -1) * nets[ 0]->batch * nets[ 0]->subdivisions; 
         for (j = 0; j < n; ++j){ 
         (nets[j]->seen) = (nets[ 0]->seen); 
         } 
         
         // 同步网络的所有layer 
         for (j = 0; j < layers; ++j) { 
         threads[j] = sync_layer_in_thread(nets, n, j); 
         } 
         for (j = 0; j < layers; ++j) { 
         pthread_join(threads[j], 0); 
         } 
         free(threads); 
         } 
         
         // n 个 GPU 训练网络 
         float train_networks(network *nets, int n, data d, int interval) 
         { 
         int i; 
         int batch = nets[ 0]->batch; 
         int subdivisions = nets[ 0]->subdivisions; 
         // 保证 d.X.rows 能够被 n 整除 
         assert(batch subdivisions * n == d.X.rows); 
         pthread_t threads = (pthread_t ) calloc(n, sizeof( pthread_t)); 
         float errors = (float ) calloc(n, sizeof( float)); 
         
         float sum = 0; 
         for(i = 0; i < n; ++i){ 
         // 多GPU分配数据 
         data p = get_data_part(d, i, n); 
         // 创建并返回训练网络的线程 
         threads[i] = train_network_in_thread(nets[i], p, errors + i); 
         } 
         
         for(i = 0; i < n; ++i){ 
         pthread_join(threads[i], 0); 
         //printf("%f\n", errors[i]); 
         sum += errors[i]; 
         } 
         //cudaDeviceSynchronize(); 
         
         // interval个batch后，同步网络 
         if (get_current_batch(nets[ 0]) % interval == 0) { 
         printf( "Syncing... "); 
         fflush( stdout); 
         sync_nets(nets, n, interval); 
         printf( "Done!\n"); 
         } 
         //cudaDeviceSynchronize(); 
         
         free(threads); 
         free(errors); 
         return ( float)sum/(n); 
         } 
         
         // 将网络输出从GPU内存拷贝到主机内存 
         void pull_network_output(network *net) 
         { 
         layer l = get_network_output_layer(net); 
         
         // 将GPU内存拷贝到主机内存 
         cuda_pull_array(l.output_gpu, l.output, l.outputs*l.batch); 
         } 
         
         #endif