fasterRCNN

以你之姓@ 2023-07-02 05:26 53阅读 0赞

# ![20191009191333910.png][] #

# [日萌社][Link 1] #

[人工智能AI：Keras PyTorch MXNet TensorFlow PaddlePaddle 深度学习实战（不定时更新）][AI_Keras PyTorch MXNet TensorFlow PaddlePaddle]

--------------------

# **github下载代码** #

[https://github.com/dragen1860/Deep-Learning-with-TensorFlow-book][https_github.com_dragen1860_Deep-Learning-with-TensorFlow-book]

[https://github.com/dragen1860/TensorFlow-2.x-Tutorials][https_github.com_dragen1860_TensorFlow-2.x-Tutorials]

--------------------

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9henVzYS5ibG9nLmNzZG4ubmV0_size_16_color_FFFFFF_t_70][]

[![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9henVzYS5ibG9nLmNzZG4ubmV0_size_16_color_FFFFFF_t_70 1][]][https_github.com_dragen1860_TensorFlow-2.x-Tutorials]

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9henVzYS5ibG9nLmNzZG4ubmV0_size_16_color_FFFFFF_t_70 2][]

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9henVzYS5ibG9nLmNzZG4ubmV0_size_16_color_FFFFFF_t_70 3][]

--------------------

# **inspect\_model.py** #

import matplotlib
    import matplotlib.pyplot as plt
    
    import os
    import tensorflow as tf
    import numpy as np
    import visualize
    
    
    
    
    print(tf.__version__)
    
    
    # tensorflow config - using one gpu and extending the GPU 
    # memory region needed by the TensorFlow process
    os.environ['CUDA_VISIBLE_DEVICES'] = '0'
    os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
    
    
    
    # #### load dataset
    
    # In[10]:
    
    
    from detection.datasets import coco, data_generator
    
    
    # In[11]:
    
    
    img_mean = (123.675, 116.28, 103.53)
    # img_std = (58.395, 57.12, 57.375)
    img_std = (1., 1., 1.)
    
    
    # In[12]:
    
    
    train_dataset = coco.CocoDataSet('/scratch/llong/datasets/coco2017/', 'train',
                                     flip_ratio=0.5,
                                     pad_mode='fixed',
                                     mean=img_mean,
                                     std=img_std,
                                     scale=(800, 1024))
    
    
    # #### display a sample
    
    # In[13]:
    
    
    img, img_meta, bboxes, labels = train_dataset[6]
    rgb_img = np.round(img + img_mean)
    print('origin shape:', img_meta[:3])
    print('image  shape:', img_meta[3:6])
    print('pad    shape:', img_meta[6:9])
    print('scale factor:', img_meta[9:10])
    print('flip  yet   :', img_meta[10:11])
    print('img    shape:', img.shape)
    print('bboxes:', bboxes.shape)
    print('labels:', labels)
    print('label2name:', train_dataset.get_categories())
    print('lenof classes:', len(train_dataset.get_categories()))
    names = [train_dataset.get_categories()[i] for i in labels]
    print('names:', names)
    # In[24]:
    
    
    plt.imshow(rgb_img.astype(np.int32))
    plt.show()
    
    # In[35]:
    
    
    
    
    # In[6]:
    
    
    visualize.display_instances(rgb_img, bboxes, labels, train_dataset.get_categories())
    
    
    # #### load model
    
    # In[36]:
    
    
    from detection.models.detectors import faster_rcnn
    
    model = faster_rcnn.FasterRCNN(num_classes=len(train_dataset.get_categories()))
    
    
    # In[37]:
    
    
    # [1, 1024, 1024, 3]
    batch_imgs = tf.Variable(np.expand_dims(img, 0))
    # [1, 11]
    batch_metas = tf.Variable(np.expand_dims(img_meta, 0))
    # [1, nnum of boxes, 4]
    batch_bboxes = tf.Variable(np.expand_dims(bboxes, 0))
    # [1, num of boxes]
    batch_labels = tf.Variable(np.expand_dims(labels, 0))
    
    _ = model((batch_imgs, batch_metas, batch_bboxes, batch_labels), training=True)
    
    
    # In[38]:
    
    
    model.load_weights('weights/faster_rcnn.h5')
    
    
    # ### Stage 1: Region Proposal Network
    # 
    # #### 1.a RPN Targets
    
    # In[39]:
    
    
    anchors, valid_flags = model.rpn_head.generator.generate_pyramid_anchors(batch_metas)
    
    rpn_target_matchs, rpn_target_deltas = model.rpn_head.anchor_target.build_targets(
                    anchors, valid_flags, batch_bboxes, batch_labels)
    
    
    # In[40]:
    
    
    positive_anchors = tf.gather(anchors, tf.where(tf.equal(rpn_target_matchs, 1))[:, 1])
    negative_anchors = tf.gather(anchors, tf.where(tf.equal(rpn_target_matchs, -1))[:, 1])
    neutral_anchors = tf.gather(anchors, tf.where(tf.equal(rpn_target_matchs, 0))[:, 1])
    positive_target_deltas = rpn_target_deltas[0, :tf.where(tf.equal(rpn_target_matchs, 1)).shape[0]]
    
    
    # In[41]:
    
    
    from detection.core.bbox import transforms
        
    refined_anchors = transforms.delta2bbox(
        positive_anchors, positive_target_deltas, (0., 0., 0., 0.), (0.1, 0.1, 0.2, 0.2))
    
    
    # In[45]:
    
    
    print('rpn_target_matchs:\t', rpn_target_matchs[0].shape.as_list())
    print('rpn_target_deltas:\t', rpn_target_deltas[0].shape.as_list())
    print('positive_anchors:\t', positive_anchors.shape.as_list())
    print('negative_anchors:\t', negative_anchors.shape.as_list())
    print('neutral_anchors:\t', neutral_anchors.shape.as_list())
    print('refined_anchors:\t', refined_anchors.shape.as_list())
    
    
    # In[44]:
    
    
    visualize.draw_boxes(rgb_img, 
                         boxes=positive_anchors.numpy(), 
                         refined_boxes=refined_anchors.numpy())
    plt.show()
    
    
    # #### 1.b RPN Predictions
    
    # In[15]:
    
    
    training = False
    C2, C3, C4, C5 = model.backbone(batch_imgs, 
                                    training=training)
    
    P2, P3, P4, P5, P6 = model.neck([C2, C3, C4, C5], 
                                    training=training)
    
    rpn_feature_maps = [P2, P3, P4, P5, P6]
    rcnn_feature_maps = [P2, P3, P4, P5]
    
    rpn_class_logits, rpn_probs, rpn_deltas = model.rpn_head(
        rpn_feature_maps, training=training)
    
    
    # In[16]:
    
    
    rpn_probs_tmp = rpn_probs[0, :, 1]
    
    
    # In[17]:
    
    
    # Show top anchors by score (before refinement)
    limit = 100
    ix = tf.nn.top_k(rpn_probs_tmp, k=limit).indices[::-1]
    
    
    # In[18]:
    
    
    visualize.draw_boxes(rgb_img, boxes=tf.gather(anchors, ix).numpy())
    
    
    # ### Stage 2:  Proposal Classification
    
    # In[19]:
    
    
    proposals_list = model.rpn_head.get_proposals(
        rpn_probs, rpn_deltas, batch_metas)
    
    
    # In[20]:
    
    
    rois_list = proposals_list
    
    pooled_regions_list = model.roi_align(
        (rois_list, rcnn_feature_maps, batch_metas), training=training)
    
    rcnn_class_logits_list, rcnn_probs_list, rcnn_deltas_list =     model.bbox_head(pooled_regions_list, training=training)
    
    
    # In[21]:
    
    
    detections_list = model.bbox_head.get_bboxes(
        rcnn_probs_list, rcnn_deltas_list, rois_list, batch_metas)
    
    
    # In[22]:
    
    
    tmp = detections_list[0][:, :4]
    
    
    # In[23]:
    
    
    visualize.draw_boxes(rgb_img, boxes=tmp.numpy())
    
    
    # ### Stage 3: Run model directly
    
    # In[24]:
    
    
    detections_list = model((batch_imgs, batch_metas), training=False)
    tmp = detections_list[0][:, :4]
    visualize.draw_boxes(rgb_img, boxes=tmp.numpy())
    
    
    # ### Stage 4: Test (Detection)
    
    # In[25]:
    
    
    from detection.datasets.utils import get_original_image
    ori_img = get_original_image(img, img_meta, img_mean)
    
    
    # In[26]:
    
    
    proposals = model.simple_test_rpn(img, img_meta)
    
    
    # In[27]:
    
    
    res = model.simple_test_bboxes(img, img_meta, proposals)
    
    
    # In[28]:
    
    
    visualize.display_instances(ori_img, res['rois'], res['class_ids'], 
                                train_dataset.get_categories(), scores=res['scores'])

#  #

--------------------

# **roi\_test.py** #

import tensorflow as tf
    import matplotlib.pyplot as plt
    
    img = plt.imread('/home/llong/Downloads/number.jpg') /255.
    img2 = plt.imread('/home/llong/Downloads/number2.jpg') /255.
    img = tf.convert_to_tensor(img, dtype=tf.float32)
    img = tf.expand_dims(img, axis=0)
    img = tf.image.resize(img, (1000,1000))
    img2 = tf.convert_to_tensor(img2, dtype=tf.float32)
    img2 = tf.expand_dims(img2, axis=0)
    img2 = tf.image.resize(img2, (1000,1000))
    
    img = tf.concat([img, img2], axis=0)
    print('img:', img.shape)
    
    a = tf.image.crop_and_resize(img, [[0.5, 0.5, 1.0, 1.0], [0.5, 0.5, 1.5, 1.5]], [0, 1], crop_size=(500, 500))
    print('a:', a.shape)
    
    plt.subplot(2,2,1)
    plt.imshow(img[0])
    plt.subplot(2,2,2)
    plt.imshow(img[1])
    plt.subplot(2,2,3)
    plt.imshow(a[0])
    plt.subplot(2,2,4)
    plt.imshow(a[1])
    plt.show()

#  #

--------------------

# **train\_model.py** #

import  os
    import  tensorflow as tf
    from    tensorflow import keras
    import  numpy as np
    from    matplotlib import pyplot as plt
    import  visualize
    
    from    detection.datasets import coco, data_generator
    from    detection.datasets.utils import get_original_image
    from    detection.models.detectors import faster_rcnn
    
    os.environ['CUDA_VISIBLE_DEVICES'] = '0'
    os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
    print(tf.__version__)
    assert tf.__version__.startswith('2.')
    tf.random.set_seed(22)
    np.random.seed(22)
    
    img_mean = (123.675, 116.28, 103.53)
    # img_std = (58.395, 57.12, 57.375)
    img_std = (1., 1., 1.)
    batch_size = 1
    train_dataset = coco.CocoDataSet('/scratch/llong/datasets/coco2017/', 'train',
                                     flip_ratio=0.5,
                                     pad_mode='fixed',
                                     mean=img_mean,
                                     std=img_std,
                                     scale=(800, 1216))
    num_classes = len(train_dataset.get_categories())
    train_generator = data_generator.DataGenerator(train_dataset)
    train_tf_dataset = tf.data.Dataset.from_generator(
        train_generator, (tf.float32, tf.float32, tf.float32, tf.int32))
    train_tf_dataset = train_tf_dataset.batch(batch_size).prefetch(100).shuffle(100)
    # train_tf_dataset = train_tf_dataset.padded_batch(
    #     batch_size, padded_shapes=([None, None, None], [None], [None, None], [None]))
    
    model = faster_rcnn.FasterRCNN(num_classes=num_classes)
    optimizer = keras.optimizers.SGD(1e-3, momentum=0.9, nesterov=True)
    
    img, img_meta, bboxes, labels = train_dataset[6] # [N, 4], shape:[N]=data:[62]
    rgb_img = np.round(img + img_mean)
    ori_img = get_original_image(img, img_meta, img_mean)
    # visualize.display_instances(rgb_img, bboxes, labels, train_dataset.get_categories())
    
    
    batch_imgs = tf.convert_to_tensor(np.expand_dims(img, 0)) # [1, 1216, 1216, 3]
    batch_metas = tf.convert_to_tensor(np.expand_dims(img_meta, 0)) # [1, 11]
    
    # dummpy forward to build network variables
    _ = model((batch_imgs, batch_metas), training=False)
    
    proposals = model.simple_test_rpn(img, img_meta)
    res = model.simple_test_bboxes(img, img_meta, proposals)
    visualize.display_instances(ori_img, res['rois'], res['class_ids'],
                                train_dataset.get_categories(), scores=res['scores'])
    plt.savefig('image_demo_random.png')
    
    model.load_weights('weights/faster_rcnn.h5', by_name=True)
    
    proposals = model.simple_test_rpn(img, img_meta)
    res = model.simple_test_bboxes(img, img_meta, proposals)
    visualize.display_instances(ori_img, res['rois'], res['class_ids'],
                                train_dataset.get_categories(), scores=res['scores'])
    plt.savefig('image_demo_ckpt.png')
    
    for epoch in range(100):
    
        loss_history = []
        for (batch, inputs) in enumerate(train_tf_dataset):
    
            batch_imgs, batch_metas, batch_bboxes, batch_labels = inputs
            with tf.GradientTape() as tape:
                rpn_class_loss, rpn_bbox_loss, rcnn_class_loss, rcnn_bbox_loss = model(
                    (batch_imgs, batch_metas, batch_bboxes, batch_labels), training=True)
    
                loss_value = rpn_class_loss + rpn_bbox_loss + rcnn_class_loss + rcnn_bbox_loss
    
            grads = tape.gradient(loss_value, model.trainable_variables)
            optimizer.apply_gradients(zip(grads, model.trainable_variables))
    
            loss_history.append(loss_value.numpy())
    
            if batch % 10 == 0:
                print('epoch', epoch, batch, np.mean(loss_history))
    
                # img, img_meta = batch_imgs[0].numpy(), batch_metas[0].numpy()
                # ori_img = get_original_image(img, img_meta, img_mean)
                # proposals = model.simple_test_rpn(img, img_meta)
                # res = model.simple_test_bboxes(img, img_meta, proposals)
                # visualize.display_instances(ori_img, res['rois'], res['class_ids'],
                #                             train_dataset.get_categories(), scores=res['scores'])
                # plt.savefig('images/%d-%d.png' % (epoch, batch))

#  #

--------------------

# **visualize.py** #

import os
    import sys
    import random
    import itertools
    import colorsys
    
    import numpy as np
    from skimage.measure import find_contours
    
    import matplotlib.pyplot as plt
    from matplotlib import patches, lines
    from matplotlib.patches import Polygon
    # import IPython.display
    
    def random_colors(N, bright=True):
        '''
        Generate random colors.
        To get visually distinct colors, generate them in HSV space then
        convert to RGB.
        '''
        brightness = 1.0 if bright else 0.7
        hsv = [(i / N, 1, brightness) for i in range(N)]
        colors = list(map(lambda c: colorsys.hsv_to_rgb(*c), hsv))
        random.shuffle(colors)
        return colors
    
    def display_instances(image, boxes, class_ids, class_names,
                          scores=None, title="",
                          figsize=(16, 16), ax=None):
        '''
        boxes: [num_instance, (y1, x1, y2, x2, class_id)] in image coordinates.
        class_ids: [num_instances]
        class_names: list of class names of the dataset
        scores: (optional) confidence scores for each box
        figsize: (optional) the size of the image.
        '''
        # Number of instances
        N = boxes.shape[0]
        if not N:
            print("\n*** No instances to display *** \n")
        # else:
        #     assert boxes.shape[0] == masks.shape[-1] == class_ids.shape[0]
    
        if not ax:
            _, ax = plt.subplots(1, figsize=figsize)
    
        # Generate random colors
        colors = random_colors(N)
    
        # Show area outside image boundaries.
        height, width = image.shape[:2]
        ax.set_ylim(height + 10, -10)
        ax.set_xlim(-10, width + 10)
        ax.axis('off')
        ax.set_title(title)
    
        masked_image = image.astype(np.uint32).copy()
        for i in range(N):
            color = colors[i]
    
            # Bounding box
            if not np.any(boxes[i]):
                # Skip this instance. Has no bbox. Likely lost in image cropping.
                continue
            y1, x1, y2, x2 = boxes[i]
            y1, x1, y2, x2 = int(y1), int(x1), int(y2), int(x2)
            p = patches.Rectangle((x1, y1), x2 - x1, y2 - y1, linewidth=2,
                                  alpha=0.7, linestyle="dashed",
                                  edgecolor=color, facecolor='none')
            ax.add_patch(p)
    
            # Label
            class_id = class_ids[i]
            score = scores[i] if scores is not None else None
            label = class_names[class_id]
            x = random.randint(x1, (x1 + x2) // 2)
            caption = "{} {:.3f}".format(label, score) if score else label
            ax.text(x1, y1 + 8, caption,
                    color='w', size=11, backgroundcolor="none")
    
    
        plt.imshow(image.astype(np.uint8))
        
    def draw_boxes(image, boxes=None, refined_boxes=None,
                   captions=None, visibilities=None,
                   title="", ax=None):
        '''Draw bounding boxes and segmentation masks with differnt
        customizations.
        boxes: [N, (y1, x1, y2, x2, class_id)] in image coordinates.
        refined_boxes: Like boxes, but draw with solid lines to show
            that they're the result of refining 'boxes'.
        captions: List of N titles to display on each box
        visibilities: (optional) List of values of 0, 1, or 2. Determine how
            prominant each bounding box should be.
        title: An optional title to show over the image
        ax: (optional) Matplotlib axis to draw on.
        '''
        # Number of boxes
        N = boxes.shape[0] if boxes is not None else refined_boxes.shape[0]
    
        # Matplotlib Axis
        if not ax:
            _, ax = plt.subplots(1, figsize=(16, 16))
    
        # Generate random colors
        colors = random_colors(N)
    
        # Show area outside image boundaries.
        margin = image.shape[0] // 10
        ax.set_ylim(image.shape[0] + margin, -margin)
        ax.set_xlim(-margin, image.shape[1] + margin)
        ax.axis('off')
    
        ax.set_title(title)
    
        for i in range(N):
            # Box visibility
            visibility = visibilities[i] if visibilities is not None else 1
            if visibility == 0:
                color = "gray"
                style = "dotted"
                alpha = 0.5
            elif visibility == 1:
                color = colors[i]
                style = "dotted"
                alpha = 1
            elif visibility == 2:
                color = colors[i]
                style = "solid"
                alpha = 1
    
            # Boxes
            if boxes is not None:
                if not np.any(boxes[i]):
                    # Skip this instance. Has no bbox. Likely lost in cropping.
                    continue
                y1, x1, y2, x2 = boxes[i]
                p = patches.Rectangle((x1, y1), x2 - x1, y2 - y1, linewidth=2,
                                      alpha=alpha, linestyle=style,
                                      edgecolor=color, facecolor='none')
                ax.add_patch(p)
    
            # Refined boxes
            if refined_boxes is not None and visibility > 0:
                ry1, rx1, ry2, rx2 = refined_boxes[i].astype(np.int32)
                p = patches.Rectangle((rx1, ry1), rx2 - rx1, ry2 - ry1, linewidth=2,
                                      edgecolor=color, facecolor='none')
                ax.add_patch(p)
                # Connect the top-left corners of the anchor and proposal
                if boxes is not None:
                    ax.add_line(lines.Line2D([x1, rx1], [y1, ry1], color=color))
    
            # Captions
            if captions is not None:
                caption = captions[i]
                # If there are refined boxes, display captions on them
                if refined_boxes is not None:
                    y1, x1, y2, x2 = ry1, rx1, ry2, rx2
                x = random.randint(x1, (x1 + x2) // 2)
                ax.text(x1, y1, caption, size=11, verticalalignment='top',
                        color='w', backgroundcolor="none",
                        bbox={'facecolor': color, 'alpha': 0.5,
                              'pad': 2, 'edgecolor': 'none'})
    
        ax.imshow(image.astype(np.uint8))

[20191009191333910.png]: https://img-blog.csdnimg.cn/20191009191333910.png
[Link 1]: http://www.rimengshe.com/
[AI_Keras PyTorch MXNet TensorFlow PaddlePaddle]: https://blog.csdn.net/zimiao552147572/article/details/88867161
[https_github.com_dragen1860_Deep-Learning-with-TensorFlow-book]: https://github.com/dragen1860/Deep-Learning-with-TensorFlow-book
[https_github.com_dragen1860_TensorFlow-2.x-Tutorials]: https://github.com/dragen1860/TensorFlow-2.x-Tutorials
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9henVzYS5ibG9nLmNzZG4ubmV0_size_16_color_FFFFFF_t_70]: https://img-blog.csdnimg.cn/20200126210539438.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9henVzYS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9henVzYS5ibG9nLmNzZG4ubmV0_size_16_color_FFFFFF_t_70 1]: https://img-blog.csdnimg.cn/202001262105293.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9henVzYS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9henVzYS5ibG9nLmNzZG4ubmV0_size_16_color_FFFFFF_t_70 2]: https://img-blog.csdnimg.cn/20200126210443258.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9henVzYS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9henVzYS5ibG9nLmNzZG4ubmV0_size_16_color_FFFFFF_t_70 3]: https://img-blog.csdnimg.cn/20200126210507532.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9henVzYS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70