【计算机视觉】DINOv2（视觉大模型）代码四个不同模型的对比，以 28 * 28 的图像为例（完整的源代码）

╰半夏微凉° 2023-10-13 20:31 1阅读 0赞

#### 文章目录 ####

*  一、ViT-S/14
 *  二、ViT-B/14
 *  三、ViT-L/14
 *  四、ViT-g/14

## 一、ViT-S/14 ##

import torch
    import torchvision.transforms as T
    import matplotlib.pyplot as plt
    import numpy as np
    import matplotlib.image as mpimg 
    from PIL import Image
    from sklearn.decomposition import PCA
    import matplotlib
     
    patch_h = 28
    patch_w = 28
    feat_dim = 384 # vits14
     
    transform = T.Compose([
        T.GaussianBlur(9, sigma=(0.1, 2.0)),
        T.Resize((patch_h * 14, patch_w * 14)),
        T.CenterCrop((patch_h * 14, patch_w * 14)),
        T.ToTensor(),
        T.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),
    ])
     
    dinov2_vits14 = torch.hub.load('', 'dinov2_vits14',source='local').cuda()
     
    features = torch.zeros(4, patch_h * patch_w, feat_dim)
    imgs_tensor = torch.zeros(4, 3, patch_h * 14, patch_w * 14).cuda()
     
    img_path = f'/kaggle/input/demo-image/1 (4).png'
    img = Image.open(img_path).convert('RGB')
    imgs_tensor[0] = transform(img)[:3]
    with torch.no_grad():
        features_dict = dinov2_vits14.forward_features(imgs_tensor)
        features = features_dict['x_norm_patchtokens']
        
    features = features.reshape(4 * patch_h * patch_w, feat_dim).cpu()
    pca = PCA(n_components = 3)
    pca.fit(features)
    pca_features = pca.transform(features)
    pca_features[:, 0] = (pca_features[:, 0] - pca_features[:, 0].min()) / (pca_features[:, 0].max() - pca_features[:, 0].min())
     
    pca_features_fg = pca_features[:, 0] > 0.3
    pca_features_bg = ~pca_features_fg
     
    b = np.where(pca_features_bg)
    
    ## 前景
    pca.fit(features[pca_features_fg])
    pca_features_rem = pca.transform(features[pca_features_fg])
    for i in range(3):
        pca_features_rem[:, i] = (pca_features_rem[:, i] - pca_features_rem[:, i].min()) / (pca_features_rem[:, i].max() - pca_features_rem[:, i].min())
        # 使用平均值和标准差进行变换，个人发现这种变换可以提供更好的可视化效果
        # pca_features_rem[:, i] = (pca_features_rem[:, i] - pca_features_rem[:, i].mean()) / (pca_features_rem[:, i].std() ** 2) + 0.5
    
    pca_features_rgb = pca_features.copy()
    pca_features_rgb[pca_features_fg] = pca_features_rem
    pca_features_rgb[b] = 0
    pca_features_rgb = pca_features_rgb.reshape(4, patch_h, patch_w, 3)
    plt.imshow(pca_features_rgb[0][...,::-1])
    plt.savefig('features_s14.png')
    plt.show()
    plt.close()

![在这里插入图片描述][33a923b85c534400940d6b1e3711b1b1.png]

print('---s14---')
    print(features)
    print('---维度---')
    print(features.shape)

![在这里插入图片描述][bba298af0b7f4ae99d05f8e987735234.png]

print('---pca_features---')
    print(pca_features)
    print('---维度---')
    print(pca_features.shape)

![在这里插入图片描述][061b544e23fe4c15ba5f9ad16c28d70b.png]

## 二、ViT-B/14 ##

patch_h = 28
    patch_w = 28
    feat_dim = 768
     
    transform = T.Compose([
        T.GaussianBlur(9, sigma=(0.1, 2.0)),
        T.Resize((patch_h * 14, patch_w * 14)),
        T.CenterCrop((patch_h * 14, patch_w * 14)),
        T.ToTensor(),
        T.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),
    ])
     
    dinov2_vitb14 = torch.hub.load('', 'dinov2_vitb14',source='local').cuda()
     
    features_b14 = torch.zeros(4, patch_h * patch_w, feat_dim)
    imgs_tensor_b14 = torch.zeros(4, 3, patch_h * 14, patch_w * 14).cuda()
     
    img_path = f'/kaggle/input/demo-image/1 (4).png'
    img = Image.open(img_path).convert('RGB')
    imgs_tensor_b14[0] = transform(img)[:3]
    with torch.no_grad():
        features_dict_b14 = dinov2_vitb14.forward_features(imgs_tensor_b14)
        features_b14 = features_dict_b14['x_norm_patchtokens']
        
    features_b14 = features_b14.reshape(4 * patch_h * patch_w, feat_dim).cpu()
    pca = PCA(n_components = 3)
    pca.fit(features_b14)
    pca_features_b14 = pca.transform(features_b14)
    pca_features_b14[:, 0] = (pca_features_b14[:, 0] - pca_features_b14[:, 0].min()) / (pca_features_b14[:, 0].max() - pca_features_b14[:, 0].min())
     
    pca_features_fg_b14 = pca_features_b14[:, 0] > 0.3
    pca_features_bg_b14 = ~pca_features_fg_b14
     
    b = np.where(pca_features_bg_b14)
    pca.fit(features_b14[pca_features_fg_b14])
    pca_features_rem_b14 = pca.transform(features_b14[pca_features_fg_b14])
    for i in range(3):
        pca_features_rem_b14[:, i] = (pca_features_rem_b14[:, i] - pca_features_rem_b14[:, i].min()) \
        / (pca_features_rem_b14[:, i].max() - pca_features_rem_b14[:, i].min())
    
    pca_features_rgb_b14 = pca_features_b14.copy()
    pca_features_rgb_b14[pca_features_fg_b14] = pca_features_rem_b14
    pca_features_rgb_b14[b] = 0
    pca_features_rgb_b14 = pca_features_rgb_b14.reshape(4, patch_h, patch_w, 3)
    plt.imshow(pca_features_rgb_b14[0][...,::-1])
    plt.savefig('features_b14.png')
    plt.show()
    plt.close()

![在这里插入图片描述][a6712f8be305409382c64928113da3a6.png]

print('---b14---')
    print(features_b14)
    print('---维度---')
    print(features_b14.shape)

![在这里插入图片描述][50e0f018419a48a0836bf68d84e49457.png]

print('---pca_features_b14---')
    print(pca_features_b14)
    print('---维度---')
    print(pca_features_b14.shape)

![在这里插入图片描述][60894e2a3126457193f9d162deffbf0c.png]

## 三、ViT-L/14 ##

patch_h = 28
    patch_w = 28
    feat_dim = 1024
     
    transform = T.Compose([
        T.GaussianBlur(9, sigma=(0.1, 2.0)),
        T.Resize((patch_h * 14, patch_w * 14)),
        T.CenterCrop((patch_h * 14, patch_w * 14)),
        T.ToTensor(),
        T.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),
    ])
     
    dinov2_vitl14 = torch.hub.load('', 'dinov2_vitl14',source='local').cuda()
     
    features_l14 = torch.zeros(4, patch_h * patch_w, feat_dim)
    imgs_tensor_l14 = torch.zeros(4, 3, patch_h * 14, patch_w * 14).cuda()
     
    img_path = f'/kaggle/input/demo-image/1 (4).png'
    img = Image.open(img_path).convert('RGB')
    imgs_tensor_l14[0] = transform(img)[:3]
    with torch.no_grad():
        features_dict_l14 = dinov2_vitl14.forward_features(imgs_tensor_l14)
        features_l14 = features_dict_l14['x_norm_patchtokens']
        
    features_l14 = features_l14.reshape(4 * patch_h * patch_w, feat_dim).cpu()
    pca = PCA(n_components = 3)
    pca.fit(features_l14)
    pca_features_l14 = pca.transform(features_l14)
    pca_features_l14[:, 0] = (pca_features_l14[:, 0] - pca_features_l14[:, 0].min()) \
    / (pca_features_l14[:, 0].max() - pca_features_l14[:, 0].min())
     
    pca_features_fg_l14 = pca_features_l14[:, 0] > 0.3
    pca_features_bg_l14 = ~pca_features_fg_l14
     
    b = np.where(pca_features_bg_l14)
    pca.fit(features_l14[pca_features_fg_l14])
    pca_features_rem_l14 = pca.transform(features_l14[pca_features_fg_l14])
    for i in range(3):
        pca_features_rem_l14[:, i] = (pca_features_rem_l14[:, i] - pca_features_rem_l14[:, i].min()) \
        / (pca_features_rem_l14[:, i].max() - pca_features_rem_l14[:, i].min())
    
    pca_features_rgb_l14 = pca_features_l14.copy()
    pca_features_rgb_l14[pca_features_fg_l14] = pca_features_rem_l14
    pca_features_rgb_l14[b] = 0
    pca_features_rgb_l14 = pca_features_rgb_l14.reshape(4, patch_h, patch_w, 3)
    plt.imshow(pca_features_rgb_l14[0][...,::-1])
    plt.savefig('features_l14.png')
    plt.show()
    plt.close()

![在这里插入图片描述][9c4919d25bd74bdb87174bef325607bd.png]

print('---l14---')
    print(features_l14)
    print('---维度---')
    print(features_l14.shape)

![在这里插入图片描述][7ff915c6008e4a67a69ac887ae352602.png]

print('---pca_features_l14---')
    print(pca_features_l14)
    print('---维度---')
    print(pca_features_l14.shape)

![在这里插入图片描述][c249c656803746b89538a09d3adb5809.png]

## 四、ViT-g/14 ##

patch_h = 28
    patch_w = 28
    feat_dim = 1536
     
    transform = T.Compose([
        T.GaussianBlur(9, sigma=(0.1, 2.0)),
        T.Resize((patch_h * 14, patch_w * 14)),
        T.CenterCrop((patch_h * 14, patch_w * 14)),
        T.ToTensor(),
        T.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),
    ])
     
    dinov2_vitg14 = torch.hub.load('', 'dinov2_vitg14',source='local').cuda()
     
    features_g14 = torch.zeros(4, patch_h * patch_w, feat_dim)
    imgs_tensor_g14 = torch.zeros(4, 3, patch_h * 14, patch_w * 14).cuda()
     
    img_path = f'/kaggle/input/demo-image/1 (4).png'
    img = Image.open(img_path).convert('RGB')
    imgs_tensor_g14[0] = transform(img)[:3]
    with torch.no_grad():
        features_dict_g14 = dinov2_vitg14.forward_features(imgs_tensor_g14)
        features_g14 = features_dict_g14['x_norm_patchtokens']
        
    features_g14 = features_g14.reshape(4 * patch_h * patch_w, feat_dim).cpu()
    pca = PCA(n_components = 3)
    pca.fit(features_g14)
    pca_features_g14 = pca.transform(features_g14)
    pca_features_g14[:, 0] = (pca_features_g14[:, 0] - pca_features_g14[:, 0].min()) \
    / (pca_features_g14[:, 0].max() - pca_features_g14[:, 0].min())
     
    pca_features_fg_g14 = pca_features_g14[:, 0] > 0.3
    pca_features_bg_g14 = ~pca_features_fg_g14
     
    b = np.where(pca_features_bg_g14)
    
    pca.fit(features_g14[pca_features_fg_g14])
    pca_features_rem_g14 = pca.transform(features_g14[pca_features_fg_g14])
    for i in range(3):
        pca_features_rem_g14[:, i] = (pca_features_rem_g14[:, i] - pca_features_rem_g14[:, i].min()) \
        / (pca_features_rem_g14[:, i].max() - pca_features_rem_g14[:, i].min())
    
    pca_features_rgb_g14 = pca_features_g14.copy()
    pca_features_rgb_g14[pca_features_fg_g14] = pca_features_rem_g14
    pca_features_rgb_g14[b] = 0
    pca_features_rgb_g14 = pca_features_rgb_g14.reshape(4, patch_h, patch_w, 3)
    plt.imshow(pca_features_rgb_g14[0][...,::-1])
    plt.savefig('features_g14.png')
    plt.show()
    plt.close()

![在这里插入图片描述][29357395052c495d9d8f3699fb3b17d2.png]

print('---g14---')
    print(features_g14)
    print('---维度---')
    print(features_g14.shape)

![在这里插入图片描述][b1b0dba4cbe74b308d3c364dab26e69a.png]

print('---pca_features_g14---')
    print(pca_features_g14)
    print('---维度---')
    print(pca_features_g14.shape)

![在这里插入图片描述][452413a25d4b46d891ed1cf7a0f100e5.png]

[33a923b85c534400940d6b1e3711b1b1.png]: https://img-blog.csdnimg.cn/33a923b85c534400940d6b1e3711b1b1.png
[bba298af0b7f4ae99d05f8e987735234.png]: https://img-blog.csdnimg.cn/bba298af0b7f4ae99d05f8e987735234.png
[061b544e23fe4c15ba5f9ad16c28d70b.png]: https://img-blog.csdnimg.cn/061b544e23fe4c15ba5f9ad16c28d70b.png
[a6712f8be305409382c64928113da3a6.png]: https://img-blog.csdnimg.cn/a6712f8be305409382c64928113da3a6.png
[50e0f018419a48a0836bf68d84e49457.png]: https://img-blog.csdnimg.cn/50e0f018419a48a0836bf68d84e49457.png
[60894e2a3126457193f9d162deffbf0c.png]: https://img-blog.csdnimg.cn/60894e2a3126457193f9d162deffbf0c.png
[9c4919d25bd74bdb87174bef325607bd.png]: https://img-blog.csdnimg.cn/9c4919d25bd74bdb87174bef325607bd.png
[7ff915c6008e4a67a69ac887ae352602.png]: https://img-blog.csdnimg.cn/7ff915c6008e4a67a69ac887ae352602.png
[c249c656803746b89538a09d3adb5809.png]: https://img-blog.csdnimg.cn/c249c656803746b89538a09d3adb5809.png
[29357395052c495d9d8f3699fb3b17d2.png]: https://img-blog.csdnimg.cn/29357395052c495d9d8f3699fb3b17d2.png
[b1b0dba4cbe74b308d3c364dab26e69a.png]: https://img-blog.csdnimg.cn/b1b0dba4cbe74b308d3c364dab26e69a.png
[452413a25d4b46d891ed1cf7a0f100e5.png]: https://img-blog.csdnimg.cn/452413a25d4b46d891ed1cf7a0f100e5.png