在图像中放置虚拟物体(python实现)

系统管理员 2022-02-24 03:47 286阅读 0赞

在图像中放置虚拟物体即是将图像作为背景，创建个虚拟三维物体投影到图像上。

本实验环境：OpenGL、pygame、python（安装见[python配置pygame和OpenGL][python_pygame_OpenGL]）

# -*- coding: utf-8 -*-
    import math
    import pickle
    import sys
    from pylab import *
    from OpenGL.GL import *
    from OpenGL.GLU import *
    from OpenGL.GLUT import *
    import pygame, pygame.image
    from pygame.locals import *
    from PCV.geometry import homography, camera
    from PCV.localdescriptors import sift
    
    def cube_points(c, wid):   # 绘制立方体的一各点列表
        """ Creates a list of points for plotting
            a cube with plot. (the first 5 points are
            the bottom square, some sides repeated). """
        p = []
        # 底部
        p.append([c[0] - wid, c[1] - wid, c[2] - wid])
        p.append([c[0] - wid, c[1] + wid, c[2] - wid])
        p.append([c[0] + wid, c[1] + wid, c[2] - wid])
        p.append([c[0] + wid, c[1] - wid, c[2] - wid])
        p.append([c[0] - wid, c[1] - wid, c[2] - wid])  # 和第一个相同
    
        # 顶部
        p.append([c[0] - wid, c[1] - wid, c[2] + wid])
        p.append([c[0] - wid, c[1] + wid, c[2] + wid])
        p.append([c[0] + wid, c[1] + wid, c[2] + wid])
        p.append([c[0] + wid, c[1] - wid, c[2] + wid])
        p.append([c[0] - wid, c[1] - wid, c[2] + wid])  # 和第一个相同
    
        # 竖直边
        p.append([c[0] - wid, c[1] - wid, c[2] + wid])
        p.append([c[0] - wid, c[1] + wid, c[2] + wid])
        p.append([c[0] - wid, c[1] + wid, c[2] - wid])
        p.append([c[0] + wid, c[1] + wid, c[2] - wid])
        p.append([c[0] + wid, c[1] + wid, c[2] + wid])
        p.append([c[0] + wid, c[1] - wid, c[2] + wid])
        p.append([c[0] + wid, c[1] - wid, c[2] - wid])
    
        return array(p).T
    
    def my_calibration(sz):
        row, col = sz
        fx = 2555 * col / 2592
        fy = 2586 * row / 1936
        K = diag([fx, fy, 1])
        K[0, 2] = 0.5 * col
        K[1, 2] = 0.5 * row
        return K
    
    def set_projection_from_camera(K):  # 获取视图
        glMatrixMode(GL_PROJECTION)
        glLoadIdentity()
        fx = K[0, 0]
        fy = K[1, 1]
        fovy = 2 * math.atan(0.5 * height / fy) * 180 / math.pi
        aspect = (width * fy) / (height * fx)
        # 定义近和远的剪裁平面
        near = 0.1
        far = 100.0
        # 设定透视
        gluPerspective(fovy, aspect, near, far)
        glViewport(0, 0, width, height)
    
    def set_modelview_from_camera(Rt):  # 获取矩阵
        glMatrixMode(GL_MODELVIEW)
        glLoadIdentity()
        # 围绕x轴将茶壶旋转90度，使z轴向上
        Rx = np.array([[1, 0, 0], [0, 0, -1], [0, 1, 0]])
        # 获得旋转的最佳逼近
        R = Rt[:, :3]
        U, S, V = np.linalg.svd(R)
        R = np.dot(U, V)
        R[0, :] = -R[0, :]  # 改变x轴的符号
        # 获得平移量
        t = Rt[:, 3]
        # 获得4*4的的模拟视图矩阵
        M = np.eye(4)
        M[:3, :3] = np.dot(R, Rx)
        M[:3, 3] = t
        # 转置并压平以获取列序数值
        M = M.T
        m = M.flatten()
        # 将模拟视图矩阵替换成新的矩阵
        glLoadMatrixf(m)
    
    def draw_background(imname):
        # 载入背景图像
        bg_image = pygame.image.load(imname).convert()
        bg_data = pygame.image.tostring(bg_image, "RGBX", 1)  # 将图像转为字符串描述
        glMatrixMode(GL_MODELVIEW)  # 将当前矩阵指定为投影矩阵
        glLoadIdentity()  # 把矩阵设为单位矩阵
    
        glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)  # 清楚颜色、深度缓冲
        glEnable(GL_TEXTURE_2D)  # 纹理映射
        glBindTexture(GL_TEXTURE_2D, glGenTextures(1))
        glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, width, height, 0, GL_RGBA, GL_UNSIGNED_BYTE, bg_data)
        glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
        glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
        # 绑定纹理
        glBegin(GL_QUADS)
        glTexCoord2f(0.0, 0.0); glVertex3f(-1.0, -1.0, -1.0)
        glTexCoord2f(1.0, 0.0); glVertex3f(1.0, -1.0, -1.0)
        glTexCoord2f(1.0, 1.0); glVertex3f(1.0, 1.0, -1.0)
        glTexCoord2f(0.0, 1.0); glVertex3f(-1.0, 1.0, -1.0)
        glEnd()
        glDeleteTextures(1)  # 清除纹理
    
    def draw_teapot(size):  # 红色茶壶
        glEnable(GL_LIGHTING)
        glEnable(GL_LIGHT0)
        glEnable(GL_DEPTH_TEST)
        glClear(GL_DEPTH_BUFFER_BIT)
        # 绘制红色茶壶
        glMaterialfv(GL_FRONT, GL_AMBIENT, [0, 0, 0, 0])
        glMaterialfv(GL_FRONT, GL_DIFFUSE, [0.5, 0.0, 0.0, 0.0])
        glMaterialfv(GL_FRONT, GL_SPECULAR, [0.7, 0.6, 0.6, 0.0])
        glMaterialf(GL_FRONT, GL_SHININESS, 0.25 * 128.0)
        glutSolidTeapot(size)
    
    def drawFunc(size):  # 白色茶壶
        glRotatef(0.5, 5, 5, 0)  # (角度,x,y,z)
        glutWireTeapot(size)
        # 刷新显示
        glFlush()
    width, height = 1000, 747
    
    
    def setup():  # 设置窗口和pygame环境
        pygame.init()
        pygame.display.set_mode((width, height), OPENGL | DOUBLEBUF)
        pygame.display.set_caption("OpenGL AR demo")
    
    # 计算特征
    sift.process_image('book_frontal.JPG', 'im0.sift')
    l0, d0 = sift.read_features_from_file('im0.sift')
    
    sift.process_image('book_perspective.JPG', 'im1.sift')
    l1, d1 = sift.read_features_from_file('im1.sift')
    
    # 匹配特征，计算单应性矩阵
    matches = sift.match_twosided(d0, d1)
    ndx = matches.nonzero()[0]
    fp = homography.make_homog(l0[ndx, :2].T)
    ndx2 = [int(matches[i]) for i in ndx]
    tp = homography.make_homog(l1[ndx2, :2].T)
    
    model = homography.RansacModel()
    H, inliers = homography.H_from_ransac(fp, tp, model)
    
    # 计算照相机标定矩阵
    K = my_calibration((747, 1000))
    # 位于边长为0.2，z=0平面上的三维点
    box = cube_points([0, 0, 0.1], 0.1)
    
    # 投影第一幅图下个上底部的正方形
    cam1 = camera.Camera(hstack((K, dot(K, array([[0], [0], [-1]])))))
    # 底部正方形上的点
    box_cam1 = cam1.project(homography.make_homog(box[:, :5]))
    
    # 使用H将点变换到第二幅图像中
    box_trans = homography.normalize(dot(H, box_cam1))
    
    # 从cam1和H中计算第二个照相机矩阵
    cam2 = camera.Camera(dot(H, cam1.P))
    A = dot(linalg.inv(K), cam2.P[:, :3])
    A = array([A[:, 0], A[:, 1], cross(A[:, 0], A[:, 1])]).T
    cam2.P[:, :3] = dot(K, A)
    # 使用第二个照相机矩阵投影
    box_cam2 = cam2.project(homography.make_homog(box))
    
    Rt = dot(linalg.inv(K), cam2.P)
    # ******************************************************************
    # 以下是显示正方体在图像上（实现以平面和标记物进行姿态估计）
    # # plotting
    # im0 = array(Image.open('book_frontal.JPG'))
    # im1 = array(Image.open('book_perspective.JPG'))
    # 
    # figure()
    # imshow(im0)
    # plot(box_cam1[0, :], box_cam1[1, :], linewidth=3)
    # title('2D projection of bottom square')
    # axis('off')
    # 
    # figure()
    # imshow(im1)
    # plot(box_trans[0, :], box_trans[1, :], linewidth=3)
    # title('2D projection transfered with H')
    # axis('off')
    # 
    # figure()
    # imshow(im1)
    # plot(box_cam2[0, :], box_cam2[1, :], linewidth=3)
    # title('3D points projected in second image')
    # axis('off')
    # 
    # show()
    # ********************************************************************
    setup()
    draw_background("book_perspective.bmp")
    set_projection_from_camera(K)
    set_modelview_from_camera(Rt)
    
    draw_teapot(0.05)  # 显示红色茶壶
    # drawFunc(0.05)  # 显示白色空心茶壶
    pygame.display.flip()
    while True:
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                sys.exit()

运行显示正方体的代码所得到的结果：

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM3Mzc0NjQz_size_16_color_FFFFFF_t_70][]![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM3Mzc0NjQz_size_16_color_FFFFFF_t_70 1][]![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM3Mzc0NjQz_size_16_color_FFFFFF_t_70 2][]

由图一和图二可以看出，矩形框所框出的位置是相同的，这是因为使用了单应性矩阵，利用图一中书作为标记物，以此计算图二矩形框投影矩阵的位置，图三显示的是三维立方体。

运行显示红色茶壶：

>     draw_teapot(0.05)  # 显示红色茶壶

则可得到结果（白色茶壶也同理，所以也附上白色茶壶图片）：

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM3Mzc0NjQz_size_16_color_FFFFFF_t_70 3][]                ![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM3Mzc0NjQz_size_16_color_FFFFFF_t_70 4][]

茶壶都是利用OpenGL画上去的虚拟物体，这里是将茶壶按照和坐标轴对齐的方式显示出来。除了茶壶之外还有很多物体可以放置在图像上，可以选择载入模型或者自己绘制，OpenGL也提供了一些方法可以直接使用，可以参考[GLUT模型绘制函数][GLUT]此篇博客，实现方式和显示茶壶大同小异，这里就不展示了。

在实现过程中，可能会遇到如下问题：

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM3Mzc0NjQz_size_16_color_FFFFFF_t_70 5][]

解决办法：据大佬指导，这个错误是因为freeglut和glut共存的缘故，它们俩定义了相同的方法，导致动态链接库的重叠，所以只要删除其中一个就好了。（我的文件是放在下图所示路径，用Anaconda安装python的在Anaconda2\\Lib\\site-packages\\OpenGL\\DLLS路径下）

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM3Mzc0NjQz_size_16_color_FFFFFF_t_70 6][]

[python_pygame_OpenGL]: https://blog.csdn.net/qq_37374643/article/details/88983248
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM3Mzc0NjQz_size_16_color_FFFFFF_t_70]: https://img-blog.csdnimg.cn/2019040316205185.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM3Mzc0NjQz,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM3Mzc0NjQz_size_16_color_FFFFFF_t_70 1]: https://img-blog.csdnimg.cn/20190403162113223.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM3Mzc0NjQz,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM3Mzc0NjQz_size_16_color_FFFFFF_t_70 2]: /images/20220224/4367b3ee5e5e41caa5cf367476bcd35a.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM3Mzc0NjQz_size_16_color_FFFFFF_t_70 3]: https://img-blog.csdnimg.cn/20190403163608970.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM3Mzc0NjQz,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM3Mzc0NjQz_size_16_color_FFFFFF_t_70 4]: /images/20220224/ed18eb9450694df2975ec9716cc2ccb5.png
[GLUT]: https://blog.csdn.net/augusdi/article/details/7076734
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM3Mzc0NjQz_size_16_color_FFFFFF_t_70 5]: /images/20220224/f872094ecf894f3a99f7c8e86684c231.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM3Mzc0NjQz_size_16_color_FFFFFF_t_70 6]: /images/20220224/5da6fabe20e4435c8dfa86c4047e8723.png