OpenCV4 调用摄像头黑屏问题

OpenCV 调用 Android 摄像头这一块，我之前研究了好几天，都是一片黑，毫无头绪。后来发现 OpenCV4 要想调用摄像头，必须继承自 OpenCV 的 CameraActivity ！！！

CameraActivity.java 的源码如下，可以看出大部分代码都是为了 Android M（6.0）以上请求权限而生的，只有两个地方非常关键

protected List<? extends CameraBridgeViewBase> getCameraViewList() { …… }
子 Activity 在继承 CameraActivity 后，需要复写该函数，把 JavaCamera2View 或 JavaCameraView 送入 List 作为返回值。

cameraBridgeViewBase.setCameraPermissionGranted()
相机视图初始情况下是黑屏的，即不工作状态。只有当权限授予完毕，调用了 setCameraPermissionGranted 之后，OpenCV 才开始调用相机并把数据输出到 SurfaceView 上。

public class CameraActivity extends Activity {

private static final int CAMERA_PERMISSION_REQUEST_CODE = 200;
protected List<? extends CameraBridgeViewBase> getCameraViewList() {
    return new ArrayList<CameraBridgeViewBase>();
}
protected void onCameraPermissionGranted() {
    List<? extends CameraBridgeViewBase> cameraViews = getCameraViewList();
    if (cameraViews == null) {
        return;
    }
    for (CameraBridgeViewBase cameraBridgeViewBase: cameraViews) {
        if (cameraBridgeViewBase != null) {
            cameraBridgeViewBase.setCameraPermissionGranted();
        }
    }
}
@Override
protected void onStart() {
    super.onStart();
    boolean havePermission = true;
    if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
        if (checkSelfPermission(CAMERA) != PackageManager.PERMISSION_GRANTED) {
            requestPermissions(new String[]{CAMERA}, CAMERA_PERMISSION_REQUEST_CODE);
            havePermission = false;
        }
    }
    if (havePermission) {
        onCameraPermissionGranted();
    }
}
@Override
@TargetApi(Build.VERSION_CODES.M)
public void onRequestPermissionsResult(int requestCode, String[] permissions, int[] grantResults) {
    if (requestCode == CAMERA_PERMISSION_REQUEST_CODE && grantResults.length > 0
            && grantResults[0] == PackageManager.PERMISSION_GRANTED) {
        onCameraPermissionGranted();
    }
    super.onRequestPermissionsResult(requestCode, permissions, grantResults);
}

}

实现要领

首先，要继承 CameraActivity，之前已经说过了。这个基类会去申请权限，然后通知 javaCameraView 已获取到权限，可以正常使用。
复写父类的 getCameraViewList 方法，将 javaCameraView 回送回去，这样当权限已被赋予时，就可以通知到预览界面开始正常工作了。
OpenCV 已经为我们实现了 Camera 和 Camera2 的函数，如果应用最低版本 minSdkVersion > 5.0，建议使用 JavaCamera2View 的相关函数，否则使用 JavaCameraView。
在 onResume 时判断 opencv 库是否加载完毕，然后启用预览视图。在 onPause 时由于界面被遮挡，此时应该暂停摄像头的预览以节省手机性能和电量损耗。
切换前后摄像头时，要先禁用，设置完后启用才会生效。
Camera2 和 Camera 的绝大部分差异 OpenCV 均已经为我们屏蔽在类的内部了，唯一的差别就是两者实现的 CvCameraViewListener 监听器里的预览函数 onCameraFrame 的参数略有不同。从下面的源码可以看出 CvCameraViewListener2 的 inputFrame 由 Mat 类型改为了 CvCameraViewFrame 类型，它额外提供了一个转化为灰度图的接口。

CvCameraViewListener

public interface CvCameraViewListener {
   /**
     * This method is invoked when camera preview has started. After this method is invoked
     * the frames will start to be delivered to client via the onCameraFrame() callback.
     * @param width -  the width of the frames that will be delivered
     * @param height - the height of the frames that will be delivered
     */
    public void onCameraViewStarted(int width, int height);
    /**
     * This method is invoked when camera preview has been stopped for some reason.
     * No frames will be delivered via onCameraFrame() callback after this method is called.
     */
    public void onCameraViewStopped();
    /**
     * This method is invoked when delivery of the frame needs to be done.
     * The returned values - is a modified frame which needs to be displayed on the screen.
     * TODO: pass the parameters specifying the format of the frame (BPP, YUV or RGB and etc)
     */
    public Mat onCameraFrame(Mat inputFrame);
}

CvCameraViewListener2

public interface CvCameraViewListener2 {
   /**
     * This method is invoked when camera preview has started. After this method is invoked
     * the frames will start to be delivered to client via the onCameraFrame() callback.
     * @param width -  the width of the frames that will be delivered
     * @param height - the height of the frames that will be delivered
     */
    public void onCameraViewStarted(int width, int height);
    /**
     * This method is invoked when camera preview has been stopped for some reason.
     * No frames will be delivered via onCameraFrame() callback after this method is called.
     */
    public void onCameraViewStopped();
    /**
     * This method is invoked when delivery of the frame needs to be done.
     * The returned values - is a modified frame which needs to be displayed on the screen.
     * TODO: pass the parameters specifying the format of the frame (BPP, YUV or RGB and etc)
     */
    public Mat onCameraFrame(CvCameraViewFrame inputFrame);
};
/**
 * This class interface is abstract representation of single frame from camera for onCameraFrame callback
 * Attention: Do not use objects, that represents this interface out of onCameraFrame callback!
 */
public interface CvCameraViewFrame {
    /**
     * This method returns RGBA Mat with frame
     */
    public Mat rgba();
    /**
     * This method returns single channel gray scale Mat with frame
     */
    public Mat gray();
};

示例程序

下面使用 Camera2 来实现拍照功能（注意：Camera2 只能用于 Android 5.0 以上的手机）

Java代码

public class OpencvCameraActivity extends CameraActivity {
    private static final String TAG = "OpencvCam";
    private JavaCamera2View javaCameraView;
    private Button switchCameraBtn;
    private int cameraId = JavaCamera2View.CAMERA_ID_ANY;
    private CameraBridgeViewBase.CvCameraViewListener2 cvCameraViewListener2 = new CameraBridgeViewBase.CvCameraViewListener2() {
        @Override
        public void onCameraViewStarted(int width, int height) {
            Log.i(TAG, "onCameraViewStarted width=" + width + ", height=" + height);
        }
        @Override
        public void onCameraViewStopped() {
            Log.i(TAG, "onCameraViewStopped");
        }
        @Override
        public Mat onCameraFrame(CameraBridgeViewBase.CvCameraViewFrame inputFrame) {
            return inputFrame.rgba();
        }
    }
    private BaseLoaderCallback baseLoaderCallback = new BaseLoaderCallback(this) {
        @Override
        public void onManagerConnected(int status) {
            Log.i(TAG, "onManagerConnected status=" + status + ", javaCameraView=" + javaCameraView);
            switch (status) {
                case LoaderCallbackInterface.SUCCESS: {
                    if (javaCameraView != null) {
                        javaCameraView.setCvCameraViewListener(cvCameraViewListener2);
                        // 禁用帧率显示
                        javaCameraView.disableFpsMeter();
                        javaCameraView.enableView();
                    }
                }
                break;
                default:
                    super.onManagerConnected(status);
                    break;
            }
        }
    };
    //复写父类的 getCameraViewList 方法，把 javaCameraView 送到父 Activity，一旦权限被授予之后，javaCameraView 的 setCameraPermissionGranted 就会自动被调用。
    @Override
    protected List<? extends CameraBridgeViewBase> getCameraViewList() {
        Log.i(TAG, "getCameraViewList");
        List<CameraBridgeViewBase> list = new ArrayList<>();
        list.add(javaCameraView);
        return list;
    }
    @Override
    protected void onCreate(Bundle savedInstanceState) {
        super.onCreate(savedInstanceState);
        setContentView(R.layout.activity_camera);
       findView();
        setListener();
    }
    private void findView() {
        javaCameraView = findViewById(R.id.javaCameraView);
        switchCameraBtn = findViewById(R.id.switchCameraBtn);
    }
    private void setListener() {
        switchCameraBtn.setOnClickListener(view -> {
            switch (cameraId) {
                case JavaCamera2View.CAMERA_ID_ANY:
                case JavaCamera2View.CAMERA_ID_BACK:
                    cameraId = JavaCamera2View.CAMERA_ID_FRONT;
                    break;
                case JavaCamera2View.CAMERA_ID_FRONT:
                    cameraId = JavaCamera2View.CAMERA_ID_BACK;
                    break;
            }
            Log.i(TAG, "cameraId : " + cameraId);
            //切换前后摄像头，要先禁用，设置完再启用才会生效
            javaCameraView.disableView();
            javaCameraView.setCameraIndex(cameraId);
            javaCameraView.enableView();
        });
    }
    @Override
    public void onPause() {
        Log.i(TAG, "onPause");
        super.onPause();
        if (javaCameraView != null) {
            javaCameraView.disableView();
        }
    }
    @Override
    public void onResume() {
        Log.i(TAG, "onResume");
        super.onResume();
        if (OpenCVLoader.initDebug()) {
            Log.i(TAG, "initDebug true");
            baseLoaderCallback.onManagerConnected(LoaderCallbackInterface.SUCCESS);
        } else {
            Log.i(TAG, "initDebug false");
            OpenCVLoader.initAsync(OpenCVLoader.OPENCV_VERSION, this, baseLoaderCallback);
        }
    }
}

布局文件

布局文件很简单，核心就是这个 JavaCamera2View 视图

<?xml version="1.0" encoding="utf-8"?>
<androidx.constraintlayout.widget.ConstraintLayout xmlns:android="http://schemas.android.com/apk/res/android"
    xmlns:app="http://schemas.android.com/apk/res-auto"
    android:layout_width="match_parent"
    android:layout_height="match_parent">
    <org.opencv.android.JavaCamera2View
        android:id="@+id/javaCameraView"
        android:layout_width="match_parent"
        android:layout_height="match_parent"
        app:show_fps="true"
        app:camera_id="any" />
    <Button
        android:id="@+id/switchCameraBtn"
        android:layout_width="match_parent"
        android:layout_height="wrap_content"
        android:text="切换摄像头"
        app:layout_constraintBottom_toBottomOf="parent" />
</androidx.constraintlayout.widget.ConstraintLayout>

全屏预览

虽然使用了上述方法，但相机的预览视图还是只占了屏幕的一小丢丢，而且还是头朝左的。

此时需要修改 OpenCV 的源码里 CameraBridgeViewBase.java 中的 deliverAndDrawFrame 方法，对图像进行旋转缩放。

/**
 * 获取屏幕旋转角度
 */
private int rotationToDegree() {
    WindowManager windowManager = (WindowManager) getContext().getSystemService(Context.WINDOW_SERVICE);
    int rotation = windowManager.getDefaultDisplay().getRotation();
    int degrees = 0;
    switch(rotation) {
        case Surface.ROTATION_0:
            if(mCameraIndex == CAMERA_ID_FRONT) {
                degrees = -90;
            } else {
                degrees = 90;
            }
            break;
        case Surface.ROTATION_90:
            break;
        case Surface.ROTATION_180:
            break;
        case Surface.ROTATION_270:
            if(mCameraIndex == CAMERA_ID_ANY || mCameraIndex == CAMERA_ID_BACK) {
                degrees = 180;
            }
            break;
    }
    return degrees;
}
/**
 * 计算得到屏幕宽高比
 */
private float calcScale(int widthSource, int heightSource, int widthTarget, int heightTarget) {
    if(widthTarget <= heightTarget) {
        return (float) heightTarget / (float) heightSource;
    } else {
        return (float) widthTarget / (float) widthSource;
    }
}
/**
  * This method shall be called by the subclasses when they have valid
  * object and want it to be delivered to external client (via callback) and
  * then displayed on the screen.
  * @param frame - the current frame to be delivered
  */
 protected void deliverAndDrawFrame(CvCameraViewFrame frame) {
     Mat modified;
     if (mListener != null) {
         modified = mListener.onCameraFrame(frame);
     } else {
         modified = frame.rgba();
     }
     boolean bmpValid = true;
     if (modified != null) {
         try {
             Utils.matToBitmap(modified, mCacheBitmap);
         } catch(Exception e) {
             Log.e(TAG, "Mat type: " + modified);
             Log.e(TAG, "Bitmap type: " + mCacheBitmap.getWidth() + "*" + mCacheBitmap.getHeight());
             Log.e(TAG, "Utils.matToBitmap() throws an exception: " + e.getMessage());
             bmpValid = false;
         }
     }
     if (bmpValid && mCacheBitmap != null) {
         Canvas canvas = getHolder().lockCanvas();
         if (canvas != null) {
             canvas.drawColor(0, android.graphics.PorterDuff.Mode.CLEAR);
             if (BuildConfig.DEBUG) Log.d(TAG, "mStretch value: " + mScale);
             //TODO 额外添加，让预览框达到全屏效果
             int degrees = rotationToDegree();
             Matrix matrix = new Matrix();
             matrix.postRotate(degrees);
             Bitmap outputBitmap = Bitmap.createBitmap(mCacheBitmap, 0, 0, mCacheBitmap.getWidth(), mCacheBitmap.getHeight(), matrix, true);
             if (outputBitmap.getWidth() <= canvas.getWidth()) {
                 mScale = calcScale(outputBitmap.getWidth(), outputBitmap.getHeight(), canvas.getWidth(), canvas.getHeight());
             } else {
                 mScale = calcScale(canvas.getWidth(), canvas.getHeight(), outputBitmap.getWidth(), outputBitmap.getHeight());
             }
             if (mScale != 0) {
                 canvas.scale(mScale, mScale, 0, 0);
             }
             Log.d(TAG, "mStretch value: " + mScale);
             canvas.drawBitmap(outputBitmap, 0, 0, null);
             /*
             if (mScale != 0) {
                 canvas.drawBitmap(mCacheBitmap, new Rect(0,0,mCacheBitmap.getWidth(), mCacheBitmap.getHeight()),
                      new Rect((int)((canvas.getWidth() - mScale*mCacheBitmap.getWidth()) / 2),
                      (int)((canvas.getHeight() - mScale*mCacheBitmap.getHeight()) / 2),
                      (int)((canvas.getWidth() - mScale*mCacheBitmap.getWidth()) / 2 + mScale*mCacheBitmap.getWidth()),
                      (int)((canvas.getHeight() - mScale*mCacheBitmap.getHeight()) / 2 + mScale*mCacheBitmap.getHeight())), null);
             } else {
                  canvas.drawBitmap(mCacheBitmap, new Rect(0,0,mCacheBitmap.getWidth(), mCacheBitmap.getHeight()),
                      new Rect((canvas.getWidth() - mCacheBitmap.getWidth()) / 2,
                      (canvas.getHeight() - mCacheBitmap.getHeight()) / 2,
                      (canvas.getWidth() - mCacheBitmap.getWidth()) / 2 + mCacheBitmap.getWidth(),
                      (canvas.getHeight() - mCacheBitmap.getHeight()) / 2 + mCacheBitmap.getHeight()), null);
             }
             */
             if (mFpsMeter != null) {
                 mFpsMeter.measure();
                 mFpsMeter.draw(canvas, 20, 30);
             }
             getHolder().unlockCanvasAndPost(canvas);
         }
     }
 }