C++ OpenCV Video Processing

Video processing refers to processing and analyzing each frame of a video sequence.

OpenCV provides powerful tools for processing video data, including video reading, frame processing, video saving, and real-time video processing.

Application Scenarios of Video Processing

Video Surveillance:

  • Use background subtraction to detect moving objects.

  • Use optical flow to analyze object motion trajectories.

Video Analysis:

  • Extract key frames from videos.

  • Analyze object behavior in videos.

Real-time Processing:

  • Real-time video filters (e.g., edge detection, blur, etc.).

  • Real-time object detection and tracking.

Reading and Displaying Videos

Reading Video Files (VideoCapture)

In OpenCV,VideoCapturethe class is used to read video frames from video files or cameras.

To read a video file, first create aVideoCaptureobject, and specify the path to the video file.

Example

#include <opencv2/opencv.hpp>
using namespace cv;

int main() {
    // Create a VideoCapture object and open the video file
    VideoCapture cap("example.mp4");

    // Check whether the video was opened successfully
    if (!cap.isOpened()) {
        std::cerr << "Error: Could not open video file." << std::endl;
        return -1;
    }

    // The video reading and display code will be introduced below
    return 0;
}

Displaying Video Frames

After reading the video file, you can read the video frame by frame in a loop, and use theimshowfunction to display each frame.

Example

Mat frame;
while (true) {
    // Read the next frame
    cap >> frame;

    // If the frame is empty, the video has ended
    if (frame.empty()) {
        break;
    }

    // Display the current frame
    imshow("Video", frame);

    // Wait 30 milliseconds, press ESC to exit
    if (waitKey(30) == 27) {
        break;
    }
}

// Release the VideoCapture object
cap.release();
// Close all windows
destroyAllWindows();

Saving Videos (VideoWriter)

If you want to save the processed video to a file, you can use theVideoWriterclass. First, you need to specify the output file name, encoding format, frame rate, and frame size.

Example

// Get the frame rate and frame size of the video
double fps = cap.get(CAP_PROP_FPS);
Size frameSize(cap.get(CAP_PROP_FRAME_WIDTH), cap.get(CAP_PROP_FRAME_HEIGHT));

// Create a VideoWriter object
VideoWriter writer("output.avi", VideoWriter::fourcc('M', 'J', 'P', 'G'), fps, frameSize);

while (true) {
    cap >> frame;
    if (frame.empty()) {
        break;
    }

    // Write frames to the output video file
    writer.write(frame);

    imshow("Video", frame);
    if (waitKey(30) == 27) {
        break;
    }
}

// Release the VideoCapture and VideoWriter objects
cap.release();
writer.release();
destroyAllWindows();

Video Frame Processing

Processing Video Frame by Frame

In video processing, it is usually necessary to perform specific processing operations on each frame. For example, you can convert each frame to grayscale, perform edge detection, and so on.

Example

while (true) {
    cap >> frame;
    if (frame.empty()) {
        break;
    }

    // Convert the frame to a grayscale image
    Mat grayFrame;
    cvtColor(frame, grayFrame, COLOR_BGR2GRAY);

    // Display the grayscale frame
    imshow("Gray Video", grayFrame);

    if (waitKey(30) == 27) {
        break;
    }
}

Real-time Processing of Video Frames

When processing video frames in real time, it is usually necessary to apply some real-time processing algorithms to each frame. For example, detect moving objects in the video in real time.

Example

Mat prevFrame, nextFrame, diffFrame;
cap >> prevFrame;
cvtColor(prevFrame, prevFrame, COLOR_BGR2GRAY);

while (true) {
    cap >> nextFrame;
    if (nextFrame.empty()) {
        break;
    }

    cvtColor(nextFrame, nextFrame, COLOR_BGR2GRAY);

    // Calculate the difference between frames
    absdiff(prevFrame, nextFrame, diffFrame);

    // Display the difference frame
    imshow("Motion Detection", diffFrame);

    // Update the previous frame
    prevFrame = nextFrame.clone();

    if (waitKey(30) == 27) {
        break;
    }
}

Real-time Camera Processing

Opening the Camera

Using theVideoCaptureclass, you can not only read video files, but also open the camera for real-time video stream processing. To open the camera, simply set theVideoCaptureparameter to 0 (which indicates the default camera).

Example

VideoCapture cap(0);

if (!cap.isOpened()) {
    std::cerr << "Error: Could not open camera." << std::endl;
    return -1;
}

Processing Real-time Video Streams

After opening the camera, you can process the real-time video stream frame by frame just like processing a video file. For example, you can apply an edge detection algorithm to the real-time video stream.

Example

while (true) {
    cap >> frame;
    if (frame.empty()) {
        break;
    }

    // Apply Canny edge detection
    Mat edges;
    Canny(frame, edges, 100, 200);

    // Display the edge detection result
    imshow("Edges", edges);

    if (waitKey(30) == 27) {
        break;
    }
}

cap.release();
destroyAllWindows();

Advanced Video Processing Techniques

Video Background Subtraction

Background subtraction is used to extract foreground objects from a video.

Example

#include <opencv2/opencv.hpp>
#include <iostream>

using namespace cv;
using namespace std;

int main() {
    // Open a video file or camera
    VideoCapture cap("video.mp4");
    if (!cap.isOpened()) {
        cout << "Error: Unable to open video file or camera!" << endl;
        return -1;
    }

    // Create a background subtractor
    Ptr<BackgroundSubtractor> bgSubtractor = createBackgroundSubtractorMOG2();

    // Process video frames
    Mat frame, fgMask;
    while (true) {
        cap >> frame;
        if (frame.empty()) break;

        // Apply background subtraction
        bgSubtractor->apply(frame, fgMask);

        // Display the result
        imshow("Frame", frame);
        imshow("Foreground Mask", fgMask);

        // Press ESC to exit
        if (waitKey(30) == 27) break;
    }

    // Release resources
    cap.release();
    destroyAllWindows();

    return 0;
}

Optical Flow Computation

Optical flow is used to calculate the motion of objects in video frames.

Sparse optical flow (Lucas-Kanade method):

Example

#include <opencv2/opencv.hpp>
#include <iostream>

using namespace cv;
using namespace std;

int main() {
    // Open a video file or camera
    VideoCapture cap("video.mp4");
    if (!cap.isOpened()) {
        cout << "Error: Unable to open video file or camera!" << endl;
        return -1;
    }

    // Read the first frame
    Mat oldFrame, oldGray;
    cap >> oldFrame;
    cvtColor(oldFrame, oldGray, COLOR_BGR2GRAY);

    // Select feature points
    vector<Point2f> oldPoints;
    goodFeaturesToTrack(oldGray, oldPoints, 100, 0.3, 7);

    // Process video frames
    Mat frame, gray;
    while (true) {
        cap >> frame;
        if (frame.empty()) break;

        // Convert to grayscale image
        cvtColor(frame, gray, COLOR_BGR2GRAY);

        // Compute optical flow
        vector<Point2f> newPoints;
        vector<uchar> status;
        vector<float> err;
        calcOpticalFlowPyrLK(oldGray, gray, oldPoints, newPoints, status, err);

        // Draw optical flow trajectories
        for (size_t i = 0; i < oldPoints.size(); i++) {
            if (status[i]) {
                line(frame, oldPoints[i], newPoints[i], Scalar(0, 255, 0), 2);
                circle(frame, newPoints[i], 3, Scalar(0, 0, 255), -1);
            }
        }

        // Update frames and feature points
        oldGray = gray.clone();
        oldPoints = newPoints;

        // Display the result
        imshow("Optical Flow", frame);

        // Press ESC to exit
        if (waitKey(30) == 27) break;
    }

    // Release resources
    cap.release();
    destroyAllWindows();

    return 0;
}
Other Extensions