Basic video filtering example

[glebignatieff]

Hello there!

I'm working on my pet project which requires some video editing, i.e. filtering. At first, I was thinking about wrapping needed FFmpeg's functionality myself, but then I try out your library first. Unfortunately, I couldn't find any video filtering example, so I was trying to recreate FFmpeg's filtering_video.c example. Since I'm writing this issue you may've guessed that I've miserably failed. I've tried to apply two filters:

1. Just a dummy null filter. As a result I get the same video but with the first second missing.
2. trim filter. I trim 5 seconds from 10-second video and as a result I get a very slow 60-second video. I figured that I have some time_base problems, but it's very confusing since I'm new to FFmpeg. Then I've noticed that output packets don't have pts and dts set and when I set them manually I get a correct video, but I don't think it should be like that, right?

I was hoping that you could help me with that by providing a correct video filtering example or at least giving some insights on what I'm doing wrong.

Here's my code.

#include <iostream>
#include <thread>

#include <av.h>
#include <averror.h>
#include <avtime.h>
#include <codeccontext.h>
#include <filters/buffersink.h>
#include <filters/filtergraph.h>
#include <formatcontext.h>

#ifdef av_err2str
#undef av_err2str
av_always_inline char *av_err2str(int errnum)
{
    static char str[AV_ERROR_MAX_STRING_SIZE];
    memset(str, 0, sizeof(str));
    return av_make_error_string(str, AV_ERROR_MAX_STRING_SIZE, errnum);
}
#endif


constexpr auto kVideoFile = "small_bunny_1080p_60fps.mp4";
constexpr auto kOutputFile = "filtered_video.mp4";
//constexpr auto kFilterDescr = "null";
constexpr auto kFilterDescr="trim=5:10,setpts=PTS-STARTPTS";

namespace {

av::FormatContext fmt_ctx;
av::FormatContext out_ctx;
av::Stream video_stream;
av::VideoDecoderContext dec_ctx;
av::VideoEncoderContext enc_ctx;
av::FilterContext buffersink_ctx;
av::FilterContext buffersrc_ctx;
av::FilterGraph filter_graph;

}


static int open_input_file(const char *filename)
{
    fmt_ctx.openInput(filename);
    fmt_ctx.findStreamInfo();

    for (int i = 0; i < fmt_ctx.streamsCount(); ++i) {
        auto stream = fmt_ctx.stream(i);
        if (stream.mediaType() == AVMEDIA_TYPE_VIDEO) {
            video_stream = fmt_ctx.stream(i);
            break;
        }
    }

    dec_ctx = av::VideoDecoderContext{video_stream};
    dec_ctx.setRefCountedFrames(true);
    dec_ctx.open();

    return 0;
}

static int open_output_file(const char *filename)
{
    out_ctx.openOutput(filename);

    auto out_codec = findEncodingCodec(out_ctx.outputFormat());
    auto out_stream = out_ctx.addStream(out_codec);
    enc_ctx = av::VideoEncoderContext{out_stream};

    enc_ctx.setWidth(dec_ctx.width());
    enc_ctx.setHeight(dec_ctx.height());
    enc_ctx.setSampleAspectRatio(dec_ctx.sampleAspectRatio());
    if (dec_ctx.pixelFormat() > -1) enc_ctx.setPixelFormat(dec_ctx.pixelFormat());
    enc_ctx.setTimeBase(av::Rational{1, 1000});
    enc_ctx.setBitRate(dec_ctx.bitRate());
    enc_ctx.addFlags(
        out_ctx.outputFormat().isFlags(AVFMT_GLOBALHEADER) ? AV_CODEC_FLAG_GLOBAL_HEADER : 0);

    // instead of codepar?
    out_stream.setFrameRate(video_stream.frameRate());
    //    out_stream.setAverageFrameRate(video_stream.frameRate());
    //    out_stream.setTimeBase(video_stream.timeBase());

    out_ctx.dump();
    enc_ctx.open();
    out_ctx.writeHeader();
    out_ctx.flush();

    return 0;
}

static int init_filters(const char *filters_descr)
{
    char args[512];

    snprintf(args, sizeof(args), "video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:pixel_aspect=%d/%d",
             dec_ctx.width(), dec_ctx.height(), dec_ctx.pixelFormat(),
             dec_ctx.timeBase().getNumerator(), dec_ctx.timeBase().getDenominator(),
             dec_ctx.sampleAspectRatio().getNumerator(),
             dec_ctx.sampleAspectRatio().getDenominator());

    const av::Filter buffersrc{"buffer"}, buffersink{"buffersink"};
    buffersrc_ctx = filter_graph.createFilter(buffersrc, "in", args);
    buffersink_ctx = filter_graph.createFilter(buffersink, "out", {});

    filter_graph.parse(filters_descr, buffersrc_ctx, buffersink_ctx);
    filter_graph.config();

    return 0;
}

static void show_frame_info(const av::VideoFrame &frame, const std::string &tag = "Frame")
{
    std::clog << tag << ": pts=" << frame.pts() << " / " << frame.pts().seconds() << " / "
              << frame.timeBase() << ", " << frame.width() << "x" << frame.height()
              << ", size=" << frame.size() << ", ref=" << frame.isReferenced() << ":"
              << frame.refCount() << " / type: " << frame.pictureType() << std::endl;
}

static void show_packet_info(const av::Packet &packet, const std::string &tag = "Packet")
{
    std::clog << tag << ": pts=" << packet.pts() << ", dts=" << packet.dts() << " / "
              << packet.pts().seconds() << " / " << packet.timeBase()
              << " / st: " << packet.streamIndex() << std::endl;
}

int main(int argc, char **argv)
{
    int ret;

    if ((ret = open_input_file(kVideoFile)) < 0) goto end;
    if ((ret = open_output_file(kOutputFile)) < 0) goto end;
    if ((ret = init_filters(kFilterDescr)) < 0) goto end;

    while (true) {
        auto packet = fmt_ctx.readPacket();
        if (!packet) {
            ret = AVERROR_EOF;
            goto end;
        }

        if (packet.streamIndex() == video_stream.index()) {
            show_packet_info(packet, "inPacket");

            auto frame = dec_ctx.decode(packet);
            if (!frame) continue;

            // It's needed until `outFrame.setTimeBase(timeBase());` in codeccontext.cpp not fixed
            //            frame.setTimeBase(video_stream.timeBase());

            // Why?
            frame.setTimeBase(enc_ctx.timeBase());
            frame.setStreamIndex(0);
            frame.setPictureType();

            show_frame_info(frame, "inFrame");

            av::BufferSrcFilterContext{buffersrc_ctx}.addVideoFrame(frame);

            while (true) {
                std::error_code ec;
                av::VideoFrame filt_frame;
                av::BufferSinkFilterContext{buffersink_ctx}.getVideoFrame(filt_frame, ec);
                filt_frame.setTimeBase(frame.timeBase());

                ret = ec.value();
                if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) break;
                if (ret < 0) goto end;

                show_frame_info(filt_frame, "outFrame");

                auto out_packet = enc_ctx.encode(filt_frame);
                out_packet.setStreamIndex(0);
                if (!out_packet) {
                    std::cerr << "Empty out packet" << std::endl;
                    continue;
                }

                show_packet_info(out_packet, "outPacket");

                out_ctx.writePacket(out_packet);
            }
        }
    }

end:
    out_ctx.writeTrailer();

    if (ret < 0 && ret != AVERROR_EOF) {
        fprintf(stderr, "Error occurred: %s\n", av_err2str(ret));
        exit(1);
    }
}

FFmpeg

ffmpeg version 4.3.1 Copyright (c) 2000-2020 the FFmpeg developers
  built with Apple clang version 11.0.3 (clang-1103.0.32.62)
  configuration: --prefix=/usr/local/Cellar/ffmpeg/4.3.1_1 --enable-shared --enable-pthreads --enable-version3 --enable-avresample --cc=clang --host-cflags= --host-ldflags= --enable-ffplay --enable-gnutls --enable-gpl --enable-libaom --enable-libbluray --enable-libdav1d --enable-libmp3lame --enable-libopus --enable-librav1e --enable-librubberband --enable-libsnappy --enable-libsrt --enable-libtesseract --enable-libtheora --enable-libvidstab --enable-libvorbis --enable-libvpx --enable-libwebp --enable-libx264 --enable-libx265 --enable-libxml2 --enable-libxvid --enable-lzma --enable-libfontconfig --enable-libfreetype --enable-frei0r --enable-libass --enable-libopencore-amrnb --enable-libopencore-amrwb --enable-libopenjpeg --enable-librtmp --enable-libspeex --enable-libsoxr --enable-videotoolbox --disable-libjack --disable-indev=jack
  libavutil      56. 51.100 / 56. 51.100
  libavcodec     58. 91.100 / 58. 91.100
  libavformat    58. 45.100 / 58. 45.100
  libavdevice    58. 10.100 / 58. 10.100
  libavfilter     7. 85.100 /  7. 85.100
  libavresample   4.  0.  0 /  4.  0.  0
  libswscale      5.  7.100 /  5.  7.100
  libswresample   3.  7.100 /  3.  7.100
  libpostproc    55.  7.100 / 55.  7.100

P.S. The video I'm using for tests I got from https://github.com/leandromoreira/ffmpeg-libav-tutorial using fetch_bbb_video.sh script.

[zacgibson21]

I wanted to follow up on this and see if there was any update on a tutorial for Filters?

[mmomtchev]

@h4tr3d what is the situation with the filtering code? I have the feeling that it is in a roofing more or less finished, some minor construction remains state?