(三) ffmpeg filter学习-编写自己的filter

#include "libavutil/opt.h"
#include "libavutil/imgutils.h"
#include "libavutil/avassert.h"
#include "avfilter.h"
#include "formats.h"
#include "internal.h"
#include "video.h"


typedef struct TransformContext {
    const AVClass *class;
    int backUp;
    //add some private data if you want
} TransformContext;

typedef struct ThreadData {
    AVFrame *in, *out;
} ThreadData;

static void image_copy_plane(uint8_t *dst, int dst_linesize,
                         const uint8_t *src, int src_linesize,
                         int bytewidth, int height)
{
    if (!dst || !src)
        return;
    av_assert0(abs(src_linesize) >= bytewidth);
    av_assert0(abs(dst_linesize) >= bytewidth);
    for (;height > 0; height--) {
        memcpy(dst, src, bytewidth);
        dst += dst_linesize;
        src += src_linesize;
    }
}

//for YUV data, frame->data[0] save Y, frame->data[1] save U, frame->data[2] save V
static int frame_copy_video(AVFrame *dst, const AVFrame *src)
{
    int i, planes;

    if (dst->width  > src->width ||
        dst->height > src->height)
        return AVERROR(EINVAL);

    planes = av_pix_fmt_count_planes(dst->format);
    //make sure data is valid
    for (i = 0; i < planes; i++)
        if (!dst->data[i] || !src->data[i])
            return AVERROR(EINVAL);

    const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(dst->format);
    int planes_nb = 0;
    for (i = 0; i < desc->nb_components; i++)
        planes_nb = FFMAX(planes_nb, desc->comp[i].plane + 1);

    for (i = 0; i < planes_nb; i++) {
        int h = dst->height;
        int bwidth = av_image_get_linesize(dst->format, dst->width, i);
        if (bwidth < 0) {
            av_log(NULL, AV_LOG_ERROR, "av_image_get_linesize failed
");
            return;
        }
        if (i == 1 || i == 2) {
            h = AV_CEIL_RSHIFT(dst->height, desc->log2_chroma_h);
        }
        image_copy_plane(dst->data[i], dst->linesize[i],
                            src->data[i], src->linesize[i],
                            bwidth, h);
    }
    return 0;
}

/**************************************************************************
* you can modify this function, do what you want here. use src frame, and blend to dst frame.
* for this demo, we just copy some part of src frame to dst frame(out_w = in_w/2, out_h = in_h/2)
***************************************************************************/
static int do_conversion(AVFilterContext *ctx, void *arg, int jobnr,
                        int nb_jobs)
{
    TransformContext *privCtx = ctx->priv;
    ThreadData *td = arg;
    AVFrame *dst = td->out;
    AVFrame *src = td->in;

    frame_copy_video(dst, src);
    return 0;
}

static int filter_frame(AVFilterLink *link, AVFrame *in)
{
    av_log(NULL, AV_LOG_WARNING, "### chenxf filter_frame, link %x, frame %x 
", link, in);
    AVFilterContext *avctx = link->dst;
    AVFilterLink *outlink = avctx->outputs[0];
    AVFrame *out;

    //allocate a new buffer, data is null
    out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
    if (!out) {
        av_frame_free(&in);
        return AVERROR(ENOMEM);
    }

    //the new output frame, property is the same as input frame, only width/height is different
    av_frame_copy_props(out, in);
    out->width  = outlink->w;
    out->height = outlink->h;

    ThreadData td;
    td.in = in;
    td.out = out;
    int res;
    if(res = avctx->internal->execute(avctx, do_conversion, &td, NULL, FFMIN(outlink->h, avctx->graph->nb_threads))) {
        return res;
    }

    av_frame_free(&in);

    return ff_filter_frame(outlink, out);
}

static av_cold int config_output(AVFilterLink *outlink)
{
    AVFilterContext *ctx = outlink->src;
    TransformContext *privCtx = ctx->priv;

    //you can modify output width/height here
    outlink->w = ctx->inputs[0]->w/2;
    outlink->h = ctx->inputs[0]->h/2;
    av_log(NULL, AV_LOG_DEBUG, "configure output, w h = (%d %d), format %d 
", outlink->w, outlink->h, outlink->format);

    return 0;
}

static av_cold int init(AVFilterContext *ctx)
{
    av_log(NULL, AV_LOG_DEBUG, "init 
");
    TransformContext *privCtx = ctx->priv;
    //init something here if you want
    return 0;
}

static av_cold void uninit(AVFilterContext *ctx)
{
    av_log(NULL, AV_LOG_DEBUG, "uninit 
");
    TransformContext *privCtx = ctx->priv;
    //uninit something here if you want
}

//currently we just support the most common YUV420, can add more if needed
static int query_formats(AVFilterContext *ctx)
{
    static const enum AVPixelFormat pix_fmts[] = {
        AV_PIX_FMT_YUV420P,
        AV_PIX_FMT_NONE
    };
    AVFilterFormats *fmts_list = ff_make_format_list(pix_fmts);
    if (!fmts_list)
        return AVERROR(ENOMEM);
    return ff_set_common_formats(ctx, fmts_list);
}


//*************
#define OFFSET(x) offsetof(TransformContext, x)
#define FLAGS AV_OPT_FLAG_VIDEO_PARAM|AV_OPT_FLAG_FILTERING_PARAM

static const AVOption transform_options[] = {
    { "backUp",         "a backup parameters, NOT use so far",          OFFSET(backUp),    AV_OPT_TYPE_STRING, {.str = "0"}, CHAR_MIN, CHAR_MAX, FLAGS },
    { NULL }

};// TODO: add something if needed

static const AVClass transform_class = {
    .class_name       = "transform",
    .item_name        = av_default_item_name,
    .option           = transform_options,
    .version          = LIBAVUTIL_VERSION_INT,
    .category         = AV_CLASS_CATEGORY_FILTER,
};

static const AVFilterPad avfilter_vf_transform_inputs[] = {
    {
        .name         = "transform_inputpad",
        .type         = AVMEDIA_TYPE_VIDEO,
        .filter_frame = filter_frame,
    },
    { NULL }
};

static const AVFilterPad avfilter_vf_transform_outputs[] = {
    {
        .name = "transform_outputpad",
        .type = AVMEDIA_TYPE_VIDEO,
        .config_props = config_output,
    },
    { NULL }
};

AVFilter ff_vf_transform = {
    .name           = "transform",
    .description    = NULL_IF_CONFIG_SMALL("cut a part of video"),
    .priv_size      = sizeof(TransformContext),
    .priv_class     = &transform_class,
    .init          = init,
    .uninit        = uninit,
    .query_formats = query_formats,
    .inputs         = avfilter_vf_transform_inputs,
    .outputs        = avfilter_vf_transform_outputs,
};

static int decode_video(InputStream *ist, AVPacket *pkt, int *got_output)
{
    AVFrame* decoded_frame, f;
    //解码
    ret = avcodec_decode_video2(ist->dec_ctx,
                                decoded_frame, got_output, pkt);
    //......
    //送给滤镜
    for (i = 0; i < ist->nb_filters; i++) {
        f = decodec_frame;
        ret = av_buffersrc_add_frame_flags(ist->filters[i]->filter, f, AV_BUFFERSRC_FLAG_PUSH);
    }
}

static int av_buffersrc_add_frame_internal(AVFilterContext *ctx,
                                           AVFrame *frame, int flags)
{
    //写FIFO
    av_fifo_generic_write(s->fifo, &copy, sizeof(copy), NULL);
    if ((flags & AV_BUFFERSRC_FLAG_PUSH))
        if ((ret = ctx->output_pads[0].request_frame(ctx->outputs[0])) < 0)
            return ret;

    return 0;
}

static int request_frame(AVFilterLink *link)
{
    BufferSourceContext *c = link->src->priv;
    AVFrame *frame;
    int ret;
    //省略......
    av_fifo_generic_read(c->fifo, &frame, sizeof(frame), NULL);
    av_log(NULL, AV_LOG_WARNING, "request_frame, frame-pts %lld 
", frame->pts);
    //这个link，是第一个link，链接当前的AVFilterContext和下一个AVFilterContext，也就是我们自己写的vf_transform.c
    ret = ff_filter_frame(link, frame);

    return ret;
}

static int ff_filter_frame_framed(AVFilterLink *link, AVFrame *frame)
{
    //定义一个函数指针filter_frame。所指向的函数，参数为AVFilterLink *, AVFrame *,返回值为int
    int (*filter_frame)(AVFilterLink *, AVFrame *);
    AVFilterContext *dstctx = link->dst;//下一个AVFilterContext，对本例来说，就是我们自己写的transform 滤镜，源码在vf_transform.c
    AVFilterPad *dst = link->dstpad;
    AVFrame *out = NULL;
    int ret;

    if (!(filter_frame = dst->filter_frame))//函数指针filter_frame，link->dstpad其实就是dstctx->input_pads，也就是transform滤镜定义的
        filter_frame = default_filter_frame;
    //省略300字

    ret = filter_frame(link, out);
    link->frame_count++;
    ff_update_link_current_pts(link, pts);
    return ret;
}

static const AVFilterPad avfilter_vf_transform_inputs[] = {
    {
        .name         = "default",
        .type         = AVMEDIA_TYPE_VIDEO,
        .filter_frame = filter_frame,
    },
    { NULL }
};

static int filter_frame(AVFilterLink *link, AVFrame *in)
{
    AVFilterContext *avctx = link->dst;//第一个link的dst AVFilterContext，其实就是当前的filter的AVFilterContext
    AVFilterLink *outlink = avctx->outputs[0];//当前的AVFilterContext，outputs[0]指向第二个AVFilterLink
    AVFrame *out;

    //分配一个空的AVFrame。
    out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
    if (!out) {
        av_frame_free(&in);
        return AVERROR(ENOMEM);
    }

    //分配的空buffer的参数和上一个基本一致，但修改宽高。当然啦，如果你愿意，不修改宽高，那就不需要下面2句。
    av_frame_copy_props(out, in);
    out->width  = outlink->w;
    out->height = outlink->h;
    out->format = outlink->format;

    ThreadData td;
    td.in = in;
    td.out = out;
    int res;
    if(res = avctx->internal->execute(avctx, do_conversion, &td, NULL, FFMIN(outlink->h, avctx->graph->nb_threads))) {
        return res;
    }//启用一个子线程，执行比较耗时的变换。do_conversion是我们要做的变换。

    av_frame_free(&in);

    return ff_filter_frame(outlink, out);//此时的ff_filter_frame，输入参数和前面buffersrc.c调用的已经不一样。outlink是第二个AVFilterLink，buffer也是做了变换的新的buffer
}

static int ff_filter_frame_framed(AVFilterLink *link, AVFrame *frame)
{
    //定义一个函数指针filter_frame。所指向的函数，参数为AVFilterLink *, AVFrame *,返回值为int
    int (*filter_frame)(AVFilterLink *, AVFrame *);
    AVFilterContext *dstctx = link->dst;//下一个AVFilterContext，对本例来说，就是系统默认的第三个滤镜，名字叫"format"，源码在vf_format.c
    AVFilterPad *dst = link->dstpad;
    AVFrame *out = NULL;
    int ret;

    if (!(filter_frame = dst->filter_frame))//vf_format.c没有实现filter函数，因为返回为空
        filter_frame = default_filter_frame;//所以函数会走到这，函数指针filter_frame 将指向default_filter_frame
    //省略300字

    ret = filter_frame(link, out);
    link->frame_count++;
    ff_update_link_current_pts(link, pts);
    return ret;
}

static int default_filter_frame(AVFilterLink *link, AVFrame *frame)
{
    //该函数没干啥，又调用ff_filter_frame了，第一个参数，换成第三个AVFilterLink了，第二个参数不变，frame默默的传递出去
    return ff_filter_frame(link->dst->outputs[0], frame);
}

static int ff_filter_frame_framed(AVFilterLink *link, AVFrame *frame)
{
    //定义一个函数指针filter_frame。所指向的函数，参数为AVFilterLink *, AVFrame *,返回值为int
    int (*filter_frame)(AVFilterLink *, AVFrame *);
    AVFilterContext *dstctx = link->dst;//下一个AVFilterContext，对本例来说，就是系统默认的最后一个滤镜，名字叫"buffersink"，源码在bufffersink.c
    AVFilterPad *dst = link->dstpad;
    AVFrame *out = NULL;
    int ret;

    if (!(filter_frame = dst->filter_frame))//指向buffersink.c实现的filte_frame函数
        filter_frame = default_filter_frame;
    //省略300字

    ret = filter_frame(link, out);
    link->frame_count++;
    ff_update_link_current_pts(link, pts);
    return ret;
}

static int filter_frame(AVFilterLink *link, AVFrame *frame)
{
    AVFilterContext *ctx = link->dst;
    BufferSinkContext *buf = link->dst->priv;
    int ret;

    if ((ret = add_buffer_ref(ctx, frame)) < 0)
        return ret;
    //省略300字
    return 0;
}

static int add_buffer_ref(AVFilterContext *ctx, AVFrame *ref)
{
    BufferSinkContext *buf = ctx->priv;

    /* cache frame */
    //把buffer存到FIFO
    av_fifo_generic_write(buf->fifo, &ref, FIFO_INIT_ELEMENT_SIZE, NULL);
    return 0;
}

static int reap_filters(int flush)
{
    AVFrame *filtered_frame = NULL;//该指针将存储一个经过滤镜处理后的buffer，并送给encoder
    int i;

    /* Reap all buffers present in the buffer sinks */
    for (i = 0; i < nb_output_streams; i++) {//一路video，一路audio，那么nb_output_streams = 2
        OutputStream *ost = output_streams[i];
        OutputFile    *of = output_files[ost->file_index];
        AVFilterContext *filter;
        AVCodecContext *enc = ost->enc_ctx;
        int ret = 0;

        if (!ost->filter)
            continue;
        filter = ost->filter->filter;//OutputStream的filter指针指向buffersink.c定义的AVFilterContext。也就是本文讨论的，最后一个AVFilterContext

        if (!ost->filtered_frame && !(ost->filtered_frame = av_frame_alloc())) {
            return AVERROR(ENOMEM);
        }
        filtered_frame = ost->filtered_frame;

        while (1) {
            double float_pts = AV_NOPTS_VALUE; // this is identical to filtered_frame.pts but with higher precision
            //av_buffersink_get_frame_flags定义在buffersink.c，用于从FIFO读出一帧
            ret = av_buffersink_get_frame_flags(filter, filtered_frame,
                                               AV_BUFFERSINK_FLAG_NO_REQUEST);
            if (ret < 0) {
                //省略，检查ret
                //如果ret<0，不是别的错误，那认为还没有数据，跳出循环
                break;
            }
            switch (filter->inputs[0]->type) {
            case AVMEDIA_TYPE_VIDEO:
                //do_video_out函数将会做video编码
                do_video_out(of->ctx, ost, filtered_frame, float_pts);
                break;
            case AVMEDIA_TYPE_AUDIO:
                //do_audio_out函数将会做audioo编码
                do_audio_out(of->ctx, ost, filtered_frame);
                break;
            default:
                // TODO support subtitle filters
                av_assert0(0);
            }

            av_frame_unref(filtered_frame);
        }
    }

    return 0;
}

static void do_video_out(AVFormatContext *s,
                         OutputStream *ost,
                         AVFrame *next_picture,
                         double sync_ipts)
{
    int ret;
    AVCodecContext *enc = ost->enc_ctx;
    int nb_frames, nb0_frames, i;
    //省略300字
    for (i = 0; i < nb_frames; i++) {
        AVFrame *in_picture;
        if (i < nb0_frames && ost->last_frame) {
            in_picture = ost->last_frame;
        } else
            in_picture = next_picture;
        //省略300字
        ost->frames_encoded++;
        //开始编码
        ret = avcodec_encode_video2(enc, &pkt, in_picture, &got_packet);
    }
    ///省略300字
}

>choose_output()函数用于选择一个OutputStream。比如有一个audio，一个video，那要根据pts策略，比如谁的pts比较小，就挑哪个OutputStream先干活。
transcode_frome_filter()函数用于选个一个InputStream，用于下一步的process_input()。
process_input()函数主要是解码，并把解码的buffer送往filter处理。
reap_filters()函数主要是，从filter的FIFO拿出buffer，并编码。