Line data Source code
1 : /*
2 : * SPDX-License-Identifier: GPL-3.0-only
3 : * MuseScore-Studio-CLA-applies
4 : *
5 : * MuseScore Studio
6 : * Music Composition & Notation
7 : *
8 : * Copyright (C) 2025 MuseScore Limited
9 : *
10 : * This program is free software: you can redistribute it and/or modify
11 : * it under the terms of the GNU General Public License version 3 as
12 : * published by the Free Software Foundation.
13 : *
14 : * This program is distributed in the hope that it will be useful,
15 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 : * GNU General Public License for more details.
18 : *
19 : * You should have received a copy of the GNU General Public License
20 : * along with this program. If not, see <https://www.gnu.org/licenses/>.
21 : */
22 : #include "videoencoder.h"
23 : #include "ffmpeglibhandler.h"
24 :
25 : #include <algorithm>
26 :
27 : #include "io/file.h"
28 : #include "io/path.h"
29 : #include "defer.h"
30 : #include "log.h"
31 :
32 : namespace muse::media::ffmpeg::v9 {
33 : struct MemoryReadContext {
34 : const uint8_t* data = nullptr;
35 : int64_t size = 0;
36 : int64_t pos = 0;
37 : };
38 :
39 0 : static int memReadPacket(void* opaque, uint8_t* buf, int buf_size)
40 : {
41 0 : auto* ctx = static_cast<MemoryReadContext*>(opaque);
42 0 : int64_t remaining = ctx->size - ctx->pos;
43 0 : int toRead = static_cast<int>(std::min(static_cast<int64_t>(buf_size), remaining));
44 0 : if (toRead <= 0) {
45 0 : return AVERROR_EOF;
46 : }
47 0 : memcpy(buf, ctx->data + ctx->pos, toRead);
48 0 : ctx->pos += toRead;
49 0 : return toRead;
50 : }
51 :
52 0 : static int64_t memSeek(void* opaque, int64_t offset, int whence)
53 : {
54 0 : auto* ctx = static_cast<MemoryReadContext*>(opaque);
55 0 : switch (whence) {
56 0 : case SEEK_SET: ctx->pos = offset;
57 0 : break;
58 0 : case SEEK_CUR: ctx->pos += offset;
59 0 : break;
60 0 : case SEEK_END: ctx->pos = ctx->size + offset;
61 0 : break;
62 0 : case AVSEEK_SIZE: return ctx->size;
63 0 : default: return -1;
64 : }
65 0 : if (ctx->pos < 0) {
66 0 : ctx->pos = 0;
67 : }
68 0 : if (ctx->pos > ctx->size) {
69 0 : ctx->pos = ctx->size;
70 : }
71 0 : return ctx->pos;
72 : }
73 :
74 : struct FFmpeg {
75 : int width = 0;
76 : int height = 0;
77 : unsigned int ptsCounter = 0;
78 :
79 : // FFmpeg stuff
80 : AVFormatContext* formatCtx = nullptr;
81 : AVStream* videoStream = nullptr;
82 : AVCodecContext* codecCtx = nullptr;
83 : const AVCodec* codec = nullptr;
84 : // Frame data
85 : AVFrame* ppicture = nullptr;
86 : uint8_t* picture_buf = nullptr;
87 : // Conversion
88 : SwsContext* img_convert_ctx = nullptr;
89 : // Packet
90 : AVPacket* pkt;
91 :
92 : bool opened = false;
93 : bool finished = false;
94 : };
95 :
96 0 : VideoEncoder::VideoEncoder()
97 : {
98 0 : m_ffmpeg = new FFmpeg();
99 0 : }
100 :
101 0 : VideoEncoder::~VideoEncoder()
102 : {
103 0 : delete m_ffmpeg;
104 0 : }
105 :
106 0 : bool VideoEncoder::load(const FFmpegLibPaths& paths)
107 : {
108 0 : auto handler = std::make_shared<FFmpegLibHandler>();
109 0 : if (!handler->loadLib(paths.avUtilPath, paths.avCodecPath, paths.avFormatPath,
110 0 : paths.swScalePath, paths.swResamplePath)
111 0 : || !handler->loadApi()) {
112 0 : LOGW() << "FFmpeg libraries not found";
113 0 : return false;
114 : }
115 0 : m_ffmpegHandler = std::move(handler);
116 0 : return true;
117 0 : }
118 :
119 0 : bool VideoEncoder::open(const muse::io::path_t& fileName, const Options& options)
120 : {
121 0 : m_ffmpeg->ptsCounter = 0;
122 :
123 0 : m_ffmpeg->width = options.width;
124 0 : m_ffmpeg->height = options.height;
125 :
126 0 : if (m_ffmpegHandler->avformat_alloc_output_context2(&m_ffmpeg->formatCtx, nullptr, options.format.c_str(), fileName.c_str()) < 0
127 0 : || !m_ffmpeg->formatCtx) {
128 0 : LOGE() << "failed to allocate output context";
129 0 : return false;
130 : }
131 :
132 : // Add the video stream
133 0 : m_ffmpeg->videoStream = m_ffmpegHandler->avformat_new_stream(m_ffmpeg->formatCtx, nullptr);
134 0 : if (!m_ffmpeg->videoStream) {
135 0 : LOGE() << "failed allocate stream";
136 0 : return false;
137 : }
138 :
139 0 : m_ffmpeg->videoStream->time_base.den = options.fps;
140 0 : m_ffmpeg->videoStream->time_base.num = 1;
141 :
142 : // find the video encoder
143 0 : m_ffmpeg->codec = m_ffmpegHandler->avcodec_find_encoder(AV_CODEC_ID_H264);
144 0 : if (!m_ffmpeg->codec) {
145 0 : LOGE() << "not found codec";
146 0 : return false;
147 : }
148 0 : m_ffmpeg->codecCtx = m_ffmpegHandler->avcodec_alloc_context3(m_ffmpeg->codec);
149 0 : if (m_ffmpeg->codecCtx == nullptr) {
150 0 : LOGE() << "failed to allocate AV context";
151 0 : return false;
152 : }
153 :
154 0 : m_ffmpeg->videoStream->codecpar->codec_type = AVMEDIA_TYPE_VIDEO;
155 0 : m_ffmpeg->videoStream->codecpar->codec_id = AV_CODEC_ID_H264;
156 0 : m_ffmpeg->videoStream->codecpar->width = options.width;
157 0 : m_ffmpeg->videoStream->codecpar->height = options.height;
158 0 : m_ffmpeg->videoStream->codecpar->format = AV_PIX_FMT_YUV420P;
159 0 : m_ffmpeg->videoStream->codecpar->bit_rate = options.bitrate;
160 :
161 0 : if (m_ffmpegHandler->avcodec_parameters_to_context(m_ffmpeg->codecCtx, m_ffmpeg->videoStream->codecpar) < 0) {
162 0 : m_ffmpegHandler->avcodec_free_context(&m_ffmpeg->codecCtx);
163 0 : LOGE() << "failed to set codec parameters from stream";
164 0 : return false;
165 : }
166 :
167 0 : m_ffmpeg->codecCtx->profile = AV_PROFILE_H264_MAIN;
168 0 : m_ffmpeg->codecCtx->time_base.den = options.fps;
169 0 : m_ffmpeg->codecCtx->time_base.num = 1;
170 0 : m_ffmpeg->codecCtx->gop_size = options.gop;
171 0 : m_ffmpeg->codecCtx->thread_count = 10;
172 0 : m_ffmpeg->codecCtx->max_b_frames = 0;
173 0 : m_ffmpegHandler->av_opt_set_int(m_ffmpeg->codecCtx, "b_strategy", 1, 0);
174 :
175 0 : m_ffmpeg->codecCtx->me_cmp = 1;
176 0 : m_ffmpeg->codecCtx->me_range = 16;
177 0 : m_ffmpegHandler->av_opt_set_int(m_ffmpeg->codecCtx, "hex", 1, 0);
178 :
179 0 : m_ffmpeg->codecCtx->qmin = 10;
180 0 : m_ffmpeg->codecCtx->qmax = 51;
181 0 : m_ffmpegHandler->av_opt_set_int(m_ffmpeg->codecCtx, "sc_threshold", 40, 0);
182 0 : m_ffmpeg->codecCtx->flags |= AV_CODEC_FLAG_LOOP_FILTER;
183 0 : m_ffmpeg->codecCtx->me_subpel_quality = 5;
184 0 : m_ffmpeg->codecCtx->i_quant_factor = 0.71f;
185 0 : m_ffmpeg->codecCtx->qcompress = 0.6f;
186 0 : m_ffmpeg->codecCtx->max_qdiff = 4;
187 :
188 0 : if (m_ffmpeg->formatCtx->oformat->flags & AVFMT_GLOBALHEADER) {
189 0 : m_ffmpeg->codecCtx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
190 : }
191 :
192 0 : if (m_ffmpegHandler->avcodec_open2(m_ffmpeg->codecCtx, m_ffmpeg->codec, nullptr) < 0) {
193 0 : LOGE() << "failed open codec";
194 0 : return false;
195 : }
196 :
197 : // Copy codec params to stream AFTER open - codec adds extradata (SPS/PPS) during open
198 0 : if (m_ffmpegHandler->avcodec_parameters_from_context(m_ffmpeg->videoStream->codecpar, m_ffmpeg->codecCtx) < 0) {
199 0 : m_ffmpegHandler->avcodec_free_context(&m_ffmpeg->codecCtx);
200 0 : LOGE() << "failed to copy codec parameters to stream";
201 0 : return false;
202 : }
203 :
204 : // Allocate the YUV frame
205 0 : m_ffmpeg->ppicture = m_ffmpegHandler->av_frame_alloc();
206 0 : if (!m_ffmpeg->ppicture) {
207 0 : LOGE() << "failed allocate frame";
208 0 : return false;
209 : }
210 :
211 0 : m_ffmpeg->ppicture->width = options.width;
212 0 : m_ffmpeg->ppicture->height = options.height;
213 0 : m_ffmpeg->ppicture->format = AV_PIX_FMT_YUV420P;
214 :
215 0 : int size = m_ffmpegHandler->av_image_get_buffer_size(AV_PIX_FMT_YUV420P, m_ffmpeg->width, m_ffmpeg->height, 1);
216 0 : m_ffmpeg->picture_buf = new uint8_t[size];
217 0 : if (!m_ffmpeg->picture_buf) {
218 0 : LOGE() << "failed allocate frame buf";
219 0 : m_ffmpegHandler->av_frame_free(&m_ffmpeg->ppicture);
220 0 : return false;
221 : }
222 :
223 : // Setup the planes
224 0 : m_ffmpegHandler->av_image_fill_arrays(m_ffmpeg->ppicture->data, m_ffmpeg->ppicture->linesize, m_ffmpeg->picture_buf,
225 0 : AV_PIX_FMT_YUV420P, m_ffmpeg->width, m_ffmpeg->height, 1);
226 :
227 0 : if (m_ffmpegHandler->avio_open(&m_ffmpeg->formatCtx->pb, fileName.c_str(), AVIO_FLAG_WRITE) < 0) {
228 0 : LOGE() << "failed open file: " << fileName;
229 0 : return false;
230 : }
231 :
232 0 : int ret = m_ffmpegHandler->avformat_write_header(m_ffmpeg->formatCtx, NULL);
233 0 : if (ret < 0) {
234 0 : LOGE() << "failed write AV header";
235 0 : return false;
236 : }
237 :
238 : //av_dump_format(m_ffmpeg->formatCtx, 0, fileName.c_str(), 1);
239 :
240 0 : m_ffmpeg->opened = true;
241 0 : m_outputPath = fileName;
242 :
243 0 : return true;
244 : }
245 :
246 0 : void VideoEncoder::finishEncode()
247 : {
248 0 : if (!m_ffmpeg->opened || m_ffmpeg->finished) {
249 0 : return;
250 : }
251 :
252 0 : int ret = m_ffmpegHandler->avcodec_send_frame(m_ffmpeg->codecCtx, NULL);
253 0 : if (ret < 0) {
254 0 : LOGE() << "failed to flush encoder buffer";
255 0 : return;
256 : }
257 :
258 : while (true) {
259 0 : m_ffmpeg->pkt = m_ffmpegHandler->av_packet_alloc();
260 0 : ret = m_ffmpegHandler->avcodec_receive_packet(m_ffmpeg->codecCtx, m_ffmpeg->pkt);
261 0 : if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
262 0 : m_ffmpegHandler->av_packet_free(&m_ffmpeg->pkt);
263 0 : break;
264 0 : } else if (ret < 0) {
265 0 : m_ffmpegHandler->av_packet_free(&m_ffmpeg->pkt);
266 0 : LOGE() << "error during encoding";
267 0 : return;
268 : }
269 :
270 0 : m_ffmpeg->pkt->pts = m_ffmpegHandler->av_rescale_q(m_ffmpeg->ptsCounter, m_ffmpeg->codecCtx->time_base,
271 0 : m_ffmpeg->videoStream->time_base);
272 0 : m_ffmpeg->pkt->dts = m_ffmpegHandler->av_rescale_q(m_ffmpeg->ptsCounter, m_ffmpeg->codecCtx->time_base,
273 0 : m_ffmpeg->videoStream->time_base);
274 :
275 0 : m_ffmpeg->ptsCounter++;
276 :
277 0 : ret = m_ffmpegHandler->av_interleaved_write_frame(m_ffmpeg->formatCtx, m_ffmpeg->pkt);
278 0 : if (ret < 0) {
279 0 : m_ffmpegHandler->av_packet_free(&m_ffmpeg->pkt);
280 0 : return;
281 : }
282 0 : m_ffmpegHandler->av_packet_free(&m_ffmpeg->pkt);
283 : }
284 :
285 0 : m_ffmpegHandler->av_write_trailer(m_ffmpeg->formatCtx);
286 :
287 0 : if (m_ffmpeg->formatCtx->pb) {
288 0 : m_ffmpegHandler->avio_close(m_ffmpeg->formatCtx->pb);
289 0 : m_ffmpeg->formatCtx->pb = nullptr;
290 : }
291 :
292 0 : m_ffmpeg->finished = true;
293 : }
294 :
295 0 : void VideoEncoder::close()
296 : {
297 0 : if (!m_ffmpeg->opened) {
298 0 : return;
299 : }
300 :
301 0 : if (!m_ffmpeg->finished) {
302 0 : finishEncode();
303 : }
304 :
305 0 : m_ffmpegHandler->avcodec_free_context(&m_ffmpeg->codecCtx);
306 :
307 0 : if (m_ffmpeg->picture_buf) {
308 0 : delete[] m_ffmpeg->picture_buf;
309 0 : m_ffmpeg->picture_buf = 0;
310 : }
311 :
312 0 : if (m_ffmpeg->ppicture) {
313 0 : m_ffmpegHandler->av_frame_free(&m_ffmpeg->ppicture);
314 0 : m_ffmpeg->ppicture = 0;
315 : }
316 :
317 0 : if (m_ffmpeg->img_convert_ctx) {
318 0 : m_ffmpegHandler->sws_freeContext(m_ffmpeg->img_convert_ctx);
319 0 : m_ffmpeg->img_convert_ctx = nullptr;
320 : }
321 :
322 0 : if (m_ffmpeg->formatCtx) {
323 0 : if (m_ffmpeg->formatCtx->pb) {
324 0 : m_ffmpegHandler->avio_close(m_ffmpeg->formatCtx->pb);
325 0 : m_ffmpeg->formatCtx->pb = nullptr;
326 : }
327 0 : m_ffmpegHandler->avformat_free_context(m_ffmpeg->formatCtx);
328 0 : m_ffmpeg->formatCtx = nullptr;
329 : }
330 0 : m_ffmpeg->opened = false;
331 0 : m_ffmpeg->finished = false;
332 :
333 0 : m_outputPath = io::path_t();
334 : }
335 :
336 0 : bool VideoEncoder::encodeImage(const QImage& img)
337 : {
338 0 : if (!m_ffmpeg->opened) {
339 0 : return false;
340 : }
341 :
342 0 : convertImage_sws(img);
343 :
344 0 : m_ffmpeg->ppicture->pts = m_ffmpegHandler->av_rescale_q(m_ffmpeg->codecCtx->frame_num, m_ffmpeg->codecCtx->time_base,
345 0 : m_ffmpeg->videoStream->time_base);
346 :
347 0 : int ret = m_ffmpegHandler->avcodec_send_frame(m_ffmpeg->codecCtx, m_ffmpeg->ppicture);
348 0 : if (ret < 0) {
349 0 : return false;
350 : }
351 :
352 0 : while (ret >= 0) {
353 0 : m_ffmpeg->pkt = m_ffmpegHandler->av_packet_alloc();
354 0 : ret = m_ffmpegHandler->avcodec_receive_packet(m_ffmpeg->codecCtx, m_ffmpeg->pkt);
355 0 : if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
356 0 : m_ffmpegHandler->av_packet_free(&m_ffmpeg->pkt);
357 0 : return true;
358 0 : } else if (ret < 0) {
359 0 : m_ffmpegHandler->av_packet_free(&m_ffmpeg->pkt);
360 0 : LOGE() << "error during encoding";
361 0 : return false;
362 : }
363 :
364 0 : m_ffmpeg->pkt->pts = m_ffmpegHandler->av_rescale_q(m_ffmpeg->ptsCounter, m_ffmpeg->codecCtx->time_base,
365 0 : m_ffmpeg->videoStream->time_base);
366 0 : m_ffmpeg->pkt->dts = m_ffmpegHandler->av_rescale_q(m_ffmpeg->ptsCounter, m_ffmpeg->codecCtx->time_base,
367 0 : m_ffmpeg->videoStream->time_base);
368 :
369 0 : m_ffmpeg->ptsCounter++;
370 :
371 0 : ret = m_ffmpegHandler->av_interleaved_write_frame(m_ffmpeg->formatCtx, m_ffmpeg->pkt);
372 0 : if (ret < 0) {
373 0 : m_ffmpegHandler->av_packet_free(&m_ffmpeg->pkt);
374 0 : return false;
375 : }
376 0 : m_ffmpegHandler->av_packet_free(&m_ffmpeg->pkt);
377 : }
378 :
379 0 : return true;
380 : }
381 :
382 0 : bool VideoEncoder::encodeVideo(const ByteArray& videoData, int maxFrames)
383 : {
384 0 : if (!m_ffmpeg->opened) {
385 0 : LOGE() << "encodeVideo: encoder not opened";
386 0 : return false;
387 : }
388 :
389 0 : MemoryReadContext memCtx;
390 0 : memCtx.data = videoData.constData();
391 0 : memCtx.size = static_cast<int64_t>(videoData.size());
392 0 : memCtx.pos = 0;
393 :
394 : static constexpr int AVIO_BUF_SIZE = 4096;
395 0 : unsigned char* avioBuf = static_cast<unsigned char*>(m_ffmpegHandler->av_malloc(AVIO_BUF_SIZE));
396 0 : if (!avioBuf) {
397 0 : LOGE() << "encodeVideo: failed to allocate avio buffer";
398 0 : return false;
399 : }
400 :
401 0 : AVIOContext* avioCtx = m_ffmpegHandler->avio_alloc_context(
402 0 : avioBuf, AVIO_BUF_SIZE, 0, &memCtx, memReadPacket, nullptr, memSeek);
403 0 : if (!avioCtx) {
404 0 : LOGE() << "encodeVideo: failed to allocate AVIOContext";
405 0 : return false;
406 : }
407 :
408 0 : AVFormatContext* inputFmtCtx = m_ffmpegHandler->avformat_alloc_context();
409 0 : if (!inputFmtCtx) {
410 0 : m_ffmpegHandler->avio_context_free(&avioCtx);
411 0 : LOGE() << "encodeVideo: failed to allocate format context";
412 0 : return false;
413 : }
414 0 : inputFmtCtx->pb = avioCtx;
415 :
416 0 : AVCodecContext* decodeCtx = nullptr;
417 0 : AVFrame* decodedFrame = nullptr;
418 0 : AVPacket* packet = nullptr;
419 :
420 0 : SwsContext* decSwsCtx = nullptr;
421 :
422 0 : DEFER {
423 0 : if (decSwsCtx) {
424 0 : m_ffmpegHandler->sws_freeContext(decSwsCtx);
425 : }
426 0 : if (inputFmtCtx) {
427 0 : m_ffmpegHandler->avformat_close_input(&inputFmtCtx);
428 : }
429 0 : if (avioCtx) {
430 0 : m_ffmpegHandler->avio_context_free(&avioCtx);
431 : }
432 0 : if (decodeCtx) {
433 0 : m_ffmpegHandler->avcodec_free_context(&decodeCtx);
434 : }
435 0 : if (decodedFrame) {
436 0 : m_ffmpegHandler->av_frame_free(&decodedFrame);
437 : }
438 0 : if (packet) {
439 0 : m_ffmpegHandler->av_packet_free(&packet);
440 : }
441 0 : };
442 :
443 0 : if (m_ffmpegHandler->avformat_open_input(&inputFmtCtx, nullptr, nullptr, nullptr) < 0) {
444 0 : LOGE() << "encodeVideo: failed to open input from memory";
445 0 : return false;
446 : }
447 :
448 0 : if (m_ffmpegHandler->avformat_find_stream_info(inputFmtCtx, nullptr) < 0) {
449 0 : LOGE() << "encodeVideo: failed to find stream info";
450 0 : return false;
451 : }
452 :
453 0 : int videoStreamIdx = -1;
454 0 : for (unsigned i = 0; i < inputFmtCtx->nb_streams; i++) {
455 0 : if (inputFmtCtx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
456 0 : videoStreamIdx = static_cast<int>(i);
457 0 : break;
458 : }
459 : }
460 :
461 0 : if (videoStreamIdx < 0) {
462 0 : LOGE() << "encodeVideo: no video stream found";
463 0 : return false;
464 : }
465 :
466 0 : AVCodecParameters* codecPar = inputFmtCtx->streams[videoStreamIdx]->codecpar;
467 :
468 0 : const AVCodec* decoder = m_ffmpegHandler->avcodec_find_decoder(codecPar->codec_id);
469 0 : if (!decoder) {
470 0 : LOGE() << "encodeVideo: decoder not found for codec_id " << codecPar->codec_id;
471 0 : return false;
472 : }
473 :
474 0 : decodeCtx = m_ffmpegHandler->avcodec_alloc_context3(decoder);
475 0 : if (!decodeCtx) {
476 0 : LOGE() << "encodeVideo: failed to allocate decoder context";
477 0 : return false;
478 : }
479 :
480 0 : m_ffmpegHandler->avcodec_parameters_to_context(decodeCtx, codecPar);
481 :
482 0 : if (m_ffmpegHandler->avcodec_open2(decodeCtx, decoder, nullptr) < 0) {
483 0 : LOGE() << "encodeVideo: failed to open decoder";
484 0 : return false;
485 : }
486 :
487 0 : decSwsCtx = m_ffmpegHandler->sws_getCachedContext(
488 : nullptr,
489 : decodeCtx->width, decodeCtx->height, decodeCtx->pix_fmt,
490 0 : m_ffmpeg->width, m_ffmpeg->height, AV_PIX_FMT_YUV420P,
491 : SWS_BICUBIC, nullptr, nullptr, nullptr);
492 :
493 0 : if (!decSwsCtx) {
494 0 : LOGE() << "encodeVideo: failed to create sws context";
495 0 : return false;
496 : }
497 :
498 0 : decodedFrame = m_ffmpegHandler->av_frame_alloc();
499 0 : packet = m_ffmpegHandler->av_packet_alloc();
500 :
501 0 : int framesEncoded = 0;
502 0 : bool done = false;
503 :
504 0 : auto encodeDecodedFrame = [&]() -> bool {
505 0 : m_ffmpegHandler->sws_scale(decSwsCtx,
506 0 : decodedFrame->data, decodedFrame->linesize,
507 0 : 0, decodedFrame->height,
508 0 : m_ffmpeg->ppicture->data, m_ffmpeg->ppicture->linesize);
509 :
510 0 : m_ffmpeg->ppicture->pts = m_ffmpegHandler->av_rescale_q(
511 0 : m_ffmpeg->codecCtx->frame_num, m_ffmpeg->codecCtx->time_base, m_ffmpeg->videoStream->time_base);
512 :
513 0 : int ret = m_ffmpegHandler->avcodec_send_frame(m_ffmpeg->codecCtx, m_ffmpeg->ppicture);
514 0 : if (ret < 0) {
515 0 : return false;
516 : }
517 :
518 0 : while (ret >= 0) {
519 0 : AVPacket* outPkt = m_ffmpegHandler->av_packet_alloc();
520 0 : ret = m_ffmpegHandler->avcodec_receive_packet(m_ffmpeg->codecCtx, outPkt);
521 0 : if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
522 0 : m_ffmpegHandler->av_packet_free(&outPkt);
523 0 : break;
524 : }
525 0 : if (ret < 0) {
526 0 : m_ffmpegHandler->av_packet_free(&outPkt);
527 0 : return false;
528 : }
529 :
530 0 : outPkt->pts = m_ffmpegHandler->av_rescale_q(m_ffmpeg->ptsCounter, m_ffmpeg->codecCtx->time_base,
531 0 : m_ffmpeg->videoStream->time_base);
532 0 : outPkt->dts = m_ffmpegHandler->av_rescale_q(m_ffmpeg->ptsCounter, m_ffmpeg->codecCtx->time_base,
533 0 : m_ffmpeg->videoStream->time_base);
534 0 : m_ffmpeg->ptsCounter++;
535 :
536 0 : m_ffmpegHandler->av_interleaved_write_frame(m_ffmpeg->formatCtx, outPkt);
537 0 : m_ffmpegHandler->av_packet_free(&outPkt);
538 : }
539 :
540 0 : framesEncoded++;
541 0 : if (maxFrames > 0 && framesEncoded >= maxFrames) {
542 0 : done = true;
543 : }
544 :
545 0 : return true;
546 0 : };
547 :
548 0 : while (!done && m_ffmpegHandler->av_read_frame(inputFmtCtx, packet) >= 0) {
549 0 : if (packet->stream_index != videoStreamIdx) {
550 0 : m_ffmpegHandler->av_packet_unref(packet);
551 0 : continue;
552 : }
553 :
554 0 : int ret = m_ffmpegHandler->avcodec_send_packet(decodeCtx, packet);
555 0 : m_ffmpegHandler->av_packet_unref(packet);
556 :
557 0 : while (!done && ret >= 0) {
558 0 : ret = m_ffmpegHandler->avcodec_receive_frame(decodeCtx, decodedFrame);
559 0 : if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
560 : break;
561 : }
562 0 : if (ret < 0) {
563 0 : LOGE() << "encodeVideo: error receiving decoded frame";
564 0 : return false;
565 : }
566 :
567 0 : if (!encodeDecodedFrame()) {
568 0 : LOGE() << "encodeVideo: error encoding frame";
569 0 : return false;
570 : }
571 : }
572 : }
573 :
574 0 : if (!done) {
575 0 : m_ffmpegHandler->avcodec_send_packet(decodeCtx, nullptr);
576 : while (true) {
577 0 : int ret = m_ffmpegHandler->avcodec_receive_frame(decodeCtx, decodedFrame);
578 0 : if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
579 : break;
580 : }
581 0 : if (ret < 0 || !encodeDecodedFrame() || done) {
582 0 : break;
583 : }
584 0 : }
585 : }
586 :
587 0 : LOGI() << "encodeVideo: encoded " << framesEncoded << " frames from memory buffer";
588 0 : return true;
589 0 : }
590 :
591 0 : bool VideoEncoder::addAudio(const io::path_t& audioPath)
592 : {
593 0 : if (m_outputPath.empty()) {
594 0 : LOGE() << "addAudio: encoder was not opened or path not set";
595 0 : return false;
596 : }
597 :
598 0 : const io::path_t videoPath = m_outputPath;
599 0 : const io::path_t tmpPath = m_outputPath + ".tmp";
600 :
601 0 : AVFormatContext* videoFmtCtx = nullptr;
602 0 : AVFormatContext* audioFmtCtx = nullptr;
603 0 : AVFormatContext* outputFmtCtx = nullptr;
604 :
605 0 : bool cleaned = false;
606 0 : auto cleanup = [&]() {
607 0 : if (cleaned) {
608 0 : return;
609 : }
610 :
611 0 : if (videoFmtCtx) {
612 0 : m_ffmpegHandler->avformat_close_input(&videoFmtCtx);
613 : }
614 0 : if (audioFmtCtx) {
615 0 : m_ffmpegHandler->avformat_close_input(&audioFmtCtx);
616 : }
617 0 : if (outputFmtCtx) {
618 0 : if (outputFmtCtx->pb) {
619 0 : m_ffmpegHandler->avio_close(outputFmtCtx->pb);
620 0 : outputFmtCtx->pb = nullptr;
621 : }
622 0 : m_ffmpegHandler->avformat_free_context(outputFmtCtx);
623 : }
624 :
625 0 : cleaned = true;
626 0 : };
627 :
628 0 : DEFER {
629 0 : cleanup();
630 0 : };
631 :
632 0 : if (m_ffmpegHandler->avformat_open_input(&videoFmtCtx, videoPath.c_str(), nullptr, nullptr) < 0) {
633 0 : LOGE() << "addAudio: failed to open video input: " << videoPath;
634 0 : return false;
635 : }
636 0 : if (m_ffmpegHandler->avformat_find_stream_info(videoFmtCtx, nullptr) < 0) {
637 0 : LOGE() << "addAudio: failed to find video stream info";
638 0 : return false;
639 : }
640 :
641 0 : if (m_ffmpegHandler->avformat_open_input(&audioFmtCtx, audioPath.c_str(), nullptr, nullptr) < 0) {
642 0 : LOGE() << "addAudio: failed to open audio input: " << audioPath;
643 0 : return false;
644 : }
645 0 : if (m_ffmpegHandler->avformat_find_stream_info(audioFmtCtx, nullptr) < 0) {
646 0 : LOGE() << "addAudio: failed to find audio stream info";
647 0 : return false;
648 : }
649 :
650 0 : if (m_ffmpegHandler->avformat_alloc_output_context2(&outputFmtCtx, nullptr, "mp4", tmpPath.c_str()) < 0 || !outputFmtCtx) {
651 0 : LOGE() << "addAudio: failed to allocate output context";
652 0 : return false;
653 : }
654 :
655 0 : int videoInIdx = -1;
656 0 : int audioInIdx = -1;
657 0 : int outVideoIdx = -1;
658 0 : int outAudioIdx = -1;
659 :
660 0 : for (unsigned i = 0; i < videoFmtCtx->nb_streams; i++) {
661 0 : if (videoFmtCtx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
662 0 : videoInIdx = static_cast<int>(i);
663 0 : AVStream* outStream = m_ffmpegHandler->avformat_new_stream(outputFmtCtx, nullptr);
664 0 : if (!outStream) {
665 0 : LOGE() << "addAudio: failed to create output video stream";
666 0 : return false;
667 : }
668 0 : m_ffmpegHandler->avcodec_parameters_copy(outStream->codecpar, videoFmtCtx->streams[i]->codecpar);
669 0 : outStream->codecpar->codec_tag = 0;
670 0 : outStream->time_base = videoFmtCtx->streams[i]->time_base;
671 0 : outVideoIdx = outStream->index;
672 0 : break;
673 : }
674 : }
675 :
676 0 : for (unsigned i = 0; i < audioFmtCtx->nb_streams; i++) {
677 0 : if (audioFmtCtx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
678 0 : audioInIdx = static_cast<int>(i);
679 0 : AVStream* outStream = m_ffmpegHandler->avformat_new_stream(outputFmtCtx, nullptr);
680 0 : if (!outStream) {
681 0 : LOGE() << "addAudio: failed to create output audio stream";
682 0 : return false;
683 : }
684 0 : m_ffmpegHandler->avcodec_parameters_copy(outStream->codecpar, audioFmtCtx->streams[i]->codecpar);
685 0 : outStream->codecpar->codec_tag = 0;
686 0 : outStream->time_base = audioFmtCtx->streams[i]->time_base;
687 0 : outAudioIdx = outStream->index;
688 0 : break;
689 : }
690 : }
691 :
692 0 : if (videoInIdx < 0 || audioInIdx < 0) {
693 0 : LOGE() << "addAudio: could not find video or audio stream";
694 0 : return false;
695 : }
696 :
697 0 : if (m_ffmpegHandler->avio_open(&outputFmtCtx->pb, tmpPath.c_str(), AVIO_FLAG_WRITE) < 0) {
698 0 : LOGE() << "addAudio: failed to open output file: " << tmpPath;
699 0 : return false;
700 : }
701 :
702 0 : if (m_ffmpegHandler->avformat_write_header(outputFmtCtx, nullptr) < 0) {
703 0 : LOGE() << "addAudio: failed to write header";
704 0 : return false;
705 : }
706 :
707 0 : AVPacket* pkt = m_ffmpegHandler->av_packet_alloc();
708 :
709 0 : while (m_ffmpegHandler->av_read_frame(videoFmtCtx, pkt) >= 0) {
710 0 : if (pkt->stream_index == videoInIdx) {
711 0 : pkt->stream_index = outVideoIdx;
712 0 : m_ffmpegHandler->av_packet_rescale_ts(pkt, videoFmtCtx->streams[videoInIdx]->time_base,
713 0 : outputFmtCtx->streams[outVideoIdx]->time_base);
714 0 : pkt->pos = -1;
715 0 : m_ffmpegHandler->av_interleaved_write_frame(outputFmtCtx, pkt);
716 : }
717 0 : m_ffmpegHandler->av_packet_unref(pkt);
718 : }
719 :
720 0 : while (m_ffmpegHandler->av_read_frame(audioFmtCtx, pkt) >= 0) {
721 0 : if (pkt->stream_index == audioInIdx) {
722 0 : pkt->stream_index = outAudioIdx;
723 0 : m_ffmpegHandler->av_packet_rescale_ts(pkt, audioFmtCtx->streams[audioInIdx]->time_base,
724 0 : outputFmtCtx->streams[outAudioIdx]->time_base);
725 0 : pkt->pos = -1;
726 0 : m_ffmpegHandler->av_interleaved_write_frame(outputFmtCtx, pkt);
727 : }
728 0 : m_ffmpegHandler->av_packet_unref(pkt);
729 : }
730 :
731 0 : m_ffmpegHandler->av_packet_free(&pkt);
732 0 : m_ffmpegHandler->av_write_trailer(outputFmtCtx);
733 :
734 0 : cleanup();
735 :
736 0 : if (!io::File::copy(tmpPath, videoPath, true)) {
737 0 : LOGE() << "addAudio: failed to replace with muxed file";
738 0 : return false;
739 : }
740 :
741 0 : io::File::remove(tmpPath);
742 :
743 0 : return true;
744 0 : }
745 :
746 0 : bool VideoEncoder::convertImage_sws(const QImage& img)
747 : {
748 : // Check if the image matches the size
749 0 : if (img.width() != m_ffmpeg->width || img.height() != m_ffmpeg->height) {
750 0 : LOGE() << "wrong image size!";
751 0 : return false;
752 : }
753 :
754 0 : if (img.format() != QImage::Format_RGB32 && img.format() != QImage::Format_ARGB32) {
755 0 : LOGE() << "wrong image format";
756 0 : return false;
757 : }
758 :
759 0 : m_ffmpeg->img_convert_ctx = m_ffmpegHandler->sws_getCachedContext(m_ffmpeg->img_convert_ctx, m_ffmpeg->width, m_ffmpeg->height,
760 : AV_PIX_FMT_BGRA,
761 0 : m_ffmpeg->width, m_ffmpeg->height, AV_PIX_FMT_YUV420P, SWS_BICUBIC,
762 : NULL, NULL, NULL);
763 :
764 0 : if (!m_ffmpeg->img_convert_ctx) {
765 0 : LOGE() << "failed initialize the conversion context";
766 0 : return false;
767 : }
768 :
769 : uint8_t* srcplanes[3];
770 0 : srcplanes[0] = (uint8_t*)img.bits();
771 0 : srcplanes[1] = 0;
772 0 : srcplanes[2] = 0;
773 :
774 : int srcstride[3];
775 0 : srcstride[0] = img.bytesPerLine();
776 0 : srcstride[1] = 0;
777 0 : srcstride[2 ]= 0;
778 :
779 0 : m_ffmpegHandler->sws_scale(m_ffmpeg->img_convert_ctx, srcplanes, srcstride, 0, m_ffmpeg->height, m_ffmpeg->ppicture->data,
780 0 : m_ffmpeg->ppicture->linesize);
781 :
782 0 : return true;
783 : }
784 : }
|
