OpenShot Library | libopenshot  0.7.0
FFmpegReader.cpp
Go to the documentation of this file.
1 
12 // Copyright (c) 2008-2024 OpenShot Studios, LLC, Fabrice Bellard
13 //
14 // SPDX-License-Identifier: LGPL-3.0-or-later
15 
16 #include <thread> // for std::this_thread::sleep_for
17 #include <chrono> // for std::chrono::milliseconds
18 #include <algorithm>
19 #include <cmath>
20 #include <sstream>
21 #include <unistd.h>
22 
23 #include <QTransform>
24 
25 #include "FFmpegUtilities.h"
26 #include "effects/CropHelpers.h"
27 
28 #include "FFmpegReader.h"
29 #include "Exceptions.h"
30 #include "MemoryTrim.h"
31 #include "Timeline.h"
32 #include "ZmqLogger.h"
33 
34 #define ENABLE_VAAPI 0
35 
36 #if USE_HW_ACCEL
37 #define MAX_SUPPORTED_WIDTH 1950
38 #define MAX_SUPPORTED_HEIGHT 1100
39 
40 #if ENABLE_VAAPI
41 #include "libavutil/hwcontext_vaapi.h"
42 
43 typedef struct VAAPIDecodeContext {
44  VAProfile va_profile;
45  VAEntrypoint va_entrypoint;
46  VAConfigID va_config;
47  VAContextID va_context;
48 
49 #if FF_API_STRUCT_VAAPI_CONTEXT
50  // FF_DISABLE_DEPRECATION_WARNINGS
51  int have_old_context;
52  struct vaapi_context *old_context;
53  AVBufferRef *device_ref;
54  // FF_ENABLE_DEPRECATION_WARNINGS
55 #endif
56 
57  AVHWDeviceContext *device;
58  AVVAAPIDeviceContext *hwctx;
59 
60  AVHWFramesContext *frames;
61  AVVAAPIFramesContext *hwfc;
62 
63  enum AVPixelFormat surface_format;
64  int surface_count;
65  } VAAPIDecodeContext;
66 #endif // ENABLE_VAAPI
67 #endif // USE_HW_ACCEL
68 
69 
70 using namespace openshot;
71 
72 int hw_de_on = 0;
73 #if USE_HW_ACCEL
74  AVPixelFormat hw_de_av_pix_fmt_global = AV_PIX_FMT_NONE;
75  AVHWDeviceType hw_de_av_device_type_global = AV_HWDEVICE_TYPE_NONE;
76 #endif
77 
78 // Normalize deprecated JPEG-range YUVJ formats before creating swscale contexts.
79 // swscale expects non-YUVJ formats plus explicit color-range metadata.
80 static AVPixelFormat NormalizeDeprecatedPixFmt(AVPixelFormat pix_fmt, bool& is_full_range) {
81  switch (pix_fmt) {
82  case AV_PIX_FMT_YUVJ420P:
83  is_full_range = true;
84  return AV_PIX_FMT_YUV420P;
85  case AV_PIX_FMT_YUVJ422P:
86  is_full_range = true;
87  return AV_PIX_FMT_YUV422P;
88  case AV_PIX_FMT_YUVJ444P:
89  is_full_range = true;
90  return AV_PIX_FMT_YUV444P;
91  case AV_PIX_FMT_YUVJ440P:
92  is_full_range = true;
93  return AV_PIX_FMT_YUV440P;
94 #ifdef AV_PIX_FMT_YUVJ411P
95  case AV_PIX_FMT_YUVJ411P:
96  is_full_range = true;
97  return AV_PIX_FMT_YUV411P;
98 #endif
99  default:
100  return pix_fmt;
101  }
102 }
103 
104 FFmpegReader::FFmpegReader(const std::string &path, bool inspect_reader)
105  : FFmpegReader(path, DurationStrategy::VideoPreferred, inspect_reader) {}
106 
107 FFmpegReader::FFmpegReader(const std::string &path, DurationStrategy duration_strategy, bool inspect_reader)
108  : path(path), pFormatCtx(NULL), videoStream(-1), audioStream(-1), pCodecCtx(NULL), aCodecCtx(NULL),
109  pStream(NULL), aStream(NULL), packet(NULL), pFrame(NULL), is_open(false), is_duration_known(false),
110  check_interlace(false), check_fps(false), duration_strategy(duration_strategy), previous_packet_location{-1, 0},
111  is_seeking(false), seeking_pts(0), seeking_frame(0), is_video_seek(true), seek_count(0),
112  seek_audio_frame_found(0), seek_video_frame_found(0), last_seek_max_frame(-1), seek_stagnant_count(0),
113  last_frame(0), largest_frame_processed(0), current_video_frame(0), audio_pts(0), video_pts(0),
114  hold_packet(false), pts_offset_seconds(0.0), audio_pts_seconds(0.0), video_pts_seconds(0.0),
115  NO_PTS_OFFSET(-99999), source_width(0), source_height(0), source_rotation(0.0), enable_seek(true) {
116 
117  // Initialize FFMpeg, and register all formats and codecs
118  AV_REGISTER_ALL
119  AVCODEC_REGISTER_ALL
120 
121  // Init timestamp offsets
122  pts_offset_seconds = NO_PTS_OFFSET;
123  video_pts_seconds = NO_PTS_OFFSET;
124  audio_pts_seconds = NO_PTS_OFFSET;
125 
126  // Init cache
127  const int init_working_cache_frames = std::max(Settings::Instance()->CACHE_MIN_FRAMES, OPEN_MP_NUM_PROCESSORS * 4);
128  const int init_final_cache_frames = std::max(Settings::Instance()->CACHE_MIN_FRAMES, OPEN_MP_NUM_PROCESSORS * 4);
129  working_cache.SetMaxBytesFromInfo(init_working_cache_frames, info.width, info.height, info.sample_rate, info.channels);
130  final_cache.SetMaxBytesFromInfo(init_final_cache_frames, info.width, info.height, info.sample_rate, info.channels);
131 
132  // Open and Close the reader, to populate its attributes (such as height, width, etc...)
133  if (inspect_reader) {
134  Open();
135  Close();
136  }
137 }
138 
139 FFmpegReader::~FFmpegReader() {
140  if (is_open)
141  // Auto close reader if not already done
142  Close();
143 }
144 
145 // This struct holds the associated video frame and starting sample # for an audio packet.
146 bool AudioLocation::is_near(AudioLocation location, int samples_per_frame, int64_t amount) {
147  // Is frame even close to this one?
148  if (abs(location.frame - frame) >= 2)
149  // This is too far away to be considered
150  return false;
151 
152  // Note that samples_per_frame can vary slightly frame to frame when the
153  // audio sampling rate is not an integer multiple of the video fps.
154  int64_t diff = samples_per_frame * (location.frame - frame) + location.sample_start - sample_start;
155  if (abs(diff) <= amount)
156  // close
157  return true;
158 
159  // not close
160  return false;
161 }
162 
163 #if USE_HW_ACCEL
164 
165 // Get hardware pix format
166 static enum AVPixelFormat get_hw_dec_format(AVCodecContext *ctx, const enum AVPixelFormat *pix_fmts)
167 {
168  const enum AVPixelFormat *p;
169 
170  // Prefer only the format matching the selected hardware decoder
171  int selected = openshot::Settings::Instance()->HARDWARE_DECODER;
172 
173  for (p = pix_fmts; *p != AV_PIX_FMT_NONE; p++) {
174  switch (*p) {
175 #if defined(__linux__)
176  // Linux pix formats
177  case AV_PIX_FMT_VAAPI:
178  if (selected == 1) {
179  hw_de_av_pix_fmt_global = AV_PIX_FMT_VAAPI;
180  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_VAAPI;
181  return *p;
182  }
183  break;
184  case AV_PIX_FMT_VDPAU:
185  if (selected == 6) {
186  hw_de_av_pix_fmt_global = AV_PIX_FMT_VDPAU;
187  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_VDPAU;
188  return *p;
189  }
190  break;
191 #endif
192 #if defined(_WIN32)
193  // Windows pix formats
194  case AV_PIX_FMT_DXVA2_VLD:
195  if (selected == 3) {
196  hw_de_av_pix_fmt_global = AV_PIX_FMT_DXVA2_VLD;
197  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_DXVA2;
198  return *p;
199  }
200  break;
201  case AV_PIX_FMT_D3D11:
202  if (selected == 4) {
203  hw_de_av_pix_fmt_global = AV_PIX_FMT_D3D11;
204  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_D3D11VA;
205  return *p;
206  }
207  break;
208 #endif
209 #if defined(__APPLE__)
210  // Apple pix formats
211  case AV_PIX_FMT_VIDEOTOOLBOX:
212  if (selected == 5) {
213  hw_de_av_pix_fmt_global = AV_PIX_FMT_VIDEOTOOLBOX;
214  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_VIDEOTOOLBOX;
215  return *p;
216  }
217  break;
218 #endif
219  // Cross-platform pix formats
220  case AV_PIX_FMT_CUDA:
221  if (selected == 2) {
222  hw_de_av_pix_fmt_global = AV_PIX_FMT_CUDA;
223  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_CUDA;
224  return *p;
225  }
226  break;
227  case AV_PIX_FMT_QSV:
228  if (selected == 7) {
229  hw_de_av_pix_fmt_global = AV_PIX_FMT_QSV;
230  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_QSV;
231  return *p;
232  }
233  break;
234  default:
235  // This is only here to silence unused-enum warnings
236  break;
237  }
238  }
239  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::get_hw_dec_format (Unable to decode this file using hardware decode)");
240  return AV_PIX_FMT_NONE;
241 }
242 
243 int FFmpegReader::IsHardwareDecodeSupported(int codecid)
244 {
245  int ret;
246  switch (codecid) {
247  case AV_CODEC_ID_H264:
248  case AV_CODEC_ID_MPEG2VIDEO:
249  case AV_CODEC_ID_VC1:
250  case AV_CODEC_ID_WMV1:
251  case AV_CODEC_ID_WMV2:
252  case AV_CODEC_ID_WMV3:
253  ret = 1;
254  break;
255  default :
256  ret = 0;
257  break;
258  }
259  return ret;
260 }
261 #endif // USE_HW_ACCEL
262 
263 void FFmpegReader::Open() {
264  // Open reader if not already open
265  if (!is_open) {
266  // Prevent async calls to the following code
267  const std::lock_guard<std::recursive_mutex> lock(getFrameMutex);
268 
269  // Initialize format context
270  pFormatCtx = NULL;
271  {
272  hw_de_on = (!force_sw_decode && openshot::Settings::Instance()->HARDWARE_DECODER != 0 ? 1 : 0);
273  hw_decode_failed = false;
274  hw_decode_error_count = 0;
275  hw_decode_succeeded = false;
276  ZmqLogger::Instance()->AppendDebugMethod("Decode hardware acceleration settings", "hw_de_on", hw_de_on, "HARDWARE_DECODER", openshot::Settings::Instance()->HARDWARE_DECODER);
277  }
278 
279  // Open video file
280  if (avformat_open_input(&pFormatCtx, path.c_str(), NULL, NULL) != 0)
281  throw InvalidFile("FFmpegReader could not open media file.", path);
282 
283  // Retrieve stream information
284  if (avformat_find_stream_info(pFormatCtx, NULL) < 0)
285  throw NoStreamsFound("No streams found in file.", path);
286 
287  videoStream = -1;
288  audioStream = -1;
289 
290  // Init end-of-file detection variables
291  packet_status.reset(true);
292 
293  // Loop through each stream, and identify the video and audio stream index
294  for (unsigned int i = 0; i < pFormatCtx->nb_streams; i++) {
295  // Is this a video stream?
296  if (AV_GET_CODEC_TYPE(pFormatCtx->streams[i]) == AVMEDIA_TYPE_VIDEO && videoStream < 0) {
297  videoStream = i;
298  packet_status.video_eof = false;
299  packet_status.packets_eof = false;
300  packet_status.end_of_file = false;
301  }
302  // Is this an audio stream?
303  if (AV_GET_CODEC_TYPE(pFormatCtx->streams[i]) == AVMEDIA_TYPE_AUDIO && audioStream < 0) {
304  audioStream = i;
305  packet_status.audio_eof = false;
306  packet_status.packets_eof = false;
307  packet_status.end_of_file = false;
308  }
309  }
310  if (videoStream == -1 && audioStream == -1)
311  throw NoStreamsFound("No video or audio streams found in this file.", path);
312 
313  // Is there a video stream?
314  if (videoStream != -1) {
315  // Set the stream index
316  info.video_stream_index = videoStream;
317 
318  // Set the codec and codec context pointers
319  pStream = pFormatCtx->streams[videoStream];
320 
321  // Find the codec ID from stream
322  const AVCodecID codecId = AV_FIND_DECODER_CODEC_ID(pStream);
323 
324  // Get codec and codec context from stream
325  const AVCodec *pCodec = avcodec_find_decoder(codecId);
326  AVDictionary *opts = NULL;
327  int retry_decode_open = 2;
328  // If hw accel is selected but hardware cannot handle repeat with software decoding
329  do {
330  pCodecCtx = AV_GET_CODEC_CONTEXT(pStream, pCodec);
331 #if USE_HW_ACCEL
332  if (hw_de_on && (retry_decode_open==2)) {
333  // Up to here no decision is made if hardware or software decode
334  hw_de_supported = IsHardwareDecodeSupported(pCodecCtx->codec_id);
335  }
336 #endif
337  retry_decode_open = 0;
338 
339  // Set number of threads equal to number of processors (not to exceed 16)
340  pCodecCtx->thread_count = std::min(FF_VIDEO_NUM_PROCESSORS, 16);
341 
342  if (pCodec == NULL) {
343  throw InvalidCodec("A valid video codec could not be found for this file.", path);
344  }
345 
346  // Init options
347  av_dict_set(&opts, "strict", "experimental", 0);
348 #if USE_HW_ACCEL
349  if (hw_de_on && hw_de_supported) {
350  // Open Hardware Acceleration
351  int i_decoder_hw = 0;
352  char adapter[256];
353  char *adapter_ptr = NULL;
354  int adapter_num;
355  adapter_num = openshot::Settings::Instance()->HW_DE_DEVICE_SET;
356  ZmqLogger::Instance()->AppendDebugMethod("Hardware decoding device number", "adapter_num", adapter_num);
357 
358  // Set hardware pix format (callback)
359  pCodecCtx->get_format = get_hw_dec_format;
360 
361  if (adapter_num < 3 && adapter_num >=0) {
362 #if defined(__linux__)
363  snprintf(adapter,sizeof(adapter),"/dev/dri/renderD%d", adapter_num+128);
364  adapter_ptr = adapter;
365  i_decoder_hw = openshot::Settings::Instance()->HARDWARE_DECODER;
366  switch (i_decoder_hw) {
367  case 1:
368  hw_de_av_device_type = AV_HWDEVICE_TYPE_VAAPI;
369  break;
370  case 2:
371  hw_de_av_device_type = AV_HWDEVICE_TYPE_CUDA;
372  break;
373  case 6:
374  hw_de_av_device_type = AV_HWDEVICE_TYPE_VDPAU;
375  break;
376  case 7:
377  hw_de_av_device_type = AV_HWDEVICE_TYPE_QSV;
378  break;
379  default:
380  hw_de_av_device_type = AV_HWDEVICE_TYPE_VAAPI;
381  break;
382  }
383 
384 #elif defined(_WIN32)
385  adapter_ptr = NULL;
386  i_decoder_hw = openshot::Settings::Instance()->HARDWARE_DECODER;
387  switch (i_decoder_hw) {
388  case 2:
389  hw_de_av_device_type = AV_HWDEVICE_TYPE_CUDA;
390  break;
391  case 3:
392  hw_de_av_device_type = AV_HWDEVICE_TYPE_DXVA2;
393  break;
394  case 4:
395  hw_de_av_device_type = AV_HWDEVICE_TYPE_D3D11VA;
396  break;
397  case 7:
398  hw_de_av_device_type = AV_HWDEVICE_TYPE_QSV;
399  break;
400  default:
401  hw_de_av_device_type = AV_HWDEVICE_TYPE_DXVA2;
402  break;
403  }
404 #elif defined(__APPLE__)
405  adapter_ptr = NULL;
406  i_decoder_hw = openshot::Settings::Instance()->HARDWARE_DECODER;
407  switch (i_decoder_hw) {
408  case 5:
409  hw_de_av_device_type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX;
410  break;
411  case 7:
412  hw_de_av_device_type = AV_HWDEVICE_TYPE_QSV;
413  break;
414  default:
415  hw_de_av_device_type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX;
416  break;
417  }
418 #endif
419 
420  } else {
421  adapter_ptr = NULL; // Just to be sure
422  }
423 
424  // Check if it is there and writable
425 #if defined(__linux__)
426  if( adapter_ptr != NULL && access( adapter_ptr, W_OK ) == 0 ) {
427 #elif defined(_WIN32)
428  if( adapter_ptr != NULL ) {
429 #elif defined(__APPLE__)
430  if( adapter_ptr != NULL ) {
431 #endif
432  ZmqLogger::Instance()->AppendDebugMethod("Decode Device present using device");
433  }
434  else {
435  adapter_ptr = NULL; // use default
436  ZmqLogger::Instance()->AppendDebugMethod("Decode Device not present using default");
437  }
438 
439  hw_device_ctx = NULL;
440  // Here the first hardware initialisations are made
441  if (av_hwdevice_ctx_create(&hw_device_ctx, hw_de_av_device_type, adapter_ptr, NULL, 0) >= 0) {
442  const char* hw_name = av_hwdevice_get_type_name(hw_de_av_device_type);
443  std::string hw_msg = "HW decode active: ";
444  hw_msg += (hw_name ? hw_name : "unknown");
445  ZmqLogger::Instance()->Log(hw_msg);
446  if (!(pCodecCtx->hw_device_ctx = av_buffer_ref(hw_device_ctx))) {
447  throw InvalidCodec("Hardware device reference create failed.", path);
448  }
449 
450  /*
451  av_buffer_unref(&ist->hw_frames_ctx);
452  ist->hw_frames_ctx = av_hwframe_ctx_alloc(hw_device_ctx);
453  if (!ist->hw_frames_ctx) {
454  av_log(avctx, AV_LOG_ERROR, "Error creating a CUDA frames context\n");
455  return AVERROR(ENOMEM);
456  }
457 
458  frames_ctx = (AVHWFramesContext*)ist->hw_frames_ctx->data;
459 
460  frames_ctx->format = AV_PIX_FMT_CUDA;
461  frames_ctx->sw_format = avctx->sw_pix_fmt;
462  frames_ctx->width = avctx->width;
463  frames_ctx->height = avctx->height;
464 
465  av_log(avctx, AV_LOG_DEBUG, "Initializing CUDA frames context: sw_format = %s, width = %d, height = %d\n",
466  av_get_pix_fmt_name(frames_ctx->sw_format), frames_ctx->width, frames_ctx->height);
467 
468 
469  ret = av_hwframe_ctx_init(pCodecCtx->hw_device_ctx);
470  ret = av_hwframe_ctx_init(ist->hw_frames_ctx);
471  if (ret < 0) {
472  av_log(avctx, AV_LOG_ERROR, "Error initializing a CUDA frame pool\n");
473  return ret;
474  }
475  */
476  }
477  else {
478  ZmqLogger::Instance()->Log("HW decode active: no (falling back to software)");
479  throw InvalidCodec("Hardware device create failed.", path);
480  }
481  }
482 #endif // USE_HW_ACCEL
483 
484  // Disable per-frame threading for album arts
485  // Using FF_THREAD_FRAME adds one frame decoding delay per thread,
486  // but there's only one frame in this case.
487  if (HasAlbumArt())
488  {
489  pCodecCtx->thread_type &= ~FF_THREAD_FRAME;
490  }
491 
492  // Open video codec
493  int avcodec_return = avcodec_open2(pCodecCtx, pCodec, &opts);
494  if (avcodec_return < 0) {
495  std::stringstream avcodec_error_msg;
496  avcodec_error_msg << "A video codec was found, but could not be opened. Error: " << av_err2string(avcodec_return);
497  throw InvalidCodec(avcodec_error_msg.str(), path);
498  }
499 
500 #if USE_HW_ACCEL
501  if (hw_de_on && hw_de_supported) {
502  AVHWFramesConstraints *constraints = NULL;
503  void *hwconfig = NULL;
504  hwconfig = av_hwdevice_hwconfig_alloc(hw_device_ctx);
505 
506 // TODO: needs va_config!
507 #if ENABLE_VAAPI
508  ((AVVAAPIHWConfig *)hwconfig)->config_id = ((VAAPIDecodeContext *)(pCodecCtx->priv_data))->va_config;
509  constraints = av_hwdevice_get_hwframe_constraints(hw_device_ctx,hwconfig);
510 #endif // ENABLE_VAAPI
511  if (constraints) {
512  if (pCodecCtx->coded_width < constraints->min_width ||
513  pCodecCtx->coded_height < constraints->min_height ||
514  pCodecCtx->coded_width > constraints->max_width ||
515  pCodecCtx->coded_height > constraints->max_height) {
516  ZmqLogger::Instance()->AppendDebugMethod("DIMENSIONS ARE TOO LARGE for hardware acceleration\n");
517  hw_de_supported = 0;
518  retry_decode_open = 1;
519  AV_FREE_CONTEXT(pCodecCtx);
520  if (hw_device_ctx) {
521  av_buffer_unref(&hw_device_ctx);
522  hw_device_ctx = NULL;
523  }
524  }
525  else {
526  // All is just peachy
527  ZmqLogger::Instance()->AppendDebugMethod("\nDecode hardware acceleration is used\n", "Min width :", constraints->min_width, "Min Height :", constraints->min_height, "MaxWidth :", constraints->max_width, "MaxHeight :", constraints->max_height, "Frame width :", pCodecCtx->coded_width, "Frame height :", pCodecCtx->coded_height);
528  retry_decode_open = 0;
529  }
530  av_hwframe_constraints_free(&constraints);
531  if (hwconfig) {
532  av_freep(&hwconfig);
533  }
534  }
535  else {
536  int max_h, max_w;
537  //max_h = ((getenv( "LIMIT_HEIGHT_MAX" )==NULL) ? MAX_SUPPORTED_HEIGHT : atoi(getenv( "LIMIT_HEIGHT_MAX" )));
538  max_h = openshot::Settings::Instance()->DE_LIMIT_HEIGHT_MAX;
539  //max_w = ((getenv( "LIMIT_WIDTH_MAX" )==NULL) ? MAX_SUPPORTED_WIDTH : atoi(getenv( "LIMIT_WIDTH_MAX" )));
540  max_w = openshot::Settings::Instance()->DE_LIMIT_WIDTH_MAX;
541  ZmqLogger::Instance()->AppendDebugMethod("Constraints could not be found using default limit\n");
542  //cerr << "Constraints could not be found using default limit\n";
543  if (pCodecCtx->coded_width < 0 ||
544  pCodecCtx->coded_height < 0 ||
545  pCodecCtx->coded_width > max_w ||
546  pCodecCtx->coded_height > max_h ) {
547  ZmqLogger::Instance()->AppendDebugMethod("DIMENSIONS ARE TOO LARGE for hardware acceleration\n", "Max Width :", max_w, "Max Height :", max_h, "Frame width :", pCodecCtx->coded_width, "Frame height :", pCodecCtx->coded_height);
548  hw_de_supported = 0;
549  retry_decode_open = 1;
550  AV_FREE_CONTEXT(pCodecCtx);
551  if (hw_device_ctx) {
552  av_buffer_unref(&hw_device_ctx);
553  hw_device_ctx = NULL;
554  }
555  }
556  else {
557  ZmqLogger::Instance()->AppendDebugMethod("\nDecode hardware acceleration is used\n", "Max Width :", max_w, "Max Height :", max_h, "Frame width :", pCodecCtx->coded_width, "Frame height :", pCodecCtx->coded_height);
558  retry_decode_open = 0;
559  }
560  }
561  } // if hw_de_on && hw_de_supported
562  else {
563  ZmqLogger::Instance()->AppendDebugMethod("\nDecode in software is used\n");
564  }
565 #else
566  retry_decode_open = 0;
567 #endif // USE_HW_ACCEL
568  } while (retry_decode_open); // retry_decode_open
569  // Free options
570  av_dict_free(&opts);
571 
572  // Update the File Info struct with video details (if a video stream is found)
573  UpdateVideoInfo();
574  }
575 
576  // Is there an audio stream?
577  if (audioStream != -1) {
578  // Set the stream index
579  info.audio_stream_index = audioStream;
580 
581  // Get a pointer to the codec context for the audio stream
582  aStream = pFormatCtx->streams[audioStream];
583 
584  // Find the codec ID from stream
585  AVCodecID codecId = AV_FIND_DECODER_CODEC_ID(aStream);
586 
587  // Get codec and codec context from stream
588  const AVCodec *aCodec = avcodec_find_decoder(codecId);
589  aCodecCtx = AV_GET_CODEC_CONTEXT(aStream, aCodec);
590 
591  // Audio encoding does not typically use more than 2 threads (most codecs use 1 thread)
592  aCodecCtx->thread_count = std::min(FF_AUDIO_NUM_PROCESSORS, 2);
593 
594  bool audio_opened = false;
595  if (aCodec != NULL) {
596  // Init options
597  AVDictionary *opts = NULL;
598  av_dict_set(&opts, "strict", "experimental", 0);
599 
600  // Open audio codec
601  audio_opened = (avcodec_open2(aCodecCtx, aCodec, &opts) >= 0);
602 
603  // Free options
604  av_dict_free(&opts);
605  }
606 
607  if (audio_opened) {
608  // Update the File Info struct with audio details (if an audio stream is found)
609  UpdateAudioInfo();
610 
611  // Disable malformed audio stream metadata (prevents divide-by-zero / invalid resampling math)
612  const bool invalid_audio_info =
613  (info.channels <= 0) ||
614  (info.sample_rate <= 0) ||
615  (info.audio_timebase.num <= 0) ||
616  (info.audio_timebase.den <= 0) ||
617  (aCodecCtx->sample_fmt == AV_SAMPLE_FMT_NONE);
618  if (invalid_audio_info) {
619  ZmqLogger::Instance()->AppendDebugMethod(
620  "FFmpegReader::Open (Disable invalid audio stream)",
621  "channels", info.channels,
622  "sample_rate", info.sample_rate,
623  "audio_timebase.num", info.audio_timebase.num,
624  "audio_timebase.den", info.audio_timebase.den,
625  "sample_fmt", static_cast<int>(aCodecCtx ? aCodecCtx->sample_fmt : AV_SAMPLE_FMT_NONE));
626  info.has_audio = false;
627  info.audio_stream_index = -1;
628  audioStream = -1;
629  packet_status.audio_eof = true;
630  if (aCodecCtx) {
631  if (avcodec_is_open(aCodecCtx)) {
632  avcodec_flush_buffers(aCodecCtx);
633  }
634  AV_FREE_CONTEXT(aCodecCtx);
635  aCodecCtx = nullptr;
636  }
637  aStream = nullptr;
638  }
639  } else {
640  // Keep decoding video, but disable bad/unsupported audio stream.
641  ZmqLogger::Instance()->AppendDebugMethod(
642  "FFmpegReader::Open (Audio codec unavailable; disabling audio)",
643  "audioStream", audioStream);
644  info.has_audio = false;
645  info.audio_stream_index = -1;
646  audioStream = -1;
647  packet_status.audio_eof = true;
648  if (aCodecCtx) {
649  AV_FREE_CONTEXT(aCodecCtx);
650  aCodecCtx = nullptr;
651  }
652  aStream = nullptr;
653  }
654  }
655 
656  // Guard invalid frame-rate / timebase values from malformed streams.
657  if (info.fps.num <= 0 || info.fps.den <= 0) {
658  ZmqLogger::Instance()->AppendDebugMethod(
659  "FFmpegReader::Open (Invalid FPS detected; applying fallback)",
660  "fps.num", info.fps.num,
661  "fps.den", info.fps.den);
662  info.fps.num = 30;
663  info.fps.den = 1;
664  }
665  if (info.video_timebase.num <= 0 || info.video_timebase.den <= 0) {
666  ZmqLogger::Instance()->AppendDebugMethod(
667  "FFmpegReader::Open (Invalid video_timebase detected; applying fallback)",
668  "video_timebase.num", info.video_timebase.num,
669  "video_timebase.den", info.video_timebase.den);
670  info.video_timebase = info.fps.Reciprocal();
671  }
672 
673  // Add format metadata (if any)
674  AVDictionaryEntry *tag = NULL;
675  while ((tag = av_dict_get(pFormatCtx->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) {
676  QString str_key = tag->key;
677  QString str_value = tag->value;
678  info.metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
679  }
680 
681  // Process video stream side data (rotation, spherical metadata, etc)
682  for (unsigned int i = 0; i < pFormatCtx->nb_streams; i++) {
683  AVStream* st = pFormatCtx->streams[i];
684  if (st->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
685  size_t side_data_size = 0;
686  const uint8_t *displaymatrix = ffmpeg_stream_get_side_data(
687  st, AV_PKT_DATA_DISPLAYMATRIX, &side_data_size);
688  if (displaymatrix &&
689  side_data_size >= 9 * sizeof(int32_t) &&
690  !info.metadata.count("rotate")) {
691  double rotation = -av_display_rotation_get(
692  reinterpret_cast<const int32_t *>(displaymatrix));
693  if (std::isnan(rotation))
694  rotation = 0;
695  info.metadata["rotate"] = std::to_string(rotation);
696  }
697 
698  const uint8_t *spherical = ffmpeg_stream_get_side_data(
699  st, AV_PKT_DATA_SPHERICAL, &side_data_size);
700  if (spherical && side_data_size >= sizeof(AVSphericalMapping)) {
701  info.metadata["spherical"] = "1";
702 
703  const AVSphericalMapping *map =
704  reinterpret_cast<const AVSphericalMapping *>(spherical);
705  const char *proj_name = av_spherical_projection_name(map->projection);
706  info.metadata["spherical_projection"] = proj_name ? proj_name : "unknown";
707 
708  auto to_deg = [](int32_t v) {
709  return static_cast<double>(v) / 65536.0;
710  };
711  info.metadata["spherical_yaw"] = std::to_string(to_deg(map->yaw));
712  info.metadata["spherical_pitch"] = std::to_string(to_deg(map->pitch));
713  info.metadata["spherical_roll"] = std::to_string(to_deg(map->roll));
714  }
715  break;
716  }
717  }
718  UpdateOrientedVideoInfo();
719 
720  // Init previous audio location to zero
721  previous_packet_location.frame = -1;
722  previous_packet_location.sample_start = 0;
723 
724  // Adjust cache size based on size of frame and audio
725  const int working_cache_frames = std::max(Settings::Instance()->CACHE_MIN_FRAMES, int(OPEN_MP_NUM_PROCESSORS * info.fps.ToDouble() * 2));
726  const int final_cache_frames = std::max(Settings::Instance()->CACHE_MIN_FRAMES, OPEN_MP_NUM_PROCESSORS * 2);
727  working_cache.SetMaxBytesFromInfo(working_cache_frames, info.width, info.height, info.sample_rate, info.channels);
728  final_cache.SetMaxBytesFromInfo(final_cache_frames, info.width, info.height, info.sample_rate, info.channels);
729 
730  // Scan PTS for any offsets (i.e. non-zero starting streams). At least 1 stream must start at zero timestamp.
731  // This method allows us to shift timestamps to ensure at least 1 stream is starting at zero.
732  UpdatePTSOffset();
733 
734  // Override an invalid framerate
735  if (info.fps.ToFloat() > 240.0f || (info.fps.num <= 0 || info.fps.den <= 0) || info.video_length <= 0) {
736  // Calculate FPS, duration, video bit rate, and video length manually
737  // by scanning through all the video stream packets
738  CheckFPS();
739  }
740 
741  // Mark as "open"
742  is_open = true;
743 
744  // Seek back to beginning of file (if not already seeking)
745  if (!is_seeking) {
746  Seek(1);
747  }
748  }
749 }
750 
751 void FFmpegReader::Close() {
752  // Close all objects, if reader is 'open'
753  if (is_open) {
754  // Prevent async calls to the following code
755  const std::lock_guard<std::recursive_mutex> lock(getFrameMutex);
756 
757  // Mark as "closed"
758  is_open = false;
759 
760  // Keep track of most recent packet
761  AVPacket *recent_packet = packet;
762 
763  // Drain any packets from the decoder
764  packet = NULL;
765  int attempts = 0;
766  int max_attempts = 128;
767  while (packet_status.packets_decoded() < packet_status.packets_read() && attempts < max_attempts) {
768  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::Close (Drain decoder loop)",
769  "packets_read", packet_status.packets_read(),
770  "packets_decoded", packet_status.packets_decoded(),
771  "attempts", attempts);
772  if (packet_status.video_decoded < packet_status.video_read) {
773  ProcessVideoPacket(info.video_length);
774  }
775  if (packet_status.audio_decoded < packet_status.audio_read) {
776  ProcessAudioPacket(info.video_length);
777  }
778  attempts++;
779  }
780 
781  // Remove packet
782  if (recent_packet) {
783  RemoveAVPacket(recent_packet);
784  }
785 
786  // Close the video codec
787  if (info.has_video) {
788  if(avcodec_is_open(pCodecCtx)) {
789  avcodec_flush_buffers(pCodecCtx);
790  }
791  AV_FREE_CONTEXT(pCodecCtx);
792 #if USE_HW_ACCEL
793  if (hw_de_on) {
794  if (hw_device_ctx) {
795  av_buffer_unref(&hw_device_ctx);
796  hw_device_ctx = NULL;
797  }
798  }
799 #endif // USE_HW_ACCEL
800  if (img_convert_ctx) {
801  sws_freeContext(img_convert_ctx);
802  img_convert_ctx = nullptr;
803  }
804  if (pFrameRGB_cached) {
805  AV_FREE_FRAME(&pFrameRGB_cached);
806  }
807  }
808 
809  // Close the audio codec
810  if (info.has_audio) {
811  if(avcodec_is_open(aCodecCtx)) {
812  avcodec_flush_buffers(aCodecCtx);
813  }
814  AV_FREE_CONTEXT(aCodecCtx);
815  if (avr_ctx) {
816  SWR_CLOSE(avr_ctx);
817  SWR_FREE(&avr_ctx);
818  avr_ctx = nullptr;
819  }
820  }
821 
822  // Clear final cache
823  final_cache.Clear();
824  working_cache.Clear();
825 
826  // Close the video file
827  avformat_close_input(&pFormatCtx);
828  av_freep(&pFormatCtx);
829 
830  // Do not trim here; trimming is handled on explicit cache clears
831 
832  // Reset some variables
833  last_frame = 0;
834  hold_packet = false;
835  largest_frame_processed = 0;
836  seek_audio_frame_found = 0;
837  seek_video_frame_found = 0;
838  current_video_frame = 0;
839  last_video_frame.reset();
840  last_final_video_frame.reset();
841  }
842 }
843 
844 bool FFmpegReader::HasAlbumArt() {
845  // Check if the video stream we use is an attached picture
846  // This won't return true if the file has a cover image as a secondary stream
847  // like an MKV file with an attached image file
848  return pFormatCtx && videoStream >= 0 && pFormatCtx->streams[videoStream]
849  && (pFormatCtx->streams[videoStream]->disposition & AV_DISPOSITION_ATTACHED_PIC);
850 }
851 
852 double FFmpegReader::PickDurationSeconds() const {
853  auto has_value = [](double value) { return value > 0.0; };
854 
855  switch (duration_strategy) {
856  case DurationStrategy::VideoPreferred:
857  if (has_value(video_stream_duration_seconds))
858  return video_stream_duration_seconds;
859  if (has_value(audio_stream_duration_seconds))
860  return audio_stream_duration_seconds;
861  if (has_value(format_duration_seconds))
862  return format_duration_seconds;
863  break;
864  case DurationStrategy::AudioPreferred:
865  if (has_value(audio_stream_duration_seconds))
866  return audio_stream_duration_seconds;
867  if (has_value(video_stream_duration_seconds))
868  return video_stream_duration_seconds;
869  if (has_value(format_duration_seconds))
870  return format_duration_seconds;
871  break;
872  case DurationStrategy::LongestStream:
873  default:
874  {
875  double longest = 0.0;
876  if (has_value(video_stream_duration_seconds))
877  longest = std::max(longest, video_stream_duration_seconds);
878  if (has_value(audio_stream_duration_seconds))
879  longest = std::max(longest, audio_stream_duration_seconds);
880  if (has_value(format_duration_seconds))
881  longest = std::max(longest, format_duration_seconds);
882  if (has_value(longest))
883  return longest;
884  }
885  break;
886  }
887 
888  if (has_value(format_duration_seconds))
889  return format_duration_seconds;
890  if (has_value(inferred_duration_seconds))
891  return inferred_duration_seconds;
892 
893  return 0.0;
894 }
895 
896 void FFmpegReader::ApplyDurationStrategy() {
897  const double fps_value = info.fps.ToDouble();
898  const double chosen_seconds = PickDurationSeconds();
899 
900  if (chosen_seconds <= 0.0 || fps_value <= 0.0) {
901  info.duration = 0.0f;
902  info.video_length = 0;
903  is_duration_known = false;
904  return;
905  }
906 
907  const int64_t frames = static_cast<int64_t>(std::llround(chosen_seconds * fps_value));
908  if (frames <= 0) {
909  info.duration = 0.0f;
910  info.video_length = 0;
911  is_duration_known = false;
912  return;
913  }
914 
915  info.video_length = frames;
916  info.duration = static_cast<float>(static_cast<double>(frames) / fps_value);
917  is_duration_known = true;
918 }
919 
920 void FFmpegReader::UpdateAudioInfo() {
921  int codec_channels =
922 #if HAVE_CH_LAYOUT
923  AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->ch_layout.nb_channels;
924 #else
925  AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels;
926 #endif
927 
928  // Set default audio channel layout (if needed)
929 #if HAVE_CH_LAYOUT
930  AVChannelLayout audio_ch_layout = ffmpeg_get_valid_channel_layout(
931  AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->ch_layout, codec_channels);
932  if (audio_ch_layout.nb_channels > 0) {
933  av_channel_layout_uninit(&(AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->ch_layout));
934  av_channel_layout_copy(&(AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->ch_layout), &audio_ch_layout);
935  codec_channels = audio_ch_layout.nb_channels;
936  }
937 #else
938  if (codec_channels > 0 && AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout == 0)
939  AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout = av_get_default_channel_layout(AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels);
940 #endif
941 
942  if (info.sample_rate > 0) {
943  // Skip init - if info struct already populated
944  return;
945  }
946 
947  auto record_duration = [](double &target, double seconds) {
948  if (seconds > 0.0)
949  target = std::max(target, seconds);
950  };
951 
952  // Set values of FileInfo struct
953  info.has_audio = true;
954  info.file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;
955  info.acodec = aCodecCtx->codec->name;
956 #if HAVE_CH_LAYOUT
957  info.channels = audio_ch_layout.nb_channels;
958  info.channel_layout = static_cast<ChannelLayout>(ffmpeg_channel_layout_mask(audio_ch_layout));
959 #else
960  info.channels = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels;
961  info.channel_layout = (ChannelLayout) AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout;
962 #endif
963 
964  // If channel layout is not set, guess based on the number of channels
965  if (info.channel_layout == 0) {
966  if (info.channels == 1) {
967  info.channel_layout = openshot::LAYOUT_MONO;
968  } else if (info.channels == 2) {
969  info.channel_layout = openshot::LAYOUT_STEREO;
970  }
971  }
972 
973  info.sample_rate = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->sample_rate;
974  info.audio_bit_rate = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->bit_rate;
975  if (info.audio_bit_rate <= 0) {
976  // Get bitrate from format
977  info.audio_bit_rate = pFormatCtx->bit_rate;
978  }
979 
980  // Set audio timebase
981  info.audio_timebase.num = aStream->time_base.num;
982  info.audio_timebase.den = aStream->time_base.den;
983 
984  // Get timebase of audio stream (if valid) and greater than the current duration
985  if (aStream->duration > 0) {
986  record_duration(audio_stream_duration_seconds, aStream->duration * info.audio_timebase.ToDouble());
987  }
988  if (pFormatCtx->duration > 0) {
989  // Use the format's duration when stream duration is missing or shorter
990  record_duration(format_duration_seconds, static_cast<double>(pFormatCtx->duration) / AV_TIME_BASE);
991  }
992 
993  // Calculate duration from filesize and bitrate (if any)
994  if (info.duration <= 0.0f && info.video_bit_rate > 0 && info.file_size > 0) {
995  // Estimate from bitrate, total bytes, and framerate
996  record_duration(inferred_duration_seconds, static_cast<double>(info.file_size) / info.video_bit_rate);
997  }
998 
999  // Set video timebase (if no video stream was found)
1000  if (!info.has_video) {
1001  // Set a few important default video settings (so audio can be divided into frames)
1002  info.fps.num = 30;
1003  info.fps.den = 1;
1004  info.video_timebase.num = 1;
1005  info.video_timebase.den = 30;
1006  if (info.width <= 0 || info.height <= 0) {
1007  info.width = 720;
1008  info.height = 480;
1009  }
1010 
1011  // Use timeline to set correct width & height (if any)
1012  Clip *parent = static_cast<Clip *>(ParentClip());
1013  if (parent) {
1014  if (parent->ParentTimeline()) {
1015  // Set max width/height based on parent clip's timeline (if attached to a timeline)
1016  info.width = parent->ParentTimeline()->preview_width;
1017  info.height = parent->ParentTimeline()->preview_height;
1018  }
1019  }
1020  }
1021 
1022  ApplyDurationStrategy();
1023 
1024  // Add audio metadata (if any found)
1025  AVDictionaryEntry *tag = NULL;
1026  while ((tag = av_dict_get(aStream->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) {
1027  QString str_key = tag->key;
1028  QString str_value = tag->value;
1029  info.metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
1030  }
1031 #if HAVE_CH_LAYOUT
1032  av_channel_layout_uninit(&audio_ch_layout);
1033 #endif
1034 }
1035 
1036 void FFmpegReader::UpdateVideoInfo() {
1037  if (info.vcodec.length() > 0) {
1038  // JSON-loaded readers may already have display-oriented info populated,
1039  // but decoding still needs the raw stream dimensions for orientation.
1040  if (info.has_video && pStream && pCodecCtx) {
1041  source_height = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->height;
1042  source_width = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->width;
1043  UpdateOrientedVideoInfo();
1044  }
1045  return;
1046  }
1047 
1048  auto record_duration = [](double &target, double seconds) {
1049  if (seconds > 0.0)
1050  target = std::max(target, seconds);
1051  };
1052 
1053  // Set values of FileInfo struct
1054  info.has_video = true;
1055  info.file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;
1056  source_height = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->height;
1057  source_width = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->width;
1058  info.height = source_height;
1059  info.width = source_width;
1060  info.vcodec = pCodecCtx->codec->name;
1061  info.video_bit_rate = (pFormatCtx->bit_rate / 8);
1062 
1063  // Frame rate from the container and codec
1064  AVRational framerate = av_guess_frame_rate(pFormatCtx, pStream, NULL);
1065  if (!check_fps) {
1066  info.fps.num = framerate.num;
1067  info.fps.den = framerate.den;
1068  }
1069 
1070  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::UpdateVideoInfo", "info.fps.num", info.fps.num, "info.fps.den", info.fps.den);
1071 
1072  // TODO: remove excessive debug info in the next releases
1073  // The debug info below is just for comparison and troubleshooting on users side during the transition period
1074  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::UpdateVideoInfo (pStream->avg_frame_rate)", "num", pStream->avg_frame_rate.num, "den", pStream->avg_frame_rate.den);
1075 
1076  if (pStream->sample_aspect_ratio.num != 0) {
1077  info.pixel_ratio.num = pStream->sample_aspect_ratio.num;
1078  info.pixel_ratio.den = pStream->sample_aspect_ratio.den;
1079  } else if (AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->sample_aspect_ratio.num != 0) {
1080  info.pixel_ratio.num = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->sample_aspect_ratio.num;
1081  info.pixel_ratio.den = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->sample_aspect_ratio.den;
1082  } else {
1083  info.pixel_ratio.num = 1;
1084  info.pixel_ratio.den = 1;
1085  }
1086  info.pixel_format = AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx);
1087 
1088  // Calculate the DAR (display aspect ratio)
1089  Fraction size(info.width * info.pixel_ratio.num, info.height * info.pixel_ratio.den);
1090 
1091  // Reduce size fraction
1092  size.Reduce();
1093 
1094  // Set the ratio based on the reduced fraction
1095  info.display_ratio.num = size.num;
1096  info.display_ratio.den = size.den;
1097 
1098  // Get scan type and order from codec context/params
1099  if (!check_interlace) {
1100  check_interlace = true;
1101  AVFieldOrder field_order = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->field_order;
1102  switch(field_order) {
1103  case AV_FIELD_PROGRESSIVE:
1104  info.interlaced_frame = false;
1105  break;
1106  case AV_FIELD_TT:
1107  case AV_FIELD_TB:
1108  info.interlaced_frame = true;
1109  info.top_field_first = true;
1110  break;
1111  case AV_FIELD_BT:
1112  case AV_FIELD_BB:
1113  info.interlaced_frame = true;
1114  info.top_field_first = false;
1115  break;
1116  case AV_FIELD_UNKNOWN:
1117  // Check again later?
1118  check_interlace = false;
1119  break;
1120  }
1121  // check_interlace will prevent these checks being repeated,
1122  // unless it was cleared because we got an AV_FIELD_UNKNOWN response.
1123  }
1124 
1125  // Set the video timebase
1126  info.video_timebase.num = pStream->time_base.num;
1127  info.video_timebase.den = pStream->time_base.den;
1128 
1129  // Set the duration in seconds, and video length (# of frames)
1130  record_duration(video_stream_duration_seconds, pStream->duration * info.video_timebase.ToDouble());
1131 
1132  // Check for valid duration (if found)
1133  if (pFormatCtx->duration >= 0) {
1134  // Use the format's duration as another candidate
1135  record_duration(format_duration_seconds, static_cast<double>(pFormatCtx->duration) / AV_TIME_BASE);
1136  }
1137 
1138  // Calculate duration from filesize and bitrate (if any)
1139  if (info.video_bit_rate > 0 && info.file_size > 0) {
1140  // Estimate from bitrate, total bytes, and framerate
1141  record_duration(inferred_duration_seconds, static_cast<double>(info.file_size) / info.video_bit_rate);
1142  }
1143 
1144  // Certain "image" formats do not have a valid duration
1145  if (video_stream_duration_seconds <= 0.0 && format_duration_seconds <= 0.0 &&
1146  pStream->duration == AV_NOPTS_VALUE && pFormatCtx->duration == AV_NOPTS_VALUE) {
1147  // Force an "image" duration
1148  record_duration(video_stream_duration_seconds, 60 * 60 * 1); // 1 hour duration
1149  info.has_single_image = true;
1150  }
1151  // Static GIFs can have no usable duration; fall back to a small default
1152  if (video_stream_duration_seconds <= 0.0 && format_duration_seconds <= 0.0 &&
1153  pFormatCtx && pFormatCtx->iformat && strcmp(pFormatCtx->iformat->name, "gif") == 0) {
1154  record_duration(video_stream_duration_seconds, 60 * 60 * 1); // 1 hour duration
1155  info.has_single_image = true;
1156  }
1157 
1158  ApplyDurationStrategy();
1159 
1160  // Normalize FFmpeg-decoded still images (e.g. JPG/JPEG) to match image-reader behavior.
1161  // This keeps timing/flags consistent regardless of which reader path was used.
1162  if (!info.has_single_image) {
1163  const AVCodecID codec_id = AV_FIND_DECODER_CODEC_ID(pStream);
1164  const bool likely_still_codec =
1165  codec_id == AV_CODEC_ID_MJPEG ||
1166  codec_id == AV_CODEC_ID_PNG ||
1167  codec_id == AV_CODEC_ID_BMP ||
1168  codec_id == AV_CODEC_ID_TIFF ||
1169  codec_id == AV_CODEC_ID_WEBP ||
1170  codec_id == AV_CODEC_ID_JPEG2000;
1171  const bool likely_image_demuxer =
1172  pFormatCtx && pFormatCtx->iformat && pFormatCtx->iformat->name &&
1173  strstr(pFormatCtx->iformat->name, "image2");
1174  const bool has_attached_pic = HasAlbumArt();
1175  const bool single_frame_stream =
1176  (pStream && pStream->nb_frames > 0 && pStream->nb_frames <= 1);
1177  const bool single_frame_clip = info.video_length <= 1;
1178 
1179  const bool is_still_image_video =
1180  has_attached_pic ||
1181  ((single_frame_stream || single_frame_clip) &&
1182  (likely_still_codec || likely_image_demuxer));
1183 
1184  if (is_still_image_video) {
1185  info.has_single_image = true;
1186 
1187  // Only force long duration for standalone images. For audio + attached-art
1188  // files, keep stream-derived duration so the cover image spans the audio.
1189  if (audioStream < 0) {
1190  record_duration(video_stream_duration_seconds, 60 * 60 * 1); // 1 hour duration
1191  }
1192 
1193  ApplyDurationStrategy();
1194  }
1195  }
1196 
1197  // Add video metadata (if any)
1198  AVDictionaryEntry *tag = NULL;
1199  while ((tag = av_dict_get(pStream->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) {
1200  QString str_key = tag->key;
1201  QString str_value = tag->value;
1202  info.metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
1203  }
1204 
1205  UpdateOrientedVideoInfo();
1206 }
1207 
1208 void FFmpegReader::UpdateOrientedVideoInfo() {
1209  if (!ApplyOrientationMetadata() || !info.has_video)
1210  return;
1211 
1212  if (source_width <= 0 || source_height <= 0) {
1213  source_width = info.width;
1214  source_height = info.height;
1215  }
1216  if (source_width <= 0 || source_height <= 0)
1217  return;
1218 
1219  source_rotation = 0.0;
1220  const auto rotate_meta = info.metadata.find("rotate");
1221  if (rotate_meta != info.metadata.end())
1222  source_rotation = strtod(rotate_meta->second.c_str(), nullptr);
1223 
1224  const double radians = source_rotation * M_PI / 180.0;
1225  const int oriented_width = static_cast<int>(std::round(
1226  std::fabs(source_width * std::cos(radians)) +
1227  std::fabs(source_height * std::sin(radians))));
1228  const int oriented_height = static_cast<int>(std::round(
1229  std::fabs(source_width * std::sin(radians)) +
1230  std::fabs(source_height * std::cos(radians))));
1231 
1232  if (oriented_width > 0 && oriented_height > 0) {
1233  info.width = oriented_width;
1234  info.height = oriented_height;
1235 
1236  Fraction size(info.width * info.pixel_ratio.num, info.height * info.pixel_ratio.den);
1237  size.Reduce();
1238  info.display_ratio.num = size.num;
1239  info.display_ratio.den = size.den;
1240  }
1241 }
1242 
1243 void FFmpegReader::ApplyFrameOrientation(std::shared_ptr<openshot::Frame> frame) {
1244  if (!ApplyOrientationMetadata() || std::fabs(source_rotation) < 0.0001 || !frame || !frame->has_image_data)
1245  return;
1246 
1247  auto image = frame->GetImage();
1248  if (!image || image->isNull())
1249  return;
1250 
1251  QImage oriented = image->transformed(QTransform().rotate(source_rotation), Qt::SmoothTransformation);
1252  frame->AddImage(std::make_shared<QImage>(oriented));
1253 }
1254 
1255 bool FFmpegReader::GetIsDurationKnown() {
1256  return this->is_duration_known;
1257 }
1258 
1259 std::shared_ptr<Frame> FFmpegReader::GetFrame(int64_t requested_frame) {
1260  last_seek_max_frame = -1;
1261  seek_stagnant_count = 0;
1262  // Check for open reader (or throw exception)
1263  if (!is_open)
1264  throw ReaderClosed("The FFmpegReader is closed. Call Open() before calling this method.", path);
1265 
1266  // Adjust for a requested frame that is too small or too large
1267  if (requested_frame < 1)
1268  requested_frame = 1;
1269  if (requested_frame > info.video_length && is_duration_known)
1270  requested_frame = info.video_length;
1271  if (info.has_video && info.video_length == 0)
1272  // Invalid duration of video file
1273  throw InvalidFile("Could not detect the duration of the video or audio stream.", path);
1274 
1275  // Debug output
1276  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "requested_frame", requested_frame, "last_frame", last_frame);
1277 
1278  // Check the cache for this frame
1279  std::shared_ptr<Frame> frame = final_cache.GetFrame(requested_frame);
1280  if (frame) {
1281  // Debug output
1282  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "returned cached frame", requested_frame);
1283  // Return the cached frame
1284  return frame;
1285  } else {
1286 
1287  // Prevent async calls to the remainder of this code
1288  const std::lock_guard<std::recursive_mutex> lock(getFrameMutex);
1289 
1290  // Check the cache a 2nd time (due to the potential previous lock)
1291  frame = final_cache.GetFrame(requested_frame);
1292  if (frame) {
1293  // Debug output
1294  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "returned cached frame on 2nd look", requested_frame);
1295  } else {
1296  // Frame is not in cache
1297  // Reset seek count
1298  seek_count = 0;
1299 
1300  // Are we within X frames of the requested frame?
1301  int64_t diff = requested_frame - last_frame;
1302  if (diff >= 1 && diff <= 20) {
1303  // Continue walking the stream
1304  frame = ReadStream(requested_frame);
1305  } else {
1306  // Greater than 30 frames away, or backwards, we need to seek to the nearest key frame
1307  if (enable_seek) {
1308  // Only seek if enabled
1309  Seek(requested_frame);
1310 
1311  } else if (!enable_seek && diff < 0) {
1312  // Start over, since we can't seek, and the requested frame is smaller than our position
1313  // Since we are seeking to frame 1, this actually just closes/re-opens the reader
1314  Seek(1);
1315  }
1316 
1317  // Then continue walking the stream
1318  frame = ReadStream(requested_frame);
1319  }
1320  }
1321  return frame;
1322  }
1323 }
1324 
1325 // Read the stream until we find the requested Frame
1326 std::shared_ptr<Frame> FFmpegReader::ReadStream(int64_t requested_frame) {
1327  // Allocate video frame
1328  bool check_seek = false;
1329  int packet_error = -1;
1330  int64_t no_progress_count = 0;
1331  int64_t prev_packets_read = packet_status.packets_read();
1332  int64_t prev_packets_decoded = packet_status.packets_decoded();
1333  int64_t prev_video_decoded = packet_status.video_decoded;
1334  double prev_video_pts_seconds = video_pts_seconds;
1335 
1336  // Debug output
1337  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream", "requested_frame", requested_frame);
1338 
1339  // Loop through the stream until the correct frame is found
1340  while (true) {
1341  // Check if working frames are 'finished'
1342  if (!is_seeking) {
1343  // Check for final frames
1344  CheckWorkingFrames(requested_frame);
1345  }
1346 
1347  // Check if requested 'final' frame is available (and break out of loop if found)
1348  bool is_cache_found = (final_cache.GetFrame(requested_frame) != NULL);
1349  if (is_cache_found) {
1350  break;
1351  }
1352 
1353  if (!hold_packet || !packet) {
1354  // Get the next packet
1355  packet_error = GetNextPacket();
1356  if (packet_error < 0 && !packet) {
1357  // No more packets to be found
1358  packet_status.packets_eof = true;
1359  }
1360  }
1361 
1362  // Debug output
1363  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream (GetNextPacket)", "requested_frame", requested_frame,"packets_read", packet_status.packets_read(), "packets_decoded", packet_status.packets_decoded(), "is_seeking", is_seeking);
1364 
1365  // Check the status of a seek (if any)
1366  if (is_seeking) {
1367  check_seek = CheckSeek();
1368  } else {
1369  check_seek = false;
1370  }
1371 
1372  if (check_seek) {
1373  // Packet may become NULL on Close inside Seek if CheckSeek returns false
1374  // Jump to the next iteration of this loop
1375  continue;
1376  }
1377 
1378  // Video packet
1379  if ((info.has_video && packet && packet->stream_index == videoStream) ||
1380  (info.has_video && packet_status.video_decoded < packet_status.video_read) ||
1381  (info.has_video && !packet && !packet_status.video_eof)) {
1382  // Process Video Packet
1383  ProcessVideoPacket(requested_frame);
1384  if (ReopenWithoutHardwareDecode(requested_frame)) {
1385  continue;
1386  }
1387  }
1388  // Audio packet
1389  if ((info.has_audio && packet && packet->stream_index == audioStream) ||
1390  (info.has_audio && !packet && packet_status.audio_decoded < packet_status.audio_read) ||
1391  (info.has_audio && !packet && !packet_status.audio_eof)) {
1392  // Process Audio Packet
1393  ProcessAudioPacket(requested_frame);
1394  }
1395 
1396  // Remove unused packets (sometimes we purposely ignore video or audio packets,
1397  // if the has_video or has_audio properties are manually overridden)
1398  if ((!info.has_video && packet && packet->stream_index == videoStream) ||
1399  (!info.has_audio && packet && packet->stream_index == audioStream)) {
1400  // Keep track of deleted packet counts
1401  if (packet->stream_index == videoStream) {
1402  packet_status.video_decoded++;
1403  } else if (packet->stream_index == audioStream) {
1404  packet_status.audio_decoded++;
1405  }
1406 
1407  // Remove unused packets (sometimes we purposely ignore video or audio packets,
1408  // if the has_video or has_audio properties are manually overridden)
1409  RemoveAVPacket(packet);
1410  packet = NULL;
1411  }
1412 
1413  // Determine end-of-stream (waiting until final decoder threads finish)
1414  // Force end-of-stream in some situations
1415  packet_status.end_of_file = packet_status.packets_eof && packet_status.video_eof && packet_status.audio_eof;
1416  if ((packet_status.packets_eof && packet_status.packets_read() == packet_status.packets_decoded()) || packet_status.end_of_file) {
1417  // Force EOF (end of file) variables to true, if decoder does not support EOF detection.
1418  // If we have no more packets, and all known packets have been decoded
1419  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream (force EOF)", "packets_read", packet_status.packets_read(), "packets_decoded", packet_status.packets_decoded(), "packets_eof", packet_status.packets_eof, "video_eof", packet_status.video_eof, "audio_eof", packet_status.audio_eof, "end_of_file", packet_status.end_of_file);
1420  if (!packet_status.video_eof) {
1421  packet_status.video_eof = true;
1422  }
1423  if (!packet_status.audio_eof) {
1424  packet_status.audio_eof = true;
1425  }
1426  packet_status.end_of_file = true;
1427  break;
1428  }
1429 
1430  // Detect decoder stalls with no progress at EOF and force completion so
1431  // missing frames can be finalized from prior image data.
1432  const bool has_progress =
1433  (packet_status.packets_read() != prev_packets_read) ||
1434  (packet_status.packets_decoded() != prev_packets_decoded) ||
1435  (packet_status.video_decoded != prev_video_decoded) ||
1436  (video_pts_seconds != prev_video_pts_seconds);
1437 
1438  if (has_progress) {
1439  no_progress_count = 0;
1440  } else {
1441  no_progress_count++;
1442  if (no_progress_count >= 2000
1443  && packet_status.packets_eof
1444  && !packet
1445  && !hold_packet) {
1446  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream (force EOF after stall)",
1447  "requested_frame", requested_frame,
1448  "no_progress_count", no_progress_count,
1449  "packets_read", packet_status.packets_read(),
1450  "packets_decoded", packet_status.packets_decoded(),
1451  "video_decoded", packet_status.video_decoded,
1452  "audio_decoded", packet_status.audio_decoded);
1453  packet_status.video_eof = true;
1454  packet_status.audio_eof = true;
1455  packet_status.end_of_file = true;
1456  break;
1457  }
1458  }
1459  prev_packets_read = packet_status.packets_read();
1460  prev_packets_decoded = packet_status.packets_decoded();
1461  prev_video_decoded = packet_status.video_decoded;
1462  prev_video_pts_seconds = video_pts_seconds;
1463  } // end while
1464 
1465  // Debug output
1466  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream (Completed)",
1467  "packets_read", packet_status.packets_read(),
1468  "packets_decoded", packet_status.packets_decoded(),
1469  "end_of_file", packet_status.end_of_file,
1470  "largest_frame_processed", largest_frame_processed,
1471  "Working Cache Count", working_cache.Count());
1472 
1473  // Have we reached end-of-stream (or the final frame)?
1474  if (!packet_status.end_of_file && requested_frame >= info.video_length) {
1475  // Force end-of-stream
1476  packet_status.end_of_file = true;
1477  }
1478  if (packet_status.end_of_file) {
1479  // Mark any other working frames as 'finished'
1480  CheckWorkingFrames(requested_frame);
1481  }
1482 
1483  // Return requested frame (if found)
1484  std::shared_ptr<Frame> frame = final_cache.GetFrame(requested_frame);
1485  if (frame)
1486  // Return prepared frame
1487  return frame;
1488  else {
1489 
1490  // Check if largest frame is still cached
1491  frame = final_cache.GetFrame(largest_frame_processed);
1492  int samples_in_frame = Frame::GetSamplesPerFrame(requested_frame, info.fps,
1493  info.sample_rate, info.channels);
1494  if (frame) {
1495  // Copy and return the largest processed frame (assuming it was the last in the video file)
1496  std::shared_ptr<Frame> f = CreateFrame(largest_frame_processed);
1497  if (frame->has_image_data) {
1498  f->AddImage(std::make_shared<QImage>(frame->GetImage()->copy()));
1499  }
1500 
1501  // Use solid color (if no image data found)
1502  if (!frame->has_image_data) {
1503  // Use solid black frame if no image data available
1504  f->AddColor(info.width, info.height, "#000");
1505  }
1506  // Silence audio data (if any), since we are repeating the last frame
1507  f->AddAudioSilence(samples_in_frame);
1508 
1509  return f;
1510  } else {
1511  // The largest processed frame is no longer in cache. Prefer the most recent
1512  // finalized image first, then decoded image, to avoid black flashes.
1513  std::shared_ptr<Frame> f = CreateFrame(largest_frame_processed);
1514  if (last_final_video_frame && last_final_video_frame->has_image_data
1515  && last_final_video_frame->number <= requested_frame) {
1516  f->AddImage(std::make_shared<QImage>(last_final_video_frame->GetImage()->copy()));
1517  } else if (last_video_frame && last_video_frame->has_image_data
1518  && last_video_frame->number <= requested_frame) {
1519  f->AddImage(std::make_shared<QImage>(last_video_frame->GetImage()->copy()));
1520  } else {
1521  f->AddColor(info.width, info.height, "#000");
1522  }
1523  f->AddAudioSilence(samples_in_frame);
1524  return f;
1525  }
1526  }
1527 
1528 }
1529 
1530 // Get the next packet (if any)
1531 int FFmpegReader::GetNextPacket() {
1532  int found_packet = 0;
1533  AVPacket *next_packet;
1534  next_packet = new AVPacket();
1535  found_packet = av_read_frame(pFormatCtx, next_packet);
1536 
1537  if (packet) {
1538  // Remove previous packet before getting next one
1539  RemoveAVPacket(packet);
1540  packet = NULL;
1541  }
1542  if (found_packet >= 0) {
1543  // Update current packet pointer
1544  packet = next_packet;
1545 
1546  // Keep track of packet stats
1547  if (packet->stream_index == videoStream) {
1548  packet_status.video_read++;
1549  } else if (packet->stream_index == audioStream) {
1550  packet_status.audio_read++;
1551  }
1552  } else {
1553  // No more packets found
1554  delete next_packet;
1555  packet = NULL;
1556  }
1557  // Return if packet was found (or error number)
1558  return found_packet;
1559 }
1560 
1561 // Get an AVFrame (if any)
1562 bool FFmpegReader::GetAVFrame() {
1563  int frameFinished = 0;
1564  auto note_hw_decode_failure = [&](int err, const char* stage) {
1565 #if USE_HW_ACCEL
1566  if (!hw_de_on || !hw_de_supported || force_sw_decode) {
1567  return;
1568  }
1569  if (err == AVERROR_INVALIDDATA && packet_status.video_decoded == 0) {
1570  hw_decode_error_count++;
1571  ZmqLogger::Instance()->AppendDebugMethod(
1572  std::string("FFmpegReader::GetAVFrame (hardware decode failure candidate during ") + stage + ")",
1573  "error_count", hw_decode_error_count,
1574  "error", err);
1575  if (hw_decode_error_count >= 3) {
1576  hw_decode_failed = true;
1577  }
1578  }
1579 #else
1580  (void) err;
1581  (void) stage;
1582 #endif
1583  };
1584 
1585  // Decode video frame
1586  AVFrame *next_frame = AV_ALLOCATE_FRAME();
1587 
1588 #if IS_FFMPEG_3_2
1589  int send_packet_err = 0;
1590  int64_t send_packet_pts = 0;
1591  if ((packet && packet->stream_index == videoStream) || !packet) {
1592  send_packet_err = avcodec_send_packet(pCodecCtx, packet);
1593 
1594  if (packet && send_packet_err >= 0) {
1595  send_packet_pts = GetPacketPTS();
1596  hold_packet = false;
1597  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (send packet succeeded)", "send_packet_err", send_packet_err, "send_packet_pts", send_packet_pts);
1598  }
1599  }
1600 
1601  #if USE_HW_ACCEL
1602  // Get the format from the variables set in get_hw_dec_format
1603  hw_de_av_pix_fmt = hw_de_av_pix_fmt_global;
1604  hw_de_av_device_type = hw_de_av_device_type_global;
1605  #endif // USE_HW_ACCEL
1606  if (send_packet_err < 0 && send_packet_err != AVERROR_EOF) {
1607  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (send packet: Not sent [" + av_err2string(send_packet_err) + "])", "send_packet_err", send_packet_err, "send_packet_pts", send_packet_pts);
1608  note_hw_decode_failure(send_packet_err, "send_packet");
1609  if (send_packet_err == AVERROR(EAGAIN)) {
1610  hold_packet = true;
1611  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (send packet: AVERROR(EAGAIN): user must read output with avcodec_receive_frame()", "send_packet_pts", send_packet_pts);
1612  }
1613  if (send_packet_err == AVERROR(EINVAL)) {
1614  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (send packet: AVERROR(EINVAL): codec not opened, it is an encoder, or requires flush", "send_packet_pts", send_packet_pts);
1615  }
1616  if (send_packet_err == AVERROR(ENOMEM)) {
1617  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (send packet: AVERROR(ENOMEM): failed to add packet to internal queue, or legitimate decoding errors", "send_packet_pts", send_packet_pts);
1618  }
1619  }
1620 
1621  // Always try and receive a packet, if not EOF.
1622  // Even if the above avcodec_send_packet failed to send,
1623  // we might still need to receive a packet.
1624  int receive_frame_err = 0;
1625  AVFrame *decoded_frame = next_frame;
1626  AVFrame *next_frame2;
1627 #if USE_HW_ACCEL
1628  if (hw_de_on && hw_de_supported) {
1629  next_frame2 = AV_ALLOCATE_FRAME();
1630  }
1631  else
1632 #endif // USE_HW_ACCEL
1633  {
1634  next_frame2 = next_frame;
1635  }
1636  pFrame = AV_ALLOCATE_FRAME();
1637  while (receive_frame_err >= 0) {
1638  receive_frame_err = avcodec_receive_frame(pCodecCtx, next_frame2);
1639 
1640  if (receive_frame_err != 0) {
1641  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (receive frame: frame not ready yet from decoder [\" + av_err2string(receive_frame_err) + \"])", "receive_frame_err", receive_frame_err, "send_packet_pts", send_packet_pts);
1642  note_hw_decode_failure(receive_frame_err, "receive_frame");
1643 
1644  if (receive_frame_err == AVERROR_EOF) {
1645  ZmqLogger::Instance()->AppendDebugMethod(
1646  "FFmpegReader::GetAVFrame (receive frame: AVERROR_EOF: EOF detected from decoder, flushing buffers)", "send_packet_pts", send_packet_pts);
1647  avcodec_flush_buffers(pCodecCtx);
1648  packet_status.video_eof = true;
1649  }
1650  if (receive_frame_err == AVERROR(EINVAL)) {
1651  ZmqLogger::Instance()->AppendDebugMethod(
1652  "FFmpegReader::GetAVFrame (receive frame: AVERROR(EINVAL): invalid frame received, flushing buffers)", "send_packet_pts", send_packet_pts);
1653  avcodec_flush_buffers(pCodecCtx);
1654  }
1655  if (receive_frame_err == AVERROR(EAGAIN)) {
1656  ZmqLogger::Instance()->AppendDebugMethod(
1657  "FFmpegReader::GetAVFrame (receive frame: AVERROR(EAGAIN): output is not available in this state - user must try to send new input)", "send_packet_pts", send_packet_pts);
1658  }
1659  if (receive_frame_err == AVERROR_INPUT_CHANGED) {
1660  ZmqLogger::Instance()->AppendDebugMethod(
1661  "FFmpegReader::GetAVFrame (receive frame: AVERROR_INPUT_CHANGED: current decoded frame has changed parameters with respect to first decoded frame)", "send_packet_pts", send_packet_pts);
1662  }
1663 
1664  // Break out of decoding loop
1665  // Nothing ready for decoding yet
1666  break;
1667  }
1668 
1669 #if USE_HW_ACCEL
1670  if (hw_de_on && hw_de_supported) {
1671  int err;
1672  if (next_frame2->format == hw_de_av_pix_fmt) {
1673  if ((err = av_hwframe_transfer_data(next_frame, next_frame2, 0)) < 0) {
1674  ZmqLogger::Instance()->AppendDebugMethod(
1675  "FFmpegReader::GetAVFrame (Failed to transfer data to output frame)",
1676  "hw_de_on", hw_de_on,
1677  "error", err);
1678  note_hw_decode_failure(AVERROR_INVALIDDATA, "hwframe_transfer");
1679  break;
1680  }
1681  if ((err = av_frame_copy_props(next_frame, next_frame2)) < 0) {
1682  ZmqLogger::Instance()->AppendDebugMethod(
1683  "FFmpegReader::GetAVFrame (Failed to copy props to output frame)",
1684  "hw_de_on", hw_de_on,
1685  "error", err);
1686  note_hw_decode_failure(AVERROR_INVALIDDATA, "hwframe_copy_props");
1687  break;
1688  }
1689  if (next_frame->format == AV_PIX_FMT_NONE) {
1690  next_frame->format = pCodecCtx->sw_pix_fmt;
1691  }
1692  if (next_frame->width <= 0) {
1693  next_frame->width = next_frame2->width;
1694  }
1695  if (next_frame->height <= 0) {
1696  next_frame->height = next_frame2->height;
1697  }
1698  decoded_frame = next_frame;
1699  } else {
1700  // Some hardware decoders can still return software-readable frames.
1701  decoded_frame = next_frame2;
1702  }
1703  }
1704  else
1705 #endif // USE_HW_ACCEL
1706  { // No hardware acceleration used -> no copy from GPU memory needed
1707  decoded_frame = next_frame2;
1708  }
1709 
1710  if (!decoded_frame->data[0]) {
1711  ZmqLogger::Instance()->AppendDebugMethod(
1712  "FFmpegReader::GetAVFrame (Decoded frame missing image data)",
1713  "format", decoded_frame->format,
1714  "width", decoded_frame->width,
1715  "height", decoded_frame->height);
1716  note_hw_decode_failure(AVERROR_INVALIDDATA, "decoded_frame_empty");
1717  break;
1718  }
1719 
1720  // TODO also handle possible further frames
1721  // Use only the first frame like avcodec_decode_video2
1722  frameFinished = 1;
1723  hw_decode_error_count = 0;
1724 #if USE_HW_ACCEL
1725  if (hw_de_on && hw_de_supported && !force_sw_decode) {
1726  hw_decode_succeeded = true;
1727  }
1728 #endif
1729  packet_status.video_decoded++;
1730 
1731  // Allocate image (align 32 for simd)
1732  AVPixelFormat decoded_pix_fmt = (AVPixelFormat)(decoded_frame->format);
1733  if (decoded_pix_fmt == AV_PIX_FMT_NONE)
1734  decoded_pix_fmt = (AVPixelFormat)(pStream->codecpar->format);
1735  const int decoded_width = decoded_frame->width > 0 ? decoded_frame->width : info.width;
1736  const int decoded_height = decoded_frame->height > 0 ? decoded_frame->height : info.height;
1737  if (AV_ALLOCATE_IMAGE(pFrame, decoded_pix_fmt, decoded_width, decoded_height) <= 0) {
1738  throw OutOfMemory("Failed to allocate image buffer", path);
1739  }
1740  av_image_copy(pFrame->data, pFrame->linesize, (const uint8_t**)decoded_frame->data, decoded_frame->linesize,
1741  decoded_pix_fmt, decoded_width, decoded_height);
1742  pFrame->format = decoded_pix_fmt;
1743  pFrame->width = decoded_width;
1744  pFrame->height = decoded_height;
1745  pFrame->color_range = decoded_frame->color_range;
1746  pFrame->colorspace = decoded_frame->colorspace;
1747  pFrame->color_primaries = decoded_frame->color_primaries;
1748  pFrame->color_trc = decoded_frame->color_trc;
1749  pFrame->chroma_location = decoded_frame->chroma_location;
1750 
1751  // Get display PTS from video frame, often different than packet->pts.
1752  // Sending packets to the decoder (i.e. packet->pts) is async,
1753  // and retrieving packets from the decoder (frame->pts) is async. In most decoders
1754  // sending and retrieving are separated by multiple calls to this method.
1755  if (decoded_frame->pts != AV_NOPTS_VALUE) {
1756  // This is the current decoded frame (and should be the pts used) for
1757  // processing this data
1758  video_pts = decoded_frame->pts;
1759  } else if (decoded_frame->pkt_dts != AV_NOPTS_VALUE) {
1760  // Some videos only set this timestamp (fallback)
1761  video_pts = decoded_frame->pkt_dts;
1762  }
1763 
1764  ZmqLogger::Instance()->AppendDebugMethod(
1765  "FFmpegReader::GetAVFrame (Successful frame received)", "video_pts", video_pts, "send_packet_pts", send_packet_pts);
1766 
1767  // break out of loop after each successful image returned
1768  break;
1769  }
1770 #if USE_HW_ACCEL
1771  if (hw_de_on && hw_de_supported && next_frame2 != next_frame) {
1772  AV_FREE_FRAME(&next_frame2);
1773  }
1774  #endif // USE_HW_ACCEL
1775 #else
1776  avcodec_decode_video2(pCodecCtx, next_frame, &frameFinished, packet);
1777 
1778  // always allocate pFrame (because we do that in the ffmpeg >= 3.2 as well); it will always be freed later
1779  pFrame = AV_ALLOCATE_FRAME();
1780 
1781  // is frame finished
1782  if (frameFinished) {
1783  // AVFrames are clobbered on the each call to avcodec_decode_video, so we
1784  // must make a copy of the image data before this method is called again.
1785  avpicture_alloc((AVPicture *) pFrame, pCodecCtx->pix_fmt, info.width, info.height);
1786  av_picture_copy((AVPicture *) pFrame, (AVPicture *) next_frame, pCodecCtx->pix_fmt, info.width,
1787  info.height);
1788  }
1789 #endif // IS_FFMPEG_3_2
1790 
1791  // deallocate the frame
1792  AV_FREE_FRAME(&next_frame);
1793 
1794  // Did we get a video frame?
1795  return frameFinished;
1796 }
1797 
1798 bool FFmpegReader::ReopenWithoutHardwareDecode(int64_t requested_frame) {
1799 #if USE_HW_ACCEL
1800  if (!hw_decode_failed || force_sw_decode) {
1801  return false;
1802  }
1803 
1804  ZmqLogger::Instance()->AppendDebugMethod(
1805  "FFmpegReader::ReopenWithoutHardwareDecode (falling back to software decode)",
1806  "requested_frame", requested_frame,
1807  "video_packets_read", packet_status.video_read,
1808  "video_packets_decoded", packet_status.video_decoded,
1809  "hw_decode_error_count", hw_decode_error_count);
1810 
1811  force_sw_decode = true;
1812  hw_decode_failed = false;
1813  hw_decode_error_count = 0;
1814 
1815  Close();
1816  Open();
1817  Seek(requested_frame);
1818  return true;
1819 #else
1820  (void) requested_frame;
1821  return false;
1822 #endif
1823 }
1824 
1825 bool FFmpegReader::HardwareDecodeSuccessful() const {
1826 #if USE_HW_ACCEL
1827  return hw_decode_succeeded;
1828 #else
1829  return false;
1830 #endif
1831 }
1832 
1833 // Check the current seek position and determine if we need to seek again
1834 bool FFmpegReader::CheckSeek() {
1835  // Are we seeking for a specific frame?
1836  if (is_seeking) {
1837  const int64_t kSeekRetryMax = 5;
1838  const int kSeekStagnantMax = 2;
1839 
1840  // Determine if both an audio and video packet have been decoded since the seek happened.
1841  // If not, allow the ReadStream method to keep looping
1842  if ((is_video_seek && !seek_video_frame_found) || (!is_video_seek && !seek_audio_frame_found))
1843  return false;
1844 
1845  // Check for both streams
1846  if ((info.has_video && !seek_video_frame_found) || (info.has_audio && !seek_audio_frame_found))
1847  return false;
1848 
1849  // Determine max seeked frame
1850  int64_t max_seeked_frame = std::max(seek_audio_frame_found, seek_video_frame_found);
1851  // Track stagnant seek results (no progress between retries)
1852  if (max_seeked_frame == last_seek_max_frame) {
1853  seek_stagnant_count++;
1854  } else {
1855  last_seek_max_frame = max_seeked_frame;
1856  seek_stagnant_count = 0;
1857  }
1858 
1859  // determine if we are "before" the requested frame
1860  if (max_seeked_frame >= seeking_frame) {
1861  // SEEKED TOO FAR
1862  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckSeek (Too far, seek again)",
1863  "is_video_seek", is_video_seek,
1864  "max_seeked_frame", max_seeked_frame,
1865  "seeking_frame", seeking_frame,
1866  "seeking_pts", seeking_pts,
1867  "seek_video_frame_found", seek_video_frame_found,
1868  "seek_audio_frame_found", seek_audio_frame_found);
1869 
1870  // Seek again... to the nearest Keyframe
1871  if (seek_count < kSeekRetryMax) {
1872  Seek(seeking_frame - (10 * seek_count * seek_count));
1873  } else if (seek_stagnant_count >= kSeekStagnantMax) {
1874  // Stagnant seek: force a much earlier target and keep seeking.
1875  Seek(seeking_frame - (10 * kSeekRetryMax * kSeekRetryMax));
1876  } else {
1877  // Retry budget exhausted: keep seeking from a conservative offset.
1878  Seek(seeking_frame - (10 * seek_count * seek_count));
1879  }
1880  } else {
1881  // SEEK WORKED
1882  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckSeek (Successful)",
1883  "is_video_seek", is_video_seek,
1884  "packet->pts", GetPacketPTS(),
1885  "seeking_pts", seeking_pts,
1886  "seeking_frame", seeking_frame,
1887  "seek_video_frame_found", seek_video_frame_found,
1888  "seek_audio_frame_found", seek_audio_frame_found);
1889 
1890  // Seek worked, and we are "before" the requested frame
1891  is_seeking = false;
1892  seeking_frame = 0;
1893  seeking_pts = -1;
1894  }
1895  }
1896 
1897  // return the pts to seek to (if any)
1898  return is_seeking;
1899 }
1900 
1901 // Process a video packet
1902 void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) {
1903  // Get the AVFrame from the current packet
1904  // This sets the video_pts to the correct timestamp
1905  int frame_finished = GetAVFrame();
1906 
1907  // Check if the AVFrame is finished and set it
1908  if (!frame_finished) {
1909  // No AVFrame decoded yet, bail out
1910  if (pFrame) {
1911  RemoveAVFrame(pFrame);
1912  }
1913  return;
1914  }
1915 
1916  // Calculate current frame #
1917  int64_t current_frame = ConvertVideoPTStoFrame(video_pts);
1918 
1919  // Track 1st video packet after a successful seek
1920  if (!seek_video_frame_found && is_seeking)
1921  seek_video_frame_found = current_frame;
1922 
1923  // Create or get the existing frame object. Requested frame needs to be created
1924  // in working_cache at least once. Seek can clear the working_cache, so we must
1925  // add the requested frame back to the working_cache here. If it already exists,
1926  // it will be moved to the top of the working_cache.
1927  working_cache.Add(CreateFrame(requested_frame));
1928 
1929  // Debug output
1930  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessVideoPacket (Before)", "requested_frame", requested_frame, "current_frame", current_frame);
1931 
1932  // Init some things local (for OpenMP)
1933  AVPixelFormat decoded_pix_fmt = (pFrame && pFrame->format != AV_PIX_FMT_NONE)
1934  ? static_cast<AVPixelFormat>(pFrame->format)
1935  : AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx);
1936  bool src_full_range = (pFrame && pFrame->color_range == AVCOL_RANGE_JPEG);
1937  AVPixelFormat src_pix_fmt = NormalizeDeprecatedPixFmt(decoded_pix_fmt, src_full_range);
1938  int src_width = (pFrame && pFrame->width > 0) ? pFrame->width : info.width;
1939  int src_height = (pFrame && pFrame->height > 0) ? pFrame->height : info.height;
1940  int height = src_height;
1941  int width = src_width;
1942  // Create or reuse a RGB Frame (since most videos are not in RGB, we must convert it)
1943  AVFrame *pFrameRGB = pFrameRGB_cached;
1944  if (!pFrameRGB) {
1945  pFrameRGB = AV_ALLOCATE_FRAME();
1946  if (pFrameRGB == nullptr)
1947  throw OutOfMemory("Failed to allocate frame buffer", path);
1948  pFrameRGB_cached = pFrameRGB;
1949  }
1950  AV_RESET_FRAME(pFrameRGB);
1951  uint8_t *buffer = nullptr;
1952 
1953  // Determine the max size of this source image (based on the timeline's size, the scaling mode,
1954  // and the scaling keyframes). This is a performance improvement, to keep the images as small as possible,
1955  // without losing quality. NOTE: We cannot go smaller than the timeline itself, or the add_layer timeline
1956  // method will scale it back to timeline size before scaling it smaller again. This needs to be fixed in
1957  // the future.
1958  int max_width = info.width;
1959  int max_height = info.height;
1960 
1961  Clip *parent = static_cast<Clip *>(ParentClip());
1962  if (parent) {
1963  if (parent->ParentTimeline()) {
1964  // Set max width/height based on parent clip's timeline (if attached to a timeline)
1965  max_width = parent->ParentTimeline()->preview_width;
1966  max_height = parent->ParentTimeline()->preview_height;
1967  }
1968  if (parent->scale == SCALE_FIT || parent->scale == SCALE_STRETCH) {
1969  // Best fit or Stretch scaling (based on max timeline size * scaling keyframes)
1970  float max_scale_x = parent->scale_x.GetMaxPoint().co.Y;
1971  float max_scale_y = parent->scale_y.GetMaxPoint().co.Y;
1972  max_width = std::max(float(max_width), max_width * max_scale_x);
1973  max_height = std::max(float(max_height), max_height * max_scale_y);
1974 
1975  } else if (parent->scale == SCALE_CROP) {
1976  // Cropping scale mode (based on max timeline size * cropped size * scaling keyframes)
1977  float max_scale_x = parent->scale_x.GetMaxPoint().co.Y;
1978  float max_scale_y = parent->scale_y.GetMaxPoint().co.Y;
1979  QSize width_size(max_width * max_scale_x,
1980  round(max_width / (float(info.width) / float(info.height))));
1981  QSize height_size(round(max_height / (float(info.height) / float(info.width))),
1982  max_height * max_scale_y);
1983  // respect aspect ratio
1984  if (width_size.width() >= max_width && width_size.height() >= max_height) {
1985  max_width = std::max(max_width, width_size.width());
1986  max_height = std::max(max_height, width_size.height());
1987  } else {
1988  max_width = std::max(max_width, height_size.width());
1989  max_height = std::max(max_height, height_size.height());
1990  }
1991 
1992  } else {
1993  // Scale video to equivalent unscaled size
1994  // Since the preview window can change sizes, we want to always
1995  // scale against the ratio of original video size to timeline size
1996  float preview_ratio = 1.0;
1997  if (parent->ParentTimeline()) {
1998  Timeline *t = (Timeline *) parent->ParentTimeline();
1999  preview_ratio = t->preview_width / float(t->info.width);
2000  }
2001  float max_scale_x = parent->scale_x.GetMaxPoint().co.Y;
2002  float max_scale_y = parent->scale_y.GetMaxPoint().co.Y;
2003  max_width = info.width * max_scale_x * preview_ratio;
2004  max_height = info.height * max_scale_y * preview_ratio;
2005  }
2006 
2007  // If a crop effect is resizing the image, request enough pixels to preserve detail
2008  ApplyCropResizeScale(parent, info.width, info.height, max_width, max_height);
2009  }
2010 
2011  if (HasMaxDecodeSize()) {
2012  QSize bounded_size(max_width, max_height);
2013  const QSize max_decode_size(MaxDecodeWidth(), MaxDecodeHeight());
2014  if (bounded_size.width() > max_decode_size.width() ||
2015  bounded_size.height() > max_decode_size.height()) {
2016  bounded_size.scale(max_decode_size, Qt::KeepAspectRatio);
2017  max_width = bounded_size.width();
2018  max_height = bounded_size.height();
2019  }
2020  }
2021 
2022  // Max-size calculations above are in display-oriented coordinates. Scaling
2023  // happens before orientation is applied, so swap the decode bounds for
2024  // quarter-turn sources to preserve detail after rotation.
2025  const int quarter_turn = ((static_cast<int>(std::round(source_rotation / 90.0)) * 90) % 360 + 360) % 360;
2026  if (ApplyOrientationMetadata() && (quarter_turn == 90 || quarter_turn == 270)) {
2027  std::swap(max_width, max_height);
2028  }
2029 
2030  // Determine if image needs to be scaled (for performance reasons)
2031  int original_height = src_height;
2032  if (max_width != 0 && max_height != 0 && max_width < width && max_height < height) {
2033  // Override width and height (but maintain aspect ratio)
2034  float ratio = float(width) / float(height);
2035  int possible_width = round(max_height * ratio);
2036  int possible_height = round(max_width / ratio);
2037 
2038  if (possible_width <= max_width) {
2039  // use calculated width, and max_height
2040  width = possible_width;
2041  height = max_height;
2042  } else {
2043  // use max_width, and calculated height
2044  width = max_width;
2045  height = possible_height;
2046  }
2047  }
2048 
2049  // Determine required buffer size and allocate buffer
2050  const int bytes_per_pixel = 4;
2051  int raw_buffer_size = (width * height * bytes_per_pixel) + 128;
2052 
2053  // Aligned memory allocation (for speed)
2054  constexpr size_t ALIGNMENT = 32; // AVX2
2055  int buffer_size = ((raw_buffer_size + ALIGNMENT - 1) / ALIGNMENT) * ALIGNMENT;
2056  buffer = (unsigned char*) aligned_malloc(buffer_size, ALIGNMENT);
2057 
2058  // Copy picture data from one AVFrame (or AVPicture) to another one.
2059  AV_COPY_PICTURE_DATA(pFrameRGB, buffer, PIX_FMT_RGBA, width, height);
2060 
2061  int scale_mode = SWS_FAST_BILINEAR;
2062  if (openshot::Settings::Instance()->HIGH_QUALITY_SCALING) {
2063  scale_mode = SWS_BICUBIC;
2064  }
2065  img_convert_ctx = sws_getCachedContext(img_convert_ctx, src_width, src_height, src_pix_fmt, width, height, PIX_FMT_RGBA, scale_mode, NULL, NULL, NULL);
2066  if (!img_convert_ctx)
2067  throw OutOfMemory("Failed to initialize sws context", path);
2068  const int *src_coeff = sws_getCoefficients(SWS_CS_DEFAULT);
2069  const int *dst_coeff = sws_getCoefficients(SWS_CS_DEFAULT);
2070  const int dst_full_range = 1; // RGB outputs are full-range
2071  sws_setColorspaceDetails(img_convert_ctx, src_coeff, src_full_range ? 1 : 0,
2072  dst_coeff, dst_full_range, 0, 1 << 16, 1 << 16);
2073 
2074  if (!pFrame || !pFrame->data[0] || pFrame->linesize[0] <= 0) {
2075 #if USE_HW_ACCEL
2076  if (hw_de_on && hw_de_supported && !force_sw_decode) {
2077  hw_decode_failed = true;
2078  ZmqLogger::Instance()->AppendDebugMethod(
2079  "FFmpegReader::ProcessVideoPacket (Invalid source frame; forcing software fallback)",
2080  "requested_frame", requested_frame,
2081  "current_frame", current_frame,
2082  "src_pix_fmt", src_pix_fmt,
2083  "src_width", src_width,
2084  "src_height", src_height);
2085  }
2086 #endif
2087  if (pFrame) {
2088  RemoveAVFrame(pFrame);
2089  pFrame = NULL;
2090  }
2091  return;
2092  }
2093 
2094  // Resize / Convert to RGB
2095  const int scaled_lines = sws_scale(img_convert_ctx, pFrame->data, pFrame->linesize, 0,
2096  original_height, pFrameRGB->data, pFrameRGB->linesize);
2097  if (scaled_lines <= 0) {
2098 #if USE_HW_ACCEL
2099  if (hw_de_on && hw_de_supported && !force_sw_decode) {
2100  hw_decode_failed = true;
2101  ZmqLogger::Instance()->AppendDebugMethod(
2102  "FFmpegReader::ProcessVideoPacket (sws_scale failed; forcing software fallback)",
2103  "requested_frame", requested_frame,
2104  "current_frame", current_frame,
2105  "scaled_lines", scaled_lines,
2106  "src_pix_fmt", src_pix_fmt,
2107  "src_width", src_width,
2108  "src_height", src_height);
2109  }
2110 #endif
2111  free(buffer);
2112  AV_RESET_FRAME(pFrameRGB);
2113  RemoveAVFrame(pFrame);
2114  pFrame = NULL;
2115  return;
2116  }
2117 
2118  // Create or get the existing frame object
2119  std::shared_ptr<Frame> f = CreateFrame(current_frame);
2120 
2121  // Add Image data to frame
2122  if (!ffmpeg_has_alpha(src_pix_fmt)) {
2123  // Add image with no alpha channel, Speed optimization
2124  f->AddImage(width, height, bytes_per_pixel, QImage::Format_RGBA8888_Premultiplied, buffer);
2125  } else {
2126  // Add image with alpha channel (this will be converted to premultipled when needed, but is slower)
2127  f->AddImage(width, height, bytes_per_pixel, QImage::Format_RGBA8888, buffer);
2128  }
2129  ApplyFrameOrientation(f);
2130 
2131  // Update working cache
2132  working_cache.Add(f);
2133 
2134  // Keep track of last last_video_frame
2135  last_video_frame = f;
2136 
2137  // Free the RGB image
2138  AV_RESET_FRAME(pFrameRGB);
2139 
2140  // Remove frame and packet
2141  RemoveAVFrame(pFrame);
2142 
2143  // Get video PTS in seconds
2144  video_pts_seconds = (double(video_pts) * info.video_timebase.ToDouble()) + pts_offset_seconds;
2145 
2146  // Debug output
2147  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessVideoPacket (After)", "requested_frame", requested_frame, "current_frame", current_frame, "f->number", f->number, "video_pts_seconds", video_pts_seconds);
2148 }
2149 
2150 // Process an audio packet
2151 void FFmpegReader::ProcessAudioPacket(int64_t requested_frame) {
2152  AudioLocation location;
2153  // Calculate location of current audio packet
2154  if (packet && packet->pts != AV_NOPTS_VALUE) {
2155  // Determine related video frame and starting sample # from audio PTS
2156  location = GetAudioPTSLocation(packet->pts);
2157 
2158  // Track 1st audio packet after a successful seek
2159  if (!seek_audio_frame_found && is_seeking)
2160  seek_audio_frame_found = location.frame;
2161  }
2162 
2163  // Create or get the existing frame object. Requested frame needs to be created
2164  // in working_cache at least once. Seek can clear the working_cache, so we must
2165  // add the requested frame back to the working_cache here. If it already exists,
2166  // it will be moved to the top of the working_cache.
2167  working_cache.Add(CreateFrame(requested_frame));
2168 
2169  // Debug output
2170  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Before)",
2171  "requested_frame", requested_frame,
2172  "target_frame", location.frame,
2173  "starting_sample", location.sample_start);
2174 
2175  // Init an AVFrame to hold the decoded audio samples
2176  int frame_finished = 0;
2177  AVFrame *audio_frame = AV_ALLOCATE_FRAME();
2178  AV_RESET_FRAME(audio_frame);
2179 
2180  int packet_samples = 0;
2181  int data_size = 0;
2182 
2183 #if IS_FFMPEG_3_2
2184  int send_packet_err = avcodec_send_packet(aCodecCtx, packet);
2185  if (send_packet_err < 0 && send_packet_err != AVERROR_EOF) {
2186  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Packet not sent)");
2187  }
2188  else {
2189  int receive_frame_err = avcodec_receive_frame(aCodecCtx, audio_frame);
2190  if (receive_frame_err >= 0) {
2191  frame_finished = 1;
2192  }
2193  if (receive_frame_err == AVERROR_EOF) {
2194  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (EOF detected from decoder)");
2195  packet_status.audio_eof = true;
2196  }
2197  if (receive_frame_err == AVERROR(EINVAL) || receive_frame_err == AVERROR_EOF) {
2198  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (invalid frame received or EOF from decoder)");
2199  avcodec_flush_buffers(aCodecCtx);
2200  }
2201  if (receive_frame_err != 0) {
2202  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (frame not ready yet from decoder)");
2203  }
2204  }
2205 #else
2206  int used = avcodec_decode_audio4(aCodecCtx, audio_frame, &frame_finished, packet);
2207 #endif
2208 
2209  if (frame_finished) {
2210  packet_status.audio_decoded++;
2211 
2212  // This can be different than the current packet, so we need to look
2213  // at the current AVFrame from the audio decoder. This timestamp should
2214  // be used for the remainder of this function
2215  audio_pts = audio_frame->pts;
2216 
2217  // Determine related video frame and starting sample # from audio PTS
2218  location = GetAudioPTSLocation(audio_pts);
2219 
2220  // determine how many samples were decoded
2221  int plane_size = -1;
2222 #if HAVE_CH_LAYOUT
2223  int nb_channels = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->ch_layout.nb_channels;
2224 #else
2225  int nb_channels = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels;
2226 #endif
2227  data_size = av_samples_get_buffer_size(&plane_size, nb_channels,
2228  audio_frame->nb_samples, (AVSampleFormat) (AV_GET_SAMPLE_FORMAT(aStream, aCodecCtx)), 1);
2229 
2230  // Calculate total number of samples
2231  packet_samples = audio_frame->nb_samples * nb_channels;
2232  } else {
2233  if (audio_frame) {
2234  // Free audio frame
2235  AV_FREE_FRAME(&audio_frame);
2236  }
2237  }
2238 
2239  // Estimate the # of samples and the end of this packet's location (to prevent GAPS for the next timestamp)
2240  int pts_remaining_samples = packet_samples / info.channels; // Adjust for zero based array
2241 
2242  // Bail if no samples found
2243  if (pts_remaining_samples == 0) {
2244  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (No samples, bailing)",
2245  "packet_samples", packet_samples,
2246  "info.channels", info.channels,
2247  "pts_remaining_samples", pts_remaining_samples);
2248  return;
2249  }
2250 
2251  while (pts_remaining_samples) {
2252  // Get Samples per frame (for this frame number)
2253  int samples_per_frame = Frame::GetSamplesPerFrame(previous_packet_location.frame, info.fps, info.sample_rate, info.channels);
2254 
2255  // Calculate # of samples to add to this frame
2256  int samples = samples_per_frame - previous_packet_location.sample_start;
2257  if (samples > pts_remaining_samples)
2258  samples = pts_remaining_samples;
2259 
2260  // Decrement remaining samples
2261  pts_remaining_samples -= samples;
2262 
2263  if (pts_remaining_samples > 0) {
2264  // next frame
2265  previous_packet_location.frame++;
2266  previous_packet_location.sample_start = 0;
2267  } else {
2268  // Increment sample start
2269  previous_packet_location.sample_start += samples;
2270  }
2271  }
2272 
2273  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (ReSample)",
2274  "packet_samples", packet_samples,
2275  "info.channels", info.channels,
2276  "info.sample_rate", info.sample_rate,
2277  "aCodecCtx->sample_fmt", AV_GET_SAMPLE_FORMAT(aStream, aCodecCtx));
2278 
2279  // Create output frame
2280  AVFrame *audio_converted = AV_ALLOCATE_FRAME();
2281  AV_RESET_FRAME(audio_converted);
2282  audio_converted->nb_samples = audio_frame->nb_samples;
2283  av_samples_alloc(audio_converted->data, audio_converted->linesize, info.channels, audio_frame->nb_samples, AV_SAMPLE_FMT_FLTP, 0);
2284 
2285  SWRCONTEXT *avr = avr_ctx;
2286  // setup resample context if needed
2287  if (!avr) {
2288  avr = SWR_ALLOC();
2289 #if HAVE_CH_LAYOUT
2290  AVChannelLayout input_layout = ffmpeg_get_valid_channel_layout(
2291  AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->ch_layout, info.channels);
2292  AVChannelLayout output_layout = ffmpeg_get_valid_channel_layout(
2293  input_layout, info.channels);
2294  int in_layout_err = av_opt_set_chlayout(avr, "in_chlayout", &input_layout, 0);
2295  int out_layout_err = av_opt_set_chlayout(avr, "out_chlayout", &output_layout, 0);
2296 #else
2297  av_opt_set_int(avr, "in_channel_layout", AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout, 0);
2298  av_opt_set_int(avr, "out_channel_layout", AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout, 0);
2299  av_opt_set_int(avr, "in_channels", info.channels, 0);
2300  av_opt_set_int(avr, "out_channels", info.channels, 0);
2301 #endif
2302  av_opt_set_int(avr, "in_sample_fmt", AV_GET_SAMPLE_FORMAT(aStream, aCodecCtx), 0);
2303  av_opt_set_int(avr, "out_sample_fmt", AV_SAMPLE_FMT_FLTP, 0);
2304  av_opt_set_int(avr, "in_sample_rate", info.sample_rate, 0);
2305  av_opt_set_int(avr, "out_sample_rate", info.sample_rate, 0);
2306  int swr_init_err = SWR_INIT(avr);
2307 #if HAVE_CH_LAYOUT
2308  av_channel_layout_uninit(&input_layout);
2309  av_channel_layout_uninit(&output_layout);
2310  if (in_layout_err < 0 || out_layout_err < 0 || swr_init_err < 0) {
2311  SWR_FREE(&avr);
2312  throw InvalidChannels("Could not initialize FFmpeg audio channel layout or resampler.", path);
2313  }
2314 #else
2315  if (swr_init_err < 0) {
2316  SWR_FREE(&avr);
2317  throw InvalidChannels("Could not initialize FFmpeg audio resampler.", path);
2318  }
2319 #endif
2320  avr_ctx = avr;
2321  }
2322 
2323  // Convert audio samples
2324  int nb_samples = SWR_CONVERT(avr, // audio resample context
2325  audio_converted->data, // output data pointers
2326  audio_converted->linesize[0], // output plane size, in bytes. (0 if unknown)
2327  audio_converted->nb_samples, // maximum number of samples that the output buffer can hold
2328  audio_frame->data, // input data pointers
2329  audio_frame->linesize[0], // input plane size, in bytes (0 if unknown)
2330  audio_frame->nb_samples); // number of input samples to convert
2331 
2332 
2333  int64_t starting_frame_number = -1;
2334  for (int channel_filter = 0; channel_filter < info.channels; channel_filter++) {
2335  // Array of floats (to hold samples for each channel)
2336  starting_frame_number = location.frame;
2337  int channel_buffer_size = nb_samples;
2338  auto *channel_buffer = (float *) (audio_converted->data[channel_filter]);
2339 
2340  // Loop through samples, and add them to the correct frames
2341  int start = location.sample_start;
2342  int remaining_samples = channel_buffer_size;
2343  while (remaining_samples > 0) {
2344  // Get Samples per frame (for this frame number)
2345  int samples_per_frame = Frame::GetSamplesPerFrame(starting_frame_number, info.fps, info.sample_rate, info.channels);
2346 
2347  // Calculate # of samples to add to this frame
2348  int samples = std::fmin(samples_per_frame - start, remaining_samples);
2349 
2350  // Create or get the existing frame object
2351  std::shared_ptr<Frame> f = CreateFrame(starting_frame_number);
2352 
2353  // Add samples for current channel to the frame.
2354  f->AddAudio(true, channel_filter, start, channel_buffer, samples, 1.0f);
2355 
2356  // Debug output
2357  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (f->AddAudio)",
2358  "frame", starting_frame_number,
2359  "start", start,
2360  "samples", samples,
2361  "channel", channel_filter,
2362  "samples_per_frame", samples_per_frame);
2363 
2364  // Add or update cache
2365  working_cache.Add(f);
2366 
2367  // Decrement remaining samples
2368  remaining_samples -= samples;
2369 
2370  // Increment buffer (to next set of samples)
2371  if (remaining_samples > 0)
2372  channel_buffer += samples;
2373 
2374  // Increment frame number
2375  starting_frame_number++;
2376 
2377  // Reset starting sample #
2378  start = 0;
2379  }
2380  }
2381 
2382  // Free AVFrames
2383  av_free(audio_converted->data[0]);
2384  AV_FREE_FRAME(&audio_converted);
2385  AV_FREE_FRAME(&audio_frame);
2386 
2387  // Get audio PTS in seconds
2388  audio_pts_seconds = (double(audio_pts) * info.audio_timebase.ToDouble()) + pts_offset_seconds;
2389 
2390  // Debug output
2391  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (After)",
2392  "requested_frame", requested_frame,
2393  "starting_frame", location.frame,
2394  "end_frame", starting_frame_number - 1,
2395  "audio_pts_seconds", audio_pts_seconds);
2396 
2397 }
2398 
2399 
2400 // Seek to a specific frame. This is not always frame accurate, it's more of an estimation on many codecs.
2401 void FFmpegReader::Seek(int64_t requested_frame) {
2402  // Adjust for a requested frame that is too small or too large
2403  if (requested_frame < 1)
2404  requested_frame = 1;
2405  if (requested_frame > info.video_length)
2406  requested_frame = info.video_length;
2407  if (requested_frame > largest_frame_processed && packet_status.end_of_file) {
2408  // Not possible to search past largest_frame once EOF is reached (no more packets)
2409  return;
2410  }
2411 
2412  // Debug output
2413  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::Seek",
2414  "requested_frame", requested_frame,
2415  "seek_count", seek_count,
2416  "last_frame", last_frame);
2417 
2418  // Clear working cache (since we are seeking to another location in the file)
2419  working_cache.Clear();
2420 
2421  // Reset the last frame variable
2422  video_pts = 0.0;
2423  video_pts_seconds = NO_PTS_OFFSET;
2424  audio_pts = 0.0;
2425  audio_pts_seconds = NO_PTS_OFFSET;
2426  hold_packet = false;
2427  last_frame = 0;
2428  current_video_frame = 0;
2429  largest_frame_processed = 0;
2430  last_final_video_frame.reset();
2431  bool has_audio_override = info.has_audio;
2432  bool has_video_override = info.has_video;
2433 
2434  // Init end-of-file detection variables
2435  packet_status.reset(false);
2436 
2437  // Increment seek count
2438  seek_count++;
2439 
2440  // If seeking near frame 1, we need to close and re-open the file (this is more reliable than seeking)
2441  int buffer_amount = 12;
2442  if (requested_frame - buffer_amount < 20) {
2443  // prevent Open() from seeking again
2444  is_seeking = true;
2445 
2446  // Close and re-open file (basically seeking to frame 1)
2447  Close();
2448  Open();
2449 
2450  // Update overrides (since closing and re-opening might update these)
2451  info.has_audio = has_audio_override;
2452  info.has_video = has_video_override;
2453 
2454  // Not actually seeking, so clear these flags
2455  is_seeking = false;
2456  if (seek_count == 1) {
2457  // Don't redefine this on multiple seek attempts for a specific frame
2458  seeking_frame = 1;
2459  seeking_pts = ConvertFrameToVideoPTS(1);
2460  }
2461  seek_audio_frame_found = 0; // used to detect which frames to throw away after a seek
2462  seek_video_frame_found = 0; // used to detect which frames to throw away after a seek
2463 
2464  } else {
2465  // Seek to nearest key-frame (aka, i-frame)
2466  bool seek_worked = false;
2467  int64_t seek_target = 0;
2468 
2469  // Seek video stream (if any), except album arts
2470  if (!seek_worked && info.has_video && !HasAlbumArt()) {
2471  seek_target = ConvertFrameToVideoPTS(requested_frame - buffer_amount);
2472  if (av_seek_frame(pFormatCtx, info.video_stream_index, seek_target, AVSEEK_FLAG_BACKWARD) < 0) {
2473  ZmqLogger::Instance()->Log(std::string(pFormatCtx->AV_FILENAME) + ": error while seeking video stream");
2474  } else {
2475  // VIDEO SEEK
2476  is_video_seek = true;
2477  seek_worked = true;
2478  }
2479  }
2480 
2481  // Seek audio stream (if not already seeked... and if an audio stream is found)
2482  if (!seek_worked && info.has_audio) {
2483  seek_target = ConvertFrameToAudioPTS(requested_frame - buffer_amount);
2484  if (av_seek_frame(pFormatCtx, info.audio_stream_index, seek_target, AVSEEK_FLAG_BACKWARD) < 0) {
2485  ZmqLogger::Instance()->Log(std::string(pFormatCtx->AV_FILENAME) + ": error while seeking audio stream");
2486  } else {
2487  // AUDIO SEEK
2488  is_video_seek = false;
2489  seek_worked = true;
2490  }
2491  }
2492 
2493  // Was the seek successful?
2494  if (seek_worked) {
2495  // Flush audio buffer
2496  if (info.has_audio)
2497  avcodec_flush_buffers(aCodecCtx);
2498 
2499  // Flush video buffer
2500  if (info.has_video)
2501  avcodec_flush_buffers(pCodecCtx);
2502 
2503  // Reset previous audio location to zero
2504  previous_packet_location.frame = -1;
2505  previous_packet_location.sample_start = 0;
2506 
2507  // init seek flags
2508  is_seeking = true;
2509  if (seek_count == 1) {
2510  // Don't redefine this on multiple seek attempts for a specific frame
2511  seeking_pts = seek_target;
2512  seeking_frame = requested_frame;
2513  }
2514  seek_audio_frame_found = 0; // used to detect which frames to throw away after a seek
2515  seek_video_frame_found = 0; // used to detect which frames to throw away after a seek
2516 
2517  } else {
2518  // seek failed
2519  seeking_pts = 0;
2520  seeking_frame = 0;
2521 
2522  // prevent Open() from seeking again
2523  is_seeking = true;
2524 
2525  // Close and re-open file (basically seeking to frame 1)
2526  Close();
2527  Open();
2528 
2529  // Not actually seeking, so clear these flags
2530  is_seeking = false;
2531 
2532  // disable seeking for this reader (since it failed)
2533  enable_seek = false;
2534 
2535  // Update overrides (since closing and re-opening might update these)
2536  info.has_audio = has_audio_override;
2537  info.has_video = has_video_override;
2538  }
2539  }
2540 }
2541 
2542 // Get the PTS for the current video packet
2543 int64_t FFmpegReader::GetPacketPTS() {
2544  if (packet) {
2545  int64_t current_pts = packet->pts;
2546  if (current_pts == AV_NOPTS_VALUE && packet->dts != AV_NOPTS_VALUE)
2547  current_pts = packet->dts;
2548 
2549  // Return adjusted PTS
2550  return current_pts;
2551  } else {
2552  // No packet, return NO PTS
2553  return AV_NOPTS_VALUE;
2554  }
2555 }
2556 
2557 // Update PTS Offset (if any)
2558 void FFmpegReader::UpdatePTSOffset() {
2559  if (pts_offset_seconds != NO_PTS_OFFSET) {
2560  // Skip this method if we have already set PTS offset
2561  return;
2562  }
2563  pts_offset_seconds = 0.0;
2564  double video_pts_offset_seconds = 0.0;
2565  double audio_pts_offset_seconds = 0.0;
2566 
2567  bool has_video_pts = false;
2568  if (!info.has_video) {
2569  // Mark as checked
2570  has_video_pts = true;
2571  }
2572  bool has_audio_pts = false;
2573  if (!info.has_audio) {
2574  // Mark as checked
2575  has_audio_pts = true;
2576  }
2577 
2578  // Loop through the stream (until a packet from all streams is found)
2579  while (!has_video_pts || !has_audio_pts) {
2580  // Get the next packet (if any)
2581  if (GetNextPacket() < 0)
2582  // Break loop when no more packets found
2583  break;
2584 
2585  // Get PTS of this packet
2586  int64_t pts = GetPacketPTS();
2587 
2588  // Video packet
2589  if (!has_video_pts && packet->stream_index == videoStream) {
2590  // Get the video packet start time (in seconds)
2591  video_pts_offset_seconds = 0.0 - (pts * info.video_timebase.ToDouble());
2592 
2593  // Is timestamp close to zero (within X seconds)
2594  // Ignore wildly invalid timestamps (i.e. -234923423423)
2595  if (std::abs(video_pts_offset_seconds) <= 10.0) {
2596  has_video_pts = true;
2597  }
2598  }
2599  else if (!has_audio_pts && packet->stream_index == audioStream) {
2600  // Get the audio packet start time (in seconds)
2601  audio_pts_offset_seconds = 0.0 - (pts * info.audio_timebase.ToDouble());
2602 
2603  // Is timestamp close to zero (within X seconds)
2604  // Ignore wildly invalid timestamps (i.e. -234923423423)
2605  if (std::abs(audio_pts_offset_seconds) <= 10.0) {
2606  has_audio_pts = true;
2607  }
2608  }
2609  }
2610 
2611  // Choose timestamp origin:
2612  // - If video exists, anchor timeline frame mapping to video start.
2613  // This avoids AAC priming / audio preroll shifting video frame 1 to frame 2.
2614  // - If no video exists (audio-only readers), use audio start.
2615  if (info.has_video && has_video_pts) {
2616  pts_offset_seconds = video_pts_offset_seconds;
2617  } else if (!info.has_video && has_audio_pts) {
2618  pts_offset_seconds = audio_pts_offset_seconds;
2619  } else if (has_video_pts && has_audio_pts) {
2620  // Fallback when stream flags are unusual but both timestamps exist.
2621  pts_offset_seconds = video_pts_offset_seconds;
2622  }
2623 }
2624 
2625 // Convert PTS into Frame Number
2626 int64_t FFmpegReader::ConvertVideoPTStoFrame(int64_t pts) {
2627  // Apply PTS offset
2628  int64_t previous_video_frame = current_video_frame;
2629  const double fps_value = (info.fps.num > 0 && info.fps.den > 0) ? info.fps.ToDouble() : 30.0;
2630  const double video_timebase_value =
2631  (info.video_timebase.num > 0 && info.video_timebase.den > 0)
2632  ? info.video_timebase.ToDouble()
2633  : (1.0 / 30.0);
2634 
2635  // Get the video packet start time (in seconds)
2636  double video_seconds = (double(pts) * video_timebase_value) + pts_offset_seconds;
2637 
2638  // Divide by the video timebase, to get the video frame number (frame # is decimal at this point)
2639  int64_t frame = round(video_seconds * fps_value) + 1;
2640 
2641  // Keep track of the expected video frame #
2642  if (current_video_frame == 0)
2643  current_video_frame = frame;
2644  else {
2645 
2646  // Sometimes frames are duplicated due to identical (or similar) timestamps
2647  if (frame == previous_video_frame) {
2648  // return -1 frame number
2649  frame = -1;
2650  } else {
2651  // Increment expected frame
2652  current_video_frame++;
2653  }
2654  }
2655 
2656  // Return frame #
2657  return frame;
2658 }
2659 
2660 // Convert Frame Number into Video PTS
2661 int64_t FFmpegReader::ConvertFrameToVideoPTS(int64_t frame_number) {
2662  const double fps_value = (info.fps.num > 0 && info.fps.den > 0) ? info.fps.ToDouble() : 30.0;
2663  const double video_timebase_value =
2664  (info.video_timebase.num > 0 && info.video_timebase.den > 0)
2665  ? info.video_timebase.ToDouble()
2666  : (1.0 / 30.0);
2667 
2668  // Get timestamp of this frame (in seconds)
2669  double seconds = (double(frame_number - 1) / fps_value) + pts_offset_seconds;
2670 
2671  // Calculate the # of video packets in this timestamp
2672  int64_t video_pts = round(seconds / video_timebase_value);
2673 
2674  // Apply PTS offset (opposite)
2675  return video_pts;
2676 }
2677 
2678 // Convert Frame Number into Video PTS
2679 int64_t FFmpegReader::ConvertFrameToAudioPTS(int64_t frame_number) {
2680  const double fps_value = (info.fps.num > 0 && info.fps.den > 0) ? info.fps.ToDouble() : 30.0;
2681  const double audio_timebase_value =
2682  (info.audio_timebase.num > 0 && info.audio_timebase.den > 0)
2683  ? info.audio_timebase.ToDouble()
2684  : (1.0 / 48000.0);
2685 
2686  // Get timestamp of this frame (in seconds)
2687  double seconds = (double(frame_number - 1) / fps_value) + pts_offset_seconds;
2688 
2689  // Calculate the # of audio packets in this timestamp
2690  int64_t audio_pts = round(seconds / audio_timebase_value);
2691 
2692  // Apply PTS offset (opposite)
2693  return audio_pts;
2694 }
2695 
2696 // Calculate Starting video frame and sample # for an audio PTS
2697 AudioLocation FFmpegReader::GetAudioPTSLocation(int64_t pts) {
2698  const double audio_timebase_value =
2699  (info.audio_timebase.num > 0 && info.audio_timebase.den > 0)
2700  ? info.audio_timebase.ToDouble()
2701  : (1.0 / 48000.0);
2702  const double fps_value = (info.fps.num > 0 && info.fps.den > 0) ? info.fps.ToDouble() : 30.0;
2703 
2704  // Get the audio packet start time (in seconds)
2705  double audio_seconds = (double(pts) * audio_timebase_value) + pts_offset_seconds;
2706 
2707  // Divide by the video timebase, to get the video frame number (frame # is decimal at this point)
2708  double frame = (audio_seconds * fps_value) + 1;
2709 
2710  // Frame # as a whole number (no more decimals)
2711  int64_t whole_frame = int64_t(frame);
2712 
2713  // Remove the whole number, and only get the decimal of the frame
2714  double sample_start_percentage = frame - double(whole_frame);
2715 
2716  // Get Samples per frame
2717  int samples_per_frame = Frame::GetSamplesPerFrame(whole_frame, info.fps, info.sample_rate, info.channels);
2718 
2719  // Calculate the sample # to start on
2720  int sample_start = round(double(samples_per_frame) * sample_start_percentage);
2721 
2722  // Protect against broken (i.e. negative) timestamps
2723  if (whole_frame < 1)
2724  whole_frame = 1;
2725  if (sample_start < 0)
2726  sample_start = 0;
2727 
2728  // Prepare final audio packet location
2729  AudioLocation location = {whole_frame, sample_start};
2730 
2731  // Compare to previous audio packet (and fix small gaps due to varying PTS timestamps)
2732  if (previous_packet_location.frame != -1) {
2733  if (location.is_near(previous_packet_location, samples_per_frame, samples_per_frame)) {
2734  int64_t orig_frame = location.frame;
2735  int orig_start = location.sample_start;
2736 
2737  // Update sample start, to prevent gaps in audio
2738  location.sample_start = previous_packet_location.sample_start;
2739  location.frame = previous_packet_location.frame;
2740 
2741  // Debug output
2742  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAudioPTSLocation (Audio Gap Detected)", "Source Frame", orig_frame, "Source Audio Sample", orig_start, "Target Frame", location.frame, "Target Audio Sample", location.sample_start, "pts", pts);
2743 
2744  } else {
2745  // Debug output
2746  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAudioPTSLocation (Audio Gap Ignored - too big)", "Previous location frame", previous_packet_location.frame, "Target Frame", location.frame, "Target Audio Sample", location.sample_start, "pts", pts);
2747  }
2748  }
2749 
2750  // Set previous location
2751  previous_packet_location = location;
2752 
2753  // Return the associated video frame and starting sample #
2754  return location;
2755 }
2756 
2757 // Create a new Frame (or return an existing one) and add it to the working queue.
2758 std::shared_ptr<Frame> FFmpegReader::CreateFrame(int64_t requested_frame) {
2759  // Check working cache
2760  std::shared_ptr<Frame> output = working_cache.GetFrame(requested_frame);
2761 
2762  if (!output) {
2763  // (re-)Check working cache
2764  output = working_cache.GetFrame(requested_frame);
2765  if(output) return output;
2766 
2767  // Create a new frame on the working cache
2768  output = std::make_shared<Frame>(requested_frame, info.width, info.height, "#000000", Frame::GetSamplesPerFrame(requested_frame, info.fps, info.sample_rate, info.channels), info.channels);
2769  output->SetPixelRatio(info.pixel_ratio.num, info.pixel_ratio.den); // update pixel ratio
2770  output->ChannelsLayout(info.channel_layout); // update audio channel layout from the parent reader
2771  output->SampleRate(info.sample_rate); // update the frame's sample rate of the parent reader
2772 
2773  working_cache.Add(output);
2774 
2775  // Set the largest processed frame (if this is larger)
2776  if (requested_frame > largest_frame_processed)
2777  largest_frame_processed = requested_frame;
2778  }
2779  // Return frame
2780  return output;
2781 }
2782 
2783 // Determine if frame is partial due to seek
2784 bool FFmpegReader::IsPartialFrame(int64_t requested_frame) {
2785 
2786  // Sometimes a seek gets partial frames, and we need to remove them
2787  bool seek_trash = false;
2788  int64_t max_seeked_frame = seek_audio_frame_found; // determine max seeked frame
2789  if (seek_video_frame_found > max_seeked_frame) {
2790  max_seeked_frame = seek_video_frame_found;
2791  }
2792  if ((info.has_audio && seek_audio_frame_found && max_seeked_frame >= requested_frame) ||
2793  (info.has_video && seek_video_frame_found && max_seeked_frame >= requested_frame)) {
2794  seek_trash = true;
2795  }
2796 
2797  return seek_trash;
2798 }
2799 
2800 // Check the working queue, and move finished frames to the finished queue
2801 void FFmpegReader::CheckWorkingFrames(int64_t requested_frame) {
2802 
2803  // Prevent async calls to the following code
2804  const std::lock_guard<std::recursive_mutex> lock(getFrameMutex);
2805 
2806  // Get a list of current working queue frames in the cache (in-progress frames)
2807  std::vector<std::shared_ptr<openshot::Frame>> working_frames = working_cache.GetFrames();
2808  std::vector<std::shared_ptr<openshot::Frame>>::iterator working_itr;
2809 
2810  // Loop through all working queue frames (sorted by frame #)
2811  for(working_itr = working_frames.begin(); working_itr != working_frames.end(); ++working_itr)
2812  {
2813  // Get working frame
2814  std::shared_ptr<Frame> f = *working_itr;
2815 
2816  // Was a frame found? Is frame requested yet?
2817  if (!f || f->number > requested_frame) {
2818  // If not, skip to next one
2819  continue;
2820  }
2821 
2822  // Calculate PTS in seconds (of working frame), and the most recent processed pts value
2823  double frame_pts_seconds = (double(f->number - 1) / info.fps.ToDouble()) + pts_offset_seconds;
2824  double recent_pts_seconds = std::max(video_pts_seconds, audio_pts_seconds);
2825 
2826  // Determine if video and audio are ready (based on timestamps)
2827  bool is_video_ready = false;
2828  bool is_audio_ready = false;
2829  double recent_pts_diff = recent_pts_seconds - frame_pts_seconds;
2830  if ((frame_pts_seconds <= video_pts_seconds)
2831  || (recent_pts_diff > 1.5)
2832  || packet_status.video_eof || packet_status.end_of_file) {
2833  // Video stream is past this frame (so it must be done)
2834  // OR video stream is too far behind, missing, or end-of-file
2835  is_video_ready = true;
2836  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (video ready)",
2837  "frame_number", f->number,
2838  "frame_pts_seconds", frame_pts_seconds,
2839  "video_pts_seconds", video_pts_seconds,
2840  "recent_pts_diff", recent_pts_diff);
2841  if (info.has_video && !f->has_image_data &&
2842  (packet_status.video_eof || packet_status.end_of_file)) {
2843  // Frame has no image data. Prefer timeline-previous frames to preserve
2844  // visual order, especially when decode/prefetch is out-of-order.
2845  std::shared_ptr<Frame> previous_frame_instance = final_cache.GetFrame(f->number - 1);
2846  if (previous_frame_instance && previous_frame_instance->has_image_data) {
2847  f->AddImage(std::make_shared<QImage>(previous_frame_instance->GetImage()->copy()));
2848  }
2849 
2850  // Fall back to last finalized timeline image (survives cache churn).
2851  if (!f->has_image_data
2852  && last_final_video_frame
2853  && last_final_video_frame->has_image_data
2854  && last_final_video_frame->number <= f->number) {
2855  f->AddImage(std::make_shared<QImage>(last_final_video_frame->GetImage()->copy()));
2856  }
2857 
2858  // Fall back to the last decoded image only when it is not from the future.
2859  if (!f->has_image_data
2860  && last_video_frame
2861  && last_video_frame->has_image_data
2862  && last_video_frame->number <= f->number) {
2863  f->AddImage(std::make_shared<QImage>(last_video_frame->GetImage()->copy()));
2864  }
2865 
2866  // Last-resort fallback if no prior image is available.
2867  if (!f->has_image_data) {
2868  ZmqLogger::Instance()->AppendDebugMethod(
2869  "FFmpegReader::CheckWorkingFrames (no previous image found; using black frame)",
2870  "frame_number", f->number);
2871  f->AddColor("#000000");
2872  }
2873  }
2874  }
2875 
2876  double audio_pts_diff = audio_pts_seconds - frame_pts_seconds;
2877  if ((frame_pts_seconds < audio_pts_seconds && audio_pts_diff > 1.0)
2878  || (recent_pts_diff > 1.5)
2879  || packet_status.audio_eof || packet_status.end_of_file) {
2880  // Audio stream is past this frame (so it must be done)
2881  // OR audio stream is too far behind, missing, or end-of-file
2882  // Adding a bit of margin here, to allow for partial audio packets
2883  is_audio_ready = true;
2884  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (audio ready)",
2885  "frame_number", f->number,
2886  "frame_pts_seconds", frame_pts_seconds,
2887  "audio_pts_seconds", audio_pts_seconds,
2888  "audio_pts_diff", audio_pts_diff,
2889  "recent_pts_diff", recent_pts_diff);
2890  }
2891  bool is_seek_trash = IsPartialFrame(f->number);
2892 
2893  // Adjust for available streams
2894  if (!info.has_video) is_video_ready = true;
2895  if (!info.has_audio) is_audio_ready = true;
2896 
2897  // Debug output
2898  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames",
2899  "frame_number", f->number,
2900  "is_video_ready", is_video_ready,
2901  "is_audio_ready", is_audio_ready,
2902  "video_eof", packet_status.video_eof,
2903  "audio_eof", packet_status.audio_eof,
2904  "end_of_file", packet_status.end_of_file);
2905 
2906  // Check if working frame is final
2907  if (info.has_video && !f->has_image_data
2908  && !packet_status.end_of_file && !is_seek_trash) {
2909  if (info.has_single_image) {
2910  // For still-image video (including attached cover art), reuse the most
2911  // recent image so playback does not stall waiting for video EOF.
2912  std::shared_ptr<Frame> previous_frame_instance = final_cache.GetFrame(f->number - 1);
2913  if (previous_frame_instance && previous_frame_instance->has_image_data) {
2914  f->AddImage(std::make_shared<QImage>(previous_frame_instance->GetImage()->copy()));
2915  }
2916  if (!f->has_image_data
2917  && last_final_video_frame
2918  && last_final_video_frame->has_image_data
2919  && last_final_video_frame->number <= f->number) {
2920  f->AddImage(std::make_shared<QImage>(last_final_video_frame->GetImage()->copy()));
2921  }
2922  if (!f->has_image_data
2923  && last_video_frame
2924  && last_video_frame->has_image_data
2925  && last_video_frame->number <= f->number) {
2926  f->AddImage(std::make_shared<QImage>(last_video_frame->GetImage()->copy()));
2927  }
2928  }
2929 
2930  // If both streams have advanced past this frame but the decoder never
2931  // produced image data for it, reuse the most recent non-future image.
2932  // This avoids stalling indefinitely on sparse/missing decoded frames.
2933  if (!f->has_image_data && is_video_ready && is_audio_ready) {
2934  std::shared_ptr<Frame> previous_frame_instance = final_cache.GetFrame(f->number - 1);
2935  if (previous_frame_instance && previous_frame_instance->has_image_data) {
2936  f->AddImage(std::make_shared<QImage>(previous_frame_instance->GetImage()->copy()));
2937  }
2938  if (!f->has_image_data
2939  && last_final_video_frame
2940  && last_final_video_frame->has_image_data
2941  && last_final_video_frame->number <= f->number) {
2942  f->AddImage(std::make_shared<QImage>(last_final_video_frame->GetImage()->copy()));
2943  }
2944  if (!f->has_image_data
2945  && last_video_frame
2946  && last_video_frame->has_image_data
2947  && last_video_frame->number <= f->number) {
2948  f->AddImage(std::make_shared<QImage>(last_video_frame->GetImage()->copy()));
2949  }
2950  }
2951 
2952  // Do not finalize non-EOF video frames without decoded image data.
2953  // This prevents repeated previous-frame fallbacks being cached as real frames.
2954  if (!f->has_image_data) {
2955  continue;
2956  }
2957  }
2958  if ((!packet_status.end_of_file && is_video_ready && is_audio_ready) || packet_status.end_of_file || is_seek_trash) {
2959  // Debug output
2960  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (mark frame as final)",
2961  "requested_frame", requested_frame,
2962  "f->number", f->number,
2963  "is_seek_trash", is_seek_trash,
2964  "Working Cache Count", working_cache.Count(),
2965  "Final Cache Count", final_cache.Count(),
2966  "end_of_file", packet_status.end_of_file);
2967 
2968  if (!is_seek_trash) {
2969  // Move frame to final cache
2970  final_cache.Add(f);
2971  if (f->has_image_data) {
2972  last_final_video_frame = f;
2973  }
2974 
2975  // Remove frame from working cache
2976  working_cache.Remove(f->number);
2977 
2978  // Update last frame processed
2979  last_frame = f->number;
2980  } else {
2981  // Seek trash, so delete the frame from the working cache, and never add it to the final cache.
2982  working_cache.Remove(f->number);
2983  }
2984 
2985  }
2986  }
2987 
2988  // Clear vector of frames
2989  working_frames.clear();
2990  working_frames.shrink_to_fit();
2991 }
2992 
2993 // Check for the correct frames per second (FPS) value by scanning the 1st few seconds of video packets.
2994 void FFmpegReader::CheckFPS() {
2995  if (check_fps) {
2996  // Do not check FPS more than 1 time
2997  return;
2998  } else {
2999  check_fps = true;
3000  }
3001 
3002  int frames_per_second[3] = {0,0,0};
3003  int max_fps_index = sizeof(frames_per_second) / sizeof(frames_per_second[0]);
3004  int fps_index = 0;
3005 
3006  int all_frames_detected = 0;
3007  int starting_frames_detected = 0;
3008 
3009  // Loop through the stream
3010  while (true) {
3011  // Get the next packet (if any)
3012  if (GetNextPacket() < 0)
3013  // Break loop when no more packets found
3014  break;
3015 
3016  // Video packet
3017  if (packet->stream_index == videoStream) {
3018  // Get the video packet start time (in seconds)
3019  double video_seconds = (double(GetPacketPTS()) * info.video_timebase.ToDouble()) + pts_offset_seconds;
3020  fps_index = int(video_seconds); // truncate float timestamp to int (second 1, second 2, second 3)
3021 
3022  // Is this video packet from the first few seconds?
3023  if (fps_index >= 0 && fps_index < max_fps_index) {
3024  // Yes, keep track of how many frames per second (over the first few seconds)
3025  starting_frames_detected++;
3026  frames_per_second[fps_index]++;
3027  }
3028 
3029  // Track all video packets detected
3030  all_frames_detected++;
3031  }
3032  }
3033 
3034  // Calculate FPS (based on the first few seconds of video packets)
3035  float avg_fps = 30.0;
3036  if (starting_frames_detected > 0 && fps_index > 0) {
3037  avg_fps = float(starting_frames_detected) / std::min(fps_index, max_fps_index);
3038  }
3039 
3040  // Verify average FPS is a reasonable value
3041  if (avg_fps < 8.0) {
3042  // Invalid FPS assumed, so switching to a sane default FPS instead
3043  avg_fps = 30.0;
3044  }
3045 
3046  // Update FPS (truncate average FPS to Integer)
3047  info.fps = Fraction(int(avg_fps), 1);
3048 
3049  // Update Duration and Length
3050  if (all_frames_detected > 0) {
3051  // Use all video frames detected to calculate # of frames
3052  info.video_length = all_frames_detected;
3053  info.duration = all_frames_detected / avg_fps;
3054  } else {
3055  // Use previous duration to calculate # of frames
3056  info.video_length = info.duration * avg_fps;
3057  }
3058 
3059  // Update video bit rate
3060  info.video_bit_rate = info.file_size / info.duration;
3061 }
3062 
3063 // Remove AVFrame from cache (and deallocate its memory)
3064 void FFmpegReader::RemoveAVFrame(AVFrame *remove_frame) {
3065  // Remove pFrame (if exists)
3066  if (remove_frame) {
3067  // Free memory
3068  av_freep(&remove_frame->data[0]);
3069 #ifndef WIN32
3070  AV_FREE_FRAME(&remove_frame);
3071 #endif
3072  }
3073 }
3074 
3075 // Remove AVPacket from cache (and deallocate its memory)
3076 void FFmpegReader::RemoveAVPacket(AVPacket *remove_packet) {
3077  // deallocate memory for packet
3078  AV_FREE_PACKET(remove_packet);
3079 
3080  // Delete the object
3081  delete remove_packet;
3082 }
3083 
3084 // Generate JSON string of this object
3085 std::string FFmpegReader::Json() const {
3086 
3087  // Return formatted string
3088  return JsonValue().toStyledString();
3089 }
3090 
3091 // Generate Json::Value for this object
3092 Json::Value FFmpegReader::JsonValue() const {
3093 
3094  // Create root json object
3095  Json::Value root = ReaderBase::JsonValue(); // get parent properties
3096  root["type"] = "FFmpegReader";
3097  root["path"] = path;
3098  switch (duration_strategy) {
3099  case DurationStrategy::VideoPreferred:
3100  root["duration_strategy"] = "VideoPreferred";
3101  break;
3102  case DurationStrategy::AudioPreferred:
3103  root["duration_strategy"] = "AudioPreferred";
3104  break;
3105  case DurationStrategy::LongestStream:
3106  default:
3107  root["duration_strategy"] = "LongestStream";
3108  break;
3109  }
3110 
3111  // return JsonValue
3112  return root;
3113 }
3114 
3115 // Load JSON string into this object
3116 void FFmpegReader::SetJson(const std::string value) {
3117 
3118  // Parse JSON string into JSON objects
3119  try {
3120  const Json::Value root = openshot::stringToJson(value);
3121  // Set all values that match
3122  SetJsonValue(root);
3123  }
3124  catch (const std::exception& e) {
3125  // Error parsing JSON (or missing keys)
3126  throw InvalidJSON("JSON is invalid (missing keys or invalid data types)");
3127  }
3128 }
3129 
3130 // Load Json::Value into this object
3131 void FFmpegReader::SetJsonValue(const Json::Value root) {
3132 
3133  // Set parent data
3134  ReaderBase::SetJsonValue(root);
3135 
3136  // Set data from Json (if key is found)
3137  if (!root["path"].isNull())
3138  path = root["path"].asString();
3139  if (!root["duration_strategy"].isNull()) {
3140  const std::string strategy = root["duration_strategy"].asString();
3141  if (strategy == "VideoPreferred") {
3142  duration_strategy = DurationStrategy::VideoPreferred;
3143  } else if (strategy == "AudioPreferred") {
3144  duration_strategy = DurationStrategy::AudioPreferred;
3145  } else {
3146  duration_strategy = DurationStrategy::LongestStream;
3147  }
3148  }
3149 }
openshot::stringToJson
const Json::Value stringToJson(const std::string value)
Definition: Json.cpp:16
openshot::CacheMemory::Clear
void Clear()
Clear the cache of all frames.
Definition: CacheMemory.cpp:224
AV_FIND_DECODER_CODEC_ID
#define AV_FIND_DECODER_CODEC_ID(av_stream)
Definition: FFmpegUtilities.h:317
openshot::ReaderInfo::sample_rate
int sample_rate
The number of audio samples per second (44100 is a common sample rate)
Definition: ReaderBase.h:60
openshot::FFmpegReader::FFmpegReader
FFmpegReader(const std::string &path, bool inspect_reader=true)
Constructor for FFmpegReader.
Definition: FFmpegReader.cpp:104
openshot::Fraction::ToFloat
float ToFloat()
Return this fraction as a float (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:35
openshot::Settings::HARDWARE_DECODER
int HARDWARE_DECODER
Use video codec for faster video decoding (if supported)
Definition: Settings.h:71
openshot::Coordinate::Y
double Y
The Y value of the coordinate (usually representing the value of the property being animated)
Definition: Coordinate.h:41
openshot::CacheMemory::Count
int64_t Count()
Count the frames in the queue.
Definition: CacheMemory.cpp:240
FFmpegUtilities.h
Header file for FFmpegUtilities.
openshot::ReaderBase::JsonValue
virtual Json::Value JsonValue() const =0
Generate Json::Value for this object.
Definition: ReaderBase.cpp:110
openshot::InvalidCodec
Exception when no valid codec is found for a file.
Definition: Exceptions.h:178
openshot::TimelineBase::preview_width
int preview_width
Optional preview width of timeline image. If your preview window is smaller than the timeline,...
Definition: TimelineBase.h:44
openshot::PacketStatus::reset
void reset(bool eof)
Definition: FFmpegReader.h:70
openshot::ReaderBase::MaxDecodeHeight
int MaxDecodeHeight() const
Return the current maximum decoded frame height (0 when unlimited).
Definition: ReaderBase.cpp:262
openshot::CacheMemory::GetFrame
std::shared_ptr< openshot::Frame > GetFrame(int64_t frame_number)
Get a frame from the cache.
Definition: CacheMemory.cpp:84
openshot::FFmpegReader::GetFrame
std::shared_ptr< openshot::Frame > GetFrame(int64_t requested_frame) override
Definition: FFmpegReader.cpp:1259
AV_COPY_PICTURE_DATA
#define AV_COPY_PICTURE_DATA(av_frame, buffer, pix_fmt, width, height)
Definition: FFmpegUtilities.h:326
openshot::CacheMemory::Add
void Add(std::shared_ptr< openshot::Frame > frame)
Add a Frame to the cache.
Definition: CacheMemory.cpp:47
AV_ALLOCATE_FRAME
#define AV_ALLOCATE_FRAME()
Definition: FFmpegUtilities.h:309
openshot::ReaderBase::SetJsonValue
virtual void SetJsonValue(const Json::Value root)=0
Load Json::Value into this object.
Definition: ReaderBase.cpp:161
SWR_CONVERT
#define SWR_CONVERT(ctx, out, linesize, out_count, in, linesize2, in_count)
Definition: FFmpegUtilities.h:258
openshot
This namespace is the default namespace for all code in the openshot library.
Definition: AnimatedCurve.h:24
openshot::Point::co
Coordinate co
This is the primary coordinate.
Definition: Point.h:66
openshot::Clip::scale_y
openshot::Keyframe scale_y
Curve representing the vertical scaling in percent (0 to 1)
Definition: Clip.h:332
openshot::AudioLocation
This struct holds the associated video frame and starting sample # for an audio packet.
Definition: AudioLocation.h:25
openshot::AudioLocation::frame
int64_t frame
Definition: AudioLocation.h:26
openshot::ZmqLogger::Log
void Log(std::string message)
Log message to all subscribers of this logger (if any)
Definition: ZmqLogger.cpp:103
openshot::Clip
This class represents a clip (used to arrange readers on the timeline)
Definition: Clip.h:89
openshot::DurationStrategy::AudioPreferred
@ AudioPreferred
Prefer the audio stream's duration, fallback to video then container.
openshot::Fraction
This class represents a fraction.
Definition: Fraction.h:30
openshot::AudioLocation::sample_start
int sample_start
Definition: AudioLocation.h:27
AV_FREE_FRAME
#define AV_FREE_FRAME(av_frame)
Definition: FFmpegUtilities.h:313
MemoryTrim.h
Cross-platform helper to encourage returning freed memory to the OS.
openshot::Keyframe::GetMaxPoint
Point GetMaxPoint() const
Get max point (by Y coordinate)
Definition: KeyFrame.cpp:245
openshot::ReaderBase::info
openshot::ReaderInfo info
Information about the current media file.
Definition: ReaderBase.h:91
openshot::ReaderInfo::interlaced_frame
bool interlaced_frame
Definition: ReaderBase.h:56
Timeline.h
Header file for Timeline class.
openshot::Clip::ParentTimeline
void ParentTimeline(openshot::TimelineBase *new_timeline) override
Set associated Timeline pointer.
Definition: Clip.cpp:450
openshot::FFmpegReader::~FFmpegReader
virtual ~FFmpegReader()
Destructor.
Definition: FFmpegReader.cpp:139
openshot::ReaderInfo::audio_bit_rate
int audio_bit_rate
The bit rate of the audio stream (in bytes)
Definition: ReaderBase.h:59
openshot::CacheMemory::Remove
void Remove(int64_t frame_number)
Remove a specific frame.
Definition: CacheMemory.cpp:158
AV_FREE_PACKET
#define AV_FREE_PACKET(av_packet)
Definition: FFmpegUtilities.h:314
openshot::ReaderInfo::duration
float duration
Length of time (in seconds)
Definition: ReaderBase.h:43
openshot::ReaderInfo::has_video
bool has_video
Determines if this file has a video stream.
Definition: ReaderBase.h:40
openshot::FFmpegReader::JsonValue
Json::Value JsonValue() const override
Generate Json::Value for this object.
Definition: FFmpegReader.cpp:3092
openshot::PacketStatus::audio_read
int64_t audio_read
Definition: FFmpegReader.h:51
openshot::ReaderInfo::width
int width
The width of the video (in pixesl)
Definition: ReaderBase.h:46
openshot::LAYOUT_STEREO
@ LAYOUT_STEREO
Definition: ChannelLayouts.h:31
openshot::FFmpegReader::SetJson
void SetJson(const std::string value) override
Load JSON string into this object.
Definition: FFmpegReader.cpp:3116
openshot::PacketStatus::packets_eof
bool packets_eof
Definition: FFmpegReader.h:57
hw_de_av_pix_fmt_global
AVPixelFormat hw_de_av_pix_fmt_global
Definition: FFmpegReader.cpp:74
openshot::PacketStatus::audio_decoded
int64_t audio_decoded
Definition: FFmpegReader.h:52
openshot::Fraction::ToDouble
double ToDouble() const
Return this fraction as a double (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:40
openshot::PacketStatus::video_read
int64_t video_read
Definition: FFmpegReader.h:49
hw_de_on
int hw_de_on
Definition: FFmpegReader.cpp:72
openshot::CacheBase::SetMaxBytesFromInfo
void SetMaxBytesFromInfo(int64_t number_of_frames, int width, int height, int sample_rate, int channels)
Set maximum bytes to a different amount based on a ReaderInfo struct.
Definition: CacheBase.cpp:28
openshot::ReaderBase::MaxDecodeWidth
int MaxDecodeWidth() const
Return the current maximum decoded frame width (0 when unlimited).
Definition: ReaderBase.cpp:258
AV_ALLOCATE_IMAGE
#define AV_ALLOCATE_IMAGE(av_frame, pix_fmt, width, height)
Definition: FFmpegUtilities.h:310
openshot::LAYOUT_MONO
@ LAYOUT_MONO
Definition: ChannelLayouts.h:30
openshot::Clip::scale_x
openshot::Keyframe scale_x
Curve representing the horizontal scaling in percent (0 to 1)
Definition: Clip.h:331
AV_GET_CODEC_ATTRIBUTES
#define AV_GET_CODEC_ATTRIBUTES(av_stream, av_context)
Definition: FFmpegUtilities.h:321
openshot::ReaderInfo::video_length
int64_t video_length
The number of frames in the video stream.
Definition: ReaderBase.h:53
hw_de_av_device_type_global
AVHWDeviceType hw_de_av_device_type_global
Definition: FFmpegReader.cpp:75
openshot::ReaderInfo::height
int height
The height of the video (in pixels)
Definition: ReaderBase.h:45
openshot::PacketStatus::video_eof
bool video_eof
Definition: FFmpegReader.h:55
openshot::Fraction::num
int num
Numerator for the fraction.
Definition: Fraction.h:32
if
if(!codec) codec
ZmqLogger.h
Header file for ZeroMQ-based Logger class.
openshot::Fraction::den
int den
Denominator for the fraction.
Definition: Fraction.h:33
OPEN_MP_NUM_PROCESSORS
#define OPEN_MP_NUM_PROCESSORS
Definition: OpenMPUtilities.h:23
AV_RESET_FRAME
#define AV_RESET_FRAME(av_frame)
Definition: FFmpegUtilities.h:312
openshot::AudioLocation::is_near
bool is_near(AudioLocation location, int samples_per_frame, int64_t amount)
Definition: FFmpegReader.cpp:146
SWR_CLOSE
#define SWR_CLOSE(ctx)
Definition: FFmpegUtilities.h:261
openshot::Fraction::Reciprocal
Fraction Reciprocal() const
Return the reciprocal as a Fraction.
Definition: Fraction.cpp:78
openshot::ReaderInfo::has_audio
bool has_audio
Determines if this file has an audio stream.
Definition: ReaderBase.h:41
openshot::Settings::DE_LIMIT_HEIGHT_MAX
int DE_LIMIT_HEIGHT_MAX
Maximum rows that hardware decode can handle.
Definition: Settings.h:86
openshot::InvalidJSON
Exception for invalid JSON.
Definition: Exceptions.h:223
openshot::FFmpegReader::enable_seek
bool enable_seek
Definition: FFmpegReader.h:270
openshot::ReaderInfo::file_size
int64_t file_size
Size of file (in bytes)
Definition: ReaderBase.h:44
openshot::Timeline
This class represents a timeline.
Definition: Timeline.h:153
openshot::FFmpegReader::Open
void Open() override
Open File - which is called by the constructor automatically.
Definition: FFmpegReader.cpp:263
openshot::OutOfMemory
Exception when memory could not be allocated.
Definition: Exceptions.h:354
openshot::SCALE_CROP
@ SCALE_CROP
Scale the clip until both height and width fill the canvas (cropping the overlap)
Definition: Enums.h:37
SWR_INIT
#define SWR_INIT(ctx)
Definition: FFmpegUtilities.h:263
SWRCONTEXT
#define SWRCONTEXT
Definition: FFmpegUtilities.h:264
openshot::PacketStatus::audio_eof
bool audio_eof
Definition: FFmpegReader.h:56
openshot::ReaderInfo::has_single_image
bool has_single_image
Determines if this file only contains a single image.
Definition: ReaderBase.h:42
openshot::FFmpegReader::final_cache
CacheMemory final_cache
Final cache object used to hold final frames.
Definition: FFmpegReader.h:266
openshot::ReaderInfo::video_timebase
openshot::Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
Definition: ReaderBase.h:55
openshot::Settings::Instance
static Settings * Instance()
Create or get an instance of this logger singleton (invoke the class with this method)
Definition: Settings.cpp:43
CropHelpers.h
Shared helpers for Crop effect scaling logic.
openshot::ReaderInfo::metadata
std::map< std::string, std::string > metadata
An optional map/dictionary of metadata for this reader.
Definition: ReaderBase.h:65
openshot::DurationStrategy::LongestStream
@ LongestStream
Use the longest value from video, audio, or container.
openshot::FFmpegReader
This class uses the FFmpeg libraries, to open video files and audio files, and return openshot::Frame...
Definition: FFmpegReader.h:103
path
path
Definition: FFmpegWriter.cpp:1481
openshot::Frame::GetSamplesPerFrame
int GetSamplesPerFrame(openshot::Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
Definition: Frame.cpp:484
openshot::InvalidFile
Exception for files that can not be found or opened.
Definition: Exceptions.h:193
openshot::ReaderInfo::audio_stream_index
int audio_stream_index
The index of the audio stream.
Definition: ReaderBase.h:63
openshot::ZmqLogger::Instance
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method)
Definition: ZmqLogger.cpp:35
openshot::DurationStrategy
DurationStrategy
This enumeration determines which duration source to favor.
Definition: Enums.h:60
openshot::ReaderInfo::audio_timebase
openshot::Fraction audio_timebase
The audio timebase determines how long each audio packet should be played.
Definition: ReaderBase.h:64
openshot::FFmpegReader::Close
void Close() override
Close File.
Definition: FFmpegReader.cpp:751
openshot::SCALE_FIT
@ SCALE_FIT
Scale the clip until either height or width fills the canvas (with no cropping)
Definition: Enums.h:38
openshot::PacketStatus::packets_read
int64_t packets_read()
Definition: FFmpegReader.h:60
openshot::ReaderInfo::pixel_format
int pixel_format
The pixel format (i.e. YUV420P, RGB24, etc...)
Definition: ReaderBase.h:47
openshot::ZmqLogger::AppendDebugMethod
void AppendDebugMethod(std::string method_name, std::string arg1_name="", float arg1_value=-1.0, std::string arg2_name="", float arg2_value=-1.0, std::string arg3_name="", float arg3_value=-1.0, std::string arg4_name="", float arg4_value=-1.0, std::string arg5_name="", float arg5_value=-1.0, std::string arg6_name="", float arg6_value=-1.0)
Append debug information.
Definition: ZmqLogger.cpp:178
openshot::ReaderInfo::vcodec
std::string vcodec
The name of the video codec used to encode / decode the video stream.
Definition: ReaderBase.h:52
openshot::PacketStatus::packets_decoded
int64_t packets_decoded()
Definition: FFmpegReader.h:65
AV_GET_CODEC_TYPE
#define AV_GET_CODEC_TYPE(av_stream)
Definition: FFmpegUtilities.h:316
openshot::ReaderClosed
Exception when a reader is closed, and a frame is requested.
Definition: Exceptions.h:369
openshot::ReaderInfo::channel_layout
openshot::ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
Definition: ReaderBase.h:62
AV_FREE_CONTEXT
#define AV_FREE_CONTEXT(av_context)
Definition: FFmpegUtilities.h:315
PIX_FMT_RGBA
#define PIX_FMT_RGBA
Definition: FFmpegUtilities.h:110
AV_GET_CODEC_PIXEL_FORMAT
#define AV_GET_CODEC_PIXEL_FORMAT(av_stream, av_context)
Definition: FFmpegUtilities.h:322
AVCODEC_REGISTER_ALL
#define AVCODEC_REGISTER_ALL
Definition: FFmpegUtilities.h:305
SWR_FREE
#define SWR_FREE(ctx)
Definition: FFmpegUtilities.h:262
openshot::Settings::DE_LIMIT_WIDTH_MAX
int DE_LIMIT_WIDTH_MAX
Maximum columns that hardware decode can handle.
Definition: Settings.h:89
openshot::ReaderBase::ApplyOrientationMetadata
bool ApplyOrientationMetadata() const
Return whether readers apply source orientation metadata to returned frames.
Definition: ReaderBase.cpp:274
openshot::ReaderInfo::fps
openshot::Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
Definition: ReaderBase.h:48
AV_GET_SAMPLE_FORMAT
#define AV_GET_SAMPLE_FORMAT(av_stream, av_context)
Definition: FFmpegUtilities.h:324
FF_AUDIO_NUM_PROCESSORS
#define FF_AUDIO_NUM_PROCESSORS
Definition: OpenMPUtilities.h:25
openshot::ReaderInfo::video_bit_rate
int video_bit_rate
The bit rate of the video stream (in bytes)
Definition: ReaderBase.h:49
openshot::PacketStatus::end_of_file
bool end_of_file
Definition: FFmpegReader.h:58
FF_VIDEO_NUM_PROCESSORS
#define FF_VIDEO_NUM_PROCESSORS
Definition: OpenMPUtilities.h:24
openshot::Clip::scale
openshot::ScaleType scale
The scale determines how a clip should be resized to fit its parent.
Definition: Clip.h:186
openshot::ReaderInfo::top_field_first
bool top_field_first
Definition: ReaderBase.h:57
openshot::InvalidChannels
Exception when an invalid # of audio channels are detected.
Definition: Exceptions.h:163
openshot::ChannelLayout
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround,...
Definition: ChannelLayouts.h:28
SWR_ALLOC
#define SWR_ALLOC()
Definition: FFmpegUtilities.h:260
openshot::ReaderInfo::pixel_ratio
openshot::Fraction pixel_ratio
The pixel ratio of the video stream as a fraction (i.e. some pixels are not square)
Definition: ReaderBase.h:50
AV_REGISTER_ALL
#define AV_REGISTER_ALL
Definition: FFmpegUtilities.h:304
openshot::DurationStrategy::VideoPreferred
@ VideoPreferred
Prefer the video stream's duration, fallback to audio then container.
openshot::CacheMemory::GetFrames
std::vector< std::shared_ptr< openshot::Frame > > GetFrames()
Get an array of all Frames.
Definition: CacheMemory.cpp:100
AV_GET_CODEC_CONTEXT
#define AV_GET_CODEC_CONTEXT(av_stream, av_codec)
Definition: FFmpegUtilities.h:318
openshot::ReaderInfo::video_stream_index
int video_stream_index
The index of the video stream.
Definition: ReaderBase.h:54
openshot::ReaderBase::HasMaxDecodeSize
bool HasMaxDecodeSize() const
Return true when a maximum decoded frame size is active.
Definition: ReaderBase.cpp:266
openshot::FFmpegReader::SetJsonValue
void SetJsonValue(const Json::Value root) override
Load Json::Value into this object.
Definition: FFmpegReader.cpp:3131
openshot::SCALE_STRETCH
@ SCALE_STRETCH
Scale the clip until both height and width fill the canvas (distort to fit)
Definition: Enums.h:39
openshot::ReaderInfo::acodec
std::string acodec
The name of the audio codec used to encode / decode the video stream.
Definition: ReaderBase.h:58
openshot::NoStreamsFound
Exception when no streams are found in the file.
Definition: Exceptions.h:291
openshot::ReaderInfo::display_ratio
openshot::Fraction display_ratio
The ratio of width to height of the video stream (i.e. 640x480 has a ratio of 4/3)
Definition: ReaderBase.h:51
openshot::ReaderInfo::channels
int channels
The number of audio channels used in the audio stream.
Definition: ReaderBase.h:61
openshot::FFmpegReader::Json
std::string Json() const override
Generate JSON string of this object.
Definition: FFmpegReader.cpp:3085
openshot::FFmpegReader::HardwareDecodeSuccessful
bool HardwareDecodeSuccessful() const override
Return true if hardware decode was requested and successfully produced at least one frame.
Definition: FFmpegReader.cpp:1825
openshot::FFmpegReader::GetIsDurationKnown
bool GetIsDurationKnown()
Return true if frame can be read with GetFrame()
Definition: FFmpegReader.cpp:1255
openshot::ApplyCropResizeScale
void ApplyCropResizeScale(Clip *clip, int source_width, int source_height, int &max_width, int &max_height)
Scale the requested max_width / max_height based on the Crop resize amount, capped by source size.
Definition: CropHelpers.cpp:40
openshot::PacketStatus::video_decoded
int64_t video_decoded
Definition: FFmpegReader.h:50
opts
AVDictionary * opts
Definition: FFmpegWriter.cpp:1492
Exceptions.h
Header file for all Exception classes.
openshot::Settings::HW_DE_DEVICE_SET
int HW_DE_DEVICE_SET
Which GPU to use to decode (0 is the first)
Definition: Settings.h:92
FFmpegReader.h
Header file for FFmpegReader class.
openshot::ReaderBase::getFrameMutex
std::recursive_mutex getFrameMutex
Mutex for multiple threads.
Definition: ReaderBase.h:79
openshot::ReaderBase::ParentClip
openshot::ClipBase * ParentClip()
Parent clip object of this reader (which can be unparented and NULL)
Definition: ReaderBase.cpp:244
Generated by   doxygen 1.8.17