31 #include "../include/FFmpegReader.h"
36 : last_frame(0), is_seeking(0), seeking_pts(0), seeking_frame(0), seek_count(0),
37 audio_pts_offset(99999), video_pts_offset(99999), path(path), is_video_seek(true), check_interlace(false),
38 check_fps(false), enable_seek(true), is_open(false), seek_audio_frame_found(0), seek_video_frame_found(0),
39 prev_samples(0), prev_pts(0), pts_total(0), pts_counter(0), is_duration_known(false), largest_frame_processed(0),
40 current_video_frame(0), has_missing_frames(false), num_packets_since_video_frame(0), num_checks_since_final(0),
41 packet(NULL), use_omp_threads(true) {
58 : last_frame(0), is_seeking(0), seeking_pts(0), seeking_frame(0), seek_count(0),
59 audio_pts_offset(99999), video_pts_offset(99999), path(path), is_video_seek(true), check_interlace(false),
60 check_fps(false), enable_seek(true), is_open(false), seek_audio_frame_found(0), seek_video_frame_found(0),
61 prev_samples(0), prev_pts(0), pts_total(0), pts_counter(0), is_duration_known(false), largest_frame_processed(0),
62 current_video_frame(0), has_missing_frames(false), num_packets_since_video_frame(0), num_checks_since_final(0),
63 packet(NULL), use_omp_threads(true) {
98 if (abs(diff) <= amount)
115 if (avformat_open_input(&pFormatCtx, path.c_str(), NULL, NULL) != 0)
116 throw InvalidFile(
"File could not be opened.", path);
119 if (avformat_find_stream_info(pFormatCtx, NULL) < 0)
125 for (
unsigned int i = 0; i < pFormatCtx->nb_streams; i++)
128 if (
AV_GET_CODEC_TYPE(pFormatCtx->streams[i]) == AVMEDIA_TYPE_VIDEO && videoStream < 0) {
132 if (
AV_GET_CODEC_TYPE(pFormatCtx->streams[i]) == AVMEDIA_TYPE_AUDIO && audioStream < 0) {
136 if (videoStream == -1 && audioStream == -1)
137 throw NoStreamsFound(
"No video or audio streams found in this file.", path);
140 if (videoStream != -1)
146 pStream = pFormatCtx->streams[videoStream];
152 AVCodec *pCodec = avcodec_find_decoder(codecId);
158 if (pCodec == NULL) {
159 throw InvalidCodec(
"A valid video codec could not be found for this file.", path);
163 AVDictionary *opts = NULL;
164 av_dict_set(&opts,
"strict",
"experimental", 0);
167 if (avcodec_open2(pCodecCtx, pCodec, &opts) < 0)
168 throw InvalidCodec(
"A video codec was found, but could not be opened.", path);
178 if (audioStream != -1)
184 aStream = pFormatCtx->streams[audioStream];
190 AVCodec *aCodec = avcodec_find_decoder(codecId);
196 if (aCodec == NULL) {
197 throw InvalidCodec(
"A valid audio codec could not be found for this file.", path);
201 AVDictionary *opts = NULL;
202 av_dict_set(&opts,
"strict",
"experimental", 0);
205 if (avcodec_open2(aCodecCtx, aCodec, &opts) < 0)
206 throw InvalidCodec(
"An audio codec was found, but could not be opened.", path);
216 AVDictionaryEntry *tag = NULL;
217 while ((tag = av_dict_get(pFormatCtx->metadata,
"", tag, AV_DICT_IGNORE_SUFFIX))) {
218 QString str_key = tag->key;
219 QString str_value = tag->value;
220 info.
metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
224 previous_packet_location.
frame = -1;
233 use_omp_threads = openshot::IsOMPEnabled();
248 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::Close",
"", -1,
"", -1,
"", -1,
"", -1,
"", -1,
"", -1);
253 avcodec_flush_buffers(pCodecCtx);
258 avcodec_flush_buffers(aCodecCtx);
264 working_cache.
Clear();
265 missing_frames.
Clear();
270 processed_video_frames.clear();
271 processed_audio_frames.clear();
272 processing_video_frames.clear();
273 processing_audio_frames.clear();
274 missing_audio_frames.clear();
275 missing_video_frames.clear();
276 missing_audio_frames_source.clear();
277 missing_video_frames_source.clear();
278 checked_frames.clear();
282 avformat_close_input(&pFormatCtx);
283 av_freep(&pFormatCtx);
287 largest_frame_processed = 0;
288 seek_audio_frame_found = 0;
289 seek_video_frame_found = 0;
290 current_video_frame = 0;
291 has_missing_frames =
false;
295 void FFmpegReader::UpdateAudioInfo()
299 info.
file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;
313 if (aStream->duration > 0.0f && aStream->duration >
info.
duration)
342 AVDictionaryEntry *tag = NULL;
343 while ((tag = av_dict_get(aStream->metadata,
"", tag, AV_DICT_IGNORE_SUFFIX))) {
344 QString str_key = tag->key;
345 QString str_value = tag->value;
346 info.
metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
350 void FFmpegReader::UpdateVideoInfo()
358 info.
file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;
368 if (pStream->sample_aspect_ratio.num != 0)
418 is_duration_known =
false;
423 is_duration_known =
true;
437 AVDictionaryEntry *tag = NULL;
438 while ((tag = av_dict_get(pStream->metadata,
"", tag, AV_DICT_IGNORE_SUFFIX))) {
439 QString str_key = tag->key;
440 QString str_value = tag->value;
441 info.
metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
450 throw ReaderClosed(
"The FFmpegReader is closed. Call Open() before calling this method.", path);
453 if (requested_frame < 1)
459 throw InvalidFile(
"Could not detect the duration of the video or audio stream.", path);
462 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::GetFrame",
"requested_frame", requested_frame,
"last_frame", last_frame,
"", -1,
"", -1,
"", -1,
"", -1);
468 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::GetFrame",
"returned cached frame", requested_frame,
"", -1,
"", -1,
"", -1,
"", -1,
"", -1);
475 #pragma omp critical (ReadStream)
478 if (has_missing_frames)
479 CheckMissingFrame(requested_frame);
483 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::GetFrame",
"returned cached frame on 2nd look", requested_frame,
"", -1,
"", -1,
"", -1,
"", -1,
"", -1);
493 if (last_frame == 0 && requested_frame != 1)
498 int64_t diff = requested_frame - last_frame;
499 if (diff >= 1 && diff <= 20)
502 frame = ReadStream(requested_frame);
509 Seek(requested_frame);
519 frame = ReadStream(requested_frame);
528 std::shared_ptr<Frame> FFmpegReader::ReadStream(int64_t requested_frame)
531 bool end_of_stream =
false;
532 bool check_seek =
false;
533 bool frame_finished =
false;
534 int packet_error = -1;
537 int packets_processed = 0;
539 int max_packets = 4096;
544 omp_set_nested(
true);
547 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ReadStream",
"requested_frame", requested_frame,
"OPEN_MP_NUM_PROCESSORS",
OPEN_MP_NUM_PROCESSORS,
"", -1,
"", -1,
"", -1,
"", -1);
557 packet_error = GetNextPacket();
559 int processing_video_frames_size = 0;
560 int processing_audio_frames_size = 0;
563 processing_video_frames_size = processing_video_frames.size();
564 processing_audio_frames_size = processing_audio_frames.size();
568 while (processing_video_frames_size + processing_audio_frames_size >= minimum_packets) {
571 processing_video_frames_size = processing_video_frames.size();
572 processing_audio_frames_size = processing_audio_frames.size();
576 if (packet_error < 0)
579 end_of_stream =
true;
584 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ReadStream (GetNextPacket)",
"requested_frame", requested_frame,
"processing_video_frames_size", processing_video_frames_size,
"processing_audio_frames_size", processing_audio_frames_size,
"minimum_packets", minimum_packets,
"packets_processed", packets_processed,
"is_seeking", is_seeking);
590 num_packets_since_video_frame = 0;
594 #pragma omp critical (openshot_seek)
595 check_seek = CheckSeek(
true);
605 frame_finished = GetAVFrame();
611 UpdatePTSOffset(
true);
614 ProcessVideoPacket(requested_frame);
616 if (!use_omp_threads) {
625 else if (
info.
has_audio && packet->stream_index == audioStream)
628 num_packets_since_video_frame++;
632 #pragma omp critical (openshot_seek)
633 check_seek = CheckSeek(
false);
643 UpdatePTSOffset(
false);
655 CheckMissingFrame(requested_frame);
658 CheckWorkingFrames(
false, requested_frame);
668 if ((is_cache_found && packets_processed >= minimum_packets) || packets_processed > max_packets)
678 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ReadStream (Completed)",
"packets_processed", packets_processed,
"end_of_stream", end_of_stream,
"largest_frame_processed", largest_frame_processed,
"Working Cache Count", working_cache.
Count(),
"", -1,
"", -1);
683 CheckWorkingFrames(end_of_stream, requested_frame);
700 std::shared_ptr<Frame> f = CreateFrame(largest_frame_processed);
709 int FFmpegReader::GetNextPacket()
711 int found_packet = 0;
712 AVPacket *next_packet =
new AVPacket();
713 found_packet = av_read_frame(pFormatCtx, next_packet);
717 RemoveAVPacket(packet);
721 if (found_packet >= 0)
724 packet = next_packet;
732 bool FFmpegReader::GetAVFrame()
734 int frameFinished = -1;
739 #pragma omp critical (packet_cache)
743 ret = avcodec_send_packet(pCodecCtx, packet);
744 if (ret < 0 || ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
745 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::GetAVFrame (Packet not sent)",
"", -1,
"", -1,
"", -1,
"", -1,
"", -1,
"", -1);
748 pFrame =
new AVFrame();
750 ret = avcodec_receive_frame(pCodecCtx, next_frame);
751 if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
756 if (frameFinished == 0 ) {
758 av_image_alloc(pFrame->data, pFrame->linesize,
info.
width,
info.
height, (AVPixelFormat)(pStream->codecpar->format), 1);
759 av_image_copy(pFrame->data, pFrame->linesize, (
const uint8_t**)next_frame->data, next_frame->linesize,
761 if (!check_interlace) {
762 check_interlace =
true;
770 avcodec_decode_video2(pCodecCtx, next_frame, &frameFinished, packet);
778 av_picture_copy((AVPicture *) pFrame, (AVPicture *) next_frame, pCodecCtx->pix_fmt,
info.
width,
782 if (!check_interlace) {
783 check_interlace =
true;
795 return frameFinished;
799 bool FFmpegReader::CheckSeek(
bool is_video)
806 if ((is_video_seek && !seek_video_frame_found) || (!is_video_seek && !seek_audio_frame_found))
814 int64_t max_seeked_frame = seek_audio_frame_found;
815 if (seek_video_frame_found > max_seeked_frame)
816 max_seeked_frame = seek_video_frame_found;
819 if (max_seeked_frame >= seeking_frame)
822 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckSeek (Too far, seek again)",
"is_video_seek", is_video_seek,
"max_seeked_frame", max_seeked_frame,
"seeking_frame", seeking_frame,
"seeking_pts", seeking_pts,
"seek_video_frame_found", seek_video_frame_found,
"seek_audio_frame_found", seek_audio_frame_found);
825 Seek(seeking_frame - (10 * seek_count * seek_count));
830 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckSeek (Successful)",
"is_video_seek", is_video_seek,
"current_pts", packet->pts,
"seeking_pts", seeking_pts,
"seeking_frame", seeking_frame,
"seek_video_frame_found", seek_video_frame_found,
"seek_audio_frame_found", seek_audio_frame_found);
844 void FFmpegReader::ProcessVideoPacket(int64_t requested_frame)
847 int64_t current_frame = ConvertVideoPTStoFrame(GetVideoPTS());
850 if (!seek_video_frame_found && is_seeking)
851 seek_video_frame_found = current_frame;
854 if ((current_frame < (requested_frame - 20)) or (current_frame == -1))
857 RemoveAVFrame(pFrame);
860 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessVideoPacket (Skipped)",
"requested_frame", requested_frame,
"current_frame", current_frame,
"", -1,
"", -1,
"", -1,
"", -1);
867 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessVideoPacket (Before)",
"requested_frame", requested_frame,
"current_frame", current_frame,
"", -1,
"", -1,
"", -1,
"", -1);
874 AVFrame *my_frame = pFrame;
878 processing_video_frames[current_frame] = current_frame;
880 #pragma omp task firstprivate(current_frame, my_frame, height, width, video_length, pix_fmt)
883 AVFrame *pFrameRGB = NULL;
885 uint8_t *buffer = NULL;
889 if (pFrameRGB == NULL)
890 throw OutOfBoundsFrame(
"Convert Image Broke!", current_frame, video_length);
895 int original_height = height;
898 float ratio = float(width) / float(height);
899 int possible_width = round(
max_height * ratio);
900 int possible_height = round(
max_width / ratio);
904 width = possible_width;
909 height = possible_height;
916 #pragma omp critical (video_buffer)
917 buffer = (uint8_t *) av_malloc(numBytes *
sizeof(uint8_t));
926 sws_scale(img_convert_ctx, my_frame->data, my_frame->linesize, 0,
927 original_height, pFrameRGB->data, pFrameRGB->linesize);
930 std::shared_ptr<Frame> f = CreateFrame(current_frame);
933 f->AddImage(width, height, 4, QImage::Format_RGBA8888, buffer);
936 working_cache.
Add(f);
939 #pragma omp critical (video_buffer)
940 last_video_frame = f;
947 RemoveAVFrame(my_frame);
948 sws_freeContext(img_convert_ctx);
953 processing_video_frames.erase(current_frame);
954 processed_video_frames[current_frame] = current_frame;
958 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessVideoPacket (After)",
"requested_frame", requested_frame,
"current_frame", current_frame,
"f->number", f->number,
"", -1,
"", -1,
"", -1);
965 void FFmpegReader::ProcessAudioPacket(int64_t requested_frame, int64_t target_frame,
int starting_sample)
968 if (!seek_audio_frame_found && is_seeking)
969 seek_audio_frame_found = target_frame;
972 if (target_frame < (requested_frame - 20))
975 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessAudioPacket (Skipped)",
"requested_frame", requested_frame,
"target_frame", target_frame,
"starting_sample", starting_sample,
"", -1,
"", -1,
"", -1);
982 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessAudioPacket (Before)",
"requested_frame", requested_frame,
"target_frame", target_frame,
"starting_sample", starting_sample,
"", -1,
"", -1,
"", -1);
985 int frame_finished = 0;
989 int packet_samples = 0;
994 #pragma omp critical (ProcessAudioPacket)
999 while((packet->size > 0 || (!packet->data && frame_finished)) && ret >= 0) {
1001 ret = avcodec_send_packet(aCodecCtx, packet);
1002 if (ret < 0 && ret != AVERROR(EINVAL) && ret != AVERROR_EOF) {
1003 avcodec_send_packet(aCodecCtx, NULL);
1008 ret = avcodec_receive_frame(aCodecCtx, audio_frame);
1011 if(ret == AVERROR(EINVAL) || ret == AVERROR_EOF) {
1012 avcodec_flush_buffers(aCodecCtx);
1016 ret = frame_finished;
1019 if (!packet->data && !frame_finished)
1024 int used = avcodec_decode_audio4(aCodecCtx, audio_frame, &frame_finished, packet);
1028 if (frame_finished) {
1032 int plane_size = -1;
1033 data_size = av_samples_get_buffer_size(&plane_size,
1035 audio_frame->nb_samples,
1043 int pts_remaining_samples = packet_samples /
info.
channels;
1046 int64_t adjusted_pts = packet->pts + audio_pts_offset;
1051 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessAudioPacket (Decode Info A)",
"pts_counter", pts_counter,
"PTS", adjusted_pts,
"Offset", audio_pts_offset,
"PTS Diff", adjusted_pts - prev_pts,
"Samples", pts_remaining_samples,
"Sample PTS ratio",
float(adjusted_pts - prev_pts) / pts_remaining_samples);
1052 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessAudioPacket (Decode Info B)",
"Sample Diff", pts_remaining_samples - prev_samples - prev_pts,
"Total", pts_total,
"PTS Seconds", audio_seconds,
"Sample Seconds", sample_seconds,
"Seconds Diff", audio_seconds - sample_seconds,
"raw samples", packet_samples);
1055 prev_pts = adjusted_pts;
1056 pts_total += pts_remaining_samples;
1058 prev_samples = pts_remaining_samples;
1063 processing_audio_frames.insert(pair<int, int>(previous_packet_location.
frame, previous_packet_location.
frame));
1066 while (pts_remaining_samples)
1072 int samples = samples_per_frame - previous_packet_location.
sample_start;
1073 if (samples > pts_remaining_samples)
1074 samples = pts_remaining_samples;
1077 pts_remaining_samples -= samples;
1079 if (pts_remaining_samples > 0) {
1081 previous_packet_location.
frame++;
1087 processing_audio_frames.insert(pair<int, int>(previous_packet_location.
frame, previous_packet_location.
frame));
1100 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessAudioPacket (ReSample)",
"packet_samples", packet_samples,
"info.channels",
info.
channels,
"info.sample_rate",
info.
sample_rate,
"aCodecCtx->sample_fmt",
AV_GET_SAMPLE_FORMAT(aStream, aCodecCtx),
"AV_SAMPLE_FMT_S16", AV_SAMPLE_FMT_S16,
"", -1);
1105 audio_converted->nb_samples = audio_frame->nb_samples;
1106 av_samples_alloc(audio_converted->data, audio_converted->linesize,
info.
channels, audio_frame->nb_samples, AV_SAMPLE_FMT_S16, 0);
1116 av_opt_set_int(avr,
"out_sample_fmt", AV_SAMPLE_FMT_S16, 0);
1125 audio_converted->data,
1126 audio_converted->linesize[0],
1127 audio_converted->nb_samples,
1129 audio_frame->linesize[0],
1130 audio_frame->nb_samples);
1133 memcpy(audio_buf, audio_converted->data[0], audio_converted->nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) *
info.
channels);
1141 av_free(audio_converted->data[0]);
1144 int64_t starting_frame_number = -1;
1145 bool partial_frame =
true;
1146 for (
int channel_filter = 0; channel_filter <
info.
channels; channel_filter++)
1149 starting_frame_number = target_frame;
1150 int channel_buffer_size = packet_samples /
info.
channels;
1151 float *channel_buffer =
new float[channel_buffer_size];
1154 for (
int z = 0; z < channel_buffer_size; z++)
1155 channel_buffer[z] = 0.0f;
1161 for (
int sample = 0; sample < packet_samples; sample++)
1164 if (channel_filter == channel)
1167 channel_buffer[position] = audio_buf[sample] * (1.0f / (1 << 15));
1183 int start = starting_sample;
1184 int remaining_samples = channel_buffer_size;
1185 float *iterate_channel_buffer = channel_buffer;
1186 while (remaining_samples > 0)
1192 int samples = samples_per_frame - start;
1193 if (samples > remaining_samples)
1194 samples = remaining_samples;
1197 std::shared_ptr<Frame> f = CreateFrame(starting_frame_number);
1200 if (samples_per_frame == start + samples)
1201 partial_frame =
false;
1203 partial_frame =
true;
1207 f->AddAudio(
true, channel_filter, start, iterate_channel_buffer, samples, 0.98f);
1210 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessAudioPacket (f->AddAudio)",
"frame", starting_frame_number,
"start", start,
"samples", samples,
"channel", channel_filter,
"partial_frame", partial_frame,
"samples_per_frame", samples_per_frame);
1213 working_cache.
Add(f);
1216 remaining_samples -= samples;
1219 if (remaining_samples > 0)
1220 iterate_channel_buffer += samples;
1223 starting_frame_number++;
1230 delete[] channel_buffer;
1231 channel_buffer = NULL;
1232 iterate_channel_buffer = NULL;
1243 for (int64_t f = target_frame; f < starting_frame_number; f++) {
1247 processing_audio_frames.erase(processing_audio_frames.find(f));
1250 if (processing_audio_frames.count(f) == 0)
1252 processed_audio_frames[f] = f;
1255 if (target_frame == starting_frame_number) {
1257 processing_audio_frames.erase(processing_audio_frames.find(target_frame));
1265 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessAudioPacket (After)",
"requested_frame", requested_frame,
"starting_frame", target_frame,
"end_frame", starting_frame_number - 1,
"", -1,
"", -1,
"", -1);
1272 void FFmpegReader::Seek(int64_t requested_frame)
1275 if (requested_frame < 1)
1276 requested_frame = 1;
1280 int processing_video_frames_size = 0;
1281 int processing_audio_frames_size = 0;
1284 processing_video_frames_size = processing_video_frames.size();
1285 processing_audio_frames_size = processing_audio_frames.size();
1289 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::Seek",
"requested_frame", requested_frame,
"seek_count", seek_count,
"last_frame", last_frame,
"processing_video_frames_size", processing_video_frames_size,
"processing_audio_frames_size", processing_audio_frames_size,
"video_pts_offset", video_pts_offset);
1292 while (processing_video_frames_size + processing_audio_frames_size > 0) {
1295 processing_video_frames_size = processing_video_frames.size();
1296 processing_audio_frames_size = processing_audio_frames.size();
1300 working_cache.
Clear();
1301 missing_frames.
Clear();
1306 processing_audio_frames.clear();
1307 processing_video_frames.clear();
1308 processed_video_frames.clear();
1309 processed_audio_frames.clear();
1310 missing_audio_frames.clear();
1311 missing_video_frames.clear();
1312 missing_audio_frames_source.clear();
1313 missing_video_frames_source.clear();
1314 checked_frames.clear();
1319 current_video_frame = 0;
1320 largest_frame_processed = 0;
1321 num_checks_since_final = 0;
1322 num_packets_since_video_frame = 0;
1323 has_missing_frames =
false;
1332 if (requested_frame - buffer_amount < 20)
1344 if (seek_count == 1) {
1347 seeking_pts = ConvertFrameToVideoPTS(1);
1349 seek_audio_frame_found = 0;
1350 seek_video_frame_found = 0;
1355 bool seek_worked =
false;
1356 int64_t seek_target = 0;
1361 seek_target = ConvertFrameToVideoPTS(requested_frame - buffer_amount);
1363 fprintf(stderr,
"%s: error while seeking video stream\n", pFormatCtx->AV_FILENAME);
1367 is_video_seek =
true;
1375 seek_target = ConvertFrameToAudioPTS(requested_frame - buffer_amount);
1377 fprintf(stderr,
"%s: error while seeking audio stream\n", pFormatCtx->AV_FILENAME);
1381 is_video_seek =
false;
1391 avcodec_flush_buffers(aCodecCtx);
1395 avcodec_flush_buffers(pCodecCtx);
1398 previous_packet_location.
frame = -1;
1403 if (seek_count == 1) {
1405 seeking_pts = seek_target;
1406 seeking_frame = requested_frame;
1408 seek_audio_frame_found = 0;
1409 seek_video_frame_found = 0;
1435 int64_t FFmpegReader::GetVideoPTS()
1437 int64_t current_pts = 0;
1438 if(packet->dts != AV_NOPTS_VALUE)
1439 current_pts = packet->dts;
1446 void FFmpegReader::UpdatePTSOffset(
bool is_video)
1452 if (video_pts_offset == 99999)
1458 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::UpdatePTSOffset (Video)",
"video_pts_offset", video_pts_offset,
"is_video", is_video,
"", -1,
"", -1,
"", -1,
"", -1);
1464 if (audio_pts_offset == 99999)
1470 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::UpdatePTSOffset (Audio)",
"audio_pts_offset", audio_pts_offset,
"is_video", is_video,
"", -1,
"", -1,
"", -1,
"", -1);
1476 int64_t FFmpegReader::ConvertVideoPTStoFrame(int64_t pts)
1479 pts = pts + video_pts_offset;
1480 int64_t previous_video_frame = current_video_frame;
1489 if (current_video_frame == 0)
1490 current_video_frame = frame;
1494 if (frame == previous_video_frame) {
1500 current_video_frame++;
1502 if (current_video_frame < frame)
1504 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ConvertVideoPTStoFrame (detected missing frame)",
"calculated frame", frame,
"previous_video_frame", previous_video_frame,
"current_video_frame", current_video_frame,
"", -1,
"", -1,
"", -1);
1509 while (current_video_frame < frame) {
1510 if (!missing_video_frames.count(current_video_frame)) {
1511 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ConvertVideoPTStoFrame (tracking missing frame)",
"current_video_frame", current_video_frame,
"previous_video_frame", previous_video_frame,
"", -1,
"", -1,
"", -1,
"", -1);
1512 missing_video_frames.insert(pair<int64_t, int64_t>(current_video_frame, previous_video_frame));
1513 missing_video_frames_source.insert(pair<int64_t, int64_t>(previous_video_frame, current_video_frame));
1517 has_missing_frames =
true;
1520 current_video_frame++;
1529 int64_t FFmpegReader::ConvertFrameToVideoPTS(int64_t frame_number)
1538 return video_pts - video_pts_offset;
1542 int64_t FFmpegReader::ConvertFrameToAudioPTS(int64_t frame_number)
1551 return audio_pts - audio_pts_offset;
1555 AudioLocation FFmpegReader::GetAudioPTSLocation(int64_t pts)
1558 pts = pts + audio_pts_offset;
1567 int64_t whole_frame = int64_t(frame);
1570 double sample_start_percentage = frame - double(whole_frame);
1576 int sample_start = round(
double(samples_per_frame) * sample_start_percentage);
1579 if (whole_frame < 1)
1581 if (sample_start < 0)
1588 if (previous_packet_location.
frame != -1) {
1589 if (location.
is_near(previous_packet_location, samples_per_frame, samples_per_frame))
1591 int64_t orig_frame = location.
frame;
1596 location.
frame = previous_packet_location.
frame;
1599 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::GetAudioPTSLocation (Audio Gap Detected)",
"Source Frame", orig_frame,
"Source Audio Sample", orig_start,
"Target Frame", location.
frame,
"Target Audio Sample", location.
sample_start,
"pts", pts,
"", -1);
1603 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::GetAudioPTSLocation (Audio Gap Ignored - too big)",
"Previous location frame", previous_packet_location.
frame,
"Target Frame", location.
frame,
"Target Audio Sample", location.
sample_start,
"pts", pts,
"", -1,
"", -1);
1606 for (int64_t audio_frame = previous_packet_location.
frame; audio_frame < location.
frame; audio_frame++) {
1607 if (!missing_audio_frames.count(audio_frame)) {
1608 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::GetAudioPTSLocation (tracking missing frame)",
"missing_audio_frame", audio_frame,
"previous_audio_frame", previous_packet_location.
frame,
"new location frame", location.
frame,
"", -1,
"", -1,
"", -1);
1609 missing_audio_frames.insert(pair<int64_t, int64_t>(previous_packet_location.
frame - 1, audio_frame));
1616 previous_packet_location = location;
1623 std::shared_ptr<Frame> FFmpegReader::CreateFrame(int64_t requested_frame)
1626 std::shared_ptr<Frame> output = working_cache.
GetFrame(requested_frame);
1635 working_cache.
Add(output);
1638 if (requested_frame > largest_frame_processed)
1639 largest_frame_processed = requested_frame;
1647 bool FFmpegReader::IsPartialFrame(int64_t requested_frame) {
1650 bool seek_trash =
false;
1651 int64_t max_seeked_frame = seek_audio_frame_found;
1652 if (seek_video_frame_found > max_seeked_frame)
1653 max_seeked_frame = seek_video_frame_found;
1654 if ((
info.
has_audio && seek_audio_frame_found && max_seeked_frame >= requested_frame) ||
1655 (
info.
has_video && seek_video_frame_found && max_seeked_frame >= requested_frame))
1662 bool FFmpegReader::CheckMissingFrame(int64_t requested_frame)
1668 int checked_count = 0;
1671 if (checked_frames.count(requested_frame) == 0)
1672 checked_frames[requested_frame] = 1;
1674 checked_frames[requested_frame]++;
1675 checked_count = checked_frames[requested_frame];
1678 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckMissingFrame",
"requested_frame", requested_frame,
"has_missing_frames", has_missing_frames,
"missing_video_frames.size()", missing_video_frames.size(),
"checked_count", checked_count,
"", -1,
"", -1);
1681 map<int64_t, int64_t>::iterator itr;
1682 bool found_missing_frame =
false;
1685 if (missing_video_frames.count(requested_frame) || missing_audio_frames.count(requested_frame)) {
1686 int64_t missing_source_frame = -1;
1687 if (missing_video_frames.count(requested_frame))
1688 missing_source_frame = missing_video_frames.find(requested_frame)->second;
1689 else if (missing_audio_frames.count(requested_frame))
1690 missing_source_frame = missing_audio_frames.find(requested_frame)->second;
1693 if (checked_frames.count(missing_source_frame) == 0)
1694 checked_frames[missing_source_frame] = 1;
1696 checked_frames[missing_source_frame]++;
1699 std::shared_ptr<Frame> parent_frame = missing_frames.
GetFrame(missing_source_frame);
1700 if (parent_frame == NULL) {
1702 if (parent_frame != NULL) {
1704 missing_frames.
Add(parent_frame);
1709 std::shared_ptr<Frame> missing_frame = CreateFrame(requested_frame);
1712 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckMissingFrame (Is Previous Video Frame Final)",
"requested_frame", requested_frame,
"missing_frame->number", missing_frame->number,
"missing_source_frame", missing_source_frame,
"", -1,
"", -1,
"", -1);
1715 if (parent_frame != NULL) {
1717 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckMissingFrame (AddImage from Previous Video Frame)",
"requested_frame", requested_frame,
"missing_frame->number", missing_frame->number,
"missing_source_frame", missing_source_frame,
"", -1,
"", -1,
"", -1);
1720 std::shared_ptr<QImage> parent_image = parent_frame->GetImage();
1722 missing_frame->AddImage(std::shared_ptr<QImage>(
new QImage(*parent_image)));
1724 processed_video_frames[missing_frame->number] = missing_frame->number;
1725 processed_audio_frames[missing_frame->number] = missing_frame->number;
1731 working_cache.
Remove(missing_frame->number);
1734 last_frame = missing_frame->number;
1740 return found_missing_frame;
1744 void FFmpegReader::CheckWorkingFrames(
bool end_of_stream, int64_t requested_frame)
1747 bool checked_count_tripped =
false;
1748 int max_checked_count = 80;
1762 working_cache.
Remove(f->number);
1766 CheckMissingFrame(f->number);
1769 int checked_count = 0;
1770 int checked_frames_size = 0;
1772 bool is_video_ready =
false;
1773 bool is_audio_ready =
false;
1776 is_video_ready = processed_video_frames.count(f->number);
1777 is_audio_ready = processed_audio_frames.count(f->number);
1780 checked_frames_size = checked_frames.size();
1781 if (!checked_count_tripped || f->number >= requested_frame)
1782 checked_count = checked_frames[f->number];
1785 checked_count = max_checked_count;
1788 if (previous_packet_location.
frame == f->number && !end_of_stream)
1789 is_audio_ready =
false;
1790 bool is_seek_trash = IsPartialFrame(f->number);
1797 if (checked_count >= max_checked_count && (!is_video_ready || !is_audio_ready)) {
1799 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckWorkingFrames (exceeded checked_count)",
"requested_frame", requested_frame,
"frame_number", f->number,
"is_video_ready", is_video_ready,
"is_audio_ready", is_audio_ready,
"checked_count", checked_count,
"checked_frames_size", checked_frames_size);
1802 checked_count_tripped =
true;
1804 if (
info.
has_video && !is_video_ready && last_video_frame) {
1806 f->AddImage(std::shared_ptr<QImage>(
new QImage(*last_video_frame->GetImage())));
1807 is_video_ready =
true;
1812 is_audio_ready =
true;
1817 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckWorkingFrames",
"requested_frame", requested_frame,
"frame_number", f->number,
"is_video_ready", is_video_ready,
"is_audio_ready", is_audio_ready,
"checked_count", checked_count,
"checked_frames_size", checked_frames_size);
1820 if ((!end_of_stream && is_video_ready && is_audio_ready) || end_of_stream || is_seek_trash)
1823 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckWorkingFrames (mark frame as final)",
"requested_frame", requested_frame,
"f->number", f->number,
"is_seek_trash", is_seek_trash,
"Working Cache Count", working_cache.
Count(),
"Final Cache Count",
final_cache.
Count(),
"end_of_stream", end_of_stream);
1830 f->AddImage(std::shared_ptr<QImage>(
new QImage(*last_video_frame->GetImage())));
1833 num_checks_since_final = 0;
1841 if (missing_video_frames_source.count(f->number)) {
1843 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckWorkingFrames (add frame to missing cache)",
"f->number", f->number,
"is_seek_trash", is_seek_trash,
"Missing Cache Count", missing_frames.
Count(),
"Working Cache Count", working_cache.
Count(),
"Final Cache Count",
final_cache.
Count(),
"", -1);
1844 missing_frames.
Add(f);
1848 checked_frames.erase(f->number);
1852 working_cache.
Remove(f->number);
1855 last_frame = f->number;
1859 working_cache.
Remove(f->number);
1869 void FFmpegReader::CheckFPS()
1873 int first_second_counter = 0;
1874 int second_second_counter = 0;
1875 int third_second_counter = 0;
1876 int forth_second_counter = 0;
1877 int fifth_second_counter = 0;
1878 int frames_detected = 0;
1885 if (GetNextPacket() < 0)
1890 if (packet->stream_index == videoStream)
1896 UpdatePTSOffset(
true);
1899 pts = GetVideoPTS();
1902 RemoveAVFrame(pFrame);
1905 pts += video_pts_offset;
1911 if (video_seconds <= 1.0)
1912 first_second_counter++;
1913 else if (video_seconds > 1.0 && video_seconds <= 2.0)
1914 second_second_counter++;
1915 else if (video_seconds > 2.0 && video_seconds <= 3.0)
1916 third_second_counter++;
1917 else if (video_seconds > 3.0 && video_seconds <= 4.0)
1918 forth_second_counter++;
1919 else if (video_seconds > 4.0 && video_seconds <= 5.0)
1920 fifth_second_counter++;
1929 if (second_second_counter != 0 && third_second_counter != 0 && forth_second_counter != 0 && fifth_second_counter != 0) {
1931 int sum_fps = second_second_counter + third_second_counter + forth_second_counter + fifth_second_counter;
1932 int avg_fps = round(sum_fps / 4.0f);
1943 }
else if (second_second_counter != 0 && third_second_counter != 0) {
1945 int sum_fps = second_second_counter;
1969 void FFmpegReader::RemoveAVFrame(AVFrame* remove_frame)
1975 av_freep(&remove_frame->data[0]);
1980 void FFmpegReader::RemoveAVPacket(AVPacket* remove_packet)
1986 delete remove_packet;
1990 int64_t FFmpegReader::GetSmallestVideoFrame()
1993 map<int64_t, int64_t>::iterator itr;
1994 int64_t smallest_frame = -1;
1996 for(itr = processing_video_frames.begin(); itr != processing_video_frames.end(); ++itr)
1998 if (itr->first < smallest_frame || smallest_frame == -1)
1999 smallest_frame = itr->first;
2003 return smallest_frame;
2007 int64_t FFmpegReader::GetSmallestAudioFrame()
2010 map<int64_t, int64_t>::iterator itr;
2011 int64_t smallest_frame = -1;
2013 for(itr = processing_audio_frames.begin(); itr != processing_audio_frames.end(); ++itr)
2015 if (itr->first < smallest_frame || smallest_frame == -1)
2016 smallest_frame = itr->first;
2020 return smallest_frame;
2035 root[
"type"] =
"FFmpegReader";
2036 root[
"path"] = path;
2047 Json::Reader reader;
2048 bool success = reader.parse( value, root );
2051 throw InvalidJSON(
"JSON could not be parsed (or is invalid)",
"");
2061 throw InvalidJSON(
"JSON is invalid (missing keys or invalid data types)",
"");
2072 if (!root[
"path"].isNull())
2073 path = root[
"path"].asString();