in src/terminal/decoder.c [943:1446]
static GF_Err MediaCodec_Process(GF_Codec *codec, u32 TimeAvailable)
{
GF_CMUnit *CU;
GF_DBUnit *AU;
GF_Channel *ch, *prev_ch;
Bool drop_late_frames = 0;
u64 now, entryTime;
u32 mmlevel, cts;
u32 first, obj_time, unit_size;
GF_MediaDecoder *mdec = (GF_MediaDecoder*)codec->decio;
GF_Err e = GF_OK;
s32 cts_diff;
CU = NULL;
/*if video codec muted don't decode (try to saves ressources)
if audio codec muted we dispatch to keep sync in place*/
if (codec->Muted && (codec->type==GF_STREAM_VISUAL) ) return GF_OK;
//cannot output frame, do nothing (we force a channel query before for pull mode)
if (codec->CB->Capacity == codec->CB->UnitCount) {
if (codec->Status==GF_ESM_CODEC_PAUSE) {
gf_term_stop_codec(codec, 1);
}
if (codec->CB->UnitCount > 1) return GF_OK;
else if (codec->direct_vout) return GF_OK;
}
entryTime = gf_sys_clock_high_res();
if (!codec->odm->term->bench_mode) {
if ((codec->odm->term->flags & GF_TERM_DROP_LATE_FRAMES) || (codec->flags & GF_ESM_CODEC_IS_LOW_LATENCY))
drop_late_frames = GF_TRUE;
}
/*fetch next AU in DTS order for this codec*/
MediaDecoder_GetNextAU(codec, &ch, &AU);
/*no active channel return*/
if (!AU || !ch) {
/*if the codec is in EOS state, assume we're done*/
if (codec->Status == GF_ESM_CODEC_EOS) {
/*if codec is reordering, try to flush it*/
if (codec->is_reordering) {
if ( LockCompositionUnit(codec, codec->last_unit_cts+1, &CU, &unit_size) == GF_OUT_OF_MEM)
return GF_OK;
assert( CU );
unit_size = 0;
if (codec->odm->term->bench_mode != 2) {
e = mdec->ProcessData(mdec, NULL, 0, 0, &CU->TS, CU->data, &unit_size, 0, 0);
if (e==GF_OK) {
e = UnlockCompositionUnit(codec, CU, unit_size);
if (unit_size) return GF_OK;
}
}
}
gf_term_stop_codec(codec, 0);
if (codec->CB) {
gf_cm_set_eos(codec->CB);
}
}
/*if no data, and channel not buffering, ABORT CB buffer (data timeout or EOS not detectable)*/
else if (ch && !ch->is_pulling && !ch->BufferOn && !ch->last_au_was_seek)
gf_cm_abort_buffering(codec->CB);
//GF_LOG(GF_LOG_DEBUG, GF_LOG_CODEC, ("[%s] ODM%d: No data in decoding buffer\n", codec->decio->module_name, codec->odm->OD->objectDescriptorID));
return GF_OK;
}
/*get the object time*/
obj_time = gf_clock_time(codec->ck);
/*Media Time for media codecs is updated in the CB*/
if (!codec->CB) {
gf_es_drop_au(ch);
return GF_BAD_PARAM;
}
/*we are still flushing our CB - keep the current pending AU and wait for CB resize*/
if (codec->force_cb_resize) {
/*we cannot destroy the CB here, as we don't know who is using its output*/
return GF_OK;
}
/*image codecs*/
if (!codec->CB->no_allocation && (codec->CB->Capacity == 1)) {
/*a SHA signature is computed for each AU. This avoids decoding/recompositing when identical (for instance streaming a carousel)*/
u8 new_unit_signature[20];
gf_sha1_csum((u8*)AU->data, AU->dataLength, new_unit_signature);
if (!memcmp(codec->last_unit_signature, new_unit_signature, sizeof(new_unit_signature))) {
codec->nb_repeted_frames++;
gf_es_drop_au(ch);
return GF_OK;
}
/*usually only one image is tolerated in the stream, but just in case force reset of CB*/
if (codec->CB->UnitCount && (obj_time>=AU->CTS)) {
gf_mx_p(codec->odm->mx);
codec->CB->output->dataLength = 0;
codec->CB->UnitCount = 0;
gf_mx_v(codec->odm->mx);
}
/*CB is already full*/
if (codec->CB->UnitCount)
return GF_OK;
codec->nb_repeted_frames = 0;
memcpy(codec->last_unit_signature, new_unit_signature, sizeof(new_unit_signature));
}
if (codec->ck->speed != codec->check_speed) {
//decrease in speed
if (ABS(codec->check_speed) > ABS(codec->ck->speed)) {
codec->decode_only_rap = 0;
codec->drop_modulo = 0;
codec->drop_count = 0;
}
codec->check_speed = codec->ck->speed;
codec->consecutive_late_frames = 0;
codec->consecutive_ontime_frames = 0;
if (codec->type==GF_STREAM_AUDIO) {
if (ABS(FIX2FLT(codec->ck->speed)) > 8) {
codec->decode_only_rap = 2;
GF_LOG(GF_LOG_WARNING, GF_LOG_CODEC, ("[%s] Speed %g too hight for audio decoder/renderer - skipping decode\n", codec->decio->module_name, FIX2FLT(codec->ck->speed)));
}
} else {
//dynamically decided based on late frames found
}
}
TimeAvailable*=1000;
/*try to refill the full buffer*/
first = 1;
while (codec->CB->Capacity > codec->CB->UnitCount) {
Bool force_skip = 0;
if (codec->decode_only_rap) {
//check whether we have to skip decoding this frame
if (AU->flags & GF_DB_AU_RAP) {
if (codec->decode_only_rap==2) {
if (AU->CTS > obj_time + 500)
return GF_OK;
force_skip = 1;
} else if (codec->drop_modulo) {
codec->drop_count ++;
if (codec->drop_count >= codec->drop_modulo) {
codec->drop_count = 0;
} else {
force_skip = 1;
}
}
} else {
force_skip = 1;
}
}
/*set media processing level*/
ch->last_au_was_seek = 0;
mmlevel = GF_CODEC_LEVEL_NORMAL;
/*explicit seek*/
if (AU->flags & GF_DB_AU_IS_SEEK) {
mmlevel = GF_CODEC_LEVEL_SEEK;
ch->last_au_was_seek = 1;
}
/*implicit seek: if the last frame had the same TS, we are seeking. Ask the codec to drop*/
else if (!ch->skip_sl && codec->last_unit_cts && (codec->last_unit_cts == AU->CTS) && !ch->esd->dependsOnESID) {
mmlevel = GF_CODEC_LEVEL_SEEK;
ch->last_au_was_seek = 1;
/*object clock is paused by media control or terminal is paused: exact frame seek*/
if (
#ifndef GPAC_DISABLE_VRML
(codec->ck->mc && codec->ck->mc->paused) ||
#endif
(codec->odm->term->play_state)
) {
gf_cm_rewind_input(codec->CB);
mmlevel = GF_CODEC_LEVEL_NORMAL;
/*force staying in step-mode*/
codec->odm->term->compositor->step_mode=1;
}
}
/*only perform drop in normal playback*/
else if (!force_skip && (codec->ck->speed >= 0) && (codec->CB->Status == CB_PLAY)) {
/*extremely late, set the level to drop
NOTE: the 100 ms safety gard is to avoid discarding audio*/
if (!ch->skip_sl && (AU->CTS + (codec->is_reordering ? 1000 : 100) < obj_time) ) {
mmlevel = GF_CODEC_LEVEL_DROP;
GF_LOG(GF_LOG_INFO, GF_LOG_CODEC, ("[%s] ODM%d: frame too late (CTS %d vs time %d) - using drop level\n", codec->decio->module_name, codec->odm->OD->objectDescriptorID, AU->CTS, obj_time));
//above 1 out of this threshold frames only shown, move to I-frame only mode. 50 = 1 frame shown out of 2 GOPs at 25hz
#define IFRAME_MODE_THRESHOLD 50
codec->consecutive_late_frames++;
codec->consecutive_ontime_frames = 0;
if (codec->check_speed > 1) {
Double speed = (Double) FIX2FLT(codec->ck->speed);
u32 nb_check_frames = codec->decode_only_rap ? 5 : 30;
if (codec->consecutive_late_frames >= nb_check_frames) {
if (!codec->decode_only_rap) {
codec->drop_modulo += 2;
codec->drop_count = 0;
GF_LOG(GF_LOG_INFO, GF_LOG_CODEC, ("[%s] ODM%d: %d consecutive late frames at speed %g - increasing frame drop modulo to %d\n", codec->decio->module_name, codec->odm->OD->objectDescriptorID, codec->consecutive_late_frames, speed, codec->drop_modulo));
if (codec->drop_modulo > IFRAME_MODE_THRESHOLD) {
GF_LOG(GF_LOG_INFO, GF_LOG_CODEC, ("[%s] ODM%d: %d consecutive late frames at speed %g with drop modulo %d - moving to I-frame only decoding\n", codec->decio->module_name, codec->odm->OD->objectDescriptorID, codec->consecutive_late_frames, speed, codec->drop_modulo));
codec->drop_modulo = 0;
codec->drop_count = 0;
codec->decode_only_rap = 1;
}
} else {
codec->drop_modulo += 2;
codec->drop_count = 0;
GF_LOG(GF_LOG_INFO, GF_LOG_CODEC, ("[%s] ODM%d: %d consecutive late frames at speed %g in I-frame only mode - decoding only one I-frame out of %d\n", codec->decio->module_name, codec->odm->OD->objectDescriptorID, codec->consecutive_late_frames, speed, codec->drop_modulo));
}
codec->consecutive_late_frames = 0;
}
//very late even in I-frame mode, drop it
else if (codec->decode_only_rap) {
if (obj_time - AU->CTS > 2000) {
force_skip = 1;
}
}
}
if (ch->resync_drift && (AU->CTS + ch->resync_drift < obj_time)) {
ch->clock->StartTime += (obj_time - AU->CTS);
GF_LOG(GF_LOG_WARNING, GF_LOG_CODEC, ("[%s] ODM%d: decoder too slow on OCR stream - rewinding clock of %d ms\n", codec->decio->module_name, codec->odm->OD->objectDescriptorID, obj_time - AU->CTS));
obj_time = gf_clock_time(codec->ck);
}
}
/*we are late according to the media manager*/
else if (codec->PriorityBoost) {
mmlevel = GF_CODEC_LEVEL_VERY_LATE;
codec->consecutive_late_frames = 0;
codec->consecutive_ontime_frames ++;
}
/*otherwise we must have an idea of the load in order to set the right level
use the composition buffer for that, only on the first frame*/
else {
codec->consecutive_late_frames = 0;
codec->consecutive_ontime_frames ++;
if (first) {
//if the CB is almost empty set to very late
if (codec->CB->UnitCount <= codec->CB->Min+1) {
mmlevel = GF_CODEC_LEVEL_VERY_LATE;
} else if (codec->CB->UnitCount * 2 <= codec->CB->Capacity) {
mmlevel = GF_CODEC_LEVEL_LATE;
}
first = 0;
}
}
if (codec->check_speed > 1) {
u32 nb_check_frames = codec->decode_only_rap ? 5 : 30;
//try to decrease the drop rate, but never switch back from I-frame mode only for a given speed
if (codec->consecutive_ontime_frames > nb_check_frames) {
if (codec->drop_modulo) {
codec->drop_modulo -= 2;
codec->drop_count = 0;
GF_LOG(GF_LOG_INFO, GF_LOG_CODEC, ("[%s] ODM%d: %d consecutive on-time frames - Decreasing drop modulo %d (I-frame only mode %d)\n", codec->decio->module_name, codec->odm->OD->objectDescriptorID, codec->consecutive_ontime_frames, codec->drop_modulo, codec->decode_only_rap));
codec->consecutive_late_frames = 0;
codec->consecutive_ontime_frames = 0;
}
}
}
}
if (ch->skip_sl) {
if (codec->bytes_per_sec) {
AU->CTS = codec->last_unit_cts + ch->ts_offset + codec->cur_audio_bytes * 1000 / codec->bytes_per_sec;
} else if (codec->fps) {
AU->CTS = codec->last_unit_cts + ch->ts_offset + (u32) (codec->cur_video_frames * 1000 / codec->fps);
}
}
if ( LockCompositionUnit(codec, AU->CTS, &CU, &unit_size) == GF_OUT_OF_MEM) {
GF_LOG(GF_LOG_DEBUG, GF_LOG_CODEC, ("[%s] Exit decode loop because no more space in composition buffer\n", codec->decio->module_name ));
return GF_OK;
}
scalable_retry:
now = gf_sys_clock_high_res();
assert( CU );
if (!CU->data && unit_size && !codec->CB->no_allocation) {
e = GF_OUT_OF_MEM;
} else if (codec->odm->term->bench_mode==2) {
unit_size = 0;
gf_cm_abort_buffering(codec->CB);
} else if (force_skip) {
unit_size = 0;
GF_LOG(GF_LOG_DEBUG, GF_LOG_CODEC, ("[%s] ODM%d: force drop requested in fast playback for AU CTS %d\n", codec->decio->module_name, codec->odm->OD->objectDescriptorID, AU->CTS));
} else {
GF_LOG(GF_LOG_DEBUG, GF_LOG_CODEC, ("[%s] ODM%d ES%d at %d decoding frame DTS %d CTS %d size %d (%d in channels)\n", codec->decio->module_name, codec->odm->OD->objectDescriptorID, ch->esd->ESID, gf_clock_real_time(ch->clock), AU->DTS, AU->CTS, AU->dataLength, ch->AU_Count));
e = mdec->ProcessData(mdec, AU->data, AU->dataLength, ch->esd->ESID, &CU->TS, CU->data, &unit_size, AU->PaddingBits, mmlevel);
}
now = gf_sys_clock_high_res() - now;
if (codec->Status == GF_ESM_CODEC_STOP) {
GF_LOG(GF_LOG_DEBUG, GF_LOG_CODEC, ("[%s] Exit decode loop because codec has been stopped\n", codec->decio->module_name));
return GF_OK;
}
/*input is too small, resize composition memory*/
switch (e) {
case GF_BUFFER_TOO_SMALL:
/*release but no dispatch*/
UnlockCompositionUnit(codec, CU, 0);
/*if we have pending media, wait untill CB is completely flushed by the compositor - this shoud be fixed by avoiding to destroy the CB ... */
if (codec->CB->LastRenderedTS) {
GF_LOG(GF_LOG_INFO, GF_LOG_CODEC, ("[%s] ODM%d ES%d: Resize output buffer requested\n", codec->decio->module_name, codec->odm->OD->objectDescriptorID, ch->esd->ESID));
codec->force_cb_resize = unit_size;
return GF_OK;
}
GF_LOG(GF_LOG_INFO, GF_LOG_CODEC, ("[%s] ODM%d ES%d: Resizing output buffer %d -> %d\n", codec->decio->module_name, codec->odm->OD->objectDescriptorID, ch->esd->ESID, codec->CB->UnitSize, unit_size));
gf_codec_resize_composition_buffer(codec, unit_size);
continue;
/*this happens a lot when using non-MPEG-4 streams (ex: ffmpeg demuxer)*/
case GF_PACKED_FRAMES:
/*in seek do dispatch output otherwise we will only see the I-frame preceding the seek point*/
if (mmlevel == GF_CODEC_LEVEL_DROP) {
if (drop_late_frames && (codec->CB->UnitCount>1) ) {
unit_size = 0;
codec->nb_dropped++;
} else
ch->clock->last_TS_rendered = codec->CB->LastRenderedTS;
}
e = UnlockCompositionUnit(codec, CU, unit_size);
GF_LOG(GF_LOG_DEBUG, GF_LOG_CODEC, ("[%s] ODM%d ES%d at %d decoded packed frame TS %d in "LLU" us\n", codec->decio->module_name, codec->odm->OD->objectDescriptorID, ch->esd->ESID, gf_clock_real_time(ch->clock), AU->CTS, now));
if (ch->skip_sl) {
if (codec->bytes_per_sec) {
codec->cur_audio_bytes += unit_size;
} else if (codec->fps && unit_size) {
codec->cur_video_frames += 1;
}
} else {
u32 deltaTS = 0;
if (codec->bytes_per_sec) {
deltaTS = unit_size * 1000 / codec->bytes_per_sec;
} /*else if (0 && codec->fps && unit_size) {
deltaTS = (u32) (1000.0f / codec->fps);
} */else {
deltaTS = (AU->DTS - codec->last_unit_dts);
}
AU->CTS += deltaTS;
}
codec_update_stats(codec, 0, now, 0, 0);
continue;
/*for all cases below, don't release the composition buffer until we are sure we are not
processing a scalable stream*/
case GF_OK:
if (unit_size) {
GF_LOG(GF_LOG_DEBUG, GF_LOG_CODEC, ("[%s] ODM%d ES%d at %d decoded frame TS %d in "LLU" us (DTS %d - size %d) - %d in CB\n", codec->decio->module_name, codec->odm->OD->objectDescriptorID, ch->esd->ESID, gf_clock_real_time(ch->clock), AU->CTS, now, AU->DTS, AU->dataLength, codec->CB->UnitCount + 1));
if (codec->direct_vout) {
e = mdec->GetOutputBuffer(mdec, ch->esd->ESID, &codec->CB->pY, &codec->CB->pU, &codec->CB->pV);
if (e==GF_OK) {
gf_sc_set_video_pending_frame(codec->odm->term->compositor);
}
}
CU->sender_ntp = AU->sender_ntp;
}
#if 0
/*if no size and the decoder is not using the composition memory - if the object is in intitial buffering resume it!!*/
else if ( (!codec->CB->UnitSize && !codec->CB->Capacity) && (codec->CB->Status == CB_BUFFER)) {
codec->nb_dispatch_skipped++;
if (codec->nb_dispatch_skipped==codec->CB->UnitCount)
gf_cm_abort_buffering(codec->CB);
}
#endif
codec_update_stats(codec, AU->dataLength, now, AU->DTS, (AU->flags & GF_DB_AU_RAP));
if (ch->skip_sl) {
if (codec->bytes_per_sec) {
codec->cur_audio_bytes += unit_size;
while (codec->cur_audio_bytes>codec->bytes_per_sec) {
codec->cur_audio_bytes -= codec->bytes_per_sec;
codec->last_unit_cts += 1000;
}
} else if (codec->fps && unit_size) {
codec->cur_video_frames += 1;
}
}
#ifndef GPAC_DISABLE_LOGS
if (codec->odm->flags & GF_ODM_PREFETCH) {
GF_LOG(GF_LOG_INFO, GF_LOG_CODEC, ("[%s] ODM%d ES%d At %d decoding frame TS %d in prefetch mode\n", codec->decio->module_name, codec->odm->OD->objectDescriptorID, ch->esd->ESID, gf_clock_real_time(ch->clock), AU->CTS));
}
#endif
break;
default:
unit_size = 0;
/*error - if the object is in intitial buffering resume it!!*/
gf_cm_abort_buffering(codec->CB);
GF_LOG(GF_LOG_INFO, GF_LOG_CODEC, ("[%s] ODM%d ES%d At %d (frame TS %d - "LLU" us ): decoded error %s\n", codec->decio->module_name, codec->odm->OD->objectDescriptorID, ch->esd->ESID, gf_clock_real_time(ch->clock), AU->CTS, now, gf_error_to_string(e) ));
e = GF_OK;
break;
}
codec->last_unit_dts = AU->DTS;
/*remember base layer timing*/
if (!ch->esd->dependsOnESID && !ch->skip_sl) {
codec->last_unit_cts = AU->CTS;
codec->first_frame_processed = 1;
}
#ifndef GPAC_DISABLE_LOG
if (unit_size) {
if (ch->is_pulling) {
GF_LOG(GF_LOG_DEBUG, GF_LOG_MEDIA, ("[%s] at %d decoded frame CTS %d in "LLU" us\n", codec->decio->module_name, gf_clock_real_time(ch->clock), AU->CTS, now));
} else {
GF_LOG(GF_LOG_DEBUG, GF_LOG_MEDIA, ("[%s] at %d decoded frame CTS %d in "LLU" us - %d AU in channel\n", codec->decio->module_name, gf_clock_real_time(ch->clock), AU->CTS, now, ch->AU_Count));
}
}
#endif
/*store current CTS - we need to have exclusive access in case a PCR discontinuity remaps timestamps while we decode*/
gf_es_lock(ch, GF_TRUE);
cts = AU->CTS;
prev_ch = ch;
gf_es_drop_au(ch);
gf_es_lock(ch, GF_FALSE);
AU = NULL;
if (e) {
UnlockCompositionUnit(codec, CU, unit_size);
GF_LOG(GF_LOG_DEBUG, GF_LOG_CODEC, ("[%s] Exit decode loop because error %s\n", codec->decio->module_name, gf_error_to_string(e) ));
return e;
}
MediaDecoder_GetNextAU(codec, &ch, &AU);
/*same CTS: same output, likely scalable stream so don't release the CB*/
if (AU && (AU->CTS == cts) && (ch != prev_ch) ) {
unit_size = codec->CB->UnitSize;
goto scalable_retry;
}
/*in seek don't dispatch any output*/
if (mmlevel >= GF_CODEC_LEVEL_DROP) {
if (drop_late_frames || (mmlevel == GF_CODEC_LEVEL_SEEK) ) {
unit_size = 0;
if (drop_late_frames) codec->nb_dropped++;
} else
prev_ch->clock->last_TS_rendered = codec->CB->LastRenderedTS;
} else {
prev_ch->clock->last_TS_rendered = 0;
}
if (!codec->decode_only_rap && codec->drop_modulo) {
codec->drop_count++;
if (codec->drop_count==codec->drop_modulo) {
codec->drop_count = 0;
} else {
unit_size = 0;
}
}
cts_diff = (s32) cts;
cts_diff -= (s32) CU->TS;
if (cts_diff < 0) cts_diff = -cts_diff;
//the decoder is dispathing CTS in the previous time base , override the timestamp ...
if (cts_diff > 20000 ) {
GF_LOG(GF_LOG_WARNING, GF_LOG_MEDIA, ("[%s] decoded frame CTS %d but input frame CTS %d, lilely due to clock discontinuity\n", codec->decio->module_name, CU->TS, cts));
CU->TS = cts;
}
UnlockCompositionUnit(codec, CU, unit_size);
if (unit_size) {
GF_LOG(GF_LOG_DEBUG, GF_LOG_MEDIA, ("[%s] at %d dispatched frame CTS %d in CB\n", codec->decio->module_name, gf_clock_real_time(prev_ch->clock), CU->TS));
}
if (!ch || !AU) {
GF_LOG(GF_LOG_DEBUG, GF_LOG_CODEC, ("[%s] Exit decode loop because no more input data\n", codec->decio->module_name));
return GF_OK;
}
if (force_skip) continue;
now = gf_sys_clock_high_res() - entryTime;
/*escape from decoding loop only if above critical limit - this is to avoid starvation on audio*/
if (!ch->esd->dependsOnESID && (codec->CB->UnitCount > codec->CB->Min)) {
if (now >= TimeAvailable) {
GF_LOG(GF_LOG_DEBUG, GF_LOG_CODEC, ("[%s] Exit decode loop because time is up: %d vs %d us available\n", codec->decio->module_name, now, TimeAvailable));
return GF_OK;
}
} else if (now >= TimeAvailable) {
GF_LOG(GF_LOG_DEBUG, GF_LOG_CODEC, ("[%s] Exit decode loop because running for too long: %d vs %d us available\n", codec->decio->module_name, now, TimeAvailable));
return GF_OK;
} else if (codec->odm->term->bench_mode) {
return GF_OK;
}
}
return GF_OK;
}