in src/utils/downloader.c [2597:3167]
static GF_Err wait_for_header_and_parse(GF_DownloadSession *sess, char * sHTTP)
{
GF_NETIO_Parameter par;
s32 bytesRead, BodyStart;
u32 res, i, buf_size;
s32 LinePos, Pos;
u32 rsp_code, ContentLength, first_byte, last_byte, total_size, range, no_range;
Bool connection_closed = GF_FALSE;
char buf[1025];
char comp[400];
GF_Err e;
char * new_location;
const char * mime_type;
assert( sess->status == GF_NETIO_WAIT_FOR_REPLY );
bytesRead = res = 0;
new_location = NULL;
if (!(sess->flags & GF_NETIO_SESSION_NOT_CACHED)) {
sess->use_cache_file = GF_TRUE;
}
if (sess->from_cache_only) {
GF_NETIO_Parameter par;
sess->reply_time = (u32) (gf_sys_clock_high_res() - sess->request_start_time);
sess->rsp_hdr_size = 0;
sess->total_size = sess->bytes_done = gf_cache_get_content_length(sess->cache_entry);;
memset(&par, 0, sizeof(GF_NETIO_Parameter));
par.msg_type = GF_NETIO_DATA_TRANSFERED;
par.error = GF_OK;
gf_dm_sess_user_io(sess, &par);
gf_dm_disconnect(sess, GF_FALSE);
return GF_OK;
}
buf_size = GF_DOWNLOAD_BUFFER_SIZE;
if (sess->dm && sess->dm->limit_data_rate)
buf_size = 1024;
//always set start time to the time at last attempt reply parsing
sess->start_time = gf_sys_clock_high_res();
sess->start_time_utc = gf_net_get_utc();
while (1) {
e = gf_dm_read_data(sess, sHTTP + bytesRead, buf_size - bytesRead, &res);
switch (e) {
case GF_IP_NETWORK_EMPTY:
if (!bytesRead) {
if (gf_sys_clock_high_res() - sess->request_start_time > 1000 * sess->dm->request_timeout) {
sess->last_error = GF_IP_NETWORK_EMPTY;
sess->status = GF_NETIO_STATE_ERROR;
return GF_IP_NETWORK_EMPTY;
}
return GF_OK;
}
continue;
/*socket has been closed while configuring, retry (not sure if the server got the GET)*/
case GF_IP_CONNECTION_CLOSED:
if (sess->http_read_type == HEAD) {
/* Some servers such as shoutcast directly close connection if HEAD or an unknown method is issued */
sess->server_only_understand_get = GF_TRUE;
}
GF_LOG(GF_LOG_INFO, GF_LOG_NETWORK, ("[HTTP] Connection closed by server when getting %s - retrying\n", sess->remote_path));
gf_dm_disconnect(sess, GF_TRUE);
if (sess->num_retry)
sess->status = GF_NETIO_SETUP;
else {
sess->last_error = e;
sess->status = GF_NETIO_STATE_ERROR;
}
return e;
case GF_OK:
if (!res) return GF_OK;
break;
default:
goto exit;
}
bytesRead += res;
/*locate body start*/
BodyStart = gf_token_find(sHTTP, 0, bytesRead, "\r\n\r\n");
if (BodyStart > 0) {
BodyStart += 4;
break;
}
BodyStart = gf_token_find(sHTTP, 0, bytesRead, "\n\n");
if (BodyStart > 0) {
BodyStart += 2;
break;
}
BodyStart=0;
}
if (bytesRead < 0) {
e = GF_REMOTE_SERVICE_ERROR;
goto exit;
}
if (!BodyStart)
BodyStart = bytesRead;
sHTTP[BodyStart-1] = 0;
GF_LOG(GF_LOG_INFO, GF_LOG_NETWORK, ("[HTTP] %s\n\n", sHTTP));
sess->reply_time = (u32) (gf_sys_clock_high_res() - sess->request_start_time);
sess->rsp_hdr_size = BodyStart;
LinePos = gf_token_get_line(sHTTP, 0, bytesRead, buf, 1024);
Pos = gf_token_get(buf, 0, " \t\r\n", comp, 400);
if (!strncmp("ICY", comp, 3)) {
sess->use_cache_file = GF_FALSE;
/*be prepared not to receive any mime type from ShoutCast servers*/
if (!gf_cache_get_mime_type(sess->cache_entry))
gf_cache_set_mime_type(sess->cache_entry, "audio/mpeg");
} else if ((strncmp("HTTP", comp, 4) != 0)) {
e = GF_REMOTE_SERVICE_ERROR;
goto exit;
}
Pos = gf_token_get(buf, Pos, " ", comp, 400);
if (Pos <= 0) {
e = GF_REMOTE_SERVICE_ERROR;
goto exit;
}
rsp_code = (u32) atoi(comp);
Pos = gf_token_get(buf, Pos, " \r\n", comp, 400);
no_range = range = ContentLength = first_byte = last_byte = total_size = 0;
/* parse headers*/
while (1) {
GF_HTTPHeader *hdrp;
char *sep, *hdr_sep, *hdr, *hdr_val;
if ( (s32) LinePos + 4 > BodyStart) break;
LinePos = gf_token_get_line(sHTTP, LinePos , bytesRead, buf, 1024);
if (LinePos < 0) break;
hdr_sep = NULL;
hdr_val = NULL;
hdr = buf;
sep = strchr(buf, ':');
if (sep) {
sep[0]=0;
hdr_val = sep+1;
while (hdr_val[0]==' ') hdr_val++;
hdr_sep = strrchr(hdr_val, '\r');
if (hdr_sep) hdr_sep[0] = 0;
}
GF_SAFEALLOC(hdrp, GF_HTTPHeader);
hdrp->name = gf_strdup(hdr);
hdrp->value = gf_strdup(hdr_val);
gf_list_add(sess->headers, hdrp);
if (sep) sep[0]=':';
if (hdr_sep) hdr_sep[0] = '\r';
}
//default pre-processing of headers - needs cleanup, not all of these have to be parsed before checking reply code
for (i=0; i<gf_list_count(sess->headers); i++) {
char *val;
GF_HTTPHeader *hdrp = (GF_HTTPHeader*)gf_list_get(sess->headers, i);
if (!stricmp(hdrp->name, "Content-Length") ) {
ContentLength = (u32) atoi(hdrp->value);
if (rsp_code<300)
gf_cache_set_content_length(sess->cache_entry, ContentLength);
}
else if (!stricmp(hdrp->name, "Content-Type")) {
char *mime = gf_strdup(hdrp->value);
while (1) {
u32 len = (u32) strlen(mime);
char c = len ? mime[len-1] : 0;
if ((c=='\r') || (c=='\n')) {
mime[len-1] = 0;
} else {
break;
}
}
val = strchr(mime, ';');
if (val) val[0] = 0;
strlwr(mime);
if (rsp_code<300) {
if (sess->cache_entry) {
gf_cache_set_mime_type(sess->cache_entry, mime);
} else {
sess->mime_type = mime;
mime = NULL;
}
}
if (mime) gf_free(mime);
}
else if (!stricmp(hdrp->name, "Content-Range")) {
range = 1;
if (!strncmp(hdrp->value, "bytes", 5)) {
val = hdrp->value + 5;
if (val[0] == ':') val += 1;
val += http_skip_space(val);
if (val[0] == '*') {
sscanf(val, "*/%u", &total_size);
} else {
sscanf(val, "%u-%u/%u", &first_byte, &last_byte, &total_size);
}
}
}
else if (!stricmp(hdrp->name, "Accept-Ranges")) {
if (strstr(hdrp->value, "none")) no_range = 1;
}
else if (!stricmp(hdrp->name, "Location"))
new_location = gf_strdup(hdrp->value);
else if (!strnicmp(hdrp->name, "ice", 3) || !strnicmp(hdrp->name, "icy", 3) ) {
/* For HTTP icy servers, we disable cache */
if (sess->icy_metaint == 0)
sess->icy_metaint = -1;
sess->use_cache_file = GF_FALSE;
if (!stricmp(hdrp->name, "icy-metaint")) {
sess->icy_metaint = atoi(hdrp->value);
}
}
else if (!stricmp(hdrp->name, "Cache-Control")) {
}
else if (!stricmp(hdrp->name, "ETag")) {
if (rsp_code<300)
gf_cache_set_etag_on_server(sess->cache_entry, hdrp->value);
}
else if (!stricmp(hdrp->name, "Last-Modified")) {
if (rsp_code<300)
gf_cache_set_last_modified_on_server(sess->cache_entry, hdrp->value);
}
else if (!stricmp(hdrp->name, "Transfer-Encoding")) {
if (!stricmp(hdrp->value, "chunked"))
sess->chunked = GF_TRUE;
}
else if (!stricmp(hdrp->name, "X-UserProfileID") ) {
if (sess->dm && sess->dm->cfg)
gf_cfg_set_key(sess->dm->cfg, "Downloader", "UserProfileID", hdrp->value);
}
else if (!stricmp(hdrp->name, "Connection") ) {
if (strstr(hdrp->value, "close"))
connection_closed = GF_TRUE;
}
if (sess->status==GF_NETIO_DISCONNECTED) return GF_OK;
}
if (no_range) first_byte = 0;
gf_cache_set_headers_processed(sess->cache_entry);
par.msg_type = GF_NETIO_PARSE_REPLY;
par.error = GF_OK;
par.reply = rsp_code;
par.value = comp;
/*
* If response is correct, it means our credentials are correct
*/
if (sess->creds && rsp_code != 304)
sess->creds->valid = GF_TRUE;
/*try to flush body */
if (rsp_code>=300) {
u32 start = gf_sys_clock();
while (BodyStart + ContentLength > (u32) bytesRead) {
e = gf_dm_read_data(sess, sHTTP + bytesRead, buf_size - bytesRead, &res);
switch (e) {
case GF_IP_NETWORK_EMPTY:
gf_sleep(0);
break;
case GF_OK:
bytesRead += res;
break;
default:
start=0;
break;
}
if (gf_sys_clock()-start>100)
break;
//does not fit in our buffer, too bad we'll kill the connection
if (bytesRead == GF_DOWNLOAD_BUFFER_SIZE)
break;
}
if (BodyStart + ContentLength > (u32) bytesRead) {
ContentLength = 0;
//cannot flush, discard socket
sess->connection_close = GF_TRUE;
}
}
//remember if we can keep the session alive after the transfer is done
sess->connection_close = connection_closed;
switch (rsp_code) {
case 200:
case 201:
case 202:
case 206:
gf_dm_sess_user_io(sess, &par);
e = GF_OK;
if (sess->proxy_enabled==2) {
sess->proxy_enabled=0;
if (sess->dm)
gf_list_add(sess->dm->skip_proxy_servers, gf_strdup(sess->server_name));
}
break;
/*redirection: extract the new location*/
case 301:
case 302:
case 307:
if (!new_location || !strlen(new_location) ) {
gf_dm_sess_user_io(sess, &par);
e = GF_URL_ERROR;
goto exit;
}
while (
(new_location[strlen(new_location)-1] == '\n')
|| (new_location[strlen(new_location)-1] == '\r') )
new_location[strlen(new_location)-1] = 0;
/*reset and reconnect*/
gf_dm_disconnect(sess, GF_TRUE);
sess->status = GF_NETIO_SETUP;
e = gf_dm_sess_setup_from_url(sess, new_location);
if (e) {
sess->status = GF_NETIO_STATE_ERROR;
sess->last_error = e;
gf_dm_sess_notify_state(sess, sess->status, e);
}
return e;
case 304:
{
sess->status = GF_NETIO_PARSE_REPLY;
assert(sess->cache_entry);
sess->total_size = gf_cache_get_cache_filesize(sess->cache_entry);
gf_dm_sess_notify_state(sess, GF_NETIO_PARSE_REPLY, GF_OK);
gf_dm_disconnect(sess, GF_FALSE);
if (sess->user_proc) {
/* For modules that do not use cache and have problems with GF_NETIO_DATA_TRANSFERED ... */
const char * filename;
FILE * f;
filename = gf_cache_get_cache_filename(sess->cache_entry);
GF_LOG(GF_LOG_INFO, GF_LOG_NETWORK, ("[HTTP] Sending data to modules from %s...\n", filename));
f = gf_fopen(filename, "rb");
assert(filename);
if (!f) {
GF_LOG(GF_LOG_INFO, GF_LOG_NETWORK, ("[HTTP] FAILED to open cache file %s for reading contents !\n", filename));
/* Ooops, no cache, redowload everything ! */
gf_dm_disconnect(sess, GF_FALSE);
sess->status = GF_NETIO_SETUP;
e = gf_dm_sess_setup_from_url(sess, sess->orig_url);
sess->total_size = gf_cache_get_cache_filesize(sess->cache_entry);
if (e) {
sess->status = GF_NETIO_STATE_ERROR;
sess->last_error = e;
gf_dm_sess_notify_state(sess, sess->status, e);
}
return e;
}
par.error = GF_OK;
par.msg_type = GF_NETIO_PARSE_HEADER;
par.name = "Content-Type";
par.value = (char *) gf_cache_get_mime_type(sess->cache_entry);
gf_dm_sess_user_io(sess, &par);
sess->status = GF_NETIO_DATA_EXCHANGE;
if (! (sess->flags & GF_NETIO_SESSION_NOT_THREADED) || sess->force_data_write_callback) {
char file_cache_buff[16384];
int read = 0;
u32 total_size = gf_cache_get_cache_filesize(sess->cache_entry);
do {
read = (u32) fread(file_cache_buff, sizeof(char), 16384, f);
if (read > 0) {
sess->bytes_done += read;
sess->total_size = total_size;
sess->bytes_per_sec = 0xFFFFFFFF;
par.size = read;
par.msg_type = GF_NETIO_DATA_EXCHANGE;
par.error = GF_EOS;
par.reply = 2;
par.data = file_cache_buff;
gf_dm_sess_user_io(sess, &par);
}
} while ( read > 0);
}
gf_fclose(f);
GF_LOG(GF_LOG_INFO, GF_LOG_NETWORK, ("[HTTP] all data has been sent to modules from %s.\n", filename));
}
/* Cache file is the most recent */
sess->status = GF_NETIO_DATA_TRANSFERED;
gf_dm_sess_notify_state(sess, GF_NETIO_DATA_TRANSFERED, GF_OK);
gf_dm_disconnect(sess, GF_FALSE);
par.msg_type = GF_NETIO_DATA_TRANSFERED;
par.error = GF_OK;
gf_dm_sess_user_io(sess, &par);
return GF_OK;
}
case 401:
{
/* Do we have a credentials struct ? */
sess->creds = gf_user_credentials_register(sess->dm, sess->server_name, NULL, NULL, GF_FALSE);
if (!sess->creds) {
/* User credentials have not been filled properly, we have to abort */
gf_dm_disconnect(sess, GF_TRUE);
sess->status = GF_NETIO_STATE_ERROR;
par.error = GF_AUTHENTICATION_FAILURE;
par.msg_type = GF_NETIO_DISCONNECTED;
gf_dm_sess_user_io(sess, &par);
e = GF_AUTHENTICATION_FAILURE;
sess->last_error = e;
goto exit;
}
gf_dm_disconnect(sess, GF_FALSE);
sess->status = GF_NETIO_SETUP;
e = gf_dm_sess_setup_from_url(sess, sess->orig_url);
if (e) {
sess->status = GF_NETIO_STATE_ERROR;
sess->last_error = e;
gf_dm_sess_notify_state(sess, sess->status, e);
}
return e;
}
case 404:
/* File not found */
gf_dm_sess_user_io(sess, &par);
if ((BodyStart < (s32) bytesRead)) {
sHTTP[bytesRead] = 0;
GF_LOG(GF_LOG_ERROR, GF_LOG_NETWORK, ("[HTTP] Failure: %s\n", sHTTP + BodyStart));
}
notify_headers(sess, sHTTP, bytesRead, BodyStart);
e = GF_URL_ERROR;
goto exit;
break;
case 416:
/* Range not accepted */
gf_dm_sess_user_io(sess, &par);
notify_headers(sess, sHTTP, bytesRead, BodyStart);
e = GF_SERVICE_ERROR;
goto exit;
break;
case 400:
case 501:
/* Method not implemented ! */
if (sess->http_read_type == HEAD) {
/* Since HEAD is not understood by this server, we use a GET instead */
sess->http_read_type = GET;
sess->flags |= GF_NETIO_SESSION_NOT_CACHED;
gf_dm_disconnect(sess, GF_FALSE);
sess->status = GF_NETIO_SETUP;
sess->server_only_understand_get = GF_TRUE;
GF_LOG(GF_LOG_INFO, GF_LOG_NETWORK, ("Method not supported, try with GET.\n"));
e = gf_dm_sess_setup_from_url(sess, sess->orig_url);
if (e) {
sess->status = GF_NETIO_STATE_ERROR;
sess->last_error = e;
gf_dm_sess_notify_state(sess, sess->status, e);
}
return e;
}
gf_dm_sess_user_io(sess, &par);
notify_headers(sess, sHTTP, bytesRead, BodyStart);
e = GF_REMOTE_SERVICE_ERROR;
goto exit;
case 503:
/*retry without proxy*/
if (sess->proxy_enabled==1) {
sess->proxy_enabled=2;
gf_dm_disconnect(sess, GF_TRUE);
sess->status = GF_NETIO_SETUP;
return GF_OK;
}
default:
gf_dm_sess_user_io(sess, &par);
notify_headers(sess, sHTTP, bytesRead, BodyStart);
e = GF_REMOTE_SERVICE_ERROR;
goto exit;
}
notify_headers(sess, NULL, bytesRead, BodyStart);
if (sess->http_read_type != GET)
sess->use_cache_file = GF_FALSE;
if (sess->http_read_type==HEAD) {
gf_dm_disconnect(sess, GF_FALSE);
gf_dm_sess_notify_state(sess, GF_NETIO_DATA_TRANSFERED, GF_OK);
sess->http_read_type = GET;
return GF_OK;
}
mime_type = gf_cache_get_mime_type(sess->cache_entry);
if (!ContentLength && mime_type && ((strstr(mime_type, "ogg") || (!strcmp(mime_type, "audio/mpeg"))))) {
if (0 == sess->icy_metaint)
sess->icy_metaint = -1;
sess->use_cache_file = GF_FALSE;
}
#ifndef GPAC_DISABLE_LOGS
if (e) {
GF_LOG(GF_LOG_ERROR, GF_LOG_NETWORK, ("[HTTP] Error processing rely from %s: %s\n", sess->server_name, gf_error_to_string(e) ) );
} else {
GF_LOG(GF_LOG_DEBUG, GF_LOG_NETWORK, ("[HTTP] Reply processed from %s\n", sess->server_name ) );
}
#endif
/*some servers may reply without content length, but we MUST have it*/
if (e) goto exit;
if (sess->icy_metaint != 0) {
assert( ! sess->use_cache_file );
GF_LOG(GF_LOG_INFO, GF_LOG_NETWORK, ("[HTTP] ICY protocol detected\n"));
if (mime_type && !stricmp(mime_type, "video/nsv")) {
gf_cache_set_mime_type(sess->cache_entry, "audio/aac");
}
sess->icy_bytes = 0;
sess->total_size = SIZE_IN_STREAM;
sess->status = GF_NETIO_DATA_EXCHANGE;
} else if (!ContentLength && !sess->chunked) {
if (sess->http_read_type == GET) {
sess->total_size = SIZE_IN_STREAM;
sess->use_cache_file = GF_FALSE;
sess->status = GF_NETIO_DATA_EXCHANGE;
sess->bytes_done = 0;
} else {
gf_dm_sess_notify_state(sess, GF_NETIO_DATA_TRANSFERED, GF_OK);
gf_dm_disconnect(sess, GF_FALSE);
return GF_OK;
}
} else {
sess->total_size = ContentLength;
if (sess->use_cache_file && sess->http_read_type == GET ) {
e = gf_cache_open_write_cache(sess->cache_entry, sess);
if (e) {
GF_LOG(GF_LOG_ERROR, GF_LOG_NETWORK, ( "[CACHE] Failed to open cache, error=%d\n", e));
goto exit;
}
}
sess->status = GF_NETIO_DATA_EXCHANGE;
sess->bytes_done = 0;
}
/* we may have existing data in this buffer ... */
if (!e && (BodyStart < (s32) bytesRead)) {
if (sess->init_data) gf_free(sess->init_data);
sess->init_data_size = 0;
sess->init_data = NULL;
gf_dm_data_received(sess, (u8 *) sHTTP + BodyStart, bytesRead - BodyStart, GF_TRUE, NULL);
}
exit:
if (e) {
GF_LOG(GF_LOG_ERROR, GF_LOG_NETWORK, ("[HTTP] Error parsing reply: %s for URL %s\n", gf_error_to_string(e), sess->orig_url ));
gf_cache_entry_set_delete_files_when_deleted(sess->cache_entry);
gf_dm_remove_cache_entry_from_session(sess);
sess->cache_entry = NULL;
gf_dm_disconnect(sess, GF_FALSE);
sess->status = GF_NETIO_STATE_ERROR;
sess->last_error = e;
gf_dm_sess_notify_state(sess, sess->status, e);
return e;
}
/*DO NOT call parse_body yet, as the final user may not be connected to our session*/
return GF_OK;
}