Skip to content

Add dl_source2: dual download source with speed test and fallback #63

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
neyoneit wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Add dl_source2: dual download source with speed test and fallback #63

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
162 changes: 162 additions & 0 deletions code/client/cl_curl.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -900,6 +900,148 @@ static size_t Com_DL_HeaderCallback( void *ptr, size_t size, size_t nmemb, void
}


/*
===============================================================
Com_DL_SpeedTest()

Run a parallel 1-second speed test against two download sources.
Returns 1 if source1 is faster/equal, 2 if source2 is faster.
Returns 0 on error (use source1 as default).
Both URLs should have %m already replaced with the actual filename.
===============================================================
*/
typedef struct {
int bytesReceived;
} speedTestData_t;

static size_t Com_DL_SpeedTestWrite( void *ptr, size_t size, size_t nmemb, void *userdata )
{
speedTestData_t *data = (speedTestData_t *)userdata;
data->bytesReceived += (int)( size * nmemb );
return size * nmemb;
}

int Com_DL_SpeedTest( download_t *dl, const char *url1, const char *url2 )
{
CURL *easy1, *easy2;
CURLM *multi;
speedTestData_t data1, data2;
int running, startTime, elapsed;
float speed1, speed2;

if ( !Com_DL_Init( dl ) ) {
Com_Printf( S_COLOR_YELLOW "Speed test: failed to init cURL\n" );
return 0;
}

easy1 = dl->func.easy_init();
easy2 = dl->func.easy_init();
multi = dl->func.multi_init();

if ( !easy1 || !easy2 || !multi ) {
Com_Printf( S_COLOR_YELLOW "Speed test: failed to create cURL handles\n" );
if ( easy1 ) dl->func.easy_cleanup( easy1 );
if ( easy2 ) dl->func.easy_cleanup( easy2 );
if ( multi ) dl->func.multi_cleanup( multi );
Com_DL_Done( dl );
return 0;
}

memset( &data1, 0, sizeof( data1 ) );
memset( &data2, 0, sizeof( data2 ) );

// setup source 1
dl->func.easy_setopt( easy1, CURLOPT_URL, url1 );
dl->func.easy_setopt( easy1, CURLOPT_WRITEFUNCTION, Com_DL_SpeedTestWrite );
dl->func.easy_setopt( easy1, CURLOPT_WRITEDATA, &data1 );
dl->func.easy_setopt( easy1, CURLOPT_FOLLOWLOCATION, 1 );
dl->func.easy_setopt( easy1, CURLOPT_MAXREDIRS, 5 );
dl->func.easy_setopt( easy1, CURLOPT_FAILONERROR, 1 );
dl->func.easy_setopt( easy1, CURLOPT_NOPROGRESS, 1 );
dl->func.easy_setopt( easy1, CURLOPT_CONNECTTIMEOUT, 3L );
dl->func.easy_setopt( easy1, CURLOPT_USERAGENT, Q3_VERSION );
#if CURL_AT_LEAST_VERSION(7, 85, 0)
dl->func.easy_setopt( easy1, CURLOPT_PROTOCOLS_STR, ALLOWED_PROTOCOLS_STR );
#else
dl->func.easy_setopt( easy1, CURLOPT_PROTOCOLS, ALLOWED_PROTOCOLS );
#endif

// setup source 2
dl->func.easy_setopt( easy2, CURLOPT_URL, url2 );
dl->func.easy_setopt( easy2, CURLOPT_WRITEFUNCTION, Com_DL_SpeedTestWrite );
dl->func.easy_setopt( easy2, CURLOPT_WRITEDATA, &data2 );
dl->func.easy_setopt( easy2, CURLOPT_FOLLOWLOCATION, 1 );
dl->func.easy_setopt( easy2, CURLOPT_MAXREDIRS, 5 );
dl->func.easy_setopt( easy2, CURLOPT_FAILONERROR, 1 );
dl->func.easy_setopt( easy2, CURLOPT_NOPROGRESS, 1 );
dl->func.easy_setopt( easy2, CURLOPT_CONNECTTIMEOUT, 3L );
dl->func.easy_setopt( easy2, CURLOPT_USERAGENT, Q3_VERSION );
#if CURL_AT_LEAST_VERSION(7, 85, 0)
dl->func.easy_setopt( easy2, CURLOPT_PROTOCOLS_STR, ALLOWED_PROTOCOLS_STR );
#else
dl->func.easy_setopt( easy2, CURLOPT_PROTOCOLS, ALLOWED_PROTOCOLS );
#endif

dl->func.multi_add_handle( multi, easy1 );
dl->func.multi_add_handle( multi, easy2 );

Com_Printf( "Speed test: testing download sources...\n" );

startTime = Sys_Milliseconds();

// run both downloads for 2 seconds
do {
dl->func.multi_perform( multi, &running );
elapsed = Sys_Milliseconds() - startTime;
if ( running == 0 )
break;
} while ( elapsed < 3000 );

elapsed = Sys_Milliseconds() - startTime;
if ( elapsed < 1 ) elapsed = 1;

speed1 = (float)data1.bytesReceived / ( (float)elapsed / 1000.0f );
speed2 = (float)data2.bytesReceived / ( (float)elapsed / 1000.0f );

// log results
if ( speed1 >= 1024.0f * 1024.0f )
Com_Printf( " dl_source: %.1f MB/s\n", speed1 / ( 1024.0f * 1024.0f ) );
else
Com_Printf( " dl_source: %.0f KB/s\n", speed1 / 1024.0f );

if ( speed2 >= 1024.0f * 1024.0f )
Com_Printf( " dl_source2: %.1f MB/s\n", speed2 / ( 1024.0f * 1024.0f ) );
else
Com_Printf( " dl_source2: %.0f KB/s\n", speed2 / 1024.0f );

// cleanup
dl->func.multi_remove_handle( multi, easy1 );
dl->func.multi_remove_handle( multi, easy2 );
dl->func.easy_cleanup( easy1 );
dl->func.easy_cleanup( easy2 );
dl->func.multi_cleanup( multi );
Com_DL_Done( dl );

if ( speed2 > speed1 && data2.bytesReceived > 0 ) {
Com_Printf( S_COLOR_GREEN "Speed test: dl_source2 is faster, using as primary.\n" );
return 2;
}

if ( data1.bytesReceived > 0 ) {
Com_Printf( S_COLOR_GREEN "Speed test: dl_source is faster, using as primary.\n" );
return 1;
}

if ( data2.bytesReceived > 0 ) {
Com_Printf( S_COLOR_YELLOW "Speed test: dl_source failed, using dl_source2.\n" );
return 2;
}

Com_Printf( S_COLOR_RED "Speed test: both sources failed.\n" );
return 0;
}


/*
===============================================================
Com_DL_Begin()
Expand Down Expand Up @@ -1128,12 +1270,32 @@ qboolean Com_DL_Perform( download_t *dl )
else
{
qboolean autoDownload = dl->mapAutoDownload;
char fallbackURL[MAX_OSPATH];
char originalName[MAX_OSPATH];

dl->func.easy_getinfo( msg->easy_handle, CURLINFO_RESPONSE_CODE, &code );
Com_Printf( S_COLOR_RED "Download Error: %s Code: %ld\n",
dl->func.easy_strerror( msg->data.result ), code );

// save fallback info before cleanup
Q_strncpyz( fallbackURL, dl->fallbackURL, sizeof( fallbackURL ) );
Q_strncpyz( originalName, dl->originalName, sizeof( originalName ) );

strcpy( name, dl->TempName );
Com_DL_Cleanup( dl );
FS_Remove( name );

// try fallback source if available
if ( fallbackURL[0] != '\0' && originalName[0] != '\0' )
{
Com_Printf( S_COLOR_YELLOW "Trying fallback source...\n" );
if ( Com_DL_Begin( dl, originalName, fallbackURL, autoDownload ) )
{
dl->fallbackURL[0] = '\0'; // don't fallback again
return qtrue;
}
}

if ( autoDownload )
{
if ( cls.state == CA_CONNECTED )
Expand Down
4 changes: 4 additions & 0 deletions code/client/cl_curl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,10 @@ typedef struct download_s {
qboolean headerCheck;
qboolean mapAutoDownload;

// fallback source support
char fallbackURL[MAX_OSPATH];
char originalName[MAX_OSPATH];

struct func_s {
char* (*version)(void);
char * (*easy_escape)(CURL *curl, const char *string, int length);
Expand Down
119 changes: 117 additions & 2 deletions code/client/cl_main.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ cvar_t *cl_lanForcePackets;
cvar_t *cl_guidServerUniq;

cvar_t *dl_source;
cvar_t *dl_source2;
cvar_t *dl_usebaseq3;

cvar_t *cl_reconnectArgs;
Expand Down Expand Up @@ -3982,7 +3983,67 @@ void CL_Init( void ) {
Cvar_SetDescription( cl_guidServerUniq, "Makes cl_guid unique for each server." );

dl_source = Cvar_Get( "dl_source", "http://ws.q3df.org/maps/download/%m", CVAR_ARCHIVE );
Cvar_SetDescription( dl_source, "Cvar must point to download location." );
Cvar_SetDescription( dl_source, "Primary download source URL. Use %m for map/pak name placeholder." );

dl_source2 = Cvar_Get( "dl_source2", "https://defrag.racing/maps/download/%m", CVAR_ARCHIVE );
Cvar_SetDescription( dl_source2, "Secondary download source URL. If set, a 3-second speed test picks the faster source automatically. Use %m for map/pak name placeholder." );

// auto-fill empty sources: ensure both known mirrors are available
{
#define DL_DEFRAG_RACING "https://defrag.racing/maps/download/%m"
#define DL_WS_Q3DF "http://ws.q3df.org/maps/download/%m"

qboolean src1_empty = ( dl_source->string[0] == '\0' );
qboolean src2_empty = ( dl_source2->string[0] == '\0' );
qboolean src1_is_defrag = ( strstr( dl_source->string, "defrag.racing" ) != NULL );
qboolean src1_is_ws = ( strstr( dl_source->string, "ws.q3df.org" ) != NULL );
qboolean src2_is_defrag = ( strstr( dl_source2->string, "defrag.racing" ) != NULL );
qboolean src2_is_ws = ( strstr( dl_source2->string, "ws.q3df.org" ) != NULL );

if ( src1_empty && src2_empty ) {
// both empty: set defaults
Cvar_Set( "dl_source", DL_WS_Q3DF );
Cvar_Set( "dl_source2", DL_DEFRAG_RACING );
} else if ( src1_empty ) {
// dl_source empty: fill with whichever mirror dl_source2 is NOT
Cvar_Set( "dl_source", src2_is_defrag ? DL_WS_Q3DF : DL_DEFRAG_RACING );
} else if ( src2_empty ) {
// dl_source2 empty: fill with whichever mirror dl_source is NOT
Cvar_Set( "dl_source2", src1_is_defrag ? DL_WS_Q3DF : DL_DEFRAG_RACING );
}

// fix duplicate sources: if both point to the same server, set the other one
if ( !src1_empty && !src2_empty ) {
// re-read after potential changes above
src1_is_defrag = ( strstr( dl_source->string, "defrag.racing" ) != NULL );
src1_is_ws = ( strstr( dl_source->string, "ws.q3df.org" ) != NULL );
src2_is_defrag = ( strstr( dl_source2->string, "defrag.racing" ) != NULL );
src2_is_ws = ( strstr( dl_source2->string, "ws.q3df.org" ) != NULL );

if ( src1_is_defrag && src2_is_defrag ) {
Cvar_Set( "dl_source2", DL_WS_Q3DF );
} else if ( src1_is_ws && src2_is_ws ) {
Cvar_Set( "dl_source2", DL_DEFRAG_RACING );
}
}

// fix common misconfiguration: ws.q3df.org only works over HTTP, not HTTPS
if ( strstr( dl_source->string, "https://ws.q3df.org" ) != NULL ) {
char fixed[MAX_CVAR_VALUE_STRING];
Q_strncpyz( fixed, dl_source->string, sizeof( fixed ) );
Q_replace( "https://ws.q3df.org", "http://ws.q3df.org", fixed, sizeof( fixed ) );
Cvar_Set( "dl_source", fixed );
}
if ( strstr( dl_source2->string, "https://ws.q3df.org" ) != NULL ) {
char fixed[MAX_CVAR_VALUE_STRING];
Q_strncpyz( fixed, dl_source2->string, sizeof( fixed ) );
Q_replace( "https://ws.q3df.org", "http://ws.q3df.org", fixed, sizeof( fixed ) );
Cvar_Set( "dl_source2", fixed );
}

#undef DL_DEFRAG_RACING
#undef DL_WS_Q3DF
}

dl_usebaseq3 = Cvar_Get( "dl_usebaseq3", "0", CVAR_ARCHIVE_ND );
Cvar_CheckRange( dl_usebaseq3, "0", "1", CV_INTEGER );
Expand Down Expand Up @@ -5037,11 +5098,27 @@ static void CL_ShowIP_f( void ) {

#ifdef USE_CURL

static int dl_preferredSource = 0; // 0=not tested, 1=dl_source, 2=dl_source2

static const char *CL_DL_BuildURL( char *buf, int bufSize, const char *sourceURL, const char *pakname )
{
// simple %m replacement without needing full CURL context
Q_strncpyz( buf, sourceURL, bufSize );
if ( !Q_replace( "%m", pakname, buf, bufSize ) ) {
if ( buf[strlen(buf)-1] != '/' )
Q_strcat( buf, bufSize, "/" );
Q_strcat( buf, bufSize, pakname );
}
return buf;
}

qboolean CL_Download( const char *cmd, const char *pakname, qboolean autoDownload )
{
char url[MAX_OSPATH];
char name[MAX_CVAR_VALUE_STRING];
const char *s;
const char *primarySource;
const char *fallbackSource;

if ( dl_source->string[0] == '\0' )
{
Expand Down Expand Up @@ -5078,7 +5155,45 @@ qboolean CL_Download( const char *cmd, const char *pakname, qboolean autoDownloa
}
}

return Com_DL_Begin( &download, pakname, dl_source->string, autoDownload );
// speed test on first download if both sources are set
if ( dl_preferredSource == 0 && dl_source2->string[0] != '\0' )
{
char url1[MAX_OSPATH], url2[MAX_OSPATH];
CL_DL_BuildURL( url1, sizeof( url1 ), dl_source->string, pakname );
CL_DL_BuildURL( url2, sizeof( url2 ), dl_source2->string, pakname );
dl_preferredSource = Com_DL_SpeedTest( &download, url1, url2 );
if ( dl_preferredSource == 0 )
dl_preferredSource = 1; // both failed, default to dl_source
}

// select primary and fallback sources
if ( dl_preferredSource == 2 ) {
primarySource = dl_source2->string;
fallbackSource = dl_source->string;
} else {
primarySource = dl_source->string;
fallbackSource = ( dl_source2->string[0] != '\0' ) ? dl_source2->string : NULL;
}

if ( Com_DL_Begin( &download, pakname, primarySource, autoDownload ) ) {
// set fallback info for mid-download failure recovery
if ( fallbackSource ) {
Q_strncpyz( download.fallbackURL, fallbackSource, sizeof( download.fallbackURL ) );
Q_strncpyz( download.originalName, pakname, sizeof( download.originalName ) );
} else {
download.fallbackURL[0] = '\0';
download.originalName[0] = '\0';
}
return qtrue;
}

// primary failed to start, try fallback
if ( fallbackSource ) {
Com_Printf( S_COLOR_YELLOW "Primary source failed, trying fallback...\n" );
return Com_DL_Begin( &download, pakname, fallbackSource, autoDownload );
}

return qfalse;
}


Expand Down
1 change: 1 addition & 0 deletions code/client/client.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -371,6 +371,7 @@ void Com_DL_Cleanup( download_t *dl );
qboolean Com_DL_Begin( download_t *dl, const char *localName, const char *remoteURL, qboolean autoDownload );
qboolean Com_DL_InProgress( const download_t *dl );
qboolean Com_DL_ValidFileName( const char *fileName );
int Com_DL_SpeedTest( download_t *dl, const char *url1, const char *url2 );
qboolean CL_Download( const char *cmd, const char *pakname, qboolean autoDownload );

#endif
Expand Down