From a0bd1242b6d86ffdf43377850ea20b935b4a287e Mon Sep 17 00:00:00 2001 From: Mark Vejvoda Date: Wed, 16 Mar 2011 23:43:31 +0000 Subject: [PATCH] - improved performance of CRC precache thread, now uses up to 3 worker threads to batch process techtree's --- source/glest_game/main/main.cpp | 8 +- .../include/platform/common/simple_threads.h | 7 +- .../platform/common/simple_threads.cpp | 140 +++++++++++++++--- 3 files changed, 131 insertions(+), 24 deletions(-) diff --git a/source/glest_game/main/main.cpp b/source/glest_game/main/main.cpp index 960c4d77..b2176f23 100644 --- a/source/glest_game/main/main.cpp +++ b/source/glest_game/main/main.cpp @@ -101,7 +101,7 @@ const char *GAME_ARGS[] = { "--log-path", "--show-ini-settings", "--convert-models", - "--convert-textures", + //"--convert-textures", "--disable-backtrace", "--disable-vbo", "--verbose" @@ -2075,8 +2075,10 @@ int glestMain(int argc, char** argv) { //int *foo = (int*)-1; // make a bad pointer //printf("%d\n", *foo); // causes segfault // END - - if(config.getBool("PreCacheCRCThread","true") == true) { + bool startCRCPrecacheThread = config.getBool("PreCacheCRCThread","true"); + //printf("### In [%s::%s Line: %d] precache thread enabled = %d SystemFlags::VERBOSE_MODE_ENABLED = %d\n",__FILE__,__FUNCTION__,__LINE__,startCRCPrecacheThread,SystemFlags::VERBOSE_MODE_ENABLED); + if(SystemFlags::VERBOSE_MODE_ENABLED) printf("In [%s::%s Line: %d] precache thread enabled = %d\n",__FILE__,__FUNCTION__,__LINE__,startCRCPrecacheThread); + if(startCRCPrecacheThread == true) { vector techDataPaths = config.getPathListForType(ptTechs); preCacheThread.reset(new FileCRCPreCacheThread()); preCacheThread->setUniqueID(__FILE__); diff --git a/source/shared_lib/include/platform/common/simple_threads.h b/source/shared_lib/include/platform/common/simple_threads.h index b7eb5a61..a5593d9c 100644 --- a/source/shared_lib/include/platform/common/simple_threads.h +++ b/source/shared_lib/include/platform/common/simple_threads.h @@ -29,14 +29,15 @@ namespace Shared { namespace PlatformCommon { class FileCRCPreCacheThread : public BaseThread { protected: - vector techDataPaths; + vector workerThreadTechPaths; public: FileCRCPreCacheThread(); - FileCRCPreCacheThread(vector techDataPaths); + FileCRCPreCacheThread(vector techDataPaths,vector workerThreadTechPaths); virtual void execute(); - void setTechDataPaths(vector techDataPaths) { this->techDataPaths = techDataPaths; } + void setTechDataPaths(vector value) { this->techDataPaths = value; } + void setWorkerThreadTechPaths(vector value) { this->workerThreadTechPaths = value; } }; // ===================================================== diff --git a/source/shared_lib/sources/platform/common/simple_threads.cpp b/source/shared_lib/sources/platform/common/simple_threads.cpp index ba3c9bc9..7592b228 100644 --- a/source/shared_lib/sources/platform/common/simple_threads.cpp +++ b/source/shared_lib/sources/platform/common/simple_threads.cpp @@ -23,7 +23,17 @@ using namespace Shared::PlatformCommon; namespace Shared { namespace PlatformCommon { +const static int MAX_FileCRCPreCacheThread_WORKER_THREADS = 3; + FileCRCPreCacheThread::FileCRCPreCacheThread() : BaseThread() { + techDataPaths.clear(); + workerThreadTechPaths.clear(); +} + +FileCRCPreCacheThread::FileCRCPreCacheThread(vector techDataPaths, + vector workerThreadTechPaths) { + this->techDataPaths = techDataPaths; + this->workerThreadTechPaths = workerThreadTechPaths; } void FileCRCPreCacheThread::execute() { @@ -36,32 +46,126 @@ void FileCRCPreCacheThread::execute() { return; } - SystemFlags::OutputDebug(SystemFlags::debugNetwork,"FILE CRC PreCache thread is running\n"); + bool threadControllerMode = (workerThreadTechPaths.size() == 0); + + if(SystemFlags::VERBOSE_MODE_ENABLED) printf("FILE CRC PreCache thread is running threadControllerMode = %d\n",threadControllerMode); + SystemFlags::OutputDebug(SystemFlags::debugNetwork,"FILE CRC PreCache thread is running threadControllerMode = %d\n",threadControllerMode); try { - //tech Tree listBox - vector techPaths; - findDirs(techDataPaths, techPaths); - if(techPaths.empty() == false) { - for(unsigned int idx = 0; idx < techPaths.size(); idx++) { - string techName = techPaths[idx]; + std::vector preCacheWorkerThreadList; + if(threadControllerMode == true) { + if(SystemFlags::VERBOSE_MODE_ENABLED) printf("********************** CRC Controller thread START **********************\n"); + time_t elapsedTime = time(NULL); - time_t elapsedTime = time(NULL); - if(SystemFlags::VERBOSE_MODE_ENABLED) printf("In [%s::%s Line: %d] caching CRC value for Tech [%s] [%d of %d]\n",__FILE__,__FUNCTION__,__LINE__,techName.c_str(),idx+1,(int)techPaths.size()); - SystemFlags::OutputDebug(SystemFlags::debugNetwork,"In [%s::%s Line: %d] caching CRC value for Tech [%s] [%d of %d]\n",__FILE__,__FUNCTION__,__LINE__,techName.c_str(),idx+1,techPaths.size()); + Checksum::clearFileCache(); - int32 techCRC = getFolderTreeContentsCheckSumRecursively(techDataPaths, string("/") + techName + string("/*"), ".xml", NULL); + vector techPaths; + findDirs(techDataPaths, techPaths); + if(techPaths.empty() == false) { - if(SystemFlags::VERBOSE_MODE_ENABLED) printf("In [%s::%s Line: %d] cached CRC value for Tech [%s] is [%d] [%d of %d] took %.3f seconds.\n",__FILE__,__FUNCTION__,__LINE__,techName.c_str(),techCRC,idx+1,(int)techPaths.size(),difftime(time(NULL),elapsedTime)); - SystemFlags::OutputDebug(SystemFlags::debugNetwork,"In [%s::%s Line: %d] cached CRC value for Tech [%s] is [%d] [%d of %d] took %.3f seconds.\n",__FILE__,__FUNCTION__,__LINE__,techName.c_str(),techCRC,idx+1,techPaths.size(),difftime(time(NULL),elapsedTime)); + int techsPerWorker = (techPaths.size() / MAX_FileCRCPreCacheThread_WORKER_THREADS); + if(techPaths.size() % MAX_FileCRCPreCacheThread_WORKER_THREADS != 0) { + techsPerWorker++; + } - if(getQuitStatus() == true) { - SystemFlags::OutputDebug(SystemFlags::debugNetwork,"In [%s::%s Line: %d]\n",__FILE__,__FUNCTION__,__LINE__); - break; - } - //sleep( 50 ); + if(SystemFlags::VERBOSE_MODE_ENABLED) printf("In [%s::%s Line: %d] techsPerWorker = %d, MAX_FileCRCPreCacheThread_WORKER_THREADS = %d, techPaths.size() = %d\n",__FILE__,__FUNCTION__,__LINE__,techsPerWorker,MAX_FileCRCPreCacheThread_WORKER_THREADS,(int)techPaths.size()); + + int consumedWorkers = 0; + for(int workerIdx = 0; workerIdx < MAX_FileCRCPreCacheThread_WORKER_THREADS; ++workerIdx) { + if(getQuitStatus() == true) { + SystemFlags::OutputDebug(SystemFlags::debugNetwork,"In [%s::%s Line: %d]\n",__FILE__,__FUNCTION__,__LINE__); + break; + } + + int currentWorkerMax = (techPaths.size() - consumedWorkers); + if(currentWorkerMax > techsPerWorker) { + currentWorkerMax = techsPerWorker; + } + + vector workerTechList; + int endConsumerIndex = consumedWorkers + currentWorkerMax; + for(unsigned int idx = consumedWorkers; idx < endConsumerIndex; idx++) { + string techName = techPaths[idx]; + workerTechList.push_back(techName); + + if(SystemFlags::VERBOSE_MODE_ENABLED) printf("In [%s::%s Line: %d] Spawning CRC thread for Tech [%s] [%d of %d]\n",__FILE__,__FUNCTION__,__LINE__,techName.c_str(),idx+1,(int)techPaths.size()); + } + + if(SystemFlags::VERBOSE_MODE_ENABLED) printf("In [%s::%s Line: %d] workerIdx = %d, currentWorkerMax = %d, endConsumerIndex = %d\n",__FILE__,__FUNCTION__,__LINE__,workerIdx,currentWorkerMax,endConsumerIndex); + + FileCRCPreCacheThread *workerThread = new FileCRCPreCacheThread(techDataPaths,workerTechList); + workerThread->setUniqueID(__FILE__); + preCacheWorkerThreadList.push_back(workerThread); + workerThread->start(); + sleep(100); + + consumedWorkers += currentWorkerMax; + if(consumedWorkers >= techPaths.size()) { + break; + } + } + + if(SystemFlags::VERBOSE_MODE_ENABLED) printf("In [%s::%s Line: %d] Waiting for Spawned CRC threads to complete = preCacheWorkerThreadList.size()\n",__FILE__,__FUNCTION__,__LINE__,(int)preCacheWorkerThreadList.size()); + bool hasRunningWorkerThread = true; + for(;hasRunningWorkerThread == true;) { + hasRunningWorkerThread = false; + for(unsigned int idx = 0; idx < preCacheWorkerThreadList.size(); idx++) { + FileCRCPreCacheThread *workerThread = preCacheWorkerThreadList[idx]; + if(workerThread != NULL && workerThread->getRunningStatus() == true) { + hasRunningWorkerThread = true; + if(getQuitStatus() == true) { + workerThread->signalQuit(); + } + } + else if(workerThread != NULL && workerThread->getRunningStatus() == false) { + delete workerThread; + preCacheWorkerThreadList[idx] = NULL; + } + } + + if( getQuitStatus() == false && + hasRunningWorkerThread == true) { + sleep(20); + } + } + if(SystemFlags::VERBOSE_MODE_ENABLED) printf("********************** CRC Controller thread took %.2f seconds END **********************\n",difftime(time(NULL),elapsedTime)); } } + else { + for(unsigned int idx = 0; idx < workerThreadTechPaths.size(); idx++) { + string techName = this->workerThreadTechPaths[idx]; + if(getQuitStatus() == true) { + break; + } + if(SystemFlags::VERBOSE_MODE_ENABLED) printf("--------------------- CRC worker thread START for tech [%s] ---------------------------\n",techName.c_str()); + if(SystemFlags::VERBOSE_MODE_ENABLED) printf("In [%s::%s Line: %d] caching CRC value for Tech [%s]\n",__FILE__,__FUNCTION__,__LINE__,techName.c_str()); + SystemFlags::OutputDebug(SystemFlags::debugNetwork,"In [%s::%s Line: %d] caching CRC value for Tech [%s]\n",__FILE__,__FUNCTION__,__LINE__,techName.c_str()); + + time_t elapsedTime = time(NULL); + // Clear existing CRC to force a CRC refresh + string pathSearchString = string("/") + techName + string("/*"); + const string filterFileExt = ".xml"; + clearFolderTreeContentsCheckSum(techDataPaths, pathSearchString, filterFileExt); + clearFolderTreeContentsCheckSumList(techDataPaths, pathSearchString, filterFileExt); + + if(getQuitStatus() == true) { + break; + } + + if(SystemFlags::VERBOSE_MODE_ENABLED) printf("--------------------- CRC worker thread running for tech [%s] ---------------------------\n",techName.c_str()); + if(getQuitStatus() == false) { + int32 techCRC = getFolderTreeContentsCheckSumRecursively(techDataPaths, string("/") + techName + string("/*"), ".xml", NULL); + + if(SystemFlags::VERBOSE_MODE_ENABLED) printf("In [%s::%s Line: %d] cached CRC value for Tech [%s] is [%d] took %.3f seconds.\n",__FILE__,__FUNCTION__,__LINE__,techName.c_str(),techCRC,difftime(time(NULL),elapsedTime)); + SystemFlags::OutputDebug(SystemFlags::debugNetwork,"In [%s::%s Line: %d] cached CRC value for Tech [%s] is [%d] took %.3f seconds.\n",__FILE__,__FUNCTION__,__LINE__,techName.c_str(),techCRC,difftime(time(NULL),elapsedTime)); + } + if(SystemFlags::VERBOSE_MODE_ENABLED) printf("--------------------- CRC worker thread END for tech [%s] ---------------------------\n",techName.c_str()); + + if(getQuitStatus() == true) { + break; + } + } + } } catch(const exception &ex) { SystemFlags::OutputDebug(SystemFlags::debugError,"In [%s::%s Line: %d] Error [%s]\n",__FILE__,__FUNCTION__,__LINE__,ex.what());