diff --git a/source/glest_game/main/main.cpp b/source/glest_game/main/main.cpp index 2f67dbb5..82c021fa 100644 --- a/source/glest_game/main/main.cpp +++ b/source/glest_game/main/main.cpp @@ -1608,6 +1608,11 @@ int glestMain(int argc, char** argv) { createDirectoryPaths(userData); } } + string crcCachePath = userData + "cache/"; + if(isdir(crcCachePath.c_str()) == false) { + createDirectoryPaths(crcCachePath); + } + setCRCCacheFilePath(crcCachePath); // Set some statics based on ini entries SystemFlags::ENABLE_THREADED_LOGGING = config.getBool("ThreadedLogging","true"); @@ -1826,13 +1831,13 @@ int glestMain(int argc, char** argv) { //printf("%d\n", *foo); // causes segfault // END - if(config.getBool("AllowGameDataSynchCheck","false") == true) { + //if(config.getBool("AllowGameDataSynchCheck","false") == true) { vector techDataPaths = config.getPathListForType(ptTechs); preCacheThread.reset(new FileCRCPreCacheThread()); preCacheThread->setUniqueID(__FILE__); preCacheThread->setTechDataPaths(techDataPaths); preCacheThread->start(); - } + //} // test //Shared::Platform::MessageBox(NULL,"Mark's test.","Test",0); diff --git a/source/glest_game/menu/menu_state_connected_game.cpp b/source/glest_game/menu/menu_state_connected_game.cpp index 3a4757b0..e4bc58a3 100644 --- a/source/glest_game/menu/menu_state_connected_game.cpp +++ b/source/glest_game/menu/menu_state_connected_game.cpp @@ -1966,36 +1966,37 @@ void MenuStateConnectedGame::FTPClient_CallbackEvent(string itemName, FTP_Client Checksum::clearFileCache(); vector paths = Config::getInstance().getPathListForType(ptTilesets); - string cacheLookupId = CacheManager::getFolderTreeContentsCheckSumRecursivelyCacheLookupKey1; - std::map &crcTreeCache = CacheManager::getCachedItem< std::map >(cacheLookupId); + //string cacheLookupId = CacheManager::getFolderTreeContentsCheckSumRecursivelyCacheLookupKey1; + //std::map &crcTreeCache = CacheManager::getCachedItem< std::map >(cacheLookupId); string pathSearchString = string("/") + itemName + string("/*"); const string filterFileExt = ".xml"; + clearFolderTreeContentsCheckSum(paths, pathSearchString, filterFileExt); + clearFolderTreeContentsCheckSumList(paths, pathSearchString, filterFileExt); - string cacheKey = ""; - size_t count = paths.size(); - for(size_t idx = 0; idx < count; ++idx) { - string path = paths[idx] + pathSearchString; +// string cacheKey = ""; +// size_t count = paths.size(); +// for(size_t idx = 0; idx < count; ++idx) { +// string path = paths[idx] + pathSearchString; +// +// cacheKey += path + "_" + filterFileExt + "_"; +// } +// if(crcTreeCache.find(cacheKey) != crcTreeCache.end()) { +// SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d] CLEARING CACHED checksum for cacheKey [%s]\n",__FILE__,__FUNCTION__,__LINE__,cacheKey.c_str()); +// crcTreeCache.erase(cacheKey); +// } - cacheKey += path + "_" + filterFileExt + "_"; - } - if(crcTreeCache.find(cacheKey) != crcTreeCache.end()) { - SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d] CLEARING CACHED checksum for cacheKey [%s]\n",__FILE__,__FUNCTION__,__LINE__,cacheKey.c_str()); - crcTreeCache.erase(cacheKey); - } - - cacheLookupId = CacheManager::getFolderTreeContentsCheckSumRecursivelyCacheLookupKey2; - std::map &crcTreeCache2 = CacheManager::getCachedItem< std::map >(cacheLookupId); - - count = paths.size(); - for(size_t idx = 0; idx < count; ++idx) { - string path = paths[idx] + pathSearchString; - - string cacheKey = path + "_" + filterFileExt; - if(crcTreeCache2.find(cacheKey) != crcTreeCache2.end()) { - SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d] CLEARING CACHED checksum for cacheKey [%s]\n",__FILE__,__FUNCTION__,__LINE__,cacheKey.c_str()); - crcTreeCache2.erase(cacheKey); - } - } +// string cacheLookupId = CacheManager::getFolderTreeContentsCheckSumRecursivelyCacheLookupKey2; +// std::map &crcTreeCache2 = CacheManager::getCachedItem< std::map >(cacheLookupId); +// +// for(size_t idx = 0; idx < paths.size(); ++idx) { +// string path = paths[idx] + pathSearchString; +// +// string cacheKey = path + "_" + filterFileExt; +// if(crcTreeCache2.find(cacheKey) != crcTreeCache2.end()) { +// SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d] CLEARING CACHED checksum for cacheKey [%s]\n",__FILE__,__FUNCTION__,__LINE__,cacheKey.c_str()); +// crcTreeCache2.erase(cacheKey); +// } +// } safeMutexFTPProgress.ReleaseLock(); // END @@ -2045,36 +2046,37 @@ void MenuStateConnectedGame::FTPClient_CallbackEvent(string itemName, FTP_Client Checksum::clearFileCache(); vector paths = Config::getInstance().getPathListForType(ptTechs); - string cacheLookupId = CacheManager::getFolderTreeContentsCheckSumRecursivelyCacheLookupKey1; - std::map &crcTreeCache = CacheManager::getCachedItem< std::map >(cacheLookupId); + //string cacheLookupId = CacheManager::getFolderTreeContentsCheckSumRecursivelyCacheLookupKey1; + //std::map &crcTreeCache = CacheManager::getCachedItem< std::map >(cacheLookupId); string pathSearchString = string("/") + itemName + string("/*"); const string filterFileExt = ".xml"; + clearFolderTreeContentsCheckSum(paths, pathSearchString, filterFileExt); + clearFolderTreeContentsCheckSumList(paths, pathSearchString, filterFileExt); - string cacheKey = ""; - size_t count = paths.size(); - for(size_t idx = 0; idx < count; ++idx) { - string path = paths[idx] + pathSearchString; +// string cacheKey = ""; +// size_t count = paths.size(); +// for(size_t idx = 0; idx < count; ++idx) { +// string path = paths[idx] + pathSearchString; +// +// cacheKey += path + "_" + filterFileExt + "_"; +// } +// if(crcTreeCache.find(cacheKey) != crcTreeCache.end()) { +// SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d] CLEARING CACHED checksum for cacheKey [%s]\n",__FILE__,__FUNCTION__,__LINE__,cacheKey.c_str()); +// crcTreeCache.erase(cacheKey); +// } - cacheKey += path + "_" + filterFileExt + "_"; - } - if(crcTreeCache.find(cacheKey) != crcTreeCache.end()) { - SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d] CLEARING CACHED checksum for cacheKey [%s]\n",__FILE__,__FUNCTION__,__LINE__,cacheKey.c_str()); - crcTreeCache.erase(cacheKey); - } - - cacheLookupId = CacheManager::getFolderTreeContentsCheckSumRecursivelyCacheLookupKey2; - std::map &crcTreeCache2 = CacheManager::getCachedItem< std::map >(cacheLookupId); - - count = paths.size(); - for(size_t idx = 0; idx < count; ++idx) { - string path = paths[idx] + pathSearchString; - - string cacheKey = path + "_" + filterFileExt; - if(crcTreeCache2.find(cacheKey) != crcTreeCache2.end()) { - SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d] CLEARING CACHED checksum for cacheKey [%s]\n",__FILE__,__FUNCTION__,__LINE__,cacheKey.c_str()); - crcTreeCache2.erase(cacheKey); - } - } +// string cacheLookupId = CacheManager::getFolderTreeContentsCheckSumRecursivelyCacheLookupKey2; +// std::map &crcTreeCache2 = CacheManager::getCachedItem< std::map >(cacheLookupId); +// +// for(size_t idx = 0; idx < paths.size(); ++idx) { +// string path = paths[idx] + pathSearchString; +// +// string cacheKey = path + "_" + filterFileExt; +// if(crcTreeCache2.find(cacheKey) != crcTreeCache2.end()) { +// SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d] CLEARING CACHED checksum for cacheKey [%s]\n",__FILE__,__FUNCTION__,__LINE__,cacheKey.c_str()); +// crcTreeCache2.erase(cacheKey); +// } +// } safeMutexFTPProgress.ReleaseLock(); // END diff --git a/source/shared_lib/include/platform/common/platform_common.h b/source/shared_lib/include/platform/common/platform_common.h index 14349e24..076b7d0d 100644 --- a/source/shared_lib/include/platform/common/platform_common.h +++ b/source/shared_lib/include/platform/common/platform_common.h @@ -114,9 +114,23 @@ void findAll(const vector &paths, const string &fileFilter, vector &results, bool cutExtension=false, bool errorOnNotFound=true); vector getFolderTreeContentsListRecursively(const string &path, const string &filterFileExt, bool includeFolders=false, vector *recursiveMap=NULL); +string getCRCCacheFilePath(); +void setCRCCacheFilePath(string path); + +std::pair getFolderTreeContentsCheckSumCacheKey(vector paths, string pathSearchString, const string filterFileExt); +void clearFolderTreeContentsCheckSum(vector paths, string pathSearchString, const string filterFileExt); int32 getFolderTreeContentsCheckSumRecursively(vector paths, string pathSearchString, const string filterFileExt, Checksum *recursiveChecksum); + +std::pair getFolderTreeContentsCheckSumCacheKey(const string &path, const string filterFileExt); +void clearFolderTreeContentsCheckSum(const string &path, const string filterFileExt); int32 getFolderTreeContentsCheckSumRecursively(const string &path, const string &filterFileExt, Checksum *recursiveChecksum); + +std::pair getFolderTreeContentsCheckSumListCacheKey(vector paths, string pathSearchString, const string filterFileExt); +void clearFolderTreeContentsCheckSumList(vector paths, string pathSearchString, const string filterFileExt); vector > getFolderTreeContentsCheckSumListRecursively(vector paths, string pathSearchString, const string filterFileExt, vector > *recursiveMap); + +std::pair getFolderTreeContentsCheckSumListCacheKey(const string &path, const string filterFileExt); +void clearFolderTreeContentsCheckSumList(const string &path, const string filterFileExt); vector > getFolderTreeContentsCheckSumListRecursively(const string &path, const string &filterFileExt, vector > *recursiveMap); void createDirectoryPaths(string Path); diff --git a/source/shared_lib/sources/platform/common/platform_common.cpp b/source/shared_lib/sources/platform/common/platform_common.cpp index 7d664537..aa2d12fd 100644 --- a/source/shared_lib/sources/platform/common/platform_common.cpp +++ b/source/shared_lib/sources/platform/common/platform_common.cpp @@ -54,7 +54,7 @@ #include "socket.h" #include #include - +#include "randomgen.h" #include "leak_dumper.h" using namespace Shared::Platform; @@ -65,6 +65,10 @@ using namespace std; namespace Shared { namespace PlatformCommon { +// Refresh every 3 days +const time_t REFRESH_CRC_DAY_SECONDS = 1 * 60 * 24; +static string crcCachePath = ""; + namespace Private { bool shouldBeFullscreen = false; @@ -401,26 +405,120 @@ bool EndsWith(const string &str, const string& key) return result; } +string getCRCCacheFilePath() { + return crcCachePath; +} + +void setCRCCacheFilePath(string path) { + crcCachePath = path; +} + +string getFormattedCRCCacheFileName(std::pair cacheKeys) { + string crcCacheFile = cacheKeys.first + cacheKeys.second; + replaceAll(crcCacheFile, "/", "_"); + replaceAll(crcCacheFile, "\\", "_"); + replaceAll(crcCacheFile, "*", "_"); + replaceAll(crcCacheFile, ".", "_"); + return getCRCCacheFilePath() + crcCacheFile; +} +std::pair getFolderTreeContentsCheckSumCacheKey(vector paths, string pathSearchString, const string filterFileExt) { + string cacheLookupId = CacheManager::getFolderTreeContentsCheckSumRecursivelyCacheLookupKey1; + + string cacheKey = ""; + for(unsigned int idx = 0; idx < paths.size(); ++idx) { + string path = paths[idx] + pathSearchString; + cacheKey += path + "_" + filterFileExt + "_"; + } + return make_pair(cacheLookupId,cacheKey); +} + +bool hasCachedFileCRCValue(string crcCacheFile, int32 &value) { + bool result = false; + if(fileExists(crcCacheFile) == true) { + FILE *fp = fopen(crcCacheFile.c_str(),"r"); + if(fp != NULL) { + time_t refreshDate = 0; + int32 crcValue = 0; + int res = fscanf(fp,"%ld,%d",&refreshDate,&crcValue); + fclose(fp); + + if(SystemFlags::VERBOSE_MODE_ENABLED) printf("Found CRC Cache file [%s] now = %ld, refreshDate = %ld, crcValue = %d\n",crcCacheFile.c_str(),time(NULL), refreshDate,crcValue); + + if( refreshDate > 0 && + time(NULL) < refreshDate) { + + result = true; + value = crcValue; + } + } + } + + return result; +} + +void writeCachedFileCRCValue(string crcCacheFile, int32 &crcValue) { + FILE *fp = fopen(crcCacheFile.c_str(),"w"); + if(fp != NULL) { + RandomGen random; + int offset = random.randRange(1, 5); + time_t refreshDate = time(NULL) + (REFRESH_CRC_DAY_SECONDS * offset); + fprintf(fp,"%ld,%d",refreshDate,crcValue); + fclose(fp); + + if(SystemFlags::VERBOSE_MODE_ENABLED) printf("Writing CRC Cache file [%s] refreshDate = %ld, crcValue = %d\n",crcCacheFile.c_str(),refreshDate,crcValue); + } +} + +void clearFolderTreeContentsCheckSum(vector paths, string pathSearchString, const string filterFileExt) { + std::pair cacheKeys = getFolderTreeContentsCheckSumCacheKey(paths, pathSearchString, filterFileExt); + string cacheLookupId = cacheKeys.first; + std::map &crcTreeCache = CacheManager::getCachedItem< std::map >(cacheLookupId); + + string cacheKey = cacheKeys.second; + if(crcTreeCache.find(cacheKey) != crcTreeCache.end()) { + crcTreeCache.erase(cacheKey); + } + for(unsigned int idx = 0; idx < paths.size(); ++idx) { + string path = paths[idx]; + clearFolderTreeContentsCheckSum(path, filterFileExt); + } + + string crcCacheFile = getFormattedCRCCacheFileName(cacheKeys); + if(fileExists(crcCacheFile) == true) { +#ifdef WIN32 + int result = _unlink(crcCacheFile.c_str()); +#else + int result = unlink(crcCacheFile.c_str()); +#endif + SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d] fileitem [%s] result = %d\n",__FILE__,__FUNCTION__,__LINE__,crcCacheFile.c_str(),result); + } +} + //finds all filenames like path and gets their checksum of all files combined int32 getFolderTreeContentsCheckSumRecursively(vector paths, string pathSearchString, const string filterFileExt, Checksum *recursiveChecksum) { //SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d]\n",__FILE__,__FUNCTION__,__LINE__); - string cacheLookupId = CacheManager::getFolderTreeContentsCheckSumRecursivelyCacheLookupKey1; + + std::pair cacheKeys = getFolderTreeContentsCheckSumCacheKey(paths, pathSearchString, filterFileExt); + string cacheLookupId = cacheKeys.first; std::map &crcTreeCache = CacheManager::getCachedItem< std::map >(cacheLookupId); - string cacheKey = ""; - size_t count = paths.size(); - for(size_t idx = 0; idx < count; ++idx) { - string path = paths[idx] + pathSearchString; - - cacheKey += path + "_" + filterFileExt + "_"; - } + string cacheKey = cacheKeys.second; if(crcTreeCache.find(cacheKey) != crcTreeCache.end()) { SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s] scanning folders found CACHED checksum = %d for cacheKey [%s]\n",__FILE__,__FUNCTION__,crcTreeCache[cacheKey],cacheKey.c_str()); return crcTreeCache[cacheKey]; } + string crcCacheFile = getFormattedCRCCacheFileName(cacheKeys); + if(SystemFlags::VERBOSE_MODE_ENABLED) printf("Looking for CRC Cache file [%s]\n",crcCacheFile.c_str()); + int32 crcValue = 0; + if(hasCachedFileCRCValue(crcCacheFile, crcValue)) { + crcTreeCache[cacheKey] = crcValue; + SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s] scanning folders found CACHED FILE checksum = %d for cacheKey [%s]\n",__FILE__,__FUNCTION__,crcTreeCache[cacheKey],cacheKey.c_str()); + return crcTreeCache[cacheKey]; + } + Checksum checksum = (recursiveChecksum == NULL ? Checksum() : *recursiveChecksum); - for(size_t idx = 0; idx < count; ++idx) { + for(unsigned int idx = 0; idx < paths.size(); ++idx) { string path = paths[idx] + pathSearchString; //SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d] path = [%s], filterFileExt = [%s]\n",__FILE__,__FUNCTION__,__LINE__,path.c_str(),filterFileExt.c_str()); @@ -438,20 +536,62 @@ int32 getFolderTreeContentsCheckSumRecursively(vector paths, string path SystemFlags::OutputDebug(SystemFlags::debugNetwork,"In [%s::%s Line: %d] Final CRC file count: %d\n",__FILE__,__FUNCTION__,__LINE__,checksum.getFileCount()); crcTreeCache[cacheKey] = checksum.getFinalFileListSum(); + + writeCachedFileCRCValue(crcCacheFile, crcTreeCache[cacheKey]); + return crcTreeCache[cacheKey]; } +std::pair getFolderTreeContentsCheckSumCacheKey(const string &path, const string filterFileExt) { + string cacheLookupId = CacheManager::getFolderTreeContentsCheckSumRecursivelyCacheLookupKey2; + + string cacheKey = path + "_" + filterFileExt; + return make_pair(cacheLookupId,cacheKey); +} + +void clearFolderTreeContentsCheckSum(const string &path, const string filterFileExt) { + std::pair cacheKeys = getFolderTreeContentsCheckSumCacheKey(path,filterFileExt); + string cacheLookupId = cacheKeys.first; + std::map &crcTreeCache = CacheManager::getCachedItem< std::map >(cacheLookupId); + + string cacheKey = cacheKeys.second; + if(crcTreeCache.find(cacheKey) != crcTreeCache.end()) { + crcTreeCache.erase(cacheKey); + } + string crcCacheFile = getFormattedCRCCacheFileName(cacheKeys); + if(fileExists(crcCacheFile) == true) { +#ifdef WIN32 + int result = _unlink(crcCacheFile.c_str()); +#else + int result = unlink(crcCacheFile.c_str()); +#endif + SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d] fileitem [%s] result = %d\n",__FILE__,__FUNCTION__,__LINE__,crcCacheFile.c_str(),result); + } +} + //finds all filenames like path and gets their checksum of all files combined int32 getFolderTreeContentsCheckSumRecursively(const string &path, const string &filterFileExt, Checksum *recursiveChecksum) { //SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d] path = [%s] filterFileExt = [%s]\n",__FILE__,__FUNCTION__,__LINE__,path.c_str(),filterFileExt.c_str()); - string cacheLookupId = CacheManager::getFolderTreeContentsCheckSumRecursivelyCacheLookupKey2; + std::pair cacheKeys = getFolderTreeContentsCheckSumCacheKey(path, filterFileExt); + + string cacheLookupId = cacheKeys.first; std::map &crcTreeCache = CacheManager::getCachedItem< std::map >(cacheLookupId); - string cacheKey = path + "_" + filterFileExt; + string cacheKey = cacheKeys.second; if(crcTreeCache.find(cacheKey) != crcTreeCache.end()) { SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s] scanning [%s] found CACHED checksum = %d for cacheKey [%s]\n",__FILE__,__FUNCTION__,path.c_str(),crcTreeCache[cacheKey],cacheKey.c_str()); return crcTreeCache[cacheKey]; } + + string crcCacheFile = getFormattedCRCCacheFileName(cacheKeys); + if(SystemFlags::VERBOSE_MODE_ENABLED) printf("Looking for CRC Cache file [%s]\n",crcCacheFile.c_str()); + int32 crcValue = 0; + if(hasCachedFileCRCValue(crcCacheFile, crcValue)) { + crcTreeCache[cacheKey] = crcValue; + SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s] scanning folders found CACHED FILE checksum = %d for cacheKey [%s]\n",__FILE__,__FUNCTION__,crcTreeCache[cacheKey],cacheKey.c_str()); + return crcTreeCache[cacheKey]; + } + bool topLevelCaller = (recursiveChecksum == NULL); Checksum checksum = (recursiveChecksum == NULL ? Checksum() : *recursiveChecksum); @@ -546,6 +686,8 @@ int32 getFolderTreeContentsCheckSumRecursively(const string &path, const string crcTreeCache[cacheKey] = checksum.getFinalFileListSum(); SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s] scanning [%s] ending checksum = %d for cacheKey [%s] fileMatchCount = %d, fileLoopCount = %d\n",__FILE__,__FUNCTION__,path.c_str(),crcTreeCache[cacheKey],cacheKey.c_str(),fileMatchCount,fileLoopCount); + writeCachedFileCRCValue(crcCacheFile, crcTreeCache[cacheKey]); + return crcTreeCache[cacheKey]; } else { @@ -553,18 +695,49 @@ int32 getFolderTreeContentsCheckSumRecursively(const string &path, const string } } -vector > getFolderTreeContentsCheckSumListRecursively(vector paths, string pathSearchString, string filterFileExt, vector > *recursiveMap) { - //SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d]\n",__FILE__,__FUNCTION__,__LINE__); + +std::pair getFolderTreeContentsCheckSumListCacheKey(vector paths, string pathSearchString, const string filterFileExt) { string cacheLookupId = CacheManager::getFolderTreeContentsCheckSumListRecursivelyCacheLookupKey1; - std::map > > &crcTreeCache = CacheManager::getCachedItem< std::map > > >(cacheLookupId); string cacheKey = ""; - size_t count = paths.size(); - for(size_t idx = 0; idx < count; ++idx) { + for(unsigned int idx = 0; idx < paths.size(); ++idx) { string path = paths[idx] + pathSearchString; - cacheKey += path + "_" + filterFileExt + "_"; } + return make_pair(cacheLookupId,cacheKey); +} + +void clearFolderTreeContentsCheckSumList(vector paths, string pathSearchString, const string filterFileExt) { + std::pair cacheKeys = getFolderTreeContentsCheckSumListCacheKey(paths, pathSearchString, filterFileExt); + string cacheLookupId = cacheKeys.first; + std::map > > &crcTreeCache = CacheManager::getCachedItem< std::map > > >(cacheLookupId); + + string cacheKey = cacheKeys.second; + if(crcTreeCache.find(cacheKey) != crcTreeCache.end()) { + crcTreeCache.erase(cacheKey); + } + for(unsigned int idx = 0; idx < paths.size(); ++idx) { + string path = paths[idx]; + clearFolderTreeContentsCheckSumList(path, filterFileExt); + } + string crcCacheFile = getFormattedCRCCacheFileName(cacheKeys); + if(fileExists(crcCacheFile) == true) { +#ifdef WIN32 + int result = _unlink(crcCacheFile.c_str()); +#else + int result = unlink(crcCacheFile.c_str()); +#endif + SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d] fileitem [%s] result = %d\n",__FILE__,__FUNCTION__,__LINE__,crcCacheFile.c_str(),result); + } +} + +vector > getFolderTreeContentsCheckSumListRecursively(vector paths, string pathSearchString, string filterFileExt, vector > *recursiveMap) { + //SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d]\n",__FILE__,__FUNCTION__,__LINE__); + std::pair cacheKeys = getFolderTreeContentsCheckSumListCacheKey(paths, pathSearchString, filterFileExt); + string cacheLookupId = cacheKeys.first; + std::map > > &crcTreeCache = CacheManager::getCachedItem< std::map > > >(cacheLookupId); + + string cacheKey = cacheKeys.second; if(crcTreeCache.find(cacheKey) != crcTreeCache.end()) { SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s] scanning folders found CACHED result for cacheKey [%s]\n",__FILE__,__FUNCTION__,cacheKey.c_str()); return crcTreeCache[cacheKey]; @@ -576,7 +749,7 @@ vector > getFolderTreeContentsCheckSumListRecursively(ve bool topLevelCaller = (recursiveMap == NULL); vector > checksumFiles = (recursiveMap == NULL ? vector >() : *recursiveMap); - for(size_t idx = 0; idx < count; ++idx) { + for(unsigned int idx = 0; idx < paths.size(); ++idx) { string path = paths[idx] + pathSearchString; checksumFiles = getFolderTreeContentsCheckSumListRecursively(path, filterFileExt, &checksumFiles); } @@ -685,14 +858,41 @@ vector getFolderTreeContentsListRecursively(const string &path, const st return resultFiles; } +std::pair getFolderTreeContentsCheckSumListCacheKey(const string &path, const string filterFileExt) { + string cacheLookupId = CacheManager::getFolderTreeContentsCheckSumListRecursivelyCacheLookupKey2; + + string cacheKey = path + "_" + filterFileExt; + return make_pair(cacheLookupId,cacheKey); +} + +void clearFolderTreeContentsCheckSumList(const string &path, const string filterFileExt) { + std::pair cacheKeys = getFolderTreeContentsCheckSumListCacheKey(path,filterFileExt); + string cacheLookupId = cacheKeys.first; + std::map > > &crcTreeCache = CacheManager::getCachedItem< std::map > > >(cacheLookupId); + + string cacheKey = cacheKeys.second; + if(crcTreeCache.find(cacheKey) != crcTreeCache.end()) { + crcTreeCache.erase(cacheKey); + } + string crcCacheFile = getFormattedCRCCacheFileName(cacheKeys); + if(fileExists(crcCacheFile) == true) { +#ifdef WIN32 + int result = _unlink(crcCacheFile.c_str()); +#else + int result = unlink(crcCacheFile.c_str()); +#endif + SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d] fileitem [%s] result = %d\n",__FILE__,__FUNCTION__,__LINE__,crcCacheFile.c_str(),result); + } +} //finds all filenames like path and gets the checksum of each file vector > getFolderTreeContentsCheckSumListRecursively(const string &path, const string &filterFileExt, vector > *recursiveMap) { //SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d] path = [%s] filterFileExt = [%s]\n",__FILE__,__FUNCTION__,__LINE__,path.c_str(),filterFileExt.c_str()); - string cacheLookupId = CacheManager::getFolderTreeContentsCheckSumListRecursivelyCacheLookupKey2; + std::pair cacheKeys = getFolderTreeContentsCheckSumListCacheKey(path, filterFileExt); + string cacheLookupId = cacheKeys.first; std::map > > &crcTreeCache = CacheManager::getCachedItem< std::map > > >(cacheLookupId); - string cacheKey = path + "_" + filterFileExt; + string cacheKey = cacheKeys.second; if(crcTreeCache.find(cacheKey) != crcTreeCache.end()) { SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s] scanning [%s] FOUND CACHED result for cacheKey [%s]\n",__FILE__,__FUNCTION__,path.c_str(),cacheKey.c_str()); return crcTreeCache[cacheKey]; @@ -1110,14 +1310,14 @@ string replaceAll(string& context, const string& from, const string& to) { size_t lookHere = 0; size_t foundHere = 0; if((foundHere = context.find(from, lookHere)) != string::npos) { - if(SystemFlags::VERBOSE_MODE_ENABLED) printf("Replacing context [%s] from [%s] to [%s]\n",context.c_str(),from.c_str(),to.c_str()); + //if(SystemFlags::VERBOSE_MODE_ENABLED) printf("Replacing context [%s] from [%s] to [%s]\n",context.c_str(),from.c_str(),to.c_str()); while((foundHere = context.find(from, lookHere)) != string::npos) { context.replace(foundHere, from.size(), to); lookHere = foundHere + to.size(); } - if(SystemFlags::VERBOSE_MODE_ENABLED) printf("New context [%s]\n",context.c_str()); + //if(SystemFlags::VERBOSE_MODE_ENABLED) printf("New context [%s]\n",context.c_str()); } return context; } diff --git a/source/shared_lib/sources/platform/common/simple_threads.cpp b/source/shared_lib/sources/platform/common/simple_threads.cpp index 62c0ad93..ba3c9bc9 100644 --- a/source/shared_lib/sources/platform/common/simple_threads.cpp +++ b/source/shared_lib/sources/platform/common/simple_threads.cpp @@ -59,7 +59,7 @@ void FileCRCPreCacheThread::execute() { SystemFlags::OutputDebug(SystemFlags::debugNetwork,"In [%s::%s Line: %d]\n",__FILE__,__FUNCTION__,__LINE__); break; } - sleep( 50 ); + //sleep( 50 ); } } }