diff --git a/src/Layers/xrRender/PSLibrary.cpp b/src/Layers/xrRender/PSLibrary.cpp index 0241631148..056b090bb9 100644 --- a/src/Layers/xrRender/PSLibrary.cpp +++ b/src/Layers/xrRender/PSLibrary.cpp @@ -7,6 +7,7 @@ #include "PSLibrary.h" #include "ParticleEffect.h" #include "ParticleGroup.h" +#include "../../xrCore/_thread_types.h" #ifdef _EDITOR # include "ParticleEffectActions.h" @@ -191,49 +192,76 @@ bool CPSLibrary::Load2() } -bool CPSLibrary::Load(const char* nm) +bool CPSLibrary::LoadDefinitions(const char* nm) { + CTimer startupTimer; + startupTimer.Start(); FS_FileSet files; string_path _path; FS.update_path(_path, "$game_particles$", ""); FS.file_list(files, _path, FS_ListFiles, "*.pe,*.pg"); - FS_FileSet::iterator it = files.begin(); - FS_FileSet::iterator it_e = files.end(); + struct PreparedParticle + { + xr_string file; + shared_str name; + bool effect = false; + std::unique_ptr effectDefinition; + std::unique_ptr groupDefinition; + }; + + xr_vector prepared(files.size()); + u32 sourceIndex = 0; + for (const FS_File& file : files) + { + string_path path; + string_path name; + string_path extension; + _splitpath(file.name.c_str(), nullptr, path, name, extension); + + PreparedParticle& result = prepared[sourceIndex++]; + result.file = file.name.c_str(); + result.name.printf("%s%s", path, name); + result.effect = 0 == stricmp(extension, ".pe"); + R_ASSERT(result.effect || 0 == stricmp(extension, ".pg")); + } - string_path p_path, p_name, p_ext; - for (; it != it_e; ++it) + xr_parallel_for(0u, static_cast(prepared.size()), [&](u32 index) { - const FS_File& f = (*it); - _splitpath(f.name.c_str(), 0, p_path, p_name, p_ext); - FS.update_path(_path, "$game_particles$", f.name.c_str()); - CInifile ini(_path, TRUE, TRUE, FALSE); + PreparedParticle& result = prepared[index]; + string_path fullPath; + FS.update_path(fullPath, "$game_particles$", result.file.c_str()); + CInifile ini(fullPath, TRUE, TRUE, FALSE); - xr_sprintf(_path, sizeof(_path), "%s%s", p_path, p_name); - if (0 == stricmp(p_ext, ".pe")) + if (result.effect) { - PS::CPEDef* def = xr_new(); - def->m_Name = _path; - if (def->Load2(ini)) - m_PEDs.push_back(def); - else - xr_delete(def); + std::unique_ptr definition = std::make_unique(); + definition->m_Name = result.name; + if (definition->Load2(ini)) + result.effectDefinition = std::move(definition); } - else if (0 == stricmp(p_ext, ".pg")) + else { - PS::CPGDef* def = xr_new(); - def->m_Name = _path; - if (def->Load2(ini)) - m_PGDs.push_back(def); - else - xr_delete(def); + std::unique_ptr definition = std::make_unique(); + definition->m_Name = result.name; + if (definition->Load2(ini)) + result.groupDefinition = std::move(definition); } - else + }); + + for (PreparedParticle& result : prepared) + { + if (result.effectDefinition) { - R_ASSERT(0); + m_PEDs.push_back(result.effectDefinition.release()); + } + else if (result.groupDefinition) + { + m_PGDs.push_back(result.groupDefinition.release()); } } + m_prepare_loose_ms = startupTimer.GetElapsed_ms(); bool bRes = true; if (FS.exist(nm)) @@ -316,14 +344,32 @@ bool CPSLibrary::Load(const char* nm) FS.r_close(F); } + m_prepare_library_ms = startupTimer.GetElapsed_ms() - m_prepare_loose_ms; std::sort(m_PEDs.begin(), m_PEDs.end(), ped_sort_pred); std::sort(m_PGDs.begin(), m_PGDs.end(), pgd_sort_pred); + m_prepare_sort_ms = startupTimer.GetElapsed_ms() - m_prepare_loose_ms - m_prepare_library_ms; + + return bRes; +} +void CPSLibrary::FinalizeLoad() +{ + CTimer shaderTimer; + shaderTimer.Start(); for (PS::PEDIt e_it = m_PEDs.begin(); e_it != m_PEDs.end(); ++e_it) (*e_it)->CreateShader(); + Msg("* [STARTUP/RENDER PARTICLES] loose=%u library=%u sort=%u shaders=%u effects=%u groups=%u total=%u ms", + m_prepare_loose_ms, m_prepare_library_ms, m_prepare_sort_ms, shaderTimer.GetElapsed_ms(), + static_cast(m_PEDs.size()), static_cast(m_PGDs.size()), + m_prepare_loose_ms + m_prepare_library_ms + m_prepare_sort_ms + shaderTimer.GetElapsed_ms()); +} - return bRes; +bool CPSLibrary::Load(const char* nm) +{ + const bool result = LoadDefinitions(nm); + FinalizeLoad(); + return result; } //---------------------------------------------------- diff --git a/src/Layers/xrRender/PSLibrary.h b/src/Layers/xrRender/PSLibrary.h index 3773a06148..b0925711aa 100644 --- a/src/Layers/xrRender/PSLibrary.h +++ b/src/Layers/xrRender/PSLibrary.h @@ -19,6 +19,12 @@ class ECORE_API CPSLibrary : public particles_systems::library_interface { PS::PEDVec m_PEDs; PS::PGDVec m_PGDs; + u32 m_prepare_loose_ms = 0; + u32 m_prepare_library_ms = 0; + u32 m_prepare_sort_ms = 0; + + bool LoadDefinitions(LPCSTR nm); + void FinalizeLoad(); #ifdef _EDITOR AnsiString m_CurrentParticles; diff --git a/src/Layers/xrRender/TextureDescrManager.cpp b/src/Layers/xrRender/TextureDescrManager.cpp index c2c575f7c5..541cf75199 100644 --- a/src/Layers/xrRender/TextureDescrManager.cpp +++ b/src/Layers/xrRender/TextureDescrManager.cpp @@ -35,24 +35,6 @@ void fix_texture_thm_name(LPSTR fn) *_ext = 0; } -struct TH_LoadTHM -{ - using map_TD = xr_map; - using map_CS = xr_map; - - LPCSTR initial; - map_TD& s_texture_details; - map_CS& s_detail_scalers; -}; - -void CTextureDescrMngr::LoadTHMThread(void* args) -{ - PROF_EVENT(); - - TH_LoadTHM* p = (TH_LoadTHM*)args; - LoadTHM(p->initial, p->s_texture_details, p->s_detail_scalers); -} - void CTextureDescrMngr::LoadTHM(LPCSTR initial, map_TD& s_texture_details, map_CS& s_detail_scalers) { PROF_EVENT(); @@ -60,79 +42,215 @@ void CTextureDescrMngr::LoadTHM(LPCSTR initial, map_TD& s_texture_details, map_C FS_FileSet flist; FS.file_list(flist, initial, FS_ListFiles, "*.thm"); - STextureParams tp; - string_path fn; + struct PreparedThm + { + xr_string file; + xr_string name; + xr_string detail_name; + xr_string bump_name; + u32 flags = STextureParams::flGenerateMipMaps | STextureParams::flDitherColor; + u32 material = STextureParams::tmBlin_Phong; + float detail_scale = 1.f; + float material_weight = 0.f; + STextureParams::ETType type = STextureParams::ttImage; + STextureParams::ETBumpMode bump_mode = STextureParams::tbmNone; + }; + CTimer timer; + timer.Start(); + xr_vector prepared(flist.size()); + u32 sourceIndex = 0; for (const FS_File& fs_iter : flist) { - FS.update_path(fn, initial, fs_iter.name.c_str()); - IReader* F = FS.r_open(fn); - xr_strcpy(fn, fs_iter.name.c_str()); - fix_texture_thm_name(fn); + PreparedThm& result = prepared[sourceIndex++]; + result.file = fs_iter.name.c_str(); + string_path name; + xr_strcpy(name, fs_iter.name.c_str()); + fix_texture_thm_name(name); + result.name = name; + } + + xr_parallel_for(0u, static_cast(prepared.size()), [&](u32 index) + { + PreparedThm& result = prepared[index]; + string_path path; + FS.update_path(path, initial, result.file.c_str()); + IReader* F = FS.r_open(path); R_ASSERT(F->find_chunk(THM_CHUNK_TYPE)); F->r_u32(); - tp.Clear(); - tp.Load(*F); + R_ASSERT(F->find_chunk(THM_CHUNK_TEXTUREPARAM)); + F->r_u32(); + result.flags = F->r_u32(); + if (F->find_chunk(THM_CHUNK_TEXTURE_TYPE)) + result.type = static_cast(F->r_u32()); + if (F->find_chunk(THM_CHUNK_DETAIL_EXT)) + { + F->r_stringZ(result.detail_name); + result.detail_scale = F->r_float(); + } + if (F->find_chunk(THM_CHUNK_MATERIAL)) + { + result.material = F->r_u32(); + result.material_weight = F->r_float(); + } + if (F->find_chunk(THM_CHUNK_BUMP)) + { + F->r_float(); + result.bump_mode = static_cast(F->r_u32()); + if (result.bump_mode < STextureParams::tbmNone) + result.bump_mode = STextureParams::tbmNone; + F->r_stringZ(result.bump_name); + } FS.r_close(F); - if (STextureParams::ttImage == tp.type || STextureParams::ttTerrain == tp.type || STextureParams::ttNormalMap == - tp.type) + }); + const u32 prepareMs = timer.GetElapsed_ms(); + + for (const PreparedThm& result : prepared) + { + if (STextureParams::ttImage == result.type || STextureParams::ttTerrain == result.type || + STextureParams::ttNormalMap == result.type) { - texture_desc& desc = s_texture_details[fn]; - cl_dt_scaler*& dts = s_detail_scalers[fn]; + texture_desc& desc = s_texture_details[result.name.c_str()]; + cl_dt_scaler*& dts = s_detail_scalers[result.name.c_str()]; - if (tp.detail_name.size() && tp.flags.is_any(STextureParams::flDiffuseDetail | STextureParams::flBumpDetail) - ) + if (!result.detail_name.empty() && + (result.flags & (STextureParams::flDiffuseDetail | STextureParams::flBumpDetail))) { if (desc.m_assoc) xr_delete(desc.m_assoc); desc.m_assoc = xr_new(); - desc.m_assoc->detail_name = tp.detail_name; + desc.m_assoc->detail_name = result.detail_name.c_str(); if (dts) - dts->scale = tp.detail_scale; + dts->scale = result.detail_scale; else - /*desc.m_assoc->cs*/dts = xr_new(tp.detail_scale); + /*desc.m_assoc->cs*/dts = xr_new(result.detail_scale); desc.m_assoc->usage = 0; - if (tp.flags.is(STextureParams::flDiffuseDetail)) + if (result.flags & STextureParams::flDiffuseDetail) desc.m_assoc->usage |= (1 << 0); - if (tp.flags.is(STextureParams::flBumpDetail)) + if (result.flags & STextureParams::flBumpDetail) desc.m_assoc->usage |= (1 << 1); } if (desc.m_spec) xr_delete(desc.m_spec); desc.m_spec = xr_new(); - desc.m_spec->m_material = tp.material + (tp.material < 4 ? tp.material_weight : 0); + desc.m_spec->m_material = + result.material + (result.material < 4 ? result.material_weight : 0); desc.m_spec->m_use_steep_parallax = false; - if (tp.bump_mode == STextureParams::tbmUse) + if (result.bump_mode == STextureParams::tbmUse) { - desc.m_spec->m_bump_name = tp.bump_name; + desc.m_spec->m_bump_name = result.bump_name.c_str(); } - else if (tp.bump_mode == STextureParams::tbmUseParallax) + else if (result.bump_mode == STextureParams::tbmUseParallax) { - desc.m_spec->m_bump_name = tp.bump_name; + desc.m_spec->m_bump_name = result.bump_name.c_str(); desc.m_spec->m_use_steep_parallax = true; } } } + Msg("* [STARTUP/THM] source=%s files=%u prepare=%u ms commit=%u ms total=%u ms", + initial, static_cast(prepared.size()), prepareMs, timer.GetElapsed_ms() - prepareMs, timer.GetElapsed_ms()); } void CTextureDescrMngr::Load() { - TH_LoadTHM* gtex = new TH_LoadTHM({"$game_textures$", m_texture_details, m_detail_scalers}); - TH_LoadTHM* lvl = new TH_LoadTHM({"$level$", m_texture_details, m_detail_scalers}); - thread_spawn(LoadTHMThread, "X-Ray THM Loader 1", 0, gtex); - thread_spawn(LoadTHMThread, "X-Ray THM Loader 2", 0, lvl); - Sleep(5); + map_TD gameDetails; + map_TD levelDetails; + map_CS gameScalers; + map_CS levelScalers; + std::exception_ptr gameFailure; + std::exception_ptr levelFailure; + + xr_task_group scans; + scans.run([&]() + { + try + { + LoadTHM("$game_textures$", gameDetails, gameScalers); + } + catch (...) + { + gameFailure = std::current_exception(); + } + }); + scans.run([&]() + { + try + { + LoadTHM("$level$", levelDetails, levelScalers); + } + catch (...) + { + levelFailure = std::current_exception(); + } + }); + scans.wait(); + + auto clearTemporary = [](map_TD& details, map_CS& scalers) + { + for (auto& item : details) + { + xr_delete(item.second.m_assoc); + xr_delete(item.second.m_spec); + } + for (auto& item : scalers) + xr_delete(item.second); + details.clear(); + scalers.clear(); + }; + + if (gameFailure || levelFailure) + { + clearTemporary(gameDetails, gameScalers); + clearTemporary(levelDetails, levelScalers); + std::rethrow_exception(gameFailure ? gameFailure : levelFailure); + } + + xrSRWLockGuard dataGuard(m_data_lock); + auto merge = [&](map_TD& details, map_CS& scalers) + { + for (auto& item : details) + { + texture_desc& destination = m_texture_details[item.first]; + xr_delete(destination.m_assoc); + xr_delete(destination.m_spec); + destination.m_assoc = item.second.m_assoc; + destination.m_spec = item.second.m_spec; + item.second.m_assoc = nullptr; + item.second.m_spec = nullptr; + + auto sourceScaler = scalers.find(item.first); + if (sourceScaler == scalers.end()) + continue; + cl_dt_scaler*& destinationScaler = m_detail_scalers[item.first]; + if (destinationScaler) + { + destinationScaler->scale = sourceScaler->second->scale; + xr_delete(sourceScaler->second); + } + else + { + destinationScaler = sourceScaler->second; + } + sourceScaler->second = nullptr; + } + clearTemporary(details, scalers); + }; + + // Level THMs override game THMs deterministically, as intended by the old two-source load. + merge(gameDetails, gameScalers); + merge(levelDetails, levelScalers); } void CTextureDescrMngr::UnLoad() { + xrSRWLockGuard dataGuard(m_data_lock); for (auto& it : m_texture_details) { xr_delete(it.second.m_assoc); @@ -143,6 +261,7 @@ void CTextureDescrMngr::UnLoad() CTextureDescrMngr::~CTextureDescrMngr() { + xrSRWLockGuard dataGuard(m_data_lock); map_CS::iterator I = m_detail_scalers.begin(); map_CS::iterator E = m_detail_scalers.end(); @@ -154,6 +273,7 @@ CTextureDescrMngr::~CTextureDescrMngr() shared_str CTextureDescrMngr::GetBumpName(const shared_str& tex_name) const { + xrSRWLockGuard dataGuard(m_data_lock, true); map_TD::const_iterator I = m_texture_details.find(tex_name); if (I != m_texture_details.end()) { @@ -167,6 +287,7 @@ shared_str CTextureDescrMngr::GetBumpName(const shared_str& tex_name) const BOOL CTextureDescrMngr::UseSteepParallax(const shared_str& tex_name) const { + xrSRWLockGuard dataGuard(m_data_lock, true); map_TD::const_iterator I = m_texture_details.find(tex_name); if (I != m_texture_details.end()) { @@ -177,9 +298,9 @@ BOOL CTextureDescrMngr::UseSteepParallax(const shared_str& tex_name) const } return FALSE; } - float CTextureDescrMngr::GetMaterial(const shared_str& tex_name) const { + xrSRWLockGuard dataGuard(m_data_lock, true); map_TD::const_iterator I = m_texture_details.find(tex_name); if (I != m_texture_details.end()) { @@ -193,6 +314,7 @@ float CTextureDescrMngr::GetMaterial(const shared_str& tex_name) const void CTextureDescrMngr::GetTextureUsage(const shared_str& tex_name, BOOL& bDiffuse, BOOL& bBump) const { + xrSRWLockGuard dataGuard(m_data_lock, true); map_TD::const_iterator I = m_texture_details.find(tex_name); if (I != m_texture_details.end()) { @@ -207,6 +329,7 @@ void CTextureDescrMngr::GetTextureUsage(const shared_str& tex_name, BOOL& bDiffu BOOL CTextureDescrMngr::GetDetailTexture(const shared_str& tex_name, LPCSTR& res, R_constant_setup* & CS) const { + xrSRWLockGuard dataGuard(m_data_lock, true); map_TD::const_iterator I = m_texture_details.find(tex_name); if (I != m_texture_details.end()) { diff --git a/src/Layers/xrRender/TextureDescrManager.h b/src/Layers/xrRender/TextureDescrManager.h index 65e4307879..7c911439cc 100644 --- a/src/Layers/xrRender/TextureDescrManager.h +++ b/src/Layers/xrRender/TextureDescrManager.h @@ -5,7 +5,6 @@ #include "ETextureParams.h" class cl_dt_scaler; -struct TH_LoadTHM; class CTextureDescrMngr { @@ -49,9 +48,9 @@ class CTextureDescrMngr private: map_TD m_texture_details; map_CS m_detail_scalers; + mutable xrSRWLock m_data_lock; static void LoadTHM(LPCSTR initial, map_TD& s_texture_details, map_CS& s_detail_scalers); - static void LoadTHMThread(void* args); public: ~CTextureDescrMngr(); diff --git a/src/xrCore/LocatorAPI.cpp b/src/xrCore/LocatorAPI.cpp index 656fa6d268..45714c2548 100644 --- a/src/xrCore/LocatorAPI.cpp +++ b/src/xrCore/LocatorAPI.cpp @@ -29,6 +29,207 @@ const u32 BIG_FILE_READER_WINDOW_SIZE = 1024 * 1024; CLocatorAPI* xr_FS = NULL; +struct CLocatorAPI::ArchiveDataView +{ + void* base; + u32 size; + + ArchiveDataView(void* mapped_base, u32 mapped_size, LPCSTR name) : base(mapped_base), size(mapped_size) + { +#ifdef FS_DEBUG + register_file_mapping(base, size, name); +#endif + } + + ~ArchiveDataView() + { +#ifdef FS_DEBUG + unregister_file_mapping(base, size); +#endif + UnmapViewOfFile(base); + } +}; + +namespace +{ +class CArchiveReader final : public IReader +{ + std::shared_ptr owner; + +public: + CArchiveReader(std::shared_ptr view, void* data, int size) + : IReader(data, size), owner(std::move(view)) + { + } +}; + +class CStartupLooseReader final : public IReader +{ + std::shared_ptr> owner; + +public: + explicit CStartupLooseReader(std::shared_ptr> data) + : IReader(const_cast(data->data()), static_cast(data->size())), owner(std::move(data)) + { + } +}; + +struct InitialFileData +{ + u32 vfs; + u32 crc; + u32 ptr; + u32 size_real; + u32 size_compressed; + u32 modif; +}; + +struct InitialNameHash +{ + size_t operator()(const xr_string& value) const + { + size_t result = sizeof(size_t) == 8 ? size_t(14695981039346656037ull) : size_t(2166136261u); + const size_t prime = sizeof(size_t) == 8 ? size_t(1099511628211ull) : size_t(16777619u); + for (const unsigned char character : value) + { + result ^= character; + result *= prime; + } + return result; + } +}; +} + +struct CLocatorAPI::StartupLooseCache +{ + enum class State : u8 + { + Queued, + Loading, + Ready, + Failed + }; + + struct Entry + { + xr_string name; + u32 size = 0; + u32 modif = 0; + State state = State::Queued; + bool valid = true; + std::shared_ptr> data; + }; + + using EntryPtr = std::shared_ptr; + + std::mutex mutex; + std::condition_variable changed; + xr_map entries; + xr_vector queue; + size_t next = 0; + bool stopping = false; + xr_task_group workers; + std::atomic workers_remaining{0}; + std::atomic prepared{0}; + std::atomic failed{0}; + std::atomic promoted{0}; + std::atomic demand_wait_ms{0}; + + static std::shared_ptr> Read(const Entry& entry) + { + HANDLE file = CreateFileA(entry.name.c_str(), GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_DELETE, nullptr, OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL | FILE_FLAG_SEQUENTIAL_SCAN, nullptr); + if (file == INVALID_HANDLE_VALUE) + return {}; + BY_HANDLE_FILE_INFORMATION before; + if (!GetFileInformationByHandle(file, &before) || before.nFileSizeHigh || before.nFileSizeLow != entry.size) + { + CloseHandle(file); + return {}; + } + + auto buffer = std::make_shared>(); + buffer->resize(entry.size); + u32 total_read = 0; + while (total_read < entry.size) + { + DWORD bytes_read = 0; + if (!ReadFile(file, buffer->data() + total_read, entry.size - total_read, &bytes_read, nullptr) || !bytes_read) + break; + total_read += bytes_read; + } + BY_HANDLE_FILE_INFORMATION after; + const bool unchanged = GetFileInformationByHandle(file, &after) && !after.nFileSizeHigh && + after.nFileSizeLow == entry.size && + CompareFileTime(&before.ftLastWriteTime, &after.ftLastWriteTime) == 0; + CloseHandle(file); + + if (total_read != entry.size || !unchanged) + return {}; + + return buffer; + } + + void Load(const EntryPtr& entry) + { + std::shared_ptr> data; + try + { + data = Read(*entry); + } + catch (...) + { + } + + { + std::lock_guard guard(mutex); + if (entry->valid && data) + { + entry->data = std::move(data); + entry->state = State::Ready; + ++prepared; + } + else + { + entry->state = State::Failed; + ++failed; + } + } + changed.notify_all(); + } + + void Worker() + { + for (;;) + { + EntryPtr entry; + { + std::lock_guard guard(mutex); + while (!stopping && next < queue.size()) + { + entry = queue[next++]; + if (entry->valid && entry->state == State::Queued) + { + entry->state = State::Loading; + break; + } + entry.reset(); + } + if (stopping || !entry) + break; + } + Load(entry); + } + + if (workers_remaining.fetch_sub(1) == 1) + { + Msg("* [STARTUP/VFS] loose-cache prepared=%u failed=%u promoted=%u demand-wait=%llu ms", + prepared.load(), failed.load(), promoted.load(), demand_wait_ms.load()); + } + } +}; + #ifdef _EDITOR # define FSLTX "fs.ltx" #else @@ -203,10 +404,14 @@ XRCORE_API void _dump_open_files(int mode) CLocatorAPI::CLocatorAPI() #ifdef PROFILE_CRITICAL_SECTIONS - :m_auth_lock(MUTEX_PROFILE_ID(CLocatorAPI::m_auth_lock)) + : m_scan_lock(MUTEX_PROFILE_ID(CLocatorAPI::m_scan_lock)), + m_auth_lock(MUTEX_PROFILE_ID(CLocatorAPI::m_auth_lock)) #endif // PROFILE_CRITICAL_SECTIONS { m_Flags.zero(); + m_initial_build = false; + m_initial_archive_index_ms = 0; + m_startup_loose_cache = nullptr; // get page size SYSTEM_INFO sys_inf; GetSystemInfo(&sys_inf); @@ -223,11 +428,28 @@ CLocatorAPI::~CLocatorAPI() void CLocatorAPI::Register(LPCSTR name, u32 vfs, u32 crc, u32 ptr, u32 size_real, u32 size_compressed, u32 modif) { - //Msg("Register[%d] [%s]",vfs,name); string256 temp_file_name; xr_strcpy(temp_file_name, sizeof(temp_file_name), name); xr_strlwr(temp_file_name); + if (m_initial_build) + { + InitialFileRecord record; + record.name = temp_file_name; + record.vfs = vfs; + record.crc = crc; + record.ptr = ptr; + record.size_real = size_real; + record.size_compressed = size_compressed; + record.modif = modif & (~u32(0x3)); + m_initial_files.push_back(std::move(record)); + return; + } + + xrCriticalSectionGuard guard(m_scan_lock); + //Msg("Register[%d] [%s]",vfs,name); + InvalidateStartupLooseCache(temp_file_name); + // Register file file desc; // desc.name = xr_strlwr(xr_strdup(name)); @@ -287,6 +509,252 @@ void CLocatorAPI::Register(LPCSTR name, u32 vfs, u32 crc, u32 ptr, u32 size_real } } +void CLocatorAPI::CommitInitialFiles(u64 scan_started_at) +{ + using InitialFileMap = xr_unordered_map; + using InitialFileMapEntry = InitialFileMap::value_type; + + m_initial_build = false; + InitialFileMap merged; + merged.reserve(m_initial_files.size()); + u32 replacements = 0; + + const u64 replay_started_at = GetTickCount64(); + for (const InitialFileRecord& record : m_initial_files) + { + InitialFileData data = { + record.vfs, record.crc, record.ptr, record.size_real, record.size_compressed, record.modif + }; + auto existing = merged.find(record.name); + if (existing != merged.end()) + { + existing->second = data; + ++replacements; + continue; + } + + merged.emplace(record.name, data); + + string_path temp; + xr_strcpy(temp, sizeof(temp), record.name.c_str()); + string_path path; + string_path folder; + u32 vfs_id = record.vfs; + while (temp[0] && temp[1]) + { + _splitpath(temp, path, folder, nullptr, nullptr); + xr_strcat(path, folder); + if (merged.find(path) == merged.end()) + { + InitialFileData folder_data = data; + folder_data.vfs = vfs_id; + folder_data.ptr = 0; + folder_data.size_real = 0; + folder_data.size_compressed = 0; + folder_data.modif = u32(-1); + merged.emplace(path, folder_data); + } + xr_strcpy(temp, sizeof(temp), path); + if (xr_strlen(temp)) + temp[xr_strlen(temp) - 1] = 0; + vfs_id = u32(-1); + } + } + const u64 replay_ms = GetTickCount64() - replay_started_at; + + const u64 sort_started_at = GetTickCount64(); + xr_vector sorted; + sorted.reserve(merged.size()); + for (const InitialFileMapEntry& entry : merged) + sorted.push_back(&entry); + std::sort(sorted.begin(), sorted.end(), [](const InitialFileMapEntry* left, const InitialFileMapEntry* right) + { + return xr_strcmp(left->first.c_str(), right->first.c_str()) < 0; + }); + const u64 sort_ms = GetTickCount64() - sort_started_at; + + const u64 commit_started_at = GetTickCount64(); + R_ASSERT(m_files.empty()); + files_set committed; + auto hint = committed.end(); + for (const InitialFileMapEntry* entry : sorted) + { + const InitialFileData& source = entry->second; + file desc = { + xr_strdup(entry->first.c_str()), source.vfs, source.crc, source.ptr, source.size_real, + source.size_compressed, source.modif + }; + hint = committed.emplace_hint(hint, desc); + } + m_files.swap(committed); + const u64 commit_ms = GetTickCount64() - commit_started_at; + + u32 catalog_hash = 0; + for (const file& entry : m_files) + { + catalog_hash = crc32(entry.name, xr_strlen(entry.name) + 1, catalog_hash); + catalog_hash = crc32(&entry.vfs, sizeof(entry.vfs), catalog_hash); + catalog_hash = crc32(&entry.crc, sizeof(entry.crc), catalog_hash); + catalog_hash = crc32(&entry.ptr, sizeof(entry.ptr), catalog_hash); + catalog_hash = crc32(&entry.size_real, sizeof(entry.size_real), catalog_hash); + catalog_hash = crc32(&entry.size_compressed, sizeof(entry.size_compressed), catalog_hash); + catalog_hash = crc32(&entry.modif, sizeof(entry.modif), catalog_hash); + } + + const u64 scan_total_ms = replay_started_at - scan_started_at; + const u64 discovery_ms = scan_total_ms > m_initial_archive_index_ms + ? scan_total_ms - m_initial_archive_index_ms + : 0; + Msg("* [STARTUP/VFS] discovery=%llu ms index=%llu ms replay=%llu ms sort=%llu ms commit=%llu ms files=%u replacements=%u catalog=%08x", + discovery_ms, m_initial_archive_index_ms, replay_ms, sort_ms, commit_ms, m_files.size(), replacements, catalog_hash); + + m_initial_files.clear_and_free(); +} + +void CLocatorAPI::StartStartupLooseCache() +{ + R_ASSERT(!m_startup_loose_cache); + auto* cache = xr_new(); + u64 total_bytes = 0; + + for (LPCSTR alias : {"$game_config$", "$game_scripts$", "$game_textures$"}) + { + const PathPairIt path = pathes.find(alias); + if (path == pathes.end()) + continue; + + const bool texture_thm = xr_strcmp(alias, "$game_textures$") == 0; + xr_string prefix = path->second->m_Path; + std::transform(prefix.begin(), prefix.end(), prefix.begin(), + [](unsigned char character) { return static_cast(tolower(character)); }); + + for (const file& desc : m_files) + { + if (desc.vfs != u32(-1) || !desc.size_real || strncmp(desc.name, prefix.c_str(), prefix.size()) != 0) + continue; + if (texture_thm) + { + LPCSTR extension = strext(desc.name); + if (!extension || xr_strcmp(extension, ".thm") != 0) + continue; + } + + auto entry = std::make_shared(); + entry->name = desc.name; + entry->size = desc.size_real; + entry->modif = desc.modif; + if (cache->entries.emplace(entry->name, entry).second) + { + cache->queue.push_back(std::move(entry)); + total_bytes += desc.size_real; + } + } + } + + if (cache->queue.empty()) + { + xr_delete(cache); + return; + } + + m_startup_loose_cache = cache; + const u32 worker_count = std::min( + std::max(2u, std::thread::hardware_concurrency() / 2), + std::min(8, static_cast(cache->queue.size()))); + cache->workers_remaining = worker_count; + Msg("* [STARTUP/VFS] loose-cache queued=%u bytes=%llu workers=%u", + static_cast(cache->queue.size()), total_bytes, worker_count); + for (u32 worker = 0; worker < worker_count; ++worker) + cache->workers.run([cache] { cache->Worker(); }); +} + +void CLocatorAPI::StopStartupLooseCache() +{ + StartupLooseCache* cache = m_startup_loose_cache; + if (!cache) + return; + + { + std::lock_guard guard(cache->mutex); + cache->stopping = true; + } + cache->changed.notify_all(); + cache->workers.wait(); + m_startup_loose_cache = nullptr; + xr_delete(cache); +} + +void CLocatorAPI::InvalidateStartupLooseCache(LPCSTR name) +{ + StartupLooseCache* cache = m_startup_loose_cache; + if (!cache) + return; + string_path normalized; + xr_strcpy(normalized, sizeof(normalized), name); + xr_strlwr(normalized); + + { + std::lock_guard guard(cache->mutex); + const auto entry = cache->entries.find(normalized); + if (entry == cache->entries.end()) + return; + entry->second->valid = false; + entry->second->state = StartupLooseCache::State::Failed; + entry->second->data.reset(); + cache->entries.erase(entry); + } + cache->changed.notify_all(); +} + +bool CLocatorAPI::OpenStartupLooseCache(IReader*& reader, LPCSTR name, const file& desc) +{ + StartupLooseCache* cache = m_startup_loose_cache; + if (!cache) + return false; + + StartupLooseCache::EntryPtr entry; + bool load = false; + { + std::unique_lock guard(cache->mutex); + const auto found = cache->entries.find(name); + if (cache->stopping || found == cache->entries.end()) + return false; + entry = found->second; + if (!entry->valid || entry->size != desc.size_real || entry->modif != desc.modif) + return false; + + if (entry->state == StartupLooseCache::State::Queued) + { + entry->state = StartupLooseCache::State::Loading; + ++cache->promoted; + load = true; + } + else if (entry->state == StartupLooseCache::State::Loading) + { + const u64 started_at = GetTickCount64(); + cache->changed.wait(guard, [&entry] + { + return !entry->valid || entry->state != StartupLooseCache::State::Loading; + }); + cache->demand_wait_ms += GetTickCount64() - started_at; + } + } + + if (load) + cache->Load(entry); + + std::shared_ptr> data; + { + std::lock_guard guard(cache->mutex); + if (!entry->valid || entry->state != StartupLooseCache::State::Ready) + return false; + data = entry->data; + } + + reader = xr_new(std::move(data)); + return true; +} + IReader* open_chunk(void* ptr, u32 ID) { BOOL res; @@ -338,6 +806,8 @@ IReader* open_chunk(void* ptr, u32 ID) void CLocatorAPI::LoadArchive(archive& A, LPCSTR entrypoint) { + const u64 initial_index_started_at = m_initial_build ? GetTickCount64() : 0; + // Create base path string_path fs_entry_point; fs_entry_point[0] = 0; @@ -432,6 +902,8 @@ void CLocatorAPI::LoadArchive(archive& A, LPCSTR entrypoint) Register(full, A.vfs_idx, crc, ptr, size_real, size_compr, 0); } hdr->close(); + if (m_initial_build) + m_initial_archive_index_ms += GetTickCount64() - initial_index_started_at; // if(g_temporary_stuff_subst) // g_temporary_stuff = g_temporary_stuff_subst; @@ -453,14 +925,48 @@ void CLocatorAPI::archive::open() void CLocatorAPI::archive::close() { - CloseHandle(hSrcMap); - hSrcMap = NULL; - CloseHandle(hSrcFile); - hSrcFile = NULL; + std::atomic_store(&data_view, std::shared_ptr()); + if (hSrcMap) + { + CloseHandle(hSrcMap); + hSrcMap = NULL; + } + if (hSrcFile) + { + CloseHandle(hSrcFile); + hSrcFile = NULL; + } +} + +std::shared_ptr CLocatorAPI::GetArchiveDataView(archive& A) +{ + if (std::shared_ptr view = std::atomic_load(&A.data_view)) + return view; + + xrCriticalSectionGuard guard(m_scan_lock); + if (std::shared_ptr view = std::atomic_load(&A.data_view)) + return view; + if (A.data_view_failed) + return {}; + + A.open(); + void* base = MapViewOfFile(A.hSrcMap, FILE_MAP_READ, 0, 0, 0); + if (!base) + { + A.data_view_failed = true; + Msg("! [STARTUP/VFS] full archive mapping failed, using per-file views: %s (%s)", + A.path.c_str(), Debug.error2string(GetLastError())); + return {}; + } + + std::shared_ptr view = std::make_shared(base, A.size, A.path.c_str()); + std::atomic_store(&A.data_view, view); + return view; } void CLocatorAPI::ProcessArchive(LPCSTR _path) { + xrCriticalSectionGuard guard(m_scan_lock); // find existing archive shared_str path = _path; @@ -499,6 +1005,7 @@ void CLocatorAPI::ProcessArchive(LPCSTR _path) void CLocatorAPI::unload_archive(CLocatorAPI::archive& A) { + xrCriticalSectionGuard guard(m_scan_lock); files_it I = m_files.begin(); for (; I != m_files.end(); ++I) { @@ -535,7 +1042,7 @@ bool CLocatorAPI::load_all_unloaded_archives() } -void CLocatorAPI::ProcessOne(LPCSTR path, const _finddata_t& entry) +void CLocatorAPI::ProcessOne(LPCSTR path, const _finddata_t& entry, u32 parallel_depth) { string_path N; xr_strcpy(N, sizeof(N), path); @@ -551,7 +1058,7 @@ void CLocatorAPI::ProcessOne(LPCSTR path, const _finddata_t& entry) if (0 == xr_strcmp(entry.name, "..")) return; xr_strcat(N, "\\"); Register(N, 0xffffffff, 0, 0, entry.size, entry.size, (u32)entry.time_write); - Recurse(N); + Recurse(N, parallel_depth ? parallel_depth - 1 : 0); } else { @@ -596,7 +1103,7 @@ bool ignore_path(const char* _path) return true; } -bool CLocatorAPI::Recurse(const char* path) +bool CLocatorAPI::Recurse(const char* path, u32 parallel_depth) { string_path scanPath; xr_strcpy(scanPath, sizeof(scanPath), path); @@ -610,8 +1117,8 @@ bool CLocatorAPI::Recurse(const char* path) intptr_t handle = _findfirst(scanPath, &findData); if (handle == -1) return false; - rec_files.reserve(256); - size_t oldSize = rec_files.size(); + FFVec files; + files.reserve(256); intptr_t done = handle; while (done != -1) { @@ -628,17 +1135,21 @@ bool CLocatorAPI::Recurse(const char* path) ignore = ignore_name(findData.name); } if (!ignore) - rec_files.push_back(findData); + files.push_back(findData); done = _findnext(handle, &findData); } _findclose(handle); - size_t newSize = rec_files.size(); - if (newSize > oldSize) + if (!files.empty()) { - std::sort(rec_files.begin() + oldSize, rec_files.end(), pred_str_ff); - for (size_t i = oldSize; i < newSize; i++) - ProcessOne(path, rec_files[i]); - rec_files.erase(rec_files.begin() + oldSize, rec_files.end()); + std::sort(files.begin(), files.end(), pred_str_ff); + if (parallel_depth && !m_initial_build) + xr_parallel_foreach(files.begin(), files.end(), [this, path, parallel_depth](const _finddata_t& entry) + { + ProcessOne(path, entry, parallel_depth); + }); + else + for (const _finddata_t& entry : files) + ProcessOne(path, entry); } // insert self if (path && path[0] != 0) @@ -755,7 +1266,10 @@ void CLocatorAPI::_initialize(u32 flags, LPCSTR target_folder, LPCSTR fs_name) size_t M1 = Memory.mem_usage(); m_Flags.set(flags, TRUE); - + m_initial_build = true; + m_initial_archive_index_ms = 0; + m_initial_files.clear_not_free(); + const u64 initial_scan_started_at = GetTickCount64(); // scan root directory bNoRecurse = TRUE; string4096 buf; @@ -780,6 +1294,8 @@ void CLocatorAPI::_initialize(u32 flags, LPCSTR target_folder, LPCSTR fs_name) const char *lp_add, *lp_def, *lp_capt; string16 b_v; string4096 temp; + xr_vector recursive_roots; + u32 skipped_scans = 0; while (!pFSltx->eof()) { @@ -829,7 +1345,23 @@ void CLocatorAPI::_initialize(u32 flags, LPCSTR target_folder, LPCSTR fs_name) FS_Path* P = new FS_Path((p_it != pathes.end()) ? p_it->second->m_Path : root, lp_add, lp_def, lp_capt, fl); bNoRecurse = !(fl & FS_Path::flRecurse); - Recurse(P->m_Path); + bool already_scanned = false; + for (const xr_string& scanned_root : recursive_roots) + { + if (!_strnicmp(P->m_Path, scanned_root.c_str(), scanned_root.size())) + { + already_scanned = true; + break; + } + } + if (already_scanned) + ++skipped_scans; + else + { + Recurse(P->m_Path, bNoRecurse ? 0 : 1); + if (fl & FS_Path::flRecurse) + recursive_roots.emplace_back(P->m_Path); + } auto I = pathes.insert(std::make_pair(xr_strdup(id), P)); #ifndef DEBUG m_Flags.set(flCacheFiles, FALSE); @@ -837,11 +1369,12 @@ void CLocatorAPI::_initialize(u32 flags, LPCSTR target_folder, LPCSTR fs_name) //CHECK_OR_EXIT (I.second,"The file 'fsgame.ltx' is corrupted (it contains duplicated lines).\nPlease reinstall the game or fix the problem manually."); } + Msg("FS: skipped %u duplicate alias scans", skipped_scans); r_close(pFSltx); R_ASSERT(path_exist("$app_data_root$")); }; - + CommitInitialFiles(initial_scan_started_at); Msg("File System Ready..."); size_t M2 = Memory.mem_usage(); Msg("FS: %d files cached %d archives, %lldKb memory used.", m_files.size(), m_archives.size(), (M2 - M1) / 1024); @@ -870,10 +1403,12 @@ void CLocatorAPI::_initialize(u32 flags, LPCSTR target_folder, LPCSTR fs_name) //----------------------------------------------------------- CreateLog(0 != strstr(Core.Params, "-nolog")); + StartStartupLooseCache(); } void CLocatorAPI::_destroy() { + StopStartupLooseCache(); CloseLog(); for (files_it I = m_files.begin(); I != m_files.end(); I++) @@ -899,6 +1434,7 @@ void CLocatorAPI::_destroy() const CLocatorAPI::file* CLocatorAPI::exist(const char* fn) { + xrCriticalSectionGuard guard(m_scan_lock); files_it it = file_find_it(fn); return (it != m_files.end()) ? &(*it) : 0; } @@ -1137,6 +1673,9 @@ void CLocatorAPI::check_cached_files(LPSTR fname, const u32& fname_size, const f void CLocatorAPI::file_from_cache_impl(IReader*& R, LPSTR fname, const file& desc) { + if (OpenStartupLooseCache(R, fname, desc)) + return; + if (desc.size_real < 16 * 1024) { R = xr_new(fname); @@ -1168,6 +1707,22 @@ void CLocatorAPI::file_from_archive(IReader*& R, LPCSTR fname, const file& desc) { // Archived one archive& A = m_archives[desc.vfs]; + if (const std::shared_ptr view = GetArchiveDataView(A)) + { + R_ASSERT3(u64(desc.ptr) + desc.size_compressed <= view->size, "archive entry is outside mapping", fname); + u8* source = static_cast(view->base) + desc.ptr; + if (desc.size_real == desc.size_compressed) + { + R = xr_new(view, source, desc.size_real); + return; + } + + u8* dest = xr_alloc(desc.size_real); + rtc_decompress(dest, desc.size_real, source, desc.size_compressed); + R = xr_new(dest, desc.size_real, 0); + return; + } + u32 start = (desc.ptr / dwAllocGranularity) * dwAllocGranularity; u32 end = (desc.ptr + desc.size_compressed) / dwAllocGranularity; if ((desc.ptr + desc.size_compressed) % dwAllocGranularity) end += 1; @@ -1317,6 +1872,7 @@ void CLocatorAPI::copy_file_to_build(T*& r, LPCSTR source_name) bool CLocatorAPI::check_for_file(LPCSTR path, LPCSTR _fname, string_path& fname, const file*& desc) { + xrCriticalSectionGuard guard(m_scan_lock); // проверить нужно ли пересканировать пути check_pathes(); @@ -1485,6 +2041,7 @@ BOOL CLocatorAPI::dir_delete(LPCSTR path, LPCSTR nm, BOOL remove_files) { // const char* entry_begin = entry.name+base_len; if (!remove_files) return FALSE; + InvalidateStartupLooseCache(entry.name); unlink(entry.name); m_files.erase(cur_item); } @@ -1518,6 +2075,7 @@ void CLocatorAPI::file_delete(LPCSTR path, LPCSTR nm) if (I != m_files.end()) { // remove file + InvalidateStartupLooseCache(I->name); unlink(I->name); char* str = LPSTR(I->name); xr_free(str); @@ -1552,6 +2110,7 @@ void CLocatorAPI::file_rename(LPCSTR src, LPCSTR dest, bool bOwerwrite) if (D != m_files.end()) { if (!bOwerwrite) return; + InvalidateStartupLooseCache(D->name); unlink(D->name); char* str = LPSTR(D->name); xr_free(str); @@ -1559,6 +2118,7 @@ void CLocatorAPI::file_rename(LPCSTR src, LPCSTR dest, bool bOwerwrite) } file new_desc = *S; + InvalidateStartupLooseCache(S->name); // remove existing item char* str = LPSTR(S->name); xr_free(str); diff --git a/src/xrCore/LocatorAPI.h b/src/xrCore/LocatorAPI.h index d1e8362806..1816e9416f 100644 --- a/src/xrCore/LocatorAPI.h +++ b/src/xrCore/LocatorAPI.h @@ -12,6 +12,7 @@ #pragma warning(pop) #include "LocatorAPI_defs.h" +#include class XRCORE_API CStreamReader; @@ -19,6 +20,8 @@ class XRCORE_API CLocatorAPI { friend class FS_Path; public: + struct ArchiveDataView; + struct file { LPCSTR name; // low-case name @@ -37,8 +40,10 @@ class XRCORE_API CLocatorAPI u32 size; CInifile* header; u32 vfs_idx; + std::shared_ptr data_view; + bool data_view_failed; - archive() : hSrcFile(NULL), hSrcMap(NULL), header(NULL), size(0), vfs_idx(u32(-1)) + archive() : hSrcFile(NULL), hSrcMap(NULL), header(NULL), size(0), vfs_idx(u32(-1)), data_view_failed(false) { } @@ -51,6 +56,19 @@ class XRCORE_API CLocatorAPI void LoadArchive(archive& A, LPCSTR entrypoint = NULL); private: + struct InitialFileRecord + { + xr_string name; + u32 vfs; + u32 crc; + u32 ptr; + u32 size_real; + u32 size_compressed; + u32 modif; + }; + + struct StartupLooseCache; + struct file_pred { IC bool operator()(const file& x, const file& y) const @@ -73,13 +91,24 @@ class XRCORE_API CLocatorAPI files_set m_files; BOOL bNoRecurse; + xrCriticalSection m_scan_lock; xrCriticalSection m_auth_lock; u64 m_auth_code; + bool m_initial_build; + u64 m_initial_archive_index_ms; + xr_vector m_initial_files; + StartupLooseCache* m_startup_loose_cache; void Register(LPCSTR name, u32 vfs, u32 crc, u32 ptr, u32 size_real, u32 size_compressed, u32 modif); + void CommitInitialFiles(u64 scan_started_at); + std::shared_ptr GetArchiveDataView(archive& archive); + void StartStartupLooseCache(); + void StopStartupLooseCache(); + void InvalidateStartupLooseCache(LPCSTR name); + bool OpenStartupLooseCache(IReader*& reader, LPCSTR name, const file& desc); void ProcessArchive(LPCSTR path); - void ProcessOne(LPCSTR path, const _finddata_t& entry); - bool Recurse(LPCSTR path); + void ProcessOne(LPCSTR path, const _finddata_t& entry, u32 parallel_depth = 0); + bool Recurse(LPCSTR path, u32 parallel_depth = 0); files_it file_find_it(LPCSTR n); public: diff --git a/src/xrEngine/Environment_misc.cpp b/src/xrEngine/Environment_misc.cpp index 5d5b47735b..6014273348 100644 --- a/src/xrEngine/Environment_misc.cpp +++ b/src/xrEngine/Environment_misc.cpp @@ -856,6 +856,12 @@ void CEnvironment::load_weather_effects() void CEnvironment::load() { + CTimer startupTimer; + startupTimer.Start(); + u32 soundPrepareMs = 0; + u32 weatherMs = 0; + u32 effectsMs = 0; + if (!CurrentEnv) create_mixer(); @@ -865,9 +871,45 @@ void CEnvironment::load() if (!eff_LensFlare) eff_LensFlare = xr_new(); if (!eff_Thunderbolt) eff_Thunderbolt = xr_new(); + if (Sound) + { + xr_vector sounds; + auto collect_list = [&sounds](CInifile* config, LPCSTR key) + { + string_path sound; + for (const auto& section : config->sections()) + { + if (!config->line_exist(section.Name, key)) + continue; + LPCSTR values = config->r_string(section.Name, key); + const u32 count = _GetItemCount(values); + for (u32 index = 0; index < count; ++index) + sounds.emplace_back(_GetItem(values, index, sound)); + } + }; + auto collect_values = [&sounds](CInifile* config, LPCSTR key) + { + for (const auto& section : config->sections()) + if (config->line_exist(section.Name, key)) + sounds.emplace_back(config->r_string(section.Name, key)); + }; + collect_list(m_sound_channels_config, "sounds"); + collect_values(m_effects_config, "sound"); + collect_values(m_thunderbolts_config, "sound"); + std::sort(sounds.begin(), sounds.end()); + sounds.erase(std::unique(sounds.begin(), sounds.end()), sounds.end()); + Sound->source_prefetch_prepare(sounds); + } + soundPrepareMs = startupTimer.GetElapsed_ms(); + load_weathers(); + weatherMs = startupTimer.GetElapsed_ms() - soundPrepareMs; load_weather_effects(); + effectsMs = startupTimer.GetElapsed_ms() - soundPrepareMs - weatherMs; load_sun(); + Msg("* [STARTUP/ENV LOAD] sounds=%u weather=%u effects=%u sun=%u total=%u ms", + soundPrepareMs, weatherMs, effectsMs, + startupTimer.GetElapsed_ms() - soundPrepareMs - weatherMs - effectsMs, startupTimer.GetElapsed_ms()); } void CEnvironment::unload() diff --git a/src/xrEngine/GameMtlLib.cpp b/src/xrEngine/GameMtlLib.cpp index 613e2216c6..98b8ddb930 100644 --- a/src/xrEngine/GameMtlLib.cpp +++ b/src/xrEngine/GameMtlLib.cpp @@ -7,6 +7,22 @@ #include "../xrCore/mezz_stringbuffer.h" +namespace +{ +xr_string NormalizeMaterialName(LPCSTR name) +{ + xr_string result = name ? name : ""; + std::transform(result.begin(), result.end(), result.begin(), + [](unsigned char character) { return static_cast(tolower(character)); }); + return result; +} + +u64 MaterialPairKey(int first, int second) +{ + return (u64(static_cast(first)) << 32) | static_cast(second); +} +} + CGameMtlLibrary GMLib; //CSound_manager_interface* Sound = NULL; #ifdef _EDITOR @@ -69,6 +85,14 @@ void SGameMtl::Load(IReader& fs) void CGameMtlLibrary::Load() { + CTimer loadTimer; + loadTimer.Start(); + u32 baseMaterialsMs = 0; + u32 materialOverridesMs = 0; + u32 basePairsMs = 0; + u32 pairOverridesMs = 0; + u32 soundResourcesMs = 0; + string_path name; if (!FS.exist(name, _game_data_, GAMEMTL_FILENAME)) { @@ -110,41 +134,49 @@ void CGameMtlLibrary::Load() } OBJ->close(); } + baseMaterialsMs = loadTimer.GetElapsed_ms(); + + xr_unordered_map materialByName; + xr_unordered_map materialIndexById; + materialByName.reserve(materials.size()); + materialIndexById.reserve(materials.size()); + int biggestMaterialId = -1; + for (u16 index = 0; index < materials.size(); ++index) + { + SGameMtl* material = materials[index]; + materialByName.emplace(NormalizeMaterialName(material->m_Name.c_str()), material); + materialIndexById.emplace(material->ID, index); + biggestMaterialId = std::max(biggestMaterialId, material->ID); + } // demonized: loose gamemtl.xr loading string_path materialsLtxName; - const int biggestIdStart = -1; if (FS.exist(materialsLtxName, _game_data_, "materials\\materials", ".ltx")) { #ifdef DEBUG_PRINT_MATERIAL Msg("found materials.ltx file %s", materialsLtxName); #endif - int biggestId = biggestIdStart; + u32 addedMaterials = 0; + u32 changedMaterials = 0; auto materialsLtx = xr_new(materialsLtxName, TRUE); for (const auto& sec : materialsLtx->sections()) { SGameMtl* M; - auto material = std::find_if(materials.begin(), materials.end(), [&sec](const SGameMtl* m) { - return xr_strcmp(m->m_Name, sec.Name) == 0; + const auto material = std::find_if(materials.begin(), materials.end(), [&sec](const SGameMtl* item) + { + return xr_strcmp(item->m_Name, sec.Name) == 0; }); if (material == materials.end()) { M = xr_new(); - if (biggestId == biggestIdStart) { - for (const auto& m : materials) { - if (m->ID > biggestId) { - biggestId = m->ID; - } - } - } - M->ID = ++biggestId; + M->ID = ++biggestMaterialId; M->m_Name = sec.Name; + materialIndexById.emplace(M->ID, static_cast(materials.size())); materials.push_back(M); - - Msg("[materials.ltx] Adding new material %s, id %d", M->m_Name.c_str(), M->ID); + materialByName.emplace(NormalizeMaterialName(M->m_Name.c_str()), M); + ++addedMaterials; } else { M = *material; - - Msg("[materials.ltx] Changing existing material %s, id %d", M->m_Name.c_str(), M->ID); + ++changedMaterials; } if (materialsLtx->line_exist(M->m_Name, "desc")) M->m_Desc = materialsLtx->r_string(M->m_Name, "desc"); @@ -181,7 +213,9 @@ void CGameMtlLibrary::Load() if (materialsLtx->line_exist(M->m_Name, "density_factor")) M->fDensityFactor = materialsLtx->r_float(M->m_Name, "density_factor"); } xr_delete(materialsLtx); + Msg("* [materials.ltx] applied: added=%u changed=%u", addedMaterials, changedMaterials); } + materialOverridesMs = loadTimer.GetElapsed_ms() - baseMaterialsMs; #ifdef DEBUG_PRINT_MATERIAL for (const auto& mat : materials) { @@ -232,6 +266,16 @@ void CGameMtlLibrary::Load() } OBJ->close(); } + basePairsMs = loadTimer.GetElapsed_ms() - baseMaterialsMs - materialOverridesMs; + + xr_unordered_map pairByMaterials; + pairByMaterials.reserve(material_pairs.size()); + int biggestPairId = -1; + for (SGameMtlPair* pair : material_pairs) + { + pairByMaterials.emplace(MaterialPairKey(pair->GetMtl0(), pair->GetMtl1()), pair); + biggestPairId = std::max(biggestPairId, pair->ID); + } string_path materialPairsLtxName; if (FS.exist(materialPairsLtxName, _game_data_, "materials\\material_pairs", ".ltx")) @@ -239,72 +283,62 @@ void CGameMtlLibrary::Load() #ifdef DEBUG_PRINT_MATERIAL Msg("found material_pairs.ltx file %s", materialPairsLtxName); #endif - int biggestId = biggestIdStart; + u32 addedPairs = 0; + u32 changedPairs = 0; auto materialsLtx = xr_new(materialPairsLtxName, TRUE); for (const auto& sec : materialsLtx->sections()) { SGameMtlPair* M; std::string secStr = sec.Name.c_str(); - auto materials = splitStringMulti(secStr, "@", false, true); - if (materials.size() < 2) { + auto pairNames = splitStringMulti(secStr, "@", false, true); + if (pairNames.size() < 2) { Msg("![material_pairs.ltx] encountered wrongly defined pair %s, two materials are required", secStr.c_str()); continue; } - int m1 = GetMaterialID(materials[0].c_str()); - int m2 = GetMaterialID(materials[1].c_str()); + const auto firstMaterial = materialByName.find(NormalizeMaterialName(pairNames[0].c_str())); + const auto secondMaterial = materialByName.find(NormalizeMaterialName(pairNames[1].c_str())); + const int m1 = firstMaterial == materialByName.end() ? GAMEMTL_NONE_ID : firstMaterial->second->ID; + const int m2 = secondMaterial == materialByName.end() ? GAMEMTL_NONE_ID : secondMaterial->second->ID; if (m1 == GAMEMTL_NONE_ID) { - Msg("![material_pairs.ltx] encountered unknown material %s in string %s, skip", materials[0].c_str(), secStr.c_str()); + Msg("![material_pairs.ltx] encountered unknown material %s in string %s, skip", pairNames[0].c_str(), secStr.c_str()); continue; } if (m2 == GAMEMTL_NONE_ID) { - Msg("![material_pairs.ltx] encountered unknown material %s in string %s, skip", materials[1].c_str(), secStr.c_str()); + Msg("![material_pairs.ltx] encountered unknown material %s in string %s, skip", pairNames[1].c_str(), secStr.c_str()); continue; } - auto material = std::find_if(material_pairs.begin(), material_pairs.end(), [&m1, &m2](SGameMtlPair* m) { - return m1 == m->GetMtl0() && m2 == m->GetMtl1(); - }); - - if (material == material_pairs.end()) { + const u64 pairKey = MaterialPairKey(m1, m2); + const auto material = pairByMaterials.find(pairKey); + if (material == pairByMaterials.end()) { M = xr_new(this); - if (biggestId == biggestIdStart) { - for (const auto& m : material_pairs) { - if (m->ID > biggestId) { - biggestId = m->ID; - } - } - } - M->ID = ++biggestId; + M->ID = ++biggestPairId; M->ID_parent = -1; M->SetPair(m1, m2); material_pairs.push_back(M); - - Msg("[material_pairs.ltx] Adding new material pair %s | %s, id %d", GetMaterialByID(M->GetMtl0())->m_Name.c_str(), GetMaterialByID(M->GetMtl1())->m_Name.c_str(), M->ID); + pairByMaterials.emplace(pairKey, M); + ++addedPairs; } else { - M = *material; - - Msg("[material_pairs.ltx] Changing existing material pair %s | %s, id %d", GetMaterialByID(M->GetMtl0())->m_Name.c_str(), GetMaterialByID(M->GetMtl1())->m_Name.c_str(), M->ID); + M = material->second; + ++changedPairs; } if (materialsLtx->line_exist(sec.Name, "breaking_sounds")) { auto s = materialsLtx->r_string(sec.Name, "breaking_sounds"); M->BreakingSoundsStr = s ? s : ""; M->OwnProps.set(SGameMtlPair::flBreakingSounds, 1); - M->CreateSoundsImpl(M->BreakingSounds, s); } if (materialsLtx->line_exist(sec.Name, "step_sounds")) { auto s = materialsLtx->r_string(sec.Name, "step_sounds"); M->StepSoundsStr = s ? s : ""; M->OwnProps.set(SGameMtlPair::flStepSounds, 1); - M->CreateSoundsImpl(M->StepSounds, s); } if (materialsLtx->line_exist(sec.Name, "collide_sounds")) { auto s = materialsLtx->r_string(sec.Name, "collide_sounds"); M->CollideSoundsStr = s ? s : ""; M->OwnProps.set(SGameMtlPair::flCollideSounds, 1); - M->CreateSoundsImpl(M->CollideSounds, s); } if (materialsLtx->line_exist(sec.Name, "collide_particles")) { auto s = materialsLtx->r_string(sec.Name, "collide_particles"); @@ -320,7 +354,10 @@ void CGameMtlLibrary::Load() } } xr_delete(materialsLtx); + Msg("* [material_pairs.ltx] applied: added=%u changed=%u", addedPairs, changedPairs); } + pairOverridesMs = loadTimer.GetElapsed_ms() - + baseMaterialsMs - materialOverridesMs - basePairsMs; #ifdef DEBUG_PRINT_MATERIAL for (const auto& mat : material_pairs) { @@ -344,13 +381,29 @@ void CGameMtlLibrary::Load() #endif // DEBUG_PRINT_MATERIAL #ifndef _EDITOR + xr_vector materialSoundNames; + for (const SGameMtlPair* pair : material_pairs) + pair->CollectSoundNames(materialSoundNames); + std::sort(materialSoundNames.begin(), materialSoundNames.end()); + materialSoundNames.erase(std::unique(materialSoundNames.begin(), materialSoundNames.end()), materialSoundNames.end()); + if (Sound) + Sound->source_prefetch_prepare(materialSoundNames); + for (SGameMtlPair* pair : material_pairs) + pair->CreateSoundResources(); + soundResourcesMs = loadTimer.GetElapsed_ms() - + baseMaterialsMs - materialOverridesMs - basePairsMs - pairOverridesMs; + material_count = (u32)materials.size(); material_pairs_rt.resize(material_count * material_count, 0); for (GameMtlPairIt p_it = material_pairs.begin(); material_pairs.end() != p_it; ++p_it) { SGameMtlPair* S = *p_it; - int idx0 = GetMaterialIdx(S->mtl0) * material_count + GetMaterialIdx(S->mtl1); - int idx1 = GetMaterialIdx(S->mtl1) * material_count + GetMaterialIdx(S->mtl0); + const auto first = materialIndexById.find(S->mtl0); + const auto second = materialIndexById.find(S->mtl1); + VERIFY(first != materialIndexById.end()); + VERIFY(second != materialIndexById.end()); + int idx0 = first->second * material_count + second->second; + int idx1 = second->second * material_count + first->second; material_pairs_rt[idx0] = S; material_pairs_rt[idx1] = S; } @@ -365,6 +418,8 @@ void CGameMtlLibrary::Load() } */ FS.r_close(F); + Msg("* [STARTUP/MATERIALS] base=%u overrides=%u pairs=%u pair-overrides=%u sounds=%u total=%u ms", + baseMaterialsMs, materialOverridesMs, basePairsMs, pairOverridesMs, soundResourcesMs, loadTimer.GetElapsed_ms()); } #ifdef GM_NON_GAME diff --git a/src/xrEngine/GameMtlLib.h b/src/xrEngine/GameMtlLib.h index d1a29f8058..33a4c6c0b6 100644 --- a/src/xrEngine/GameMtlLib.h +++ b/src/xrEngine/GameMtlLib.h @@ -193,6 +193,8 @@ struct MTL_EXPORT_API SGameMtlPair void CreateSoundsImpl(SoundVec& sounds, LPCSTR str); void CreateParticlesImpl(PSVec& particles, LPCSTR str); void CreateMarksImpl(IWallMarkArray* marks, LPCSTR str); + void CollectSoundNames(xr_vector& sounds) const; + void CreateSoundResources(); #ifdef GM_NON_GAME diff --git a/src/xrEngine/GameMtlLib_Engine.cpp b/src/xrEngine/GameMtlLib_Engine.cpp index 4a07b01760..68048f05a1 100644 --- a/src/xrEngine/GameMtlLib_Engine.cpp +++ b/src/xrEngine/GameMtlLib_Engine.cpp @@ -24,7 +24,7 @@ void DestroyPSs(PSVec& lst) // Device.Resources->Delete(*it); } -void CreateSounds(SoundVec& lst, LPCSTR buf) +void CollectSoundNames(xr_vector& sounds, LPCSTR buf) { string128 tmp; int cnt = _GetItemCount(buf); @@ -45,21 +45,32 @@ void CreateSounds(SoundVec& lst, LPCSTR buf) string128 name; xr_strcpy(name, sizeof(name), (*it).name.c_str()); *strext(name) = 0; - - ref_sound snd; - snd.create(name, st_Effect, sg_SourceType); - lst.push_back(snd); + xr_strlwr(name); + sounds.emplace_back(name); } } else { - ref_sound snd; - snd.create(tmp, st_Effect, sg_SourceType); - lst.push_back(snd); + if (strext(tmp)) + *strext(tmp) = 0; + xr_strlwr(tmp); + sounds.emplace_back(tmp); } } } +void CreateSounds(SoundVec& lst, LPCSTR buf) +{ + xr_vector names; + CollectSoundNames(names, buf); + for (const xr_string& name : names) + { + ref_sound snd; + snd.create(name.c_str(), st_Effect, sg_SourceType); + lst.push_back(snd); + } +} + /* void CreateMarks(ShaderVec& lst, LPCSTR buf) { @@ -118,17 +129,14 @@ void SGameMtlPair::Load(IReader& fs) R_ASSERT(fs.find_chunk(GAMEMTLPAIR_CHUNK_BREAKING)); fs.r_stringZ(buf); BreakingSoundsStr = buf.c_str(); - CreateSounds(BreakingSounds, *buf); R_ASSERT(fs.find_chunk(GAMEMTLPAIR_CHUNK_STEP)); fs.r_stringZ(buf); StepSoundsStr = buf.c_str(); - CreateSounds(StepSounds, *buf); R_ASSERT(fs.find_chunk(GAMEMTLPAIR_CHUNK_COLLIDE)); fs.r_stringZ(buf); CollideSoundsStr = buf.c_str(); - CreateSounds(CollideSounds, *buf); fs.r_stringZ(buf); CollideParticlesStr = buf.c_str(); CreatePSs(CollideParticles, *buf); @@ -152,3 +160,17 @@ void SGameMtlPair::CreateMarksImpl(IWallMarkArray* marks, LPCSTR str) { marks->clear(); CreateMarks(marks, str); } + +void SGameMtlPair::CollectSoundNames(xr_vector& sounds) const +{ + ::CollectSoundNames(sounds, BreakingSoundsStr.c_str()); + ::CollectSoundNames(sounds, StepSoundsStr.c_str()); + ::CollectSoundNames(sounds, CollideSoundsStr.c_str()); +} + +void SGameMtlPair::CreateSoundResources() +{ + CreateSoundsImpl(BreakingSounds, BreakingSoundsStr.c_str()); + CreateSoundsImpl(StepSounds, StepSoundsStr.c_str()); + CreateSoundsImpl(CollideSounds, CollideSoundsStr.c_str()); +} diff --git a/src/xrEngine/x_ray.cpp b/src/xrEngine/x_ray.cpp index 6247ca4de3..4684a03dd4 100644 --- a/src/xrEngine/x_ray.cpp +++ b/src/xrEngine/x_ray.cpp @@ -66,6 +66,20 @@ rpc_info discord_gameinfo; rpc_strings discord_strings; float discord_update_rate = .5f; +static ULONGLONG startup_begin_time; + +void LogStartupMenuReady() +{ + static bool logged = false; + if (!logged) + { + logged = true; + Msg("* [STARTUP] total to main menu: %llu ms", GetTickCount64() - startup_begin_time); + } + if (Sound) + Sound->source_prefetch_start(); +} + //UTF-8 (ICU) #pragma comment(lib, "icuuc.lib") //#pragma comment(lib, "sicuuc.lib") @@ -633,6 +647,8 @@ void Startup() } // Initialize APP + if (Sound) + Sound->source_prefetch_pause(); Device.Create(); LALib.OnCreate(); @@ -660,6 +676,8 @@ void Startup() Memory.mem_usage(); Device.Run(); + if (Sound) + Sound->source_prefetch_stop(); // Discord clearDiscordPresence(); @@ -1245,6 +1263,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, char* lpCmdLine, int nCmdShow) { + startup_begin_time = GetTickCount64(); // Initialize LuaJIT low-memory pool FIRST, before any DLLs load and fragment // the lower 2GB address space. XR_EARLY_INIT(); @@ -1417,6 +1436,8 @@ void CApplication::OnEvent(EVENT E, u64 P1, u64 P2) { if (E == eQuit) { + if (Sound) + Sound->source_prefetch_stop(); g_SASH.EndBenchmark(); PostQuitMessage(0); @@ -1533,6 +1554,8 @@ void CApplication::LoadBegin() ll_dwReference++; if (1 == ll_dwReference) { + if (Sound) + Sound->source_prefetch_pause(); g_appLoaded = FALSE; //AVO: @@ -1568,6 +1591,8 @@ void CApplication::destroy_loading_shaders() //AVO: g_bootComplete = TRUE; + if (Sound) + Sound->source_prefetch_start(); //-AVO //hLevelLogo.destroy (); @@ -1628,6 +1653,8 @@ void CApplication::OnFrame() PROF_EVENT(); Engine.Event.OnFrame(); + if (Sound) + Sound->source_prefetch_poll(); g_SpatialSpace->update(); g_SpatialSpacePhysic->update(); if (g_pGameLevel) diff --git a/src/xrEngine/x_ray.h b/src/xrEngine/x_ray.h index 4d9e580e40..e4517c96de 100644 --- a/src/xrEngine/x_ray.h +++ b/src/xrEngine/x_ray.h @@ -117,6 +117,7 @@ extern ENGINE_API void updateDiscordPresence(); extern ENGINE_API rpc_info discord_gameinfo; extern ENGINE_API rpc_strings discord_strings; extern ENGINE_API float discord_update_rate; +extern ENGINE_API void LogStartupMenuReady(); LPCSTR xr_ToUTF8(LPCSTR input, int max_length = 128); diff --git a/src/xrGame/MainMenu.cpp b/src/xrGame/MainMenu.cpp index 2467fe93f0..607cdebf25 100644 --- a/src/xrGame/MainMenu.cpp +++ b/src/xrGame/MainMenu.cpp @@ -228,6 +228,7 @@ void CMainMenu::Activate(bool bActivate) CCameraManager::ResetPP(); }; Device.seqRender.Add(this, 4); // 1-console 2-cursor 3-tutorial + LogStartupMenuReady(); Console->Execute("stat_memory"); } diff --git a/src/xrSound/Sound.h b/src/xrSound/Sound.h index 98d57177bc..62dae62166 100644 --- a/src/xrSound/Sound.h +++ b/src/xrSound/Sound.h @@ -2,6 +2,7 @@ #define SoundH #pragma once + #ifdef XRSOUND_EXPORTS #define XRSOUND_API //__declspec(dllexport) @@ -431,6 +432,11 @@ class XRSOUND_API CSound_manager_interface virtual void object_relcase(CObject* obj) = 0; virtual const Fvector& listener_position() = 0; + virtual void source_prefetch_start() = 0; + virtual void source_prefetch_pause() = 0; + virtual void source_prefetch_prepare(const xr_vector& sources) = 0; + virtual void source_prefetch_stop() = 0; + virtual void source_prefetch_poll() = 0; #ifdef __BORLANDC__ virtual SoundEnvironment_LIB* get_env_library () = 0; virtual void refresh_env_library () = 0; diff --git a/src/xrSound/SoundRender_Core.cpp b/src/xrSound/SoundRender_Core.cpp index ea3f448cb1..42eeb8df58 100644 --- a/src/xrSound/SoundRender_Core.cpp +++ b/src/xrSound/SoundRender_Core.cpp @@ -82,7 +82,8 @@ void CSoundRender_Core::_initialize(int stage) if (strstr(Core.Params, "-prefetch_sounds")) { - i_create_all_sources(); + build_source_prefetch_manifest(); + source_prefetch_start(); } } @@ -91,6 +92,8 @@ extern xr_vector g_target_temp_data_16; void CSoundRender_Core::_clear() { + source_prefetch_stop(); + clear_source_prefetch(); bReady = FALSE; cache.destroy(); env_unload(); diff --git a/src/xrSound/SoundRender_Core.h b/src/xrSound/SoundRender_Core.h index 97aaae02c7..c4b7fbf89c 100644 --- a/src/xrSound/SoundRender_Core.h +++ b/src/xrSound/SoundRender_Core.h @@ -3,11 +3,66 @@ #include "SoundRender.h" #include "SoundRender_Environment.h" #include "SoundRender_Cache.h" +#include "SoundRender_Source.h" + +#include +#include +#include class CNotificationClient; class CSoundRender_Core : public CSound_manager_interface { + enum class ESourcePrefetchState : u8 + { + Queued, + Preparing, + Ready, + Committed, + Failed + }; + + struct SoundPrefetchJob + { + xr_string id; + xr_string path; + xr_string error; + u32 vfs = u32(-1); + u32 offset = 0; + ESourcePrefetchState state = ESourcePrefetchState::Queued; + PreparedSoundSource prepared; + }; + + xr_vector m_source_prefetch_jobs; + xr_vector m_source_prefetch_order; + xr_unordered_map m_source_prefetch_by_id; + std::mutex m_source_prefetch_mutex; + std::condition_variable m_source_prefetch_changed; + std::thread m_source_prefetch_thread; + std::atomic m_source_prefetch_idle_ms{0}; + size_t m_source_prefetch_cursor = 0; + u32 m_source_prefetch_started_at = 0; + u32 m_source_prefetch_prepared = 0; + u32 m_source_prefetch_promoted = 0; + u32 m_source_prefetch_waited_ms = 0; + u32 m_source_prefetch_failed = 0; + u32 m_source_prefetch_remaining = 0; + bool m_source_prefetch_enabled = false; + bool m_source_prefetch_pause = true; + bool m_source_prefetch_running = false; + bool m_source_prefetch_shutdown = false; + bool m_source_prefetch_completion_pending = false; + bool m_source_prefetch_completion_logged = false; + SoundPrefetchJob* m_source_prefetch_failure = nullptr; + + void source_prefetch_worker(); + void build_source_prefetch_manifest(); + void clear_source_prefetch(); + void finish_source_prepare(SoundPrefetchJob& job, PreparedSoundSource&& prepared, xr_string&& error); + CSoundRender_Source* commit_source_locked(SoundPrefetchJob& job); + u32 source_prefetch_remaining_locked() const; + u64 source_prefetch_hash_locked() const; + volatile BOOL bLocked; protected: virtual void _create_data(ref_sound_data& S, LPCSTR fName, esound_type sound_type, int game_type); @@ -125,7 +180,11 @@ class CSoundRender_Core : public CSound_manager_interface virtual BOOL is_ready() { return bReady; } virtual void object_relcase(CObject* obj); - void i_create_all_sources(); + virtual void source_prefetch_start() override; + virtual void source_prefetch_pause() override; + virtual void source_prefetch_prepare(const xr_vector& sources) override; + virtual void source_prefetch_stop() override; + virtual void source_prefetch_poll() override; virtual float get_occlusion_to(const Fvector& hear_pt, const Fvector& snd_pt, float dispersion = 0.2f); float get_occlusion(Fvector& P, float R, Fvector* occ) override; diff --git a/src/xrSound/SoundRender_Core_SourceManager.cpp b/src/xrSound/SoundRender_Core_SourceManager.cpp index 3d78d8d2cb..634be9b837 100644 --- a/src/xrSound/SoundRender_Core_SourceManager.cpp +++ b/src/xrSound/SoundRender_Core_SourceManager.cpp @@ -3,27 +3,95 @@ #include "SoundRender_Core.h" #include "SoundRender_Source.h" -#include "../xrCore/ScopeLock.hpp" -#include +#include + +namespace +{ +constexpr u32 StartupSoundIdleMs = 250; +constexpr u32 RuntimeSoundIdleMs = 10; + +void NormalizeSourceName(LPCSTR name, string256& id) +{ + xr_strcpy(id, name); + xr_strlwr(id); + if (strext(id)) + *strext(id) = 0; +} +} CSoundRender_Source* CSoundRender_Core::i_create_source(LPCSTR name) { - // Search string256 id; - xr_strcpy(id, name); - strlwr(id); - if (strext(id)) *strext(id) = 0; - auto it = s_sources.find(id); - if (it != s_sources.end()) + NormalizeSourceName(name, id); + { - return it->second; + std::lock_guard lock(m_source_prefetch_mutex); + const auto source = s_sources.find(id); + if (source != s_sources.end()) + return source->second; } - // Load a _new one - CSoundRender_Source* S = xr_new(); - S->load(id); - s_sources.insert({id, S}); - return S; + xr_string path; + CSoundRender_Source::resolve_path(id, path); + std::unique_lock lock(m_source_prefetch_mutex); + const auto existing = s_sources.find(id); + if (existing != s_sources.end()) + return existing->second; + + const auto found = m_source_prefetch_by_id.find(id); + SoundPrefetchJob* job = found == m_source_prefetch_by_id.end() ? nullptr : found->second; + if (!job || _stricmp(job->path.c_str(), path.c_str())) + { + // Level-local and dynamically discovered sounds retain the original synchronous path. + CSoundRender_Source* source = xr_new(); + source->load(id); + s_sources.insert({id, source}); + return source; + } + + if (job->state == ESourcePrefetchState::Queued) + { + job->state = ESourcePrefetchState::Preparing; + ++m_source_prefetch_promoted; + lock.unlock(); + + PreparedSoundSource prepared; + xr_string error; + try + { + CSoundRender_Source::prepare(job->path.c_str(), prepared, error); + } + catch (...) + { + error = make_string("Unhandled exception while preparing sound: %s", job->path.c_str()).c_str(); + } + finish_source_prepare(*job, std::move(prepared), std::move(error)); + lock.lock(); + } + else if (job->state == ESourcePrefetchState::Preparing) + { + CTimer wait_timer; + wait_timer.Start(); + m_source_prefetch_changed.wait(lock, [job]() + { + return job->state != ESourcePrefetchState::Preparing; + }); + m_source_prefetch_waited_ms += wait_timer.GetElapsed_ms(); + } + + if (job->state == ESourcePrefetchState::Ready) + return commit_source_locked(*job); + if (job->state == ESourcePrefetchState::Committed) + { + const auto source = s_sources.find(id); + R_ASSERT(source != s_sources.end()); + return source->second; + } + + xr_string error = job->error; + lock.unlock(); + R_ASSERT3(false, "Can't prepare sound source", error.c_str()); + return nullptr; } void CSoundRender_Core::i_destroy_source(CSoundRender_Source* S) @@ -31,43 +99,366 @@ void CSoundRender_Core::i_destroy_source(CSoundRender_Source* S) // No actual destroy at all } -void CSoundRender_Core::i_create_all_sources() +void CSoundRender_Core::build_source_prefetch_manifest() { - PROF_EVENT(); - CTimer T; - T.Start(); + CTimer timer; + timer.Start(); - FS_FileSet flist; - FS.file_list(flist, "$game_sounds$", FS_ListFiles, "*.ogg"); - const size_t sizeBefore = s_sources.size(); - - Lock lock; - const auto processFile = [&](const FS_File& file) + FS_FileSet files; + FS.file_list(files, "$game_sounds$", FS_ListFiles, "*.ogg"); + for (const FS_File& file : files) { string256 id; - xr_strcpy(id, file.name.c_str()); + NormalizeSourceName(file.name.c_str(), id); + if (m_source_prefetch_by_id.find(id) != m_source_prefetch_by_id.end()) + continue; - xr_strlwr(id); - if (strext(id)) - *strext(id) = 0; + SoundPrefetchJob* job = xr_new(); + job->id = id; + string_path path; + FS.update_path(path, "$game_sounds$", file.name.c_str()); + job->path = path; + if (const CLocatorAPI::file* descriptor = FS.exist(path)) + { + job->vfs = descriptor->vfs; + job->offset = descriptor->ptr; + } + m_source_prefetch_jobs.push_back(job); + m_source_prefetch_order.push_back(job); + m_source_prefetch_by_id.insert({job->id, job}); + } + m_source_prefetch_remaining = u32(m_source_prefetch_jobs.size()); + std::sort(m_source_prefetch_order.begin(), m_source_prefetch_order.end(), + [](const SoundPrefetchJob* left, const SoundPrefetchJob* right) { - ScopeLock scope(&lock); - const auto it = s_sources.find(id); - if (it != s_sources.end()) - return; - UNUSED(scope); + if (left->vfs != right->vfs) + return left->vfs < right->vfs; + if (left->offset != right->offset) + return left->offset < right->offset; + return left->id < right->id; + }); + m_source_prefetch_enabled = !m_source_prefetch_jobs.empty(); + Msg("* [SOUND PREFETCH] manifest: %u sources in %u ms", + u32(m_source_prefetch_jobs.size()), timer.GetElapsed_ms()); +} + +void CSoundRender_Core::source_prefetch_worker() +{ + _initialize_cpu_thread(); + thread_name("Sound prefetch"); + if (!SetThreadPriority(GetCurrentThread(), THREAD_MODE_BACKGROUND_BEGIN)) + SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_BELOW_NORMAL); + + for (;;) + { + SoundPrefetchJob* job = nullptr; + { + std::lock_guard lock(m_source_prefetch_mutex); + if (m_source_prefetch_pause || m_source_prefetch_shutdown || m_source_prefetch_failure) + break; + while (m_source_prefetch_cursor < m_source_prefetch_order.size()) + { + SoundPrefetchJob* candidate = m_source_prefetch_order[m_source_prefetch_cursor++]; + if (candidate->state == ESourcePrefetchState::Queued) + { + candidate->state = ESourcePrefetchState::Preparing; + job = candidate; + break; + } + } + if (!job) + break; + } + + PreparedSoundSource prepared; + xr_string error; + try + { + CSoundRender_Source::prepare(job->path.c_str(), prepared, error, false); + } + catch (...) + { + error = make_string("Unhandled exception while preparing sound: %s", job->path.c_str()).c_str(); + } + const bool succeeded = error.empty(); + finish_source_prepare(*job, std::move(prepared), std::move(error)); + if (succeeded) + { + std::unique_lock lock(m_source_prefetch_mutex); + m_source_prefetch_changed.wait_for(lock, + std::chrono::milliseconds(m_source_prefetch_idle_ms.load(std::memory_order_relaxed)), [this]() + { + return m_source_prefetch_pause || m_source_prefetch_shutdown || m_source_prefetch_failure; + }); + } + } + + { + std::lock_guard lock(m_source_prefetch_mutex); + m_source_prefetch_running = false; + if (!m_source_prefetch_failure && source_prefetch_remaining_locked() == 0) + m_source_prefetch_completion_pending = true; + } + m_source_prefetch_changed.notify_all(); +} + +void CSoundRender_Core::finish_source_prepare( + SoundPrefetchJob& job, PreparedSoundSource&& prepared, xr_string&& error) +{ + { + std::lock_guard lock(m_source_prefetch_mutex); + R_ASSERT(job.state == ESourcePrefetchState::Preparing); + R_ASSERT(m_source_prefetch_remaining); + --m_source_prefetch_remaining; + if (error.empty()) + { + job.prepared = std::move(prepared); + job.state = ESourcePrefetchState::Ready; + ++m_source_prefetch_prepared; + } + else + { + job.error = std::move(error); + job.state = ESourcePrefetchState::Failed; + m_source_prefetch_failure = &job; + m_source_prefetch_pause = true; + ++m_source_prefetch_failed; } + if (!m_source_prefetch_failure && source_prefetch_remaining_locked() == 0) + m_source_prefetch_completion_pending = true; + } + m_source_prefetch_changed.notify_all(); +} + +CSoundRender_Source* CSoundRender_Core::commit_source_locked(SoundPrefetchJob& job) +{ + const auto existing = s_sources.find(job.id); + if (existing != s_sources.end()) + { + job.state = ESourcePrefetchState::Committed; + return existing->second; + } + + CSoundRender_Source* source = xr_new(); + switch (job.prepared.warning) + { + case PreparedSoundSource::Warning::InvalidRate: + Msg("! Warning: Invalid source rate: %s", job.path.c_str()); + break; + case PreparedSoundSource::Warning::InvalidComment: + Log("! Invalid ogg-comment version, file: ", job.path.c_str()); + break; + case PreparedSoundSource::Warning::MissingComment: + Log("! Missing ogg-comment, file: ", job.path.c_str()); + break; + default: + break; + } + source->load_prepared(job.id.c_str(), job.prepared); + s_sources.insert({job.id, source}); + job.state = ESourcePrefetchState::Committed; + return source; +} - CSoundRender_Source* S = new CSoundRender_Source(); - S->load(id); +u32 CSoundRender_Core::source_prefetch_remaining_locked() const +{ + return m_source_prefetch_remaining; +} - lock.Enter(); - s_sources.insert({ id, S }); - lock.Leave(); +u64 CSoundRender_Core::source_prefetch_hash_locked() const +{ + u64 hash = 14695981039346656037ULL; + const auto append = [&hash](const void* data, size_t size) + { + const u8* bytes = static_cast(data); + for (size_t i = 0; i < size; ++i) + { + hash ^= bytes[i]; + hash *= 1099511628211ULL; + } }; + for (const SoundPrefetchJob* job : m_source_prefetch_jobs) + { + append(job->id.data(), job->id.size()); + append(&job->prepared.format, sizeof(job->prepared.format)); + append(&job->prepared.time_total, sizeof(job->prepared.time_total)); + append(&job->prepared.bytes_total, sizeof(job->prepared.bytes_total)); + append(&job->prepared.base_volume, sizeof(job->prepared.base_volume)); + append(&job->prepared.min_distance, sizeof(job->prepared.min_distance)); + append(&job->prepared.max_distance, sizeof(job->prepared.max_distance)); + append(&job->prepared.max_ai_distance, sizeof(job->prepared.max_ai_distance)); + append(&job->prepared.game_type, sizeof(job->prepared.game_type)); + } + return hash; +} + +void CSoundRender_Core::source_prefetch_start() +{ + std::unique_lock lock(m_source_prefetch_mutex); + if (!m_source_prefetch_enabled || m_source_prefetch_shutdown || + m_source_prefetch_failure || source_prefetch_remaining_locked() == 0) + return; + m_source_prefetch_idle_ms.store( + m_source_prefetch_started_at ? RuntimeSoundIdleMs : StartupSoundIdleMs, std::memory_order_relaxed); + if (m_source_prefetch_running) + return; - tbb::parallel_for_each(flist, processFile); + if (m_source_prefetch_thread.joinable()) + { + std::thread completed = std::move(m_source_prefetch_thread); + m_source_prefetch_running = true; // Reserve restart while the completed worker is reaped. + lock.unlock(); + completed.join(); + lock.lock(); + m_source_prefetch_running = false; + if (m_source_prefetch_shutdown || m_source_prefetch_failure || source_prefetch_remaining_locked() == 0) + return; + } - Msg("Finished creating %d sound sources. Duration: %d ms", s_sources.size() - sizeBefore, T.GetElapsed_ms()); + m_source_prefetch_pause = false; + m_source_prefetch_running = true; + if (!m_source_prefetch_started_at) + m_source_prefetch_started_at = GetTickCount(); + try + { + m_source_prefetch_thread = std::thread(&CSoundRender_Core::source_prefetch_worker, this); + } + catch (...) + { + m_source_prefetch_pause = true; + m_source_prefetch_running = false; + throw; + } + Msg("* [SOUND PREFETCH] started: remaining=%u", source_prefetch_remaining_locked()); +} + +void CSoundRender_Core::source_prefetch_pause() +{ + { + std::lock_guard lock(m_source_prefetch_mutex); + m_source_prefetch_pause = true; + } + m_source_prefetch_changed.notify_all(); + if (m_source_prefetch_thread.joinable()) + m_source_prefetch_thread.join(); + + std::lock_guard lock(m_source_prefetch_mutex); + m_source_prefetch_running = false; + if (m_source_prefetch_enabled) + Msg("* [SOUND PREFETCH] paused: prepared=%u, promoted=%u, remaining=%u", + m_source_prefetch_prepared, m_source_prefetch_promoted, source_prefetch_remaining_locked()); +} + +void CSoundRender_Core::source_prefetch_prepare(const xr_vector& sources) +{ + if (!m_source_prefetch_enabled || sources.empty()) + return; + { + std::lock_guard lock(m_source_prefetch_mutex); + if (m_source_prefetch_shutdown || m_source_prefetch_failure) + return; + } + + CTimer timer; + timer.Start(); + xr_parallel_for(0u, static_cast(sources.size()), [&](u32 index) + { + string256 id; + NormalizeSourceName(sources[index].c_str(), id); + + SoundPrefetchJob* job = nullptr; + bool producer = false; + { + std::unique_lock lock(m_source_prefetch_mutex); + const auto found = m_source_prefetch_by_id.find(id); + if (found == m_source_prefetch_by_id.end()) + return; + + job = found->second; + if (job->state == ESourcePrefetchState::Queued) + { + job->state = ESourcePrefetchState::Preparing; + ++m_source_prefetch_promoted; + producer = true; + } + else if (job->state == ESourcePrefetchState::Preparing) + return; + else + return; + } + + if (!producer) + return; + + PreparedSoundSource prepared; + xr_string error; + try + { + CSoundRender_Source::prepare(job->path.c_str(), prepared, error, false); + } + catch (...) + { + error = make_string("Unhandled exception while preparing sound: %s", job->path.c_str()).c_str(); + } + finish_source_prepare(*job, std::move(prepared), std::move(error)); + }); + Msg("* [SOUND PREFETCH] requested sources prepared: requested=%u time=%u ms", + static_cast(sources.size()), timer.GetElapsed_ms()); +} + +void CSoundRender_Core::source_prefetch_stop() +{ + { + std::lock_guard lock(m_source_prefetch_mutex); + m_source_prefetch_shutdown = true; + m_source_prefetch_pause = true; + } + m_source_prefetch_changed.notify_all(); + if (m_source_prefetch_thread.joinable()) + m_source_prefetch_thread.join(); + std::lock_guard lock(m_source_prefetch_mutex); + m_source_prefetch_running = false; +} + +void CSoundRender_Core::source_prefetch_poll() +{ + xr_string failed_path; + xr_string failure; + { + std::lock_guard lock(m_source_prefetch_mutex); + if (m_source_prefetch_shutdown) + return; + if (m_source_prefetch_failure) + { + failed_path = m_source_prefetch_failure->path; + failure = m_source_prefetch_failure->error; + } + else if (m_source_prefetch_completion_pending && !m_source_prefetch_completion_logged) + { + m_source_prefetch_completion_pending = false; + m_source_prefetch_completion_logged = true; + Msg("* [SOUND PREFETCH] complete: prepared=%u, promoted=%u, demand_wait=%u ms, " + "duration=%u ms, metadata_hash=%016llx", + m_source_prefetch_prepared, m_source_prefetch_promoted, m_source_prefetch_waited_ms, + GetTickCount() - m_source_prefetch_started_at, source_prefetch_hash_locked()); + } + } + if (!failure.empty()) + { + Msg("! [SOUND PREFETCH] failed: %s (%s)", failed_path.c_str(), failure.c_str()); + R_ASSERT3(false, failure.c_str(), failed_path.c_str()); + } +} + +void CSoundRender_Core::clear_source_prefetch() +{ + for (SoundPrefetchJob* job : m_source_prefetch_jobs) + xr_delete(job); + m_source_prefetch_jobs.clear(); + m_source_prefetch_order.clear(); + m_source_prefetch_by_id.clear(); + m_source_prefetch_cursor = 0; + m_source_prefetch_remaining = 0; + m_source_prefetch_idle_ms.store(0, std::memory_order_relaxed); } diff --git a/src/xrSound/SoundRender_Source.h b/src/xrSound/SoundRender_Source.h index 8c5a2747b2..1d97528fe2 100644 --- a/src/xrSound/SoundRender_Source.h +++ b/src/xrSound/SoundRender_Source.h @@ -7,6 +7,29 @@ // refs struct OggVorbis_File; +struct PreparedSoundSource +{ + enum class Warning : u8 + { + None, + InvalidRate, + InvalidComment, + MissingComment + }; + + xr_string path; + WAVEFORMATEX format{}; + float time_total = 0.f; + u32 bytes_total = 0; + float base_volume = 1.f; + float min_distance = 1.f; + float max_distance = 300.f; + float max_ai_distance = 300.f; + u32 game_type = 0; + bool loaded = false; + Warning warning = Warning::None; +}; + class XRSOUND_EDITOR_API CSoundRender_Source : public CSound_source { public: @@ -32,8 +55,11 @@ class XRSOUND_EDITOR_API CSoundRender_Source : public CSound_source ~CSoundRender_Source(); void load(LPCSTR name); + void load_prepared(LPCSTR name, const PreparedSoundSource& prepared); void unload(); void decompress(u32 line, OggVorbis_File* ovf); + static bool prepare(LPCSTR path, PreparedSoundSource& prepared, xr_string& error, bool log_warnings = true); + static void resolve_path(LPCSTR name, xr_string& path); virtual float length_sec() const { return fTimeTotal; } virtual u32 game_type() const { return m_uGameType; } diff --git a/src/xrSound/SoundRender_Source_loader.cpp b/src/xrSound/SoundRender_Source_loader.cpp index 7b56b235d4..bfbb165034 100644 --- a/src/xrSound/SoundRender_Source_loader.cpp +++ b/src/xrSound/SoundRender_Source_loader.cpp @@ -6,6 +6,158 @@ #include "SoundRender_Core.h" #include "SoundRender_Source.h" +namespace +{ +constexpr size_t OggPageHeaderSize = 27; +constexpr size_t OggMaximumPageSize = OggPageHeaderSize + 255 + 255 * 255; + +bool ReadOggPage(const u8* data, size_t size, size_t offset, ogg_page& page, size_t& next) +{ + if (offset > size || size - offset < OggPageHeaderSize || memcmp(data + offset, "OggS", 4) || data[offset + 4]) + return false; + + const size_t segment_count = data[offset + 26]; + const size_t header_size = OggPageHeaderSize + segment_count; + if (header_size > size - offset) + return false; + + size_t body_size = 0; + for (size_t i = 0; i < segment_count; ++i) + body_size += data[offset + OggPageHeaderSize + i]; + if (body_size > size - offset - header_size) + return false; + + page.header = const_cast(data + offset); + page.header_len = static_cast(header_size); + page.body = const_cast(data + offset + header_size); + page.body_len = static_cast(body_size); + next = offset + header_size + body_size; + + unsigned char checked_header[OggPageHeaderSize + 255]; + CopyMemory(checked_header, page.header, header_size); + ogg_page checked_page = page; + checked_page.header = checked_header; + ogg_page_checksum_set(&checked_page); + if (memcmp(checked_header + 22, page.header + 22, 4)) + return false; + return true; +} + +bool ReadSingleStreamPcmTotal( + const u8* data, size_t size, const OggVorbis_File& vorbis_file, vorbis_info& info, s64& pcm_total) +{ + if (!data || size < OggPageHeaderSize) + return false; + + const int serial = vorbis_file.current_serialno; + ogg_stream_state stream{}; + if (ogg_stream_init(&stream, serial)) + return false; + + bool valid = true; + u32 header_packets = 0; + long last_block = -1; + s64 accumulated = 0; + s64 initial_pcm = -1; + size_t offset = 0; + while (offset < size) + { + ogg_page page{}; + size_t next = 0; + if (!ReadOggPage(data, size, offset, page, next)) + { + valid = false; + break; + } + offset = next; + if (header_packets >= 3 && ogg_page_bos(&page)) + { + valid = false; + break; + } + if (ogg_page_serialno(&page) != serial) + continue; + if (ogg_stream_pagein(&stream, &page)) + { + valid = false; + break; + } + + bool setup_completed_on_page = false; + for (;;) + { + ogg_packet packet{}; + const int packet_result = ogg_stream_packetout(&stream, &packet); + if (!packet_result) + break; + if (packet_result < 0) + { + valid = false; + break; + } + if (header_packets < 3) + { + ++header_packets; + setup_completed_on_page = header_packets == 3; + continue; + } + if (setup_completed_on_page) + { + valid = false; + break; + } + + const long block = vorbis_packet_blocksize(&info, &packet); + if (block < 0) + { + valid = false; + break; + } + if (last_block != -1) + accumulated += (last_block + block) >> 2; + last_block = block; + } + if (!valid) + break; + + const s64 granule = ogg_page_granulepos(&page); + if (header_packets >= 3 && last_block != -1 && granule >= 0) + { + initial_pcm = _max(s64(0), granule - accumulated); + break; + } + } + ogg_stream_clear(&stream); + if (!valid || initial_pcm < 0) + return false; + + const size_t tail_begin = size > OggMaximumPageSize ? size - OggMaximumPageSize : 0; + s64 final_pcm = -1; + for (size_t tail = size - OggPageHeaderSize;; --tail) + { + if (data[tail] == 'O') + { + ogg_page page{}; + size_t next = 0; + if (ReadOggPage(data, size, tail, page, next) && next == size && ogg_page_eos(&page)) + { + if (ogg_page_serialno(&page) != serial) + return false; + final_pcm = ogg_page_granulepos(&page); + break; + } + } + if (tail == tail_begin) + break; + } + if (final_pcm < initial_pcm) + return false; + + pcm_total = final_pcm - initial_pcm; + return true; +} +} + // SEEK_SET 0 File beginning // SEEK_CUR 1 Current file pointer position // SEEK_END 2 End-of-file @@ -61,102 +213,205 @@ void CSoundRender_Source::decompress(u32 line, OggVorbis_File* ovf) i_decompress_fr(ovf, dest, left); } -bool CSoundRender_Source::LoadWave(LPCSTR pName) +bool CSoundRender_Source::prepare( + LPCSTR path, PreparedSoundSource& prepared, xr_string& error, bool log_warnings) { - pname = pName; + PROF_EVENT("Sound: Load ogg"); + prepared = PreparedSoundSource{}; + prepared.path = path; // Load file into memory and parse WAV-format OggVorbis_File ovf; ov_callbacks ovc = {ov_read_func, ov_seek_func, ov_close_func, ov_tell_func}; - IReader* wave = FS.r_open(pname.c_str()); - R_ASSERT3(wave&&wave->length(), "Can't open wave file:", pname.c_str()); - ov_open_callbacks(wave, &ovf,NULL, 0, ovc); + IReader* wave = FS.r_open(path); + if (!wave || !wave->length()) + { + if (wave) + FS.r_close(wave); + error = make_string("Can't open wave file: %s", path).c_str(); + return false; + } - vorbis_info* ovi = ov_info(&ovf, -1); - // verify - R_ASSERT3(ovi, "Invalid source info:", pname.c_str()); + const u8* wave_data = static_cast(wave->pointer()); + const size_t wave_size = wave->length(); + // Headers and comments do not require libvorbisfile's full seekable-open pass. + const int open_result = ov_test_callbacks(wave, &ovf, NULL, 0, ovc); + if (open_result) + { + FS.r_close(wave); + error = make_string("Invalid OGG stream (%d): %s", open_result, path).c_str(); + return false; + } + vorbis_info* ovi = ov_info(&ovf, -1); + if (!ovi) + { + ov_clear(&ovf); + FS.r_close(wave); + error = make_string("Invalid source info: %s", path).c_str(); + return false; + } + s64 pcm_total = 0; + if (!ReadSingleStreamPcmTotal(wave_data, wave_size, ovf, *ovi, pcm_total)) + { + // Chained, multiplexed and unusual streams retain the original full parser. + const int finish_result = ov_test_open(&ovf); + if (finish_result) + { + FS.r_close(wave); + error = make_string("Invalid OGG stream (%d): %s", finish_result, path).c_str(); + return false; + } + ovi = ov_info(&ovf, -1); + if (!ovi) + { + ov_clear(&ovf); + FS.r_close(wave); + error = make_string("Invalid source info: %s", path).c_str(); + return false; + } + pcm_total = ov_pcm_total(&ovf, -1); + } + //R_ASSERT3(ovi->rate == 44100, "Invalid source rate:", pname.c_str()); if (ovi->rate != 44100) { - Msg("! Warning: Invalid source rate: %s", pname.c_str()); + if (log_warnings) + Msg("! Warning: Invalid source rate: %s", path); + else + prepared.warning = PreparedSoundSource::Warning::InvalidRate; + ov_clear(&ovf); + FS.r_close(wave); + return true; + } + if (pcm_total < 0) + { ov_clear(&ovf); FS.r_close(wave); + error = make_string("Invalid PCM length: %s", path).c_str(); return false; } #ifdef DEBUG - if (ovi->channels == 2) + if (log_warnings && ovi->channels == 2) { - Msg("stereo sound source [%s]", pname.c_str()); + Msg("stereo sound source [%s]", path); } #endif // #ifdef DEBUG - ZeroMemory(&m_wformat, sizeof( WAVEFORMATEX )); + ZeroMemory(&prepared.format, sizeof(WAVEFORMATEX)); + prepared.format.nSamplesPerSec = ovi->rate; //44100; + prepared.format.wFormatTag = WAVE_FORMAT_PCM; + prepared.format.nChannels = u16(ovi->channels); + prepared.format.wBitsPerSample = 16; + prepared.format.nBlockAlign = prepared.format.wBitsPerSample / 8 * prepared.format.nChannels; + prepared.format.nAvgBytesPerSec = prepared.format.nSamplesPerSec * prepared.format.nBlockAlign; - m_wformat.nSamplesPerSec = (ovi->rate); //44100; - m_wformat.wFormatTag = WAVE_FORMAT_PCM; - m_wformat.nChannels = u16(ovi->channels); - m_wformat.wBitsPerSample = 16; - - m_wformat.nBlockAlign = m_wformat.wBitsPerSample / 8 * m_wformat.nChannels; - m_wformat.nAvgBytesPerSec = m_wformat.nSamplesPerSec * m_wformat.nBlockAlign; - - s64 pcm_total = ov_pcm_total(&ovf, -1); - dwBytesTotal = u32(pcm_total * m_wformat.nBlockAlign); - fTimeTotal = s_f_def_source_footer + dwBytesTotal / float(m_wformat.nAvgBytesPerSec); + prepared.bytes_total = u32(pcm_total * prepared.format.nBlockAlign); + prepared.time_total = s_f_def_source_footer + prepared.bytes_total / float(prepared.format.nAvgBytesPerSec); vorbis_comment* ovm = ov_comment(&ovf, -1); - if (ovm->comments) + if (ovm && ovm->comments && ovm->user_comments[0] && ovm->comment_lengths[0] >= sizeof(u32)) { IReader F(ovm->user_comments[0], ovm->comment_lengths[0]); u32 vers = F.r_u32(); - if (vers == 0x0001) + if (vers == 0x0001 && ovm->comment_lengths[0] >= 16) { - m_fMinDist = F.r_float(); - m_fMaxDist = F.r_float(); - m_fBaseVolume = 1.0f; - m_uGameType = F.r_u32(); - m_fMaxAIDist = m_fMaxDist; + prepared.min_distance = F.r_float(); + prepared.max_distance = F.r_float(); + prepared.base_volume = 1.0f; + prepared.game_type = F.r_u32(); + prepared.max_ai_distance = prepared.max_distance; } - else if (vers == 0x0002) + else if (vers == 0x0002 && ovm->comment_lengths[0] >= 20) { - m_fMinDist = F.r_float(); - m_fMaxDist = F.r_float(); - m_fBaseVolume = F.r_float(); - m_uGameType = F.r_u32(); - m_fMaxAIDist = m_fMaxDist; + prepared.min_distance = F.r_float(); + prepared.max_distance = F.r_float(); + prepared.base_volume = F.r_float(); + prepared.game_type = F.r_u32(); + prepared.max_ai_distance = prepared.max_distance; } - else if (vers == OGG_COMMENT_VERSION) + else if (vers == OGG_COMMENT_VERSION && ovm->comment_lengths[0] >= 24) { - m_fMinDist = F.r_float(); - m_fMaxDist = F.r_float(); - m_fBaseVolume = F.r_float(); - m_uGameType = F.r_u32(); - m_fMaxAIDist = F.r_float(); + prepared.min_distance = F.r_float(); + prepared.max_distance = F.r_float(); + prepared.base_volume = F.r_float(); + prepared.game_type = F.r_u32(); + prepared.max_ai_distance = F.r_float(); } else { - if (strstr(Core.Params, "-dbg")) + if (log_warnings && strstr(Core.Params, "-dbg")) { - Log("! Invalid ogg-comment version, file: ", pname.c_str()); + Log("! Invalid ogg-comment version, file: ", path); } + else if (!log_warnings && strstr(Core.Params, "-dbg")) + prepared.warning = PreparedSoundSource::Warning::InvalidComment; } } else { - if (strstr(Core.Params, "-dbg")) + if (log_warnings && strstr(Core.Params, "-dbg")) { - Log("! Missing ogg-comment, file: ", pname.c_str()); + Log("! Missing ogg-comment, file: ", path); } + else if (!log_warnings && strstr(Core.Params, "-dbg")) + prepared.warning = PreparedSoundSource::Warning::MissingComment; + } + if (prepared.max_ai_distance < 0.1f || prepared.max_distance < 0.1f) + { + ov_clear(&ovf); + FS.r_close(wave); + error = make_string("Invalid max distance: %s", path).c_str(); + return false; } - R_ASSERT3((m_fMaxAIDist >= 0.1f) && (m_fMaxDist >= 0.1f), "Invalid max distance.", pname.c_str()); ov_clear(&ovf); FS.r_close(wave); + prepared.loaded = true; return true; } +bool CSoundRender_Source::LoadWave(LPCSTR path) +{ + PreparedSoundSource prepared; + xr_string error; + R_ASSERT3(prepare(path, prepared, error), "Can't prepare sound source", error.c_str()); + load_prepared(fname.c_str(), prepared); + return prepared.loaded; +} + +void CSoundRender_Source::resolve_path(LPCSTR name, xr_string& path) +{ + string_path fn; + strconcat(sizeof(fn), fn, name, ".ogg"); + if (!FS.exist("$level$", fn)) + FS.update_path(fn, "$game_sounds$", fn); + + if (!FS.exist(fn)) + { + Msg("! Can't find sound '%s'", name); + FS.update_path(fn, "$game_sounds$", "$no_sound.ogg"); + } + path = fn; +} + +void CSoundRender_Source::load_prepared(LPCSTR name, const PreparedSoundSource& prepared) +{ + fname = name; + pname = prepared.path.c_str(); + m_wformat = prepared.format; + fTimeTotal = prepared.time_total; + dwBytesTotal = prepared.bytes_total; + m_fBaseVolume = prepared.base_volume; + m_fMinDist = prepared.min_distance; + m_fMaxDist = prepared.max_distance; + m_fMaxAIDist = prepared.max_ai_distance; + m_uGameType = prepared.game_type; + if (prepared.loaded) + SoundRender->cache.cat_create(CAT, dwBytesTotal); +} + void CSoundRender_Source::load(LPCSTR name) { string_path fn, N; @@ -165,20 +420,12 @@ void CSoundRender_Source::load(LPCSTR name) if (strext(N)) *strext(N) = 0; fname = N; - - strconcat(sizeof(fn), fn, N, ".ogg"); - if (!FS.exist("$level$", fn)) - FS.update_path(fn, "$game_sounds$", fn); - - if (!FS.exist(fn)){ - { - Msg("! Can't find sound '%s'", name); - FS.update_path(fn, "$game_sounds$", "$no_sound.ogg"); - } - } + xr_string path; + resolve_path(N, path); + xr_strcpy(fn, path.c_str()); if (LoadWave(fn)) - SoundRender->cache.cat_create(CAT, dwBytesTotal); + return; } void CSoundRender_Source::unload()