From f7f6d54c545f4dbf0929494c8a61070d20955f31 Mon Sep 17 00:00:00 2001 From: ecker Date: Sun, 9 Aug 2026 23:28:11 -0500 Subject: [PATCH] re-enabled internal allocator for STL containers (because of lines being swapped), fixed async asset loading (for OpenGL, Vulkan still doesn't like it), grouped OpenGL draw calls by material type for a speedup, some other stuff --- bin/data/config.json | 2 +- bin/data/entities/model.json | 2 +- bin/data/entities/player.json | 2 +- engine/inc/uf/engine/asset/asset.h | 35 ++++++- engine/inc/uf/engine/graph/graph.h | 2 + engine/inc/uf/utils/memory/deque.h | 4 +- engine/inc/uf/utils/memory/fifo_map.h | 4 +- engine/inc/uf/utils/memory/map.h | 4 +- engine/inc/uf/utils/memory/unordered_map.h | 4 +- engine/inc/uf/utils/memory/unordered_set.h | 4 +- engine/inc/uf/utils/memory/vector.h | 4 +- engine/src/engine/asset/asset.cpp | 88 ++++++++--------- engine/src/engine/ext/ext.cpp | 2 +- engine/src/engine/graph/animation.cpp | 10 +- engine/src/engine/graph/convert.cpp | 2 + engine/src/engine/graph/decode.cpp | 104 +++++++++++---------- engine/src/engine/graph/graph.cpp | 3 +- engine/src/ext/lua/lua.cpp | 5 + engine/src/ext/opengl/commands.cpp | 43 ++++++--- engine/src/ext/opengl/graphic.cpp | 11 ++- 20 files changed, 200 insertions(+), 135 deletions(-) diff --git a/bin/data/config.json b/bin/data/config.json index a438a8d5..0f6108a0 100644 --- a/bin/data/config.json +++ b/bin/data/config.json @@ -437,7 +437,7 @@ }, "loader": { "assert": "auto", - "async": "auto" // unironically works on opengl, not on vulkan + "async": "auto" }, "hooks": { "defer lazy calls": true diff --git a/bin/data/entities/model.json b/bin/data/entities/model.json index 2373e157..6bec24be 100644 --- a/bin/data/entities/model.json +++ b/bin/data/entities/model.json @@ -79,7 +79,7 @@ "stream": { "tag": "worldspawn", "player": "info_player_start", - "enabled": "auto", + "enabled": false, // "auto", "radius": 50, "every": 1 } diff --git a/bin/data/entities/player.json b/bin/data/entities/player.json index cfff2bb6..a17e0da2 100644 --- a/bin/data/entities/player.json +++ b/bin/data/entities/player.json @@ -12,7 +12,7 @@ "./scripts/player.lua" ], "system": { - "noclipped": true, + // "noclipped": true, "hot reload": { "enabled": true } diff --git a/engine/inc/uf/engine/asset/asset.h b/engine/inc/uf/engine/asset/asset.h index 780dd7c7..114db61b 100644 --- a/engine/inc/uf/engine/asset/asset.h +++ b/engine/inc/uf/engine/asset/asset.h @@ -13,6 +13,30 @@ namespace uf { namespace asset { typedef uf::Hooks::name_t callback_t; + struct Job { + typedef uf::stl::vector container_t; + + uf::asset::callback_t callback = ""; + uf::stl::string type = ""; + uf::asset::Payload payload = {}; + }; + struct Read { + typedef uf::stl::unordered_map> container_t; + + size_t offset; + size_t length; + uint8_t* dest; + std::function callback; + std::function&&)> callbackBuffered; + }; + struct Stream { + typedef uf::stl::unordered_map> container_t; + + size_t offset; + size_t length; + size_t chunkSize; + std::function callback; + }; #if UF_COMPONENT_POINTERED_USERDATA typedef pod::PointeredUserdata userdata_t; @@ -22,6 +46,7 @@ namespace uf { extern UF_API bool assertionLoad; extern UF_API bool asyncQueue; + extern UF_API bool deferQueues; extern UF_API uf::stl::unordered_map map; @@ -33,14 +58,22 @@ namespace uf { // URL or file path void UF_API processQueue(); void UF_API processIO(); + + void UF_API processIO( const uf::asset::Read::container_t& pendingReads, bool async = uf::asset::asyncQueue, bool wait = false ); + void UF_API processIO( const uf::asset::Stream::container_t& pendingStreams, bool async = uf::asset::asyncQueue, bool wait = false ); + void UF_API processIO( const uf::asset::Read::container_t& pendingReads, const uf::asset::Stream::container_t& pendingStreams, bool async = uf::asset::asyncQueue, bool wait = false ); void UF_API cache( const uf::asset::callback_t&, const uf::asset::Payload& ); void UF_API load( const uf::asset::callback_t&, const uf::asset::Payload& ); void UF_API read( const uf::stl::string& filename, size_t offset, size_t length, uint8_t* dest, std::function callback = {} ); - void UF_API read( const uf::stl::string& filename, size_t offset, size_t length, std::function&&)> callback = {} ); + void UF_API read( const uf::stl::string& filename, size_t offset, size_t length, std::function&&)> callback ); void UF_API stream( const uf::stl::string& filename, size_t offset, size_t length, size_t chunkSize, std::function callback ); + void UF_API read( uf::asset::Read::container_t& container, const uf::stl::string& filename, size_t offset, size_t length, uint8_t* dest, std::function callback = {} ); + void UF_API read( uf::asset::Read::container_t& container, const uf::stl::string& filename, size_t offset, size_t length, std::function&&)> callback ); + void UF_API stream( uf::asset::Stream::container_t& container, const uf::stl::string& filename, size_t offset, size_t length, size_t chunkSize, std::function callback ); + uf::stl::string UF_API cache( uf::asset::Payload& ); uf::stl::string UF_API load( uf::asset::Payload& ); diff --git a/engine/inc/uf/engine/graph/graph.h b/engine/inc/uf/engine/graph/graph.h index 6bda8fbf..79be582f 100644 --- a/engine/inc/uf/engine/graph/graph.h +++ b/engine/inc/uf/engine/graph/graph.h @@ -81,6 +81,8 @@ namespace pod { uf::stl::unordered_map meshes; uf::stl::unordered_map images; uf::stl::unordered_map bvhs; + + uf::asset::Read::container_t queue; } streams; // Local storage, used for save/load diff --git a/engine/inc/uf/utils/memory/deque.h b/engine/inc/uf/utils/memory/deque.h index 2511a026..5809840c 100644 --- a/engine/inc/uf/utils/memory/deque.h +++ b/engine/inc/uf/utils/memory/deque.h @@ -10,9 +10,9 @@ namespace uf { template< class T, #if UF_MEMORYPOOL_USE_ALLOCATOR - class Allocator = std::allocator - #else class Allocator = uf::Allocator + #else + class Allocator = std::allocator #endif > using deque = std::deque; diff --git a/engine/inc/uf/utils/memory/fifo_map.h b/engine/inc/uf/utils/memory/fifo_map.h index cdeac170..08a235bc 100644 --- a/engine/inc/uf/utils/memory/fifo_map.h +++ b/engine/inc/uf/utils/memory/fifo_map.h @@ -12,9 +12,9 @@ namespace uf { class T, class Compare = nlohmann::fifo_map_compare, #if UF_MEMORYPOOL_USE_ALLOCATOR - class Allocator = std::allocator> - #else class Allocator = uf::Allocator> + #else + class Allocator = std::allocator> #endif > using fifo_map = nlohmann::fifo_map; diff --git a/engine/inc/uf/utils/memory/map.h b/engine/inc/uf/utils/memory/map.h index a9d2b77f..8b85de46 100644 --- a/engine/inc/uf/utils/memory/map.h +++ b/engine/inc/uf/utils/memory/map.h @@ -12,9 +12,9 @@ namespace uf { class T, class Compare = std::less, #if UF_MEMORYPOOL_USE_ALLOCATOR - class Allocator = std::allocator> - #else class Allocator = uf::Allocator> + #else + class Allocator = std::allocator> #endif > using map = std::map; diff --git a/engine/inc/uf/utils/memory/unordered_map.h b/engine/inc/uf/utils/memory/unordered_map.h index 137f94cb..60ccb3ad 100644 --- a/engine/inc/uf/utils/memory/unordered_map.h +++ b/engine/inc/uf/utils/memory/unordered_map.h @@ -15,9 +15,9 @@ namespace uf { class Hash = uf::algo::hasher, class KeyEqual = std::equal_to, #if UF_MEMORYPOOL_USE_ALLOCATOR - class Allocator = std::allocator> - #else class Allocator = uf::Allocator> + #else + class Allocator = std::allocator> #endif > using unordered_map = std::unordered_map; diff --git a/engine/inc/uf/utils/memory/unordered_set.h b/engine/inc/uf/utils/memory/unordered_set.h index f080a50a..cd285708 100644 --- a/engine/inc/uf/utils/memory/unordered_set.h +++ b/engine/inc/uf/utils/memory/unordered_set.h @@ -13,9 +13,9 @@ namespace uf { class Hash = std::hash, class KeyEqual = std::equal_to, #if UF_MEMORYPOOL_USE_ALLOCATOR - class Allocator = std::allocator - #else class Allocator = uf::Allocator + #else + class Allocator = std::allocator #endif > using unordered_set = std::unordered_set; diff --git a/engine/inc/uf/utils/memory/vector.h b/engine/inc/uf/utils/memory/vector.h index bcc96ba5..4cc92eaf 100644 --- a/engine/inc/uf/utils/memory/vector.h +++ b/engine/inc/uf/utils/memory/vector.h @@ -10,9 +10,9 @@ namespace uf { template< class T, #if UF_MEMORYPOOL_USE_ALLOCATOR - class Allocator = std::allocator - #else class Allocator = uf::Allocator + #else + class Allocator = std::allocator #endif > using vector = std::vector; diff --git a/engine/src/engine/asset/asset.cpp b/engine/src/engine/asset/asset.cpp index 5a430f91..cad2cab7 100644 --- a/engine/src/engine/asset/asset.cpp +++ b/engine/src/engine/asset/asset.cpp @@ -39,70 +39,43 @@ namespace { } namespace jobs { - struct Job { - typedef uf::stl::vector container_t; - - uf::asset::callback_t callback = ""; - uf::stl::string type = ""; - uf::asset::Payload payload = {}; - }; - std::mutex mutex; - Job::container_t queue; - Job::container_t finished; + uf::asset::Job::container_t queue; + uf::asset::Job::container_t finished; }; namespace io_read { - struct Job { - typedef uf::stl::unordered_map> container_t; - - size_t offset; - size_t length; - uint8_t* dest; - std::function callback; - std::function&&)> callbackBuffered; - }; - std::mutex mutex; - Job::container_t queue; - Job::container_t finished; + uf::asset::Read::container_t queue; + uf::asset::Read::container_t finished; }; namespace io_stream { - struct Job { - typedef uf::stl::unordered_map> container_t; - - size_t offset; - size_t length; - size_t chunkSize; - std::function callback; - }; - std::mutex mutex; - Job::container_t queue; - Job::container_t finished; + uf::asset::Stream::container_t queue; + uf::asset::Stream::container_t finished; }; } // uf::asset uf::asset::masterAssetLoader; bool uf::asset::assertionLoad = true; bool uf::asset::asyncQueue = true; +bool uf::asset::deferQueues = false; uf::stl::unordered_map uf::asset::map; uf::Serializer uf::asset::metadata; void uf::asset::processQueue() { - if ( ::jobs::queue.empty() && ::jobs::finished.empty() ) return; - - STATIC_THREAD_LOCAL(::jobs::Job::container_t, jobs); - ::jobs::Job::container_t finishedJobs; + STATIC_THREAD_LOCAL(uf::asset::Job::container_t, jobs); + uf::asset::Job::container_t finishedJobs; ::jobs::mutex.lock(); std::swap( jobs, ::jobs::queue ); std::swap( finishedJobs, ::jobs::finished ); ::jobs::mutex.unlock(); + + if ( jobs.empty() && finishedJobs.empty() ) return; - bool async = uf::asset::asyncQueue; // a bit buggy - auto tasks = uf::thread::schedule(async ? uf::thread::asyncThreadName : uf::thread::mainThreadName, !true); + auto tasks = uf::thread::schedule(uf::asset::asyncQueue ? uf::thread::asyncThreadName : uf::thread::mainThreadName, false); if ( !finishedJobs.empty() ) { tasks.queue([jobs = std::move(finishedJobs)]() { @@ -131,8 +104,8 @@ void uf::asset::processQueue() { } void uf::asset::processIO() { - STATIC_THREAD_LOCAL(::io_read::Job::container_t, pendingReads); - STATIC_THREAD_LOCAL(::io_stream::Job::container_t, pendingStreams); + STATIC_THREAD_LOCAL(uf::asset::Read::container_t, pendingReads); + STATIC_THREAD_LOCAL(uf::asset::Stream::container_t, pendingStreams); ::io_read::mutex.lock(); std::swap(pendingReads, ::io_read::queue); @@ -143,8 +116,16 @@ void uf::asset::processIO() { if ( pendingReads.empty() && pendingStreams.empty() ) return; - bool async = uf::asset::asyncQueue; - auto tasks = uf::thread::schedule(async ? uf::thread::asyncThreadName : uf::thread::mainThreadName, false); + uf::asset::processIO( pendingReads, pendingStreams, uf::asset::asyncQueue, false ); +} +void uf::asset::processIO( const uf::asset::Read::container_t& pendingReads, bool async, bool wait ) { + return uf::asset::processIO( pendingReads, {}, async, wait ); +} +void uf::asset::processIO( const uf::asset::Stream::container_t& pendingStreams, bool async, bool wait ) { + return uf::asset::processIO( {}, pendingStreams, async, wait ); +} +void uf::asset::processIO( const uf::asset::Read::container_t& pendingReads, const uf::asset::Stream::container_t& pendingStreams, bool async, bool wait ) { + auto tasks = uf::thread::schedule(async ? uf::thread::asyncThreadName : uf::thread::mainThreadName, wait); for ( auto& [filename, requests] : pendingReads ) { tasks.queue([filename = filename, requests = std::move(requests)]() { @@ -215,25 +196,36 @@ void uf::asset::processIO() { void uf::asset::cache( const uf::asset::callback_t& callback, const uf::asset::Payload& payload ) { std::lock_guard lock(::jobs::mutex); - ::jobs::queue.emplace_back(::jobs::Job{ callback, "cache", payload }); + ::jobs::queue.emplace_back(uf::asset::Job{ callback, "cache", payload }); } void uf::asset::load( const uf::asset::callback_t& callback, const uf::asset::Payload& payload ) { std::lock_guard lock(::jobs::mutex); - ::jobs::queue.emplace_back(::jobs::Job{ callback, "load", payload }); + ::jobs::queue.emplace_back(uf::asset::Job{ callback, "load", payload }); } void uf::asset::read( const uf::stl::string& filename, size_t offset, size_t length, uint8_t* dest, std::function callback ) { std::lock_guard lock(::io_read::mutex); - ::io_read::queue[filename].emplace_back(::io_read::Job{ offset, length, dest, callback }); + uf::asset::read( ::io_read::queue, filename, offset, length, dest, callback ); } void uf::asset::read( const uf::stl::string& filename, size_t offset, size_t length, std::function&&)> callback ) { std::lock_guard lock(::io_read::mutex); - ::io_read::queue[filename].emplace_back(::io_read::Job{ offset, length, nullptr, nullptr, callback }); + uf::asset::read( ::io_read::queue, filename, offset, length, callback ); } void uf::asset::stream( const uf::stl::string& filename, size_t offset, size_t length, size_t chunkSize, std::function callback ) { std::lock_guard lock(::io_stream::mutex); - ::io_stream::queue[filename].emplace_back(::io_stream::Job{ offset, length, chunkSize, callback }); + uf::asset::stream( ::io_stream::queue, filename, offset, length, chunkSize, callback ); +} + +void uf::asset::read( uf::asset::Read::container_t& container, const uf::stl::string& filename, size_t offset, size_t length, uint8_t* dest, std::function callback ) { + container[filename].emplace_back(uf::asset::Read{ offset, length, dest, callback }); +} +void uf::asset::read( uf::asset::Read::container_t& container, const uf::stl::string& filename, size_t offset, size_t length, std::function&&)> callback ) { + container[filename].emplace_back(uf::asset::Read{ offset, length, nullptr, nullptr, callback }); +} + +void uf::asset::stream( uf::asset::Stream::container_t& container, const uf::stl::string& filename, size_t offset, size_t length, size_t chunkSize, std::function callback ) { + container[filename].emplace_back(uf::asset::Stream{ offset, length, chunkSize, callback }); } uf::asset::Payload uf::asset::resolveToPayload( const uf::stl::string& uri, const uf::stl::string& mime ) { diff --git a/engine/src/engine/ext/ext.cpp b/engine/src/engine/ext/ext.cpp index ce18df9a..2aab643a 100644 --- a/engine/src/engine/ext/ext.cpp +++ b/engine/src/engine/ext/ext.cpp @@ -208,7 +208,7 @@ void UF_API uf::load( ext::json::Value& json ) { uf::asset::assertionLoad = configEngineDebugJson["loader"]["assert"].as( uf::asset::assertionLoad ); } if ( configEngineDebugJson["loader"]["async"].as() == "auto" ) { - uf::asset::asyncQueue = false; // to-do: fix + uf::asset::asyncQueue = false; // to-do: find out why vulkan really does not like this #if UF_USE_OPENGL && !UF_ENV_DREAMCAST uf::asset::asyncQueue = true; #endif diff --git a/engine/src/engine/graph/animation.cpp b/engine/src/engine/graph/animation.cpp index 6c188b8d..7f869fbe 100644 --- a/engine/src/engine/graph/animation.cpp +++ b/engine/src/engine/graph/animation.cpp @@ -30,8 +30,8 @@ namespace { auto& animation = storage.animations.map[name]; auto& animStream = graph.streams.animations[name]; - bool needsIO = false; + uf::asset::Read::container_t queue; for ( size_t i = 0; i < animation.samplers.size(); ++i ) { auto& sampler = animation.samplers[i]; auto& stream = animStream.samplers[i]; @@ -40,18 +40,16 @@ namespace { if ( stream.inputs.length > 0 ) { sampler.inputs.resize( stream.inputs.length / sizeof(float) ); - uf::asset::read( stream.inputs.filename, stream.inputs.offset, stream.inputs.length, (uint8_t*)(sampler.inputs.data()) ); - needsIO = true; + uf::asset::read( queue, stream.inputs.filename, stream.inputs.offset, stream.inputs.length, (uint8_t*)(sampler.inputs.data()) ); } if ( stream.outputs.length > 0 ) { sampler.outputs.resize( stream.outputs.length / sizeof(pod::Vector4f) ); - uf::asset::read( stream.outputs.filename, stream.outputs.offset, stream.outputs.length, (uint8_t*)(sampler.outputs.data()) ); - needsIO = true; + uf::asset::read( queue, stream.outputs.filename, stream.outputs.offset, stream.outputs.length, (uint8_t*)(sampler.outputs.data()) ); } } - if ( needsIO ) uf::asset::processIO(); + uf::asset::processIO( queue ); } void unloadAnimation( pod::Graph& graph, const uf::stl::string& name ) { diff --git a/engine/src/engine/graph/convert.cpp b/engine/src/engine/graph/convert.cpp index 70456a4a..bb5af9b3 100644 --- a/engine/src/engine/graph/convert.cpp +++ b/engine/src/engine/graph/convert.cpp @@ -361,6 +361,7 @@ void uf::graph::import( pod::Graph::Storage& target, pod::Graph::Storage& storag target.primitives.merge(std::move(storage.primitives)); target.instances.merge(std::move(storage.instances)); target.meshes.merge(std::move(storage.meshes)); + target.bvhs.merge(std::move(storage.bvhs)); target.images.merge(std::move(storage.images)); target.materials.merge(std::move(storage.materials)); target.textures.merge(std::move(storage.textures)); @@ -387,6 +388,7 @@ void uf::graph::import( pod::Graph::Storage& target, pod::Graph::Storage& storag target.primitives.import(storage.primitives); target.instances.import(storage.instances); target.meshes.import(storage.meshes); + target.bvhs.import(storage.bvhs); target.images.import(storage.images); target.materials.import(storage.materials); target.textures.import(storage.textures); diff --git a/engine/src/engine/graph/decode.cpp b/engine/src/engine/graph/decode.cpp index ff39bbe9..8a70a9f3 100644 --- a/engine/src/engine/graph/decode.cpp +++ b/engine/src/engine/graph/decode.cpp @@ -25,14 +25,10 @@ #define UF_DEBUG_TIMER_MULTITRACE_END(...) #endif -namespace { - struct PendingImage { - uf::stl::string name; - uf::stl::vector buffer; - uf::stl::string extension; - size_t layers; - }; +// to force deferred loading +#define DEFERRED(...) __VA_ARGS__ +namespace { size_t deduceFormat( const uf::stl::string& format ) { if ( format == "ARGB4444" ) return uf::renderer::enums::Format::R4G4B4A4_UNORM_PACK16; if ( format == "RGB565" ) return uf::renderer::enums::Format::R5G6B5_UNORM_PACK16; @@ -86,17 +82,17 @@ namespace { } uf::stl::string fullPath = uf::io::directory( graph.name ) + "/" + filename; + auto& storage = uf::graph::getStorage(graph); + graph.streams.images[key] = { fullPath, offset, length }; - if ( graph.settings.stream.textures ) { - auto& storage = uf::graph::getStorage(graph); - graph.streams.images[key] = { fullPath, offset, length }; - } else { + bool deferred = DEFERRED(graph.settings.stream.textures); + if ( !deferred ) { size_t readLen = length > 0 ? length : uf::io::size( fullPath ); if ( readLen > 0 ) { - uf::asset::read( fullPath, offset, readLen, [&graph, key, extension, layers]( uf::stl::vector&& buffer ) { + uf::asset::read( graph.streams.queue, fullPath, offset, readLen, [&graph, key, extension, layers]( uf::stl::vector&& buffer ) { auto& storage = uf::graph::getStorage(graph); auto& image = storage.images[key].data; - + uf::image::open( image, buffer, extension, false ); uf::image::layers( image, layers ); } ); @@ -123,6 +119,7 @@ namespace { auto& storage = uf::graph::getStorage(graph); auto& animStream = graph.streams.animations[animName]; + bool deferred = DEFERRED(graph.settings.stream.animations); ext::json::forEach( json["samplers"], [&]( ext::json::Value& value ){ auto& sampler = animation.samplers.emplace_back(); sampler.interpolator = value["interpolator"].as(sampler.interpolator); @@ -135,19 +132,19 @@ namespace { size_t outputsOffset = value["outputs"]["offset"].as(); size_t outputsLen = value["outputs"]["length"].as(); - if ( graph.settings.stream.animations ) { - pod::AnimationStream::SamplerStream sStream; - sStream.inputs = { binPath, inputsOffset, inputsLen }; - sStream.outputs = { binPath, outputsOffset, outputsLen }; - animStream.samplers.emplace_back(sStream); - } else { + pod::AnimationStream::SamplerStream sStream; + sStream.inputs = { binPath, inputsOffset, inputsLen }; + sStream.outputs = { binPath, outputsOffset, outputsLen }; + animStream.samplers.emplace_back(sStream); + + if ( !deferred ) { if ( inputsLen > 0 ) { sampler.inputs.resize(inputsCount); - uf::asset::read( binPath, inputsOffset, inputsLen, (uint8_t*)(sampler.inputs.data()) ); + uf::asset::read( graph.streams.queue, binPath, inputsOffset, inputsLen, (uint8_t*)(sampler.inputs.data()) ); } if ( outputsLen > 0 ) { sampler.outputs.resize(outputsCount); - uf::asset::read( binPath, outputsOffset, outputsLen, (uint8_t*)(sampler.outputs.data()) ); + uf::asset::read( graph.streams.queue, binPath, outputsOffset, outputsLen, (uint8_t*)(sampler.outputs.data()) ); } } }); @@ -172,6 +169,7 @@ namespace { skin.joints.emplace_back( value.as() ); }); + bool deferred = DEFERRED(graph.settings.stream.enabled); if ( json["inverseBindMatrices"].isObject() ) { auto& invJson = json["inverseBindMatrices"]; size_t count = invJson["count"].as(); @@ -181,13 +179,12 @@ namespace { auto& storage = uf::graph::getStorage(graph); auto& skinStream = graph.streams.skins[skinName]; + skinStream.inverseBindMatrices = { binPath, offset, length }; - if ( graph.settings.stream.enabled ) { - skinStream.inverseBindMatrices = { binPath, offset, length }; - } else { + if ( !deferred ) { if ( length > 0 ) { skin.inverseBindMatrices.resize(count); - uf::asset::read( binPath, offset, length, (uint8_t*)(skin.inverseBindMatrices.data()) ); + uf::asset::read( graph.streams.queue, binPath, offset, length, (uint8_t*)(skin.inverseBindMatrices.data()) ); } } } @@ -195,9 +192,7 @@ namespace { return skin; } - uf::Mesh decodeMesh( ext::json::Value& json, pod::Graph& graph, const uf::stl::string& meshName ) { - uf::Mesh mesh; - + void decodeMesh( ext::json::Value& json, pod::Graph& graph, const uf::stl::string& meshName, uf::Mesh& mesh ) { #define DESERIALIZE_MESH(N) {\ auto& input = json["inputs"][#N];\ mesh.N.attributes.reserve( input["attributes"].size() );\ @@ -230,7 +225,7 @@ namespace { auto& meshStream = graph.streams.meshes[meshName]; mesh.buffers.reserve( json["buffers"].size() ); - bool deferred = graph.settings.stream.enabled; + bool deferred = DEFERRED(graph.settings.stream.enabled); ext::json::forEach( json["buffers"], [&]( ext::json::Value& value ){ uf::stl::string filename; @@ -258,7 +253,7 @@ namespace { if ( region.length == 0 ) continue; mesh.buffers[attr.buffer].resize(region.length); - uf::asset::read( region.filename, region.offset, region.length, mesh.buffers[attr.buffer].data() ); + uf::asset::read( graph.streams.queue, region.filename, region.offset, region.length, mesh.buffers[attr.buffer].data() ); } }; @@ -270,11 +265,9 @@ namespace { } mesh.updateDescriptor(); - return mesh; } - pod::BVH decodeBvh( ext::json::Value& json, pod::Graph& graph, const uf::stl::string& key ) { - pod::BVH bvh; + void decodeBvh( ext::json::Value& json, pod::Graph& graph, const uf::stl::string& key, pod::BVH& bvh ) { auto& storage = uf::graph::getStorage(graph); auto& bvhStream = graph.streams.bvhs[key]; @@ -285,19 +278,17 @@ namespace { uf::stl::string fullPath = uf::io::directory( graph.name ) + "/" + filename; bvhStream.buffer = pod::StreamRegion{ fullPath, offset, length }; - bool deferred = graph.settings.stream.enabled; + bool deferred = DEFERRED(graph.settings.stream.enabled); if ( !deferred ) { size_t readLen = length > 0 ? length : uf::io::size( fullPath ); if ( readLen > 0 ) { - uf::asset::read( fullPath, offset, readLen, [&graph, key]( uf::stl::vector&& buffer ) { + uf::asset::read( graph.streams.queue, fullPath, offset, readLen, [&graph, key]( uf::stl::vector&& buffer ) { auto& storage = uf::graph::getStorage(graph); auto& bvh = storage.bvhs[key]; - uf::bvh::deserialize( bvh, buffer ); + auto res = uf::bvh::deserialize( bvh, buffer ); } ); } } - - return bvh; } pod::Node decodeNode( ext::json::Value& json, pod::Graph& graph ) { @@ -405,9 +396,7 @@ void uf::graph::load( pod::Graph& graph, const uf::stl::string& filename, const uf::stl::string key = graph.metadata["key"].as(""); if ( key != "" ) key += ":"; - uf::stl::vector pendingImages; uf::stl::vector meshesToMinify; - tasks.queue([&]{ UF_DEBUG_TIMER_MULTITRACE("Reading material information..."); auto& node = serializer["materials"]; @@ -420,6 +409,8 @@ void uf::graph::load( pod::Graph& graph, const uf::stl::string& filename, const auto names = node["names"].as>(); graph.materials.reserve(names.size()); + storage.materials.reserve(names.size()); + for (size_t i = 0; i < names.size(); ++i) { auto name = key + names[i]; storage.materials[name] = rawMaterials[i]; @@ -442,6 +433,8 @@ void uf::graph::load( pod::Graph& graph, const uf::stl::string& filename, const auto names = node["names"].as>(); graph.textures.reserve(names.size()); + storage.textures.reserve(names.size()); + for (size_t i = 0; i < names.size(); ++i) { auto name = key + names[i]; storage.textures[name] = rawTextures[i]; @@ -464,6 +457,8 @@ void uf::graph::load( pod::Graph& graph, const uf::stl::string& filename, const auto names = node["names"].as>(); graph.samplers.reserve(names.size()); + storage.samplers.reserve(names.size()); + for (size_t i = 0; i < names.size(); ++i) { auto name = key + names[i]; storage.samplers[name] = rawSamplers[i]; @@ -486,6 +481,8 @@ void uf::graph::load( pod::Graph& graph, const uf::stl::string& filename, const auto names = node["names"].as>(); graph.lights.reserve(names.size()); + storage.lights.reserve(names.size()); + for (size_t i = 0; i < names.size(); ++i) { auto name = key + names[i]; graph.lights[name] = rawLights[i]; @@ -505,10 +502,11 @@ void uf::graph::load( pod::Graph& graph, const uf::stl::string& filename, const allPrimitives = reinterpret_cast(ioBuf.data()); } - auto& primNode = serializer["primitives"]; - graph.primitives.reserve( primNode.size() ); + auto count = serializer["primitives"].size(); + graph.primitives.reserve( count ); + storage.primitives.reserve( count ); - ext::json::forEach( primNode, [&]( ext::json::Value& value ){ + ext::json::forEach( serializer["primitives"], [&]( ext::json::Value& value ){ auto name = key + value["name"].as(); graph.primitives.emplace_back(name); @@ -531,7 +529,9 @@ void uf::graph::load( pod::Graph& graph, const uf::stl::string& filename, const tasks.queue([&]{ UF_DEBUG_TIMER_MULTITRACE("Reading images..."); - graph.images.reserve( serializer["images"].size() ); + auto count = serializer["images"].size(); + graph.images.reserve( count ); + storage.images.reserve( count ); ext::json::forEach( serializer["images"], [&]( ext::json::Value& value ){ auto name = key + value["name"].as(); @@ -549,6 +549,7 @@ void uf::graph::load( pod::Graph& graph, const uf::stl::string& filename, const tasks.queue([&]{ UF_DEBUG_TIMER_MULTITRACE("Reading meshes..."); graph.meshes.reserve( serializer["meshes"].size() ); + storage.meshes.reserve( serializer["meshes"].size() ); #if UF_USE_OPENGL bool preferMinified = true; @@ -562,7 +563,7 @@ void uf::graph::load( pod::Graph& graph, const uf::stl::string& filename, const bool hasMinifiedAsset = value["min"].isObject(); auto& json = ( preferMinified && hasMinifiedAsset ) ? value["min"] : value; - storage.meshes[name] = decodeMesh( json, graph, name ); + decodeMesh( json, graph, name, storage.meshes[name] ); graph.meshes.emplace_back(name); if ( preferMinified && !hasMinifiedAsset && !graph.settings.stream.enabled ) @@ -574,9 +575,10 @@ void uf::graph::load( pod::Graph& graph, const uf::stl::string& filename, const tasks.queue([&]{ UF_DEBUG_TIMER_MULTITRACE("Reading BVHs..."); + storage.bvhs.reserve( serializer["bvhs"].size() ); ext::json::forEach( serializer["bvhs"], [&]( ext::json::Value& value ){ auto name = key + value["name"].as(); - storage.bvhs[name] = decodeBvh( value, graph, name ); + decodeBvh( value, graph, name, storage.bvhs[name] ); }); UF_DEBUG_TIMER_MULTITRACE("Read BVHs"); @@ -587,7 +589,8 @@ void uf::graph::load( pod::Graph& graph, const uf::stl::string& filename, const auto& animNode = serializer["animations"]; if ( animNode.isObject() ) { - storage.animations.map.reserve( animNode.size() ); + storage.animations.reserve( animNode.size() ); + ext::json::forEach( animNode, [&]( const uf::stl::string& rawName, ext::json::Value& value ){ auto name = key + rawName; storage.animations[name] = decodeAnimation( value, graph, name ); @@ -595,7 +598,8 @@ void uf::graph::load( pod::Graph& graph, const uf::stl::string& filename, const }); } else if ( animNode.isArray() ) { - storage.animations.map.reserve( animNode.size() ); + storage.animations.reserve( animNode.size() ); + ext::json::forEach( animNode, [&]( ext::json::Value& value ){ uf::stl::string path = directory + "/" + value.as(); uf::Serializer json; @@ -612,6 +616,7 @@ void uf::graph::load( pod::Graph& graph, const uf::stl::string& filename, const tasks.queue([&]{ UF_DEBUG_TIMER_MULTITRACE("Reading skinning information..."); graph.skins.reserve( serializer["skins"].size() ); + storage.skins.reserve( serializer["skins"].size() ); ext::json::forEach( serializer["skins"], [&]( ext::json::Value& value ){ auto name = key + value["name"].as(); @@ -639,8 +644,7 @@ void uf::graph::load( pod::Graph& graph, const uf::stl::string& filename, const UF_DEBUG_TIMER_MULTITRACE("Executing tasks"); uf::thread::execute( tasks ); UF_DEBUG_TIMER_MULTITRACE("Processing IO"); - uf::asset::processIO(); - + uf::asset::processIO( graph.streams.queue, false, true ); // process meshes that need to be minified because I can't easily tie it to the callback UF_DEBUG_TIMER_MULTITRACE("Processing meshes for minification"); diff --git a/engine/src/engine/graph/graph.cpp b/engine/src/engine/graph/graph.cpp index c7e2e338..896dc69e 100644 --- a/engine/src/engine/graph/graph.cpp +++ b/engine/src/engine/graph/graph.cpp @@ -2317,8 +2317,9 @@ void uf::graph::reload( pod::Graph& graph ) { auto& bvh = storage.bvhs.map[meshName]; auto& bvhStream = graph.streams.bvhs[meshName]; - if ( bvh.flatNodes.empty() && bvh.nodes.empty() && bvhStream.buffer.length > 0 ) { + if ( bvh.indices.empty() && bvhStream.buffer.length > 0 ) { work.needsBvhLoad = true; + UF_MSG_DEBUG("Queued BVH load={}", meshName); } if ( radius > 0 && mesh.indirect.count && mesh.indirect.count <= primitives.size() ) { diff --git a/engine/src/ext/lua/lua.cpp b/engine/src/ext/lua/lua.cpp index 58019440..63f4a78d 100644 --- a/engine/src/ext/lua/lua.cpp +++ b/engine/src/ext/lua/lua.cpp @@ -313,6 +313,10 @@ namespace binds { } } +namespace { + std::mutex mutex; +} + void ext::lua::initialize() { if ( !ext::lua::enabled ) return; @@ -477,6 +481,7 @@ pod::LuaScript ext::lua::script( const uf::stl::string& filename ) { } void ext::lua::script( const uf::stl::string& filename, pod::LuaScript& script ) { if ( !ext::lua::enabled ) return; + std::lock_guard lock(::mutex); script.file = filename; script.env = sol::environment( ext::lua::state, sol::create, ext::lua::state.globals() ); } diff --git a/engine/src/ext/opengl/commands.cpp b/engine/src/ext/opengl/commands.cpp index ab9ef805..bc47236b 100644 --- a/engine/src/ext/opengl/commands.cpp +++ b/engine/src/ext/opengl/commands.cpp @@ -463,20 +463,36 @@ void ext::opengl::CommandBuffer::drawIndexed( const ext::opengl::CommandBuffer:: ::shadowState.projectionDirty = false; } - if ( drawInfo.blend.modeAlpha > 0 ) { - if ( !::shadowState.alphaTestEnabled ) { - GL_ERROR_CHECK(glEnable(GL_ALPHA_TEST)); - ::shadowState.alphaTestEnabled = true; - } - if ( ::shadowState.alphaCutoff != drawInfo.blend.alphaCutoff ) { - GL_ERROR_CHECK(glAlphaFunc(GL_GREATER, drawInfo.blend.alphaCutoff)); - ::shadowState.alphaCutoff = drawInfo.blend.alphaCutoff; + bool needsBlend = drawInfo.blend.modeAlpha > 0; + // this might not be necessary +#if UF_ENV_DREAMCAST + if ( drawInfo.textures.secondary.image && drawInfo.attributes.st.pointer ) needsBlend = true; +#endif + + if ( needsBlend ) { + if ( drawInfo.blend.modeAlpha > 0 ) { + if ( !::shadowState.alphaTestEnabled ) { + GL_ERROR_CHECK(glEnable(GL_ALPHA_TEST)); + ::shadowState.alphaTestEnabled = true; + } + if ( ::shadowState.alphaCutoff != drawInfo.blend.alphaCutoff ) { + GL_ERROR_CHECK(glAlphaFunc(GL_GREATER, drawInfo.blend.alphaCutoff)); + ::shadowState.alphaCutoff = drawInfo.blend.alphaCutoff; + } + } else { + if ( ::shadowState.alphaTestEnabled ) { + GL_ERROR_CHECK(glDisable(GL_ALPHA_TEST)); + ::shadowState.alphaTestEnabled = false; + } } + if ( !::shadowState.blendEnabled ) { GL_ERROR_CHECK(glEnable(GL_BLEND)); - GL_ERROR_CHECK(glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA)); ::shadowState.blendEnabled = true; } + + GL_ERROR_CHECK(glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA)); + } else { if ( ::shadowState.alphaTestEnabled ) { GL_ERROR_CHECK(glDisable(GL_ALPHA_TEST)); @@ -543,7 +559,6 @@ void ext::opengl::CommandBuffer::drawIndexed( const ext::opengl::CommandBuffer:: GL_ERROR_CHECK(glColor4f( color[0], color[1], color[2], color[3] )); ::shadowState.color = color; } - GLenum vertexType = GL_FLOAT; switch ( drawInfo.attributes.position.descriptor.type ) { @@ -665,6 +680,7 @@ void ext::opengl::CommandBuffer::drawIndexed( const ext::opengl::CommandBuffer:: GL_ERROR_CHECK(glDisableClientState(GL_COLOR_ARRAY)); ::shadowState.colorArrayEnabled = false; } + if ( drawInfo.textures.primary.image && drawInfo.attributes.uv.pointer ) { GL_ERROR_CHECK(glClientActiveTexture(GL_TEXTURE0)); GL_ERROR_CHECK(glActiveTexture(GL_TEXTURE0)); @@ -690,7 +706,6 @@ void ext::opengl::CommandBuffer::drawIndexed( const ext::opengl::CommandBuffer:: GLenum texEnv = (drawInfo.attributes.color.pointer || drawInfo.color.enabled) ? GL_MODULATE : GL_REPLACE; GL_ERROR_CHECK(glTexCoordPointer(2, uvType, uvStride, uvPtr)); GL_ERROR_CHECK(glTexEnvi(GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, texEnv)); - //GL_ERROR_CHECK(glTexEnvi(GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, drawInfo.attributes.color.pointer ? GL_MODULATE : GL_REPLACE)); } else { if ( ::shadowState.tex0Enabled ) { GL_ERROR_CHECK(glClientActiveTexture(GL_TEXTURE0)); @@ -743,6 +758,12 @@ void ext::opengl::CommandBuffer::drawIndexed( const ext::opengl::CommandBuffer:: } } + // this probably isn't necessary +#if UF_ENV_DREAMCAST + GL_ERROR_CHECK(glClientActiveTexture(GL_TEXTURE0)); + GL_ERROR_CHECK(glActiveTexture(GL_TEXTURE0)); +#endif + { GL_ERROR_CHECK(glEnableClientState(GL_VERTEX_ARRAY)); GL_ERROR_CHECK(glVertexPointer(3, vertexType, vertexStride, vertexPtr)); diff --git a/engine/src/ext/opengl/graphic.cpp b/engine/src/ext/opengl/graphic.cpp index baa3b4d7..e9f67a36 100644 --- a/engine/src/ext/opengl/graphic.cpp +++ b/engine/src/ext/opengl/graphic.cpp @@ -486,7 +486,9 @@ void ext::opengl::Graphic::record( CommandBuffer& commandBuffer, const GraphicDe auto& material = materials[materialID]; auto textureID = material.indexAlbedo; - auto& infos = pool[textureID]; + uint32_t drawHash = ( (uint32_t)(textureID & 0xFFFF) ) | ( ((uint32_t)(lightmapID & 0xFFFF)) << 16 ); + + auto& infos = pool[drawHash]; CommandBuffer::InfoDraw& drawCommandInfo = infos.emplace_back( drawCommandInfoBase ); // CommandBuffer::InfoDraw drawCommandInfo = drawCommandInfoBase; @@ -511,6 +513,7 @@ void ext::opengl::Graphic::record( CommandBuffer& commandBuffer, const GraphicDe drawCommandInfo.color.enabled = drawCommandInfo.color.value != pod::Vector4f{1.0f, 1.0f, 1.0f, 1.0f}; if ( drawCommandInfo.color.value.w == 0.0f ) { + infos.pop_back(); continue; } @@ -526,7 +529,11 @@ void ext::opengl::Graphic::record( CommandBuffer& commandBuffer, const GraphicDe if ( drawCommandInfo.blend.modeAlpha == pod::Material::AlphaMode::BLEND ) { drawCommandInfo.descriptor.renderTarget = 1; } - commandBuffer.record( drawCommandInfo ); + } + } + for ( auto& [ drawHash, infos ] : pool ) { + for ( auto& info : infos ) { + commandBuffer.record( info ); } } } else {