From e3ae6d6b85493e072eaf16876decef37673c7197 Mon Sep 17 00:00:00 2001 From: ecker Date: Thu, 6 Aug 2026 20:45:41 -0500 Subject: [PATCH] mesh streaming works with interleaved meshes, overhauled mesh streaming to work asynchronously (ironically only x86_64 OpenGL works well with it), reverted dtex conversion to 32x32 because any larger size doesn't actually work --- bin/data/config.json | 4 +- bin/data/entities/model.json | 2 +- bin/dreamcast/data/config.json | 4 +- engine/src/engine/ext/ext.cpp | 24 +- engine/src/engine/graph/decode.cpp | 2 - engine/src/engine/graph/encode.cpp | 4 +- engine/src/engine/graph/graph.cpp | 746 +++++++++++++++-------------- engine/src/ext/lz4/lz4.cpp | 13 +- engine/src/ext/zlib/zlib.cpp | 13 +- engine/src/utils/io/file.cpp | 18 +- engine/src/utils/io/vfs.cpp | 5 +- engine/src/utils/thread/thread.cpp | 9 + 12 files changed, 456 insertions(+), 388 deletions(-) diff --git a/bin/data/config.json b/bin/data/config.json index d8ec805b..587f0f86 100644 --- a/bin/data/config.json +++ b/bin/data/config.json @@ -424,8 +424,8 @@ "auto validate": false }, "loader": { - "assert": false, - "async": false // to-do: fix + "assert": "auto", + "async": "auto" // unironically works on opengl, not on vulkan }, "hooks": { "defer lazy calls": true diff --git a/bin/data/entities/model.json b/bin/data/entities/model.json index cf98d35d..2373e157 100644 --- a/bin/data/entities/model.json +++ b/bin/data/entities/model.json @@ -81,7 +81,7 @@ "player": "info_player_start", "enabled": "auto", "radius": 50, - "every": 4 + "every": 1 } } } diff --git a/bin/dreamcast/data/config.json b/bin/dreamcast/data/config.json index 1fb40fc1..4e9a42d5 100644 --- a/bin/dreamcast/data/config.json +++ b/bin/dreamcast/data/config.json @@ -148,8 +148,8 @@ "auto validate": false }, "loader": { - "assert": true, - "async": false + "assert": "auto", + "async": "auto" }, "hooks": { "defer lazy calls": true diff --git a/engine/src/engine/ext/ext.cpp b/engine/src/engine/ext/ext.cpp index b194dea9..6e25c157 100644 --- a/engine/src/engine/ext/ext.cpp +++ b/engine/src/engine/ext/ext.cpp @@ -193,9 +193,27 @@ void UF_API uf::load( ext::json::Value& json ) { uf::Entity::deleteChildrenOnDestroy = configEngineDebugJson["entity"]["delete children on destroy"].as( uf::Entity::deleteChildrenOnDestroy ); uf::Entity::deleteComponentsOnDestroy = configEngineDebugJson["entity"]["delete components on destroy"].as( uf::Entity::deleteComponentsOnDestroy ); - uf::Object::assertionLoad = configEngineDebugJson["loader"]["assert"].as( uf::Object::assertionLoad ); - uf::asset::assertionLoad = configEngineDebugJson["loader"]["assert"].as( uf::asset::assertionLoad ); - uf::asset::asyncQueue = configEngineDebugJson["loader"]["async"].as( uf::asset::asyncQueue ); + // debug builds should signal a missing asset with a crash, release should just gracefully emit an error + if ( configEngineDebugJson["loader"]["assert"].as() == "auto" ) { + #if UF_DEBUG + uf::Object::assertionLoad = true; + uf::asset::assertionLoad = true; + #else + uf::Object::assertionLoad = false; + uf::asset::assertionLoad = false; + #endif + } else { + uf::Object::assertionLoad = configEngineDebugJson["loader"]["assert"].as( uf::Object::assertionLoad ); + uf::asset::assertionLoad = configEngineDebugJson["loader"]["assert"].as( uf::asset::assertionLoad ); + } + if ( configEngineDebugJson["loader"]["async"].as() == "auto" ) { + uf::asset::asyncQueue = false; // to-do: fix + #if UF_USE_OPENGL && !UF_ENV_DREAMCAST + uf::asset::asyncQueue = true; + #endif + } else { + uf::asset::asyncQueue = configEngineDebugJson["loader"]["async"].as( uf::asset::asyncQueue ); + } uf::userdata::autoDestruct = configEngineDebugJson["userdata"]["auto destruct"].as( uf::userdata::autoDestruct ); uf::userdata::autoValidate = configEngineDebugJson["userdata"]["auto validate"].as( uf::userdata::autoValidate ); diff --git a/engine/src/engine/graph/decode.cpp b/engine/src/engine/graph/decode.cpp index ef4e2384..1d232401 100644 --- a/engine/src/engine/graph/decode.cpp +++ b/engine/src/engine/graph/decode.cpp @@ -536,8 +536,6 @@ void uf::graph::load( pod::Graph& graph, const uf::stl::string& filename, const bool preferMinified = false; #endif - if ( graph.settings.stream.enabled ) preferMinified = false; - ext::json::forEach( serializer["meshes"], [&]( ext::json::Value& value ){ auto name = key + value["name"].as(); diff --git a/engine/src/engine/graph/encode.cpp b/engine/src/engine/graph/encode.cpp index 4d2a2ac4..8762538f 100644 --- a/engine/src/engine/graph/encode.cpp +++ b/engine/src/engine/graph/encode.cpp @@ -429,8 +429,8 @@ uf::stl::string uf::graph::save( const pod::Graph& graph, const uf::stl::string& #if UF_USE_DC_TEXCONV auto img = uf::Image(image); - pod::Vector2ui size = { 64, 64 }; - uf::stl::string filter = "linear"; // "nearest" + pod::Vector2ui size = { 32, 32 }; + uf::stl::string filter = "nearest"; uf::stl::string dtexFormat = "ARGB4444"; if ( name.starts_with("lightmap") || img.getFormat() == uf::renderer::enums::Format::R8G8B8A8_RGBE ) { size = { 128, 128 }; diff --git a/engine/src/engine/graph/graph.cpp b/engine/src/engine/graph/graph.cpp index 71934b45..2559e5c7 100644 --- a/engine/src/engine/graph/graph.cpp +++ b/engine/src/engine/graph/graph.cpp @@ -101,16 +101,43 @@ namespace { shader.metadata.aliases.buffers.clear(); } - void bindTextures( pod::Graph& graph, uf::renderer::Graphic& graphic ) { - graphic.material.textures.clear(); - + bool bindTextures( pod::Graph& graph, uf::renderer::Graphic& graphic ) { auto& storage = uf::graph::getStorage( graph ); + bool changed = false; + size_t activeCount = 0; for ( auto& key : storage.images.keys ) { auto& texture = storage.images.map[key].handle; if ( texture.viewType != uf::renderer::enums::Image::VIEW_TYPE_2D ) continue; - graphic.material.textures.emplace_back().aliasTexture( texture ); + activeCount++; } + + if ( graphic.material.textures.size() != activeCount ) { + changed = true; + } else { + size_t idx = 0; + for ( auto& key : storage.images.keys ) { + auto& texture = storage.images.map[key].handle; + if ( texture.viewType != uf::renderer::enums::Image::VIEW_TYPE_2D ) continue; + + if ( graphic.material.textures[idx].image != texture.image ) { + changed = true; + break; + } + idx++; + } + } + + if ( changed ) { + graphic.material.textures.clear(); + for ( auto& key : storage.images.keys ) { + auto& texture = storage.images.map[key].handle; + if ( texture.viewType != uf::renderer::enums::Image::VIEW_TYPE_2D ) continue; + graphic.material.textures.emplace_back().aliasTexture( texture ); + } + } + + return changed; } void bindShaders( pod::Graph& graph, uf::Object& entity, uf::Mesh& mesh, uf::stl::vector& primitives ) { @@ -746,7 +773,6 @@ void uf::graph::initializeGraphics( pod::Graph& graph, uf::Object& entity, uf::M graphic.initialize(); graphic.initializeMesh( mesh ); - graphic.device = &uf::renderer::device; graphic.material.device = &uf::renderer::device; graphic.descriptor.frontFace = graphMetadataJson["renderer"]["invert"].as(true) ? uf::renderer::enums::Face::CW : uf::renderer::enums::Face::CCW; @@ -806,6 +832,7 @@ void uf::graph::initializeGraphics( pod::Graph& graph, uf::Object& entity, uf::M ::bindAddresses( graph, graphic, mesh, primitives ); graphic.process = true; + storage.stale = true; } void uf::graph::process( pod::Graph& graph ) { @@ -1088,6 +1115,7 @@ void uf::graph::process( pod::Graph& graph ) { auto& childTransform = child.getComponent>(); auto flatten = uf::transform::flatten( node.transform ); + flatten.position.y += 1; childTransform = flatten; graph.settings.stream.player = spawnID; @@ -1287,46 +1315,6 @@ void uf::graph::process( pod::Graph& graph ) { } } } -/* - if ( graphMetadataJson["debug"]["print"]["lights"].as() ) { - UF_MSG_DEBUG("Lights: {}", graph.lights.size()); - for ( auto& pair : graph.lights ) { - UF_MSG_DEBUG("\tLight: {}", pair.first); - } - } - if ( graphMetadataJson["debug"]["print"]["meshes"].as() ) { - UF_MSG_DEBUG("Meshs: {}", graph.meshes.size()); - for ( auto& name : graph.meshes ) { - UF_MSG_DEBUG("\tMesh: {}", name); - } - } - if ( graphMetadataJson["debug"]["print"]["materials"].as() ) { - UF_MSG_DEBUG("Materials: {}", graph.materials.size()); - for ( auto& name : graph.materials ) { - auto& material = storage.materials[name]; - UF_MSG_DEBUG("\tMaterial: {} | {}", name, material.indexAlbedo); - } - } - if ( graphMetadataJson["debug"]["print"]["textures"].as() ) { - UF_MSG_DEBUG("Textures: {}", graph.textures.size()); - for ( auto& name : graph.textures ) { - auto& texture = storage.textures[name]; - UF_MSG_DEBUG("\tTexture: {} | {}", name, texture.index); - } - } - if ( graphMetadataJson["debug"]["print"]["images"].as() ) { - UF_MSG_DEBUG("Images: {}", graph.images.size()); - for ( auto& name : graph.images ) { - UF_MSG_DEBUG("\tImage: {}", name); - } - } - if ( graphMetadataJson["debug"]["print"]["animations"].as() ) { - UF_MSG_DEBUG("Animations: {}", graph.animations.size()); - for ( auto& name : graph.animations ) { - UF_MSG_DEBUG("\tAnimation: {}", name); - } - } -*/ UF_DEBUG_TIMER_MULTITRACE("Updating master graph"); #if UF_GRAPH_EXTENDED @@ -2209,14 +2197,14 @@ void uf::graph::reload( pod::Graph& graph, pod::Node& node ) { if ( exists ) { auto& graphic = entity.getComponent(); bool rebuild = graphic.updateMesh( mesh ); - // update texture descriptors - ::bindTextures( graph, graphic ); - // update buffers if any of them were resized (because my aliasing system is weak) + if ( ::bindTextures( graph, graphic ) ) { + rebuild = true; + } if ( rebuild ) { ::bindBuffers( graph, graphic, mesh ); - ::bindAddresses( graph, graphic, mesh, primitives ); uf::renderer::states::rebuild = true; + storage.stale = true; } } else { uf::graph::initializeGraphics( graph, entity, mesh, primitives ); @@ -2260,19 +2248,34 @@ void uf::graph::reload( pod::Graph& graph ) { auto& graphMetadataJson = graph.metadata; pod::Vector3f controllerPosition = {}; - auto& controller = scene.getController(); { + auto& controller = scene.getController(); + + // bind if there's a non-scene controller + if ( controller.getUid() != scene.getUid() ) { auto& controllerTransform = controller.getComponent>(); controllerPosition = controllerTransform.position; + // fallback to spawn node + } else if ( 0 <= graph.settings.stream.player ) { + auto& node = graph.nodes[graph.settings.stream.player]; + controllerPosition = node.transform.position; } uf::stl::unordered_map textureDescriptors; + uf::stl::unordered_map> textureDependentNodes; + + struct PendingMeshWork { + bool needsFullLoad = false; + bool needsSparseUpdate = false; + bool needsFallbackBinding = false; + uf::stl::vector queuedLODs; + uf::stl::vector dependentNodes; + }; + uf::stl::unordered_map pendingMeshes; - uf::stl::unordered_set pendingMeshNodes; - uf::stl::unordered_set pendingTextureNodes; - uf::stl::unordered_map pendingTextures; - uf::stl::unordered_map pendingMeshes; auto oldHash = graph.settings.stream.hash; + auto newHash = oldHash; + // populate work for ( auto& node : graph.nodes ) { if ( !(0 <= node.mesh && node.mesh < graph.meshes.size()) ) continue; if ( !node.entity ) continue; @@ -2282,286 +2285,115 @@ void uf::graph::reload( pod::Graph& graph ) { float radiusSquared = radius * radius; auto& entity = node.entity->as(); - - auto& metadata = entity.getComponent(); - auto& metadataJson = entity.getComponent(); auto& transform = entity.getComponent>(); - auto model = uf::transform::model( transform ); auto meshName = graph.meshes[node.mesh]; auto& mesh = storage.meshes.map[meshName]; auto& meshStream = graph.streams.meshes[meshName]; auto& primitives = storage.primitives.map[graph.primitives[node.mesh]]; - auto tag = ext::json::find( node.name, graphMetadataJson["tags"] ); - // disable if not tagged for streaming - if ( node.index == graph.settings.stream.world ) { - isStreamable = true; + if ( node.index == graph.settings.stream.world ) isStreamable = true; + if ( !isStreamable || meshStream.buffers.empty() ) radius = 0; + + bool needsGraphicBinding = !entity.hasComponent(); + auto& work = pendingMeshes[node.mesh]; + work.dependentNodes.push_back(node.index); + + if ( work.queuedLODs.empty() && primitives.size() > 0 ) { + work.queuedLODs.resize( primitives.size(), -1 ); } - if ( !isStreamable ) { - radius = 0; - } - - if ( meshStream.buffers.empty() ) { - radius = 0; - } - - // force update if entity isn't already bound to the graphic - if ( !entity.hasComponent() ) { - pendingMeshes[node.mesh] = {}; - pendingMeshNodes.insert( node.index ); - } - - uf::stl::unordered_set processedBuffers; if ( radius > 0 && mesh.indirect.count && mesh.indirect.count <= primitives.size() ) { - // deduce draw command (indirect) buffer to write to - auto& attribute = mesh.indirect.attributes.front(); - auto& buffer = mesh.buffers[attribute.buffer]; - pod::DrawCommand* drawCommands = (pod::DrawCommand*) buffer.data(); - uf::stl::vector queuedLODs( primitives.size(), -1 ); // this is to maintain draw command order because apparently my code requires draw commands to stay in order - // fallbacks for when no draw calls are requested (mainly for the collision mesh) float closestDistance = std::numeric_limits::max(); size_t closestDrawID = 0; bool found = false; - // iterate through meshlets and cull if out of radius for ( size_t drawID = 0; drawID < primitives.size(); ++drawID ) { auto& primitive = primitives[drawID]; - auto& instance = primitive.instance; - auto& drawCommand = primitive.drawCommand; + pod::Vector3f center = uf::matrix::multiply( model, primitive.instance.bounds.center, 1.0f ); + float distanceSquared = uf::vector::distanceSquared( center, controllerPosition ); - pod::Vector3f center = uf::matrix::multiply( model, instance.bounds.center, 1.0f ); // transform the center of the draw call - float distanceSquared = uf::vector::distanceSquared( center, controllerPosition ); // saves a sqrt() - - // store closest draw call if ( distanceSquared < closestDistance ) { closestDistance = distanceSquared; closestDrawID = drawID; } - // queue if we're within the radius + if ( distanceSquared <= radiusSquared ) { found = true; - int8_t lodLevel = 0; - // deduce a simple ratio [0.0 to 1.0] of how far we are into the streaming radius float distRatio = distanceSquared / radiusSquared; - if ( distRatio > 0.6f ) lodLevel = 3; + if ( distRatio > 0.6f ) lodLevel = 3; else if ( distRatio > 0.3f ) lodLevel = 2; else if ( distRatio > 0.1f ) lodLevel = 1; - while ( lodLevel > 0 && primitive.lod.levels[lodLevel].indices == 0 ) { - lodLevel--; - } + while ( lodLevel > 0 && primitive.lod.levels[lodLevel].indices == 0 ) lodLevel--; - queuedLODs[drawID] = lodLevel; + if ( work.queuedLODs[drawID] < lodLevel ) { + work.queuedLODs[drawID] = lodLevel; + } } } - // insert closest primitive if all are out of range (because of cringe logic) - if ( !found /*&& node.index == graph.settings.stream.world*/ ) { - queuedLODs[closestDrawID] = 3; + if ( !found ) { + if ( work.queuedLODs[closestDrawID] < 3 ) work.queuedLODs[closestDrawID] = 3; } - // bail if no update is detected - auto drawCommandHash = uf::algo::fnv1a(queuedLODs); + auto drawCommandHash = uf::algo::fnv1a(work.queuedLODs); graph.settings.stream.lastUpdate = uf::physics::time::current; - if ( drawCommandHash == oldHash ) { - continue; - } - graph.settings.stream.hash = drawCommandHash; - auto& pending = pendingMeshes[node.mesh]; - pendingMeshNodes.insert( node.index ); - - // read from disk - #if UF_GRAPH_SPARSE_READ_MESH - uint32_t currentVertexCount = 0; - uint32_t currentIndexCount = 0; - - struct ActiveDraw { - size_t drawID; - int8_t lodLevel; - uint32_t fileVertexID; - }; - - /*static thread_local*/ uf::stl::unordered_map bufferSizes; /*bufferSizes.clear();*/ - /*static thread_local*/ uf::stl::unordered_map bufferWriteOffsets; /*bufferWriteOffsets.clear();*/ - - STATIC_THREAD_LOCAL(uf::stl::vector, activeDraws); - activeDraws.reserve(queuedLODs.size()); - - for ( size_t drawID = 0; drawID < queuedLODs.size(); ++drawID ) { - auto lodLevel = queuedLODs[drawID]; - if ( lodLevel >= 0 ) { - auto& lod = primitives[drawID].lod.levels[lodLevel]; - activeDraws.emplace_back(ActiveDraw{drawID, lodLevel, lod.vertexID}); - } else { - auto& drawCommand = drawCommands[drawID]; - drawCommand.vertices = 0; - drawCommand.indices = 0; - drawCommand.vertexID = 0; - drawCommand.indexID = 0; - primitives[drawID].drawCommand = drawCommand; - } + if ( drawCommandHash != oldHash ) { + work.needsSparseUpdate = true; + newHash = drawCommandHash; + } else if ( needsGraphicBinding ) { + work.needsFallbackBinding = true; } - std::sort(activeDraws.begin(), activeDraws.end(), [](const ActiveDraw& a, const ActiveDraw& b) { - return a.fileVertexID < b.fileVertexID; - }); - - for ( auto& active : activeDraws ) { - auto& lod = primitives[active.drawID].lod.levels[active.lodLevel]; - for ( auto& attr : mesh.index.attributes ) { - size_t stride = attr.stride > 0 ? attr.stride : attr.descriptor.size; - bufferSizes[attr.buffer] += lod.indices * stride; - } - for ( auto& attr : mesh.vertex.attributes ) { - size_t stride = attr.stride > 0 ? attr.stride : attr.descriptor.size; - bufferSizes[attr.buffer] += lod.vertices * stride; - } - } - - auto& buffers = mesh.buffers; // to-do: probably deduce when to mesh.buffers vs pending.buffers - for ( auto& [ b, size ] : bufferSizes ) buffers[b].resize( size ); - - for ( auto& active : activeDraws ) { - auto& primitive = primitives[active.drawID]; - auto& lod = primitive.lod.levels[active.lodLevel]; - auto& drawCommand = drawCommands[active.drawID]; - - drawCommand.vertices = lod.vertices; - drawCommand.indices = lod.indices; - drawCommand.vertexID = currentVertexCount; - drawCommand.indexID = currentIndexCount; - primitive.drawCommand = drawCommand; - - for ( auto& attr : mesh.index.attributes ) { - size_t stride = attr.stride > 0 ? attr.stride : attr.descriptor.size; - auto& region = meshStream.buffers[attr.buffer]; - size_t readBytes = lod.indices * stride; - - uf::asset::read( region.filename, region.offset + attr.offset + (lod.indexID * stride), readBytes, buffers[attr.buffer].data() + bufferWriteOffsets[attr.buffer] ); - bufferWriteOffsets[attr.buffer] += readBytes; - } - - for ( auto& attr : mesh.vertex.attributes ) { - size_t stride = attr.stride > 0 ? attr.stride : attr.descriptor.size; - auto& region = meshStream.buffers[attr.buffer]; - size_t readBytes = lod.vertices * stride; - - uf::asset::read( region.filename, region.offset + attr.offset + (lod.vertexID * stride), readBytes, buffers[attr.buffer].data() + bufferWriteOffsets[attr.buffer] ); - bufferWriteOffsets[attr.buffer] += readBytes; - } - - currentVertexCount += lod.vertices; - currentIndexCount += lod.indices; - } - - mesh.vertex.count = currentVertexCount; - mesh.index.count = currentIndexCount; - - for ( auto& attr : mesh.vertex.attributes ) attr.offset = 0; - for ( auto& attr : mesh.index.attributes ) attr.offset = 0; - - // keep the vertex data intact - #else - // disable remaining draw commands - for ( auto drawID = 0; drawID < primitives.size(); ++drawID ) { - int8_t lodLevel = queuedLODs[drawID]; - // reset from LOD0 - //primitives[drawID].drawCommand.instances = 1; - primitives[drawID].drawCommand.indices = primitives[drawID].lod.levels[0].indices; - primitives[drawID].drawCommand.indexID = primitives[drawID].lod.levels[0].indexID; - primitives[drawID].drawCommand.vertexID = primitives[drawID].lod.levels[0].vertexID; - primitives[drawID].drawCommand.vertices = primitives[drawID].lod.levels[0].vertices; - - // copy from primitive - drawCommands[drawID] = primitives[drawID].drawCommand; - - if ( lodLevel < 0 ) { - //drawCommands[drawID].instances = 0; - drawCommands[drawID].vertices = 0; - drawCommands[drawID].indices = 0; - drawCommands[drawID].indexID = 0; - drawCommands[drawID].vertexID = 0; - } else { - auto& lod = primitives[drawID].lod.levels[lodLevel]; - - drawCommands[drawID].indexID = lod.indexID; - drawCommands[drawID].indices = lod.indices; - drawCommands[drawID].vertexID = lod.vertexID; - drawCommands[drawID].vertices = lod.vertices; - } - - // synchronize primitive - primitives[drawID].drawCommand = drawCommands[drawID]; - } - - #define STREAM_MESH_DATA( N ) \ - for ( auto& attribute : mesh.N.attributes ) {\ - if ( !mesh.buffers[attribute.buffer].empty() || meshStream.buffers.empty() ) continue;\ - auto& region = meshStream.buffers[attribute.buffer];\ - mesh.buffers[attribute.buffer].resize( region.length );\ - uf::asset::read( region.filename, region.offset, region.length, mesh.buffers[attribute.buffer].data() );\ - } - - STREAM_MESH_DATA( index ); - STREAM_MESH_DATA( vertex ); - #endif } else { - // load mesh if not already loaded - #define LOAD_MESH_DATA( N ) \ - for ( auto& attribute : mesh.N.attributes ) {\ - if ( processedBuffers.count(attribute.buffer) ) continue; \ - if ( !mesh.buffers[attribute.buffer].empty() || meshStream.buffers.empty() ) continue;\ - pendingMeshes[node.mesh] = {};\ - pendingMeshNodes.insert( node.index );\ - processedBuffers.insert( attribute.buffer ); \ - auto& region = meshStream.buffers[attribute.buffer];\ - mesh.buffers[attribute.buffer].resize( region.length );\ - uf::asset::read( region.filename, region.offset, region.length, mesh.buffers[attribute.buffer].data() );\ - } + bool buffersEmpty = false; + for ( auto& attribute : mesh.index.attributes ) { + if ( mesh.buffers[attribute.buffer].empty() && !meshStream.buffers.empty() ) buffersEmpty = true; + } + for ( auto& attribute : mesh.vertex.attributes ) { + if ( mesh.buffers[attribute.buffer].empty() && !meshStream.buffers.empty() ) buffersEmpty = true; + } - LOAD_MESH_DATA( index ); - LOAD_MESH_DATA( vertex ); + if ( buffersEmpty ) work.needsFullLoad = true; + else if ( needsGraphicBinding ) work.needsFallbackBinding = true; } + // gather textures if ( graph.settings.stream.textures ) { #define INCREMENT_TEXTURE_REFCOUNT( ID, isSRGB ) if ( 0 <= ID && ID < graph.textures.size() ) {\ auto& key = graph.textures[ID];\ textureDescriptors[ID].srgb = isSRGB;\ textureDescriptors[ID].references += visible ? 1 : 0;\ textureDescriptors[ID].layers = 1;\ - auto& handle = storage.images[key].handle;\ - if ( visible && (!handle.generated() || handle.aliased) ) pendingTextureNodes.insert(node.index);\ - else if ( !visible && (handle.generated() && !handle.aliased) ) pendingTextureNodes.insert(node.index);\ + textureDependentNodes[ID].push_back(node.index);\ } for ( size_t drawID = 0; drawID < primitives.size(); ++drawID ) { auto& primitive = primitives[drawID]; - auto& instance = primitive.instance; - bool visible = primitive.drawCommand.instances > 0; - INCREMENT_TEXTURE_REFCOUNT(instance.lightmapID, false); - if ( !(0 <= instance.materialID && instance.materialID < graph.materials.size()) ) { - continue; + INCREMENT_TEXTURE_REFCOUNT(primitive.instance.lightmapID, false); + if ( 0 <= primitive.instance.materialID && primitive.instance.materialID < graph.materials.size() ) { + auto& material = storage.materials[graph.materials[primitive.instance.materialID]]; + INCREMENT_TEXTURE_REFCOUNT(material.indexAlbedo, true); + INCREMENT_TEXTURE_REFCOUNT(material.indexNormal, true); + INCREMENT_TEXTURE_REFCOUNT(material.indexEmissive, true); + INCREMENT_TEXTURE_REFCOUNT(material.indexOcclusion, true); + INCREMENT_TEXTURE_REFCOUNT(material.indexMetallicRoughness, true); } - auto& material = storage.materials[graph.materials[instance.materialID]]; - INCREMENT_TEXTURE_REFCOUNT(material.indexAlbedo, true); - INCREMENT_TEXTURE_REFCOUNT(material.indexNormal, true); - INCREMENT_TEXTURE_REFCOUNT(material.indexEmissive, true); - INCREMENT_TEXTURE_REFCOUNT(material.indexOcclusion, true); - INCREMENT_TEXTURE_REFCOUNT(material.indexMetallicRoughness, true); } - #undef INCREMENT_TEXTURE_REFCOUNT } } - // iterate through our ref counts + graph.settings.stream.hash = newHash; + storage.stale = true; + + // dispatch texture loads for ( auto& [ imageID, descriptor ] : textureDescriptors ) { auto& key = graph.images[imageID]; auto& image = storage.images[key].data; @@ -2569,17 +2401,77 @@ void uf::graph::reload( pod::Graph& graph ) { bool visible = descriptor.references > 0; if ( visible && (!texture.generated() || texture.aliased) ) { - auto& pending = pendingTextures[imageID]; + if ( !image.getPixels().empty() ) continue; + auto& imgStream = graph.streams.images[key]; + size_t readLen = imgStream.buffer.length > 0 ? imgStream.buffer.length : uf::io::size(imgStream.buffer.filename); - if ( image.getPixels().empty() ) { - auto& imgStream = graph.streams.images[key]; + if ( readLen <= 0 ) continue; + texture.layers = descriptor.layers; + texture.srgb = descriptor.srgb; - size_t readLen = imgStream.buffer.length > 0 ? imgStream.buffer.length : uf::io::size(imgStream.buffer.filename); - if ( readLen > 0 ) { - pending.buffer.resize( readLen ); - uf::asset::read( imgStream.buffer.filename, imgStream.buffer.offset, readLen, pending.buffer.data() ); - } + auto filter = uf::renderer::enums::Filter::LINEAR; + auto tag = ext::json::find( key, graphMetadataJson["tags"] ); + if ( !ext::json::isObject( tag ) ) tag["renderer"] = graphMetadataJson["renderer"]; + + if ( tag["renderer"]["filter"].is() ) { + const auto mode = uf::string::lowercase( tag["renderer"]["filter"].as("linear") ); + if ( mode == "linear" ) filter = uf::renderer::enums::Filter::LINEAR; + else if ( mode == "nearest" ) filter = uf::renderer::enums::Filter::NEAREST; } + + texture.sampler.descriptor.filter.min = filter; + texture.sampler.descriptor.filter.mag = filter; + + uf::asset::read( imgStream.buffer.filename, imgStream.buffer.offset, readLen, + [key, graphPtr = &graph, deps = std::move(textureDependentNodes[imageID])](uf::stl::vector&& buffer) mutable { + auto& graph = *graphPtr; + auto& storage = uf::graph::getStorage( *graphPtr ); + + auto isLightmap = key.starts_with("lightmap"); + auto& image = storage.images[key].data; + auto& imgStream = graph.streams.images[key]; + + uf::stl::string formatHint = uf::io::extension(image.getFilename()); + if ( imgStream.buffer.filename.find(".dtex") != uf::stl::string::npos ) formatHint = "dtex"; + + uf::Image decodedImage; + uf::image::open( decodedImage, buffer, formatHint, false ); + + #if UF_USE_OPENGL && !UF_ENV_DREAMCAST + if ( isLightmap ) ::convertLightmap( decodedImage ); + #endif + + uf::thread::queue( uf::thread::mainThreadName, [key, graphPtr, img = std::move(decodedImage), deps = std::move(deps)]() mutable { + auto& storage = uf::graph::getStorage( *graphPtr ); + auto& texture = storage.images[key].handle; + storage.images[key].data = std::move(img); + + if ( texture.aliased ) { + texture.aliased = false; + #if UF_USE_OPENGL + texture.image = 0; + #else + texture.image = {}; texture.view = {}; + #endif + } + texture.loadFromImage( storage.images[key].data ); + + for ( int32_t nodeID : deps ) { + auto& node = graphPtr->nodes[nodeID]; + if ( node.entity && node.entity->hasComponent() ) { + if ( ::bindTextures( *graphPtr, node.entity->getComponent() ) ) { + uf::renderer::states::rebuild = true; + storage.stale = true; + } + } + } + + #if UF_ENV_DREAMCAST + storage.images[key].data.clear(); + #endif + }); + } + ); } else if ( !visible && (texture.generated() && !texture.aliased) ) { image.clear(); texture.destroy( true ); @@ -2587,97 +2479,229 @@ void uf::graph::reload( pod::Graph& graph ) { } } + // dispatch mesh streams + struct StreamUpdateState { + uint32_t vertexCount = 0; + uint32_t indexCount = 0; + struct DrawCmdUpdate { size_t drawID; pod::DrawCommand cmd; }; + uf::stl::vector updatedCommands; + }; - storage.stale = true; - uf::asset::processIO(); - - for ( auto& [ imageID, pending ] : pendingTextures ) { - auto& key = graph.images[imageID]; - auto& image = storage.images[key].data; - auto& texture = storage.images[key].handle; - auto& descriptor = textureDescriptors[imageID]; - auto& imgStream = graph.streams.images[key]; - - if ( !pending.buffer.empty() ) { - uf::stl::string formatHint = uf::io::extension(image.getFilename()); - if ( imgStream.buffer.filename.find(".dtex") != uf::stl::string::npos ) formatHint = "dtex"; - - uf::image::open( image, pending.buffer, formatHint, false ); - - #if UF_USE_OPENGL && !UF_ENV_DREAMCAST - if ( key.starts_with("lightmap") ) { - ::convertLightmap( image ); - } - #endif - pending.buffer.clear(); - } - - auto filter = uf::renderer::enums::Filter::LINEAR; - auto tag = ext::json::find( key, graphMetadataJson["tags"] ); - if ( !ext::json::isObject( tag ) ) { - tag["renderer"] = graphMetadataJson["renderer"]; - } - if ( tag["renderer"]["filter"].is() ) { - const auto mode = uf::string::lowercase( tag["renderer"]["filter"].as("linear") ); - if ( mode == "linear" ) filter = uf::renderer::enums::Filter::LINEAR; - else if ( mode == "nearest" ) filter = uf::renderer::enums::Filter::NEAREST; - } - - if ( texture.aliased ) { - texture.aliased = false; - #if UF_USE_OPENGL - texture.image = 0; - #else - texture.image = {}; - texture.view = {}; - #endif - } - - texture.sampler.descriptor.filter.min = filter; - texture.sampler.descriptor.filter.mag = filter; - texture.layers = descriptor.layers; - texture.srgb = descriptor.srgb; - - texture.loadFromImage( image ); - - #if UF_ENV_DREAMCAST - image.clear(); - #endif - } - - for ( auto& [ meshID, pending ] : pendingMeshes ) { + for ( auto& [ meshID, work ] : pendingMeshes ) { auto& key = graph.meshes[meshID]; auto& mesh = storage.meshes.map[key]; + auto& meshStream = graph.streams.meshes[key]; + auto& primitives = storage.primitives.map[graph.primitives[meshID]]; - if ( !pending.buffers.empty() ) { - for ( auto& [b, buf] : pending.buffers ) mesh.buffers[b] = std::move( buf ); + if ( work.needsSparseUpdate ) { + struct ActiveDraw { size_t drawID; int8_t lodLevel; uint32_t fileVertexID; }; + uf::stl::vector activeDraws; + activeDraws.reserve(work.queuedLODs.size()); + + struct Chunk { size_t offset; uf::stl::vector data; }; + struct SparseContext { + int32_t meshID; + uf::stl::vector deps; + + uint32_t vertexCount = 0; + uint32_t indexCount = 0; + uf::stl::vector> updatedCommands; + + uf::stl::unordered_map bufferSizes; + uf::stl::unordered_map> chunks; + + std::atomic pendingReads{0}; + }; + + auto ctx = std::make_shared(); + ctx->meshID = meshID; + ctx->deps = std::move(work.dependentNodes); + + auto& attribute = mesh.indirect.attributes.front(); + pod::DrawCommand* drawCommands = (pod::DrawCommand*) mesh.buffers[attribute.buffer].data(); + + for ( size_t drawID = 0; drawID < work.queuedLODs.size(); ++drawID ) { + auto lodLevel = work.queuedLODs[drawID]; + if ( lodLevel >= 0 ) { + activeDraws.emplace_back(ActiveDraw{drawID, lodLevel, primitives[drawID].lod.levels[lodLevel].vertexID}); + } else { + pod::DrawCommand cmd = {}; + if ( drawID < mesh.indirect.count ) cmd = drawCommands[drawID]; + cmd.vertices = 0; cmd.indices = 0; cmd.vertexID = 0; cmd.indexID = 0; + ctx->updatedCommands.push_back({drawID, cmd}); + } + } + + std::sort(activeDraws.begin(), activeDraws.end(), [](const ActiveDraw& a, const ActiveDraw& b) { + return a.fileVertexID < b.fileVertexID; + }); + + uf::stl::unordered_map uniqueVertexBuffers; + for ( auto& attr : mesh.vertex.attributes ) { + uniqueVertexBuffers[attr.buffer] = attr.stride > 0 ? attr.stride : attr.descriptor.size; + } + + uf::stl::unordered_map uniqueIndexBuffers; + for ( auto& attr : mesh.index.attributes ) { + uniqueIndexBuffers[attr.buffer] = attr.stride > 0 ? attr.stride : attr.descriptor.size; + } + + int actualReadsToPerform = 0; + for ( auto& active : activeDraws ) { + auto& lod = primitives[active.drawID].lod.levels[active.lodLevel]; + + if ( lod.indices > 0 ) { + actualReadsToPerform += uniqueIndexBuffers.size(); + for ( auto& [bufID, stride] : uniqueIndexBuffers ) { + ctx->bufferSizes[bufID] += lod.indices * stride; + } + } + if ( lod.vertices > 0 ) { + actualReadsToPerform += uniqueVertexBuffers.size(); + for ( auto& [bufID, stride] : uniqueVertexBuffers ) { + ctx->bufferSizes[bufID] += lod.vertices * stride; + } + } + } + + ctx->pendingReads.store(actualReadsToPerform); + + auto finalizeStream = [ctx, graphPtr = &graph]() { + auto& currentGraph = *graphPtr; + auto& storage = uf::graph::getStorage(currentGraph); + auto& targetMesh = storage.meshes.map[currentGraph.meshes[ctx->meshID]]; + auto& targetPrimitives = storage.primitives.map[currentGraph.primitives[ctx->meshID]]; + + for (auto& [b, size] : ctx->bufferSizes) { + if (size == 0) continue; + targetMesh.buffers[b].resize(size); + for (auto& chunk : ctx->chunks[b]) { + std::memcpy(targetMesh.buffers[b].data() + chunk.offset, chunk.data.data(), chunk.data.size()); + } + } + + targetMesh.vertex.count = ctx->vertexCount; + targetMesh.index.count = ctx->indexCount; + + auto& indirectAttr = targetMesh.indirect.attributes.front(); + pod::DrawCommand* targetDrawCommands = (pod::DrawCommand*) targetMesh.buffers[indirectAttr.buffer].data(); + for ( auto& [drawID, cmd] : ctx->updatedCommands ) { + targetPrimitives[drawID].drawCommand = cmd; + targetDrawCommands[drawID] = cmd; + } + + targetMesh.updateDescriptor(); + for ( auto nodeID : ctx->deps ) uf::graph::reload( currentGraph, currentGraph.nodes[nodeID] ); + + #if UF_USE_OPENGL + uf::renderer::states::rebuild = true; + #endif + }; + + if ( actualReadsToPerform > 0 ) { + uf::stl::unordered_map bufferWriteOffsets; + + for ( auto& active : activeDraws ) { + auto& lod = primitives[active.drawID].lod.levels[active.lodLevel]; + + pod::DrawCommand cmd = {}; + if ( active.drawID < mesh.indirect.count ) cmd = drawCommands[active.drawID]; + cmd.vertices = lod.vertices; cmd.indices = lod.indices; + cmd.vertexID = ctx->vertexCount; cmd.indexID = ctx->indexCount; + ctx->updatedCommands.push_back({active.drawID, cmd}); + + auto dispatchInterleavedRead = [&](const auto& uniqueBuffers, uint32_t count, uint32_t id) { + if ( count == 0 ) return; + + for ( auto& [bufID, stride] : uniqueBuffers ) { + size_t readBytes = count * stride; + size_t writeOffset = bufferWriteOffsets[bufID]; + + auto cb = [ctx, finalizeStream, bufID, writeOffset](uf::stl::vector&& buffer) mutable { + uf::thread::queue( uf::thread::mainThreadName, [ctx, finalizeStream, bufID, writeOffset, data = std::move(buffer)]() mutable { + ctx->chunks[bufID].push_back({writeOffset, std::move(data)}); + if ( ctx->pendingReads.fetch_sub(1) == 1 ) finalizeStream(); + }); + }; + + auto& region = meshStream.buffers[bufID]; + + uf::asset::read( + region.filename, + region.offset + (id * stride), + readBytes, + cb + ); + bufferWriteOffsets[bufID] += readBytes; + } + }; + + dispatchInterleavedRead(uniqueIndexBuffers, lod.indices, lod.indexID); + dispatchInterleavedRead(uniqueVertexBuffers, lod.vertices, lod.vertexID); + + ctx->vertexCount += lod.vertices; + ctx->indexCount += lod.indices; + } + + } else { + uf::thread::queue( uf::thread::mainThreadName, finalizeStream ); + } } + else if ( work.needsFullLoad ) { + int buffersToLoad = 0; + uf::stl::unordered_set processedBuffers; - mesh.updateDescriptor(); + for ( auto& attr : mesh.index.attributes ) if ( !processedBuffers.count(attr.buffer) && (mesh.buffers[attr.buffer].empty() && !meshStream.buffers.empty()) ) { buffersToLoad++; processedBuffers.insert(attr.buffer); } + for ( auto& attr : mesh.vertex.attributes ) if ( !processedBuffers.count(attr.buffer) && (mesh.buffers[attr.buffer].empty() && !meshStream.buffers.empty()) ) { buffersToLoad++; processedBuffers.insert(attr.buffer); } - // necessary for OpenGL because recorded descriptors have invalidated pointers - // Vulkan doesn't care about the CPU-side mesh data - #if UF_USE_OPENGL - uf::renderer::states::rebuild = true; - #endif + if ( buffersToLoad > 0 ) { + auto pendingBufferCount = std::make_shared>(buffersToLoad); + auto newBuffers = std::make_shared>>(); - } - - for ( auto& nodeID : pendingMeshNodes ) { - auto& node = graph.nodes[nodeID]; - uf::graph::reload( graph, node ); + auto finishedCallback = std::make_shared([newBuffers, graphPtr = &graph, meshID, deps = std::move(work.dependentNodes)]() mutable { + auto& currentGraph = *graphPtr; + auto& targetMesh = uf::graph::getStorage(currentGraph).meshes.map[currentGraph.meshes[meshID]]; + + for (auto& [b, buf] : *newBuffers) targetMesh.buffers[b] = std::move(buf); + targetMesh.updateDescriptor(); + for ( auto nodeID : deps ) uf::graph::reload( currentGraph, currentGraph.nodes[nodeID] ); + }); + + processedBuffers.clear(); + auto dispatchFullRead = [&](auto& attributes) { + for ( auto& attr : attributes ) { + if ( processedBuffers.count(attr.buffer) || (!mesh.buffers[attr.buffer].empty() || meshStream.buffers.empty()) ) continue; + processedBuffers.insert( attr.buffer ); + + auto cb = [attrBuf = attr.buffer, newBufs = newBuffers, counter = pendingBufferCount, finishCb = finishedCallback](uf::stl::vector&& buffer) mutable { + uf::thread::queue( uf::thread::mainThreadName, [attrBuf, newBufs, counter, finishCb, data = std::move(buffer)]() mutable { + (*newBufs)[attrBuf] = std::move(data); + if ( counter->fetch_sub(1) == 1 ) (*finishCb)(); + }); + }; + uf::asset::read( meshStream.buffers[attr.buffer].filename, meshStream.buffers[attr.buffer].offset, meshStream.buffers[attr.buffer].length, cb ); + } + }; + + dispatchFullRead(mesh.index.attributes); + dispatchFullRead(mesh.vertex.attributes); + } else if ( work.needsFallbackBinding ) { + uf::thread::queue( uf::thread::mainThreadName, [graphPtr = &graph, deps = std::move(work.dependentNodes)]() { + for ( auto nodeID : deps ) uf::graph::reload( *graphPtr, graphPtr->nodes[nodeID] ); + }); + } + } + else if ( work.needsFallbackBinding ) { + uf::thread::queue( uf::thread::mainThreadName, [graphPtr = &graph, deps = std::move(work.dependentNodes)]() { + for ( auto nodeID : deps ) uf::graph::reload( *graphPtr, graphPtr->nodes[nodeID] ); + }); + } } - for ( auto& nodeID : pendingTextureNodes ) { - if ( pendingMeshNodes.count(nodeID) ) continue; - - auto& node = graph.nodes[nodeID]; - if ( !node.entity || !node.entity->hasComponent() ) continue; - - auto& graphic = node.entity->getComponent(); - ::bindTextures( graph, graphic ); - } +// uf::asset::processIO(); +// uf::thread::process( uf::thread::get(uf::thread::mainThreadName) ); } + void uf::graph::reload() { switch ( uf::graph::storageMode ) { case pod::Graph::Storage::SCENE: { diff --git a/engine/src/ext/lz4/lz4.cpp b/engine/src/ext/lz4/lz4.cpp index 62a72ead..62340725 100644 --- a/engine/src/ext/lz4/lz4.cpp +++ b/engine/src/ext/lz4/lz4.cpp @@ -175,17 +175,22 @@ bool ext::lz4::decompressScatter( const uf::stl::string& filename, uf::stl::vect for ( size_t i = currentReqIdx; i < requests.size(); ++i ) { auto& req = requests[i]; + + if ( req.start >= chunkEnd ) break; + if ( chunkEnd > req.start && chunkStart < req.start + req.len ) { size_t copyStart = (chunkStart < req.start) ? (req.start - chunkStart) : 0; size_t copyLen = std::min(dstSize - copyStart, (req.start + req.len) - (chunkStart + copyStart)); - size_t destOffset = (chunkStart + copyStart) - req.start; + std::memcpy(req.dest + destOffset, outBuffer + copyStart, copyLen); } - if ( chunkEnd >= req.start + req.len && i == currentReqIdx ) { - currentReqIdx++; - } } + + while ( currentReqIdx < requests.size() && chunkEnd >= requests[currentReqIdx].start + requests[currentReqIdx].len ) { + currentReqIdx++; + } + uncompressedOffset += dstSize; if ( currentReqIdx >= requests.size() ) return false; } diff --git a/engine/src/ext/zlib/zlib.cpp b/engine/src/ext/zlib/zlib.cpp index 845962c3..cf362c12 100644 --- a/engine/src/ext/zlib/zlib.cpp +++ b/engine/src/ext/zlib/zlib.cpp @@ -201,19 +201,24 @@ bool ext::zlib::decompressScatter( const uf::stl::string& filename, uf::stl::vec for ( size_t i = currentReqIdx; i < requests.size(); ++i ) { auto& req = requests[i]; + + if ( req.start >= chunkEnd ) break; + if ( chunkEnd > req.start && chunkStart < req.start + req.len ) { size_t copyStart = (chunkStart < req.start) ? (req.start - chunkStart) : 0; size_t copyLen = std::min(have - copyStart, (req.start + req.len) - (chunkStart + copyStart)); - size_t destOffset = (chunkStart + copyStart) - req.start; + std::memcpy(req.dest + destOffset, outBuffer + copyStart, copyLen); } - if ( chunkEnd >= req.start + req.len && i == currentReqIdx ) { - currentReqIdx++; - } + } + + while ( currentReqIdx < requests.size() && chunkEnd >= requests[currentReqIdx].start + requests[currentReqIdx].len ) { + currentReqIdx++; } uncompressedOffset += have; + if ( currentReqIdx >= requests.size() ) return false; } while (strm.avail_out == 0); diff --git a/engine/src/utils/io/file.cpp b/engine/src/utils/io/file.cpp index af1bb40f..eb5b1f2f 100644 --- a/engine/src/utils/io/file.cpp +++ b/engine/src/utils/io/file.cpp @@ -201,8 +201,9 @@ bool uf::io::readScatter( const uf::stl::string& filename, uf::stl::vector temp; - uf::vfs::readRange( filename, req.start, req.len, temp ); - std::memcpy(req.dest, temp.data(), temp.size()); + if ( req.len == 0 ) continue; + + if ( file.seek(file.handle, req.start, SEEK_SET) ) { + file.read(file.handle, req.dest, req.len); + } } + + file.close(file.handle); return true; } diff --git a/engine/src/utils/io/vfs.cpp b/engine/src/utils/io/vfs.cpp index fd751029..2f41fc69 100644 --- a/engine/src/utils/io/vfs.cpp +++ b/engine/src/utils/io/vfs.cpp @@ -68,7 +68,7 @@ namespace { } return false; #else - static struct stat buffer; + static thread_local struct stat buffer; return stat(path.c_str(), &buffer) == 0; #endif } @@ -81,7 +81,7 @@ namespace { } size_t vfs_mtime( pod::Mount& mount, const uf::stl::string& file ) { uf::stl::string path = mount.path + file; - static struct stat buffer; + static thread_local struct stat buffer; if ( stat(path.c_str(), &buffer) != 0 ) return 0; return buffer.st_mtime; } @@ -177,6 +177,7 @@ namespace { size_t currentOffset = 0; for (const auto& r : ranges) { + is.clear(); is.seekg(r.start, std::ios::beg); is.read((char*)(buffer.data() + currentOffset), r.len); currentOffset += static_cast(is.gcount()); diff --git a/engine/src/utils/thread/thread.cpp b/engine/src/utils/thread/thread.cpp index ff6e717e..08c6a87f 100644 --- a/engine/src/utils/thread/thread.cpp +++ b/engine/src/utils/thread/thread.cpp @@ -137,6 +137,15 @@ void uf::thread::process( pod::Thread& thread ) { if ( !uf::thread::has(thread.n STATIC_THREAD_LOCAL(pod::Thread::container_t, local_queue); STATIC_THREAD_LOCAL(pod::Thread::container_t, local_container); + // hardcoded cringe + if ( thread.name == uf::thread::mainThreadName ) { + if ( thread.queue.empty() ) return; + std::unique_lock lock(thread.mutex); + std::swap( local_queue, thread.queue ); + for ( auto& function : local_queue ) function(); + return; + } + #if UF_THREAD_METRICS uint32_t tasksThisFrame = 0; auto frameStart = std::chrono::high_resolution_clock::now();