From 2a02e083839ae2c851c345b02566ee35f4190723 Mon Sep 17 00:00:00 2001 From: ecker Date: Sat, 11 Jul 2026 17:48:07 -0500 Subject: [PATCH] fixes for streaming in meshes/textures (sped up scene loading because I was thrashing I/O with re-walking a gunzip file constantly the moment i swapped to storing all mesh data inside one file......), some other things I don't remember since it was a week ago --- bin/data/shaders/common/structs.h | 2 +- bin/data/shaders/graph/base/vert.h | 2 +- bin/data/shaders/graph/cull/comp.glsl | 12 +- .../shaders/graph/skinning/skinning.comp.glsl | 14 +- engine/inc/uf/ext/meshopt/meshopt.h | 1 + engine/inc/uf/ext/zlib/zlib.h | 1 + engine/inc/uf/utils/io/file.h | 7 + engine/inc/uf/utils/io/vfs.h | 4 + engine/inc/uf/utils/mesh/mesh.h | 5 + engine/src/engine/graph/decode.cpp | 283 ++++++-------- engine/src/engine/graph/encode.cpp | 19 +- engine/src/engine/graph/graph.cpp | 359 +++++++++--------- engine/src/ext/meshopt/meshopt.cpp | 101 +++++ engine/src/ext/opengl/commands.cpp | 79 ++-- engine/src/ext/vulkan/graphic.cpp | 32 +- .../src/ext/vulkan/rendermodes/deferred.cpp | 14 + engine/src/ext/zlib/zlib.cpp | 150 +++++--- engine/src/utils/io/file.cpp | 34 ++ engine/src/utils/io/vfs.cpp | 50 ++- engine/src/utils/mesh/mesh.cpp | 42 ++ 20 files changed, 753 insertions(+), 458 deletions(-) diff --git a/bin/data/shaders/common/structs.h b/bin/data/shaders/common/structs.h index fbfa0fe8..a93f0566 100644 --- a/bin/data/shaders/common/structs.h +++ b/bin/data/shaders/common/structs.h @@ -105,8 +105,8 @@ struct Bounds { struct LOD { uint indices; - uint vertexID; uint indexID; + uint vertexID; uint vertices; }; diff --git a/bin/data/shaders/graph/base/vert.h b/bin/data/shaders/graph/base/vert.h index 453ad413..378b22a4 100644 --- a/bin/data/shaders/graph/base/vert.h +++ b/bin/data/shaders/graph/base/vert.h @@ -26,7 +26,7 @@ layout (binding = 0) uniform Camera { Viewport viewport[6]; } camera; -layout (std140, binding = 1) readonly buffer DrawCommands { +layout (std430, binding = 1) readonly buffer DrawCommands { DrawCommand drawCommands[]; }; layout (std140, binding = 2) readonly buffer Instances { diff --git a/bin/data/shaders/graph/cull/comp.glsl b/bin/data/shaders/graph/cull/comp.glsl index 446caf14..d749dbbe 100644 --- a/bin/data/shaders/graph/cull/comp.glsl +++ b/bin/data/shaders/graph/cull/comp.glsl @@ -10,7 +10,7 @@ layout (local_size_x = 64, local_size_y = 1, local_size_z = 1) in; #define COMPUTE 1 #define QUERY_MIPMAPS 1 #define DEPTH_BIAS 0.00005 -#define FRUSTUM_CULLING 1 +#define FRUSTUM_CULLING 0 #define OCCLUSION_CULLING 0 // currently whack #define LODS 1 #define MAX_LODS 4 @@ -111,6 +111,8 @@ void main() { const DrawCommand drawCommand = drawCommands[gID]; if ( drawCommand.indices == 0 || drawCommand.vertices == 0 ) return; +// if ( drawCommand.instances > 1 ) return; + const Instance instance = instances[drawCommand.instanceID]; const Object object = objects[instance.objectID]; @@ -181,6 +183,9 @@ void main() { } } #endif +#if FRUSTUM_CULLING || OCCLUSION_CULLING + drawCommands[gID].instances = isVisible ? 1 : 0; +#endif #if LODS if ( isVisible ) { vec3 viewCenter = (camera.viewport[0].view * vec4(worldCenter, 1.0)).xyz; @@ -189,6 +194,7 @@ void main() { float projectedSize = (worldRadius * P11) / max(dist, 0.001); uint lodLevel = 0; + /* if ( projectedSize < 0.20 ) lodLevel = 1; if ( projectedSize < 0.08 ) lodLevel = 2; if ( projectedSize < 0.02 ) lodLevel = 3; @@ -196,8 +202,9 @@ void main() { while ( lodLevel > 0 && lodMetadata[drawCommand.instanceID].levels[lodLevel].indices == 0 ) { lodLevel--; } + */ - LOD lod = lodMetadata[drawCommand.instanceID].levels[lodLevel]; + LOD lod = lodMetadata[gID].levels[lodLevel]; if ( lod.indices > 0 ) { drawCommands[gID].indices = lod.indices; @@ -207,5 +214,4 @@ void main() { } } #endif - drawCommands[gID].instances = isVisible ? 1 : 0; } \ No newline at end of file diff --git a/bin/data/shaders/graph/skinning/skinning.comp.glsl b/bin/data/shaders/graph/skinning/skinning.comp.glsl index b1eeb078..f6245d7e 100644 --- a/bin/data/shaders/graph/skinning/skinning.comp.glsl +++ b/bin/data/shaders/graph/skinning/skinning.comp.glsl @@ -21,7 +21,7 @@ uvec4 uvec2_16x4( uvec2 i ) { layout (push_constant) uniform SkinningPush { uint jointID; - uint vertexOffset; + uint triangleCount; } push; layout (std140, binding = 0) readonly buffer Joints { @@ -45,18 +45,18 @@ layout (binding = 4) buffer VertexOutputPosition { void main() { const uint i = gl_GlobalInvocationID.x; - if ( i * 3 >= verticesInPos.length() || i * 3 >= verticesOutPos.length() ) return; + if ( i >= push.triangleCount || i >= push.triangleCount ) return; const vec3 inPos = vec3( verticesInPos[i * 3 + 0], verticesInPos[i * 3 + 1], verticesInPos[i * 3 + 2] ); const uvec4 inJoints = uvec2_16x4(verticesInJoints[i]); const vec4 inWeights = verticesInWeights[i]; - const mat4 skinned = inWeights.x * joints[push.jointID + int(inJoints.x)] - + inWeights.y * joints[push.jointID + int(inJoints.y)] - + inWeights.z * joints[push.jointID + int(inJoints.z)] - + inWeights.w * joints[push.jointID + int(inJoints.w)]; + vec4 inPos4 = vec4(inPos, 1.0); + vec3 outPos = (joints[push.jointID + int(inJoints.x)] * inPos4).xyz * inWeights.x + + (joints[push.jointID + int(inJoints.y)] * inPos4).xyz * inWeights.y + + (joints[push.jointID + int(inJoints.z)] * inPos4).xyz * inWeights.z + + (joints[push.jointID + int(inJoints.w)] * inPos4).xyz * inWeights.w; - const vec3 outPos = vec3(skinned * vec4(inPos, 1)); verticesOutPos[i * 3 + 0] = outPos[0]; verticesOutPos[i * 3 + 1] = outPos[1]; verticesOutPos[i * 3 + 2] = outPos[2]; diff --git a/engine/inc/uf/ext/meshopt/meshopt.h b/engine/inc/uf/ext/meshopt/meshopt.h index d4b9ec2b..307ad18c 100644 --- a/engine/inc/uf/ext/meshopt/meshopt.h +++ b/engine/inc/uf/ext/meshopt/meshopt.h @@ -10,6 +10,7 @@ namespace ext { namespace meshopt { bool UF_API optimize( uf::Mesh&, float simplify = 1.0f, size_t = SIZE_MAX, bool verbose = false ); + bool UF_API simplify( uf::Mesh&, float simplify = 1.0f ); uf::stl::vector computeLODs( size_t count, size_t maxLODs = 4, size_t minIndices = 3 ); uf::stl::vector UF_API generateLODs( uf::Mesh&, const uf::stl::vector&, bool verbose = false ); diff --git a/engine/inc/uf/ext/zlib/zlib.h b/engine/inc/uf/ext/zlib/zlib.h index 18fe6c45..51d8f899 100644 --- a/engine/inc/uf/ext/zlib/zlib.h +++ b/engine/inc/uf/ext/zlib/zlib.h @@ -26,6 +26,7 @@ namespace ext { bool UF_API decompressFromFile( uf::stl::vector&, const uf::stl::string& filename, size_t start, size_t len ); bool UF_API decompressFromFile( uf::stl::vector&, const uf::stl::string& filename, const uf::stl::vector& ranges ); bool UF_API decompressFromMemory( uf::stl::vector&, const void*, size_t, size_t ); + bool UF_API decompressScatter( const uf::stl::string& filename, uf::stl::vector& requests ); size_t UF_API compressToFile( const uf::stl::string&, const void*, size_t ); bool UF_API directory( const uf::stl::vector& buffer, uf::stl::unordered_map& entries ); diff --git a/engine/inc/uf/utils/io/file.h b/engine/inc/uf/utils/io/file.h index 7e605b95..37a7e0e1 100644 --- a/engine/inc/uf/utils/io/file.h +++ b/engine/inc/uf/utils/io/file.h @@ -10,6 +10,11 @@ namespace pod { size_t start; size_t len; }; + struct ScatterRequest { + size_t start; + size_t len; + uint8_t* dest; + }; } namespace uf { @@ -54,6 +59,8 @@ namespace uf { return buffer; } + bool UF_API readScatter( const uf::stl::string& filename, uf::stl::vector& requests ); + size_t UF_API write( const uf::stl::string& filename, const void*, size_t = SIZE_MAX ); template inline size_t write( const uf::stl::string& filename, const uf::stl::vector& buffer, size_t size = SIZE_MAX ) { return write( filename, buffer.data(), std::min( buffer.size(), size ) ); diff --git a/engine/inc/uf/utils/io/vfs.h b/engine/inc/uf/utils/io/vfs.h index f0f0aa76..e71385f2 100644 --- a/engine/inc/uf/utils/io/vfs.h +++ b/engine/inc/uf/utils/io/vfs.h @@ -25,6 +25,8 @@ namespace pod { std::function&)> readRange; std::function&, uf::stl::vector&)> readRanges; + + std::function)> stream; }; } @@ -47,6 +49,8 @@ namespace uf { bool UF_API readRange( const uf::stl::string& path, size_t start, size_t len, uf::stl::vector& buffer ); bool UF_API readRanges( const uf::stl::string& path, const uf::stl::vector& ranges, uf::stl::vector& buffer ); + + bool UF_API stream( const uf::stl::string& path, size_t chunkSize, std::function callback ); pod::Mount UF_API createDiskMount( const uf::stl::string& uri, int priority = 0 ); uf::stl::string UF_API resolveBase( const uf::stl::string& path ); diff --git a/engine/inc/uf/utils/mesh/mesh.h b/engine/inc/uf/utils/mesh/mesh.h index 069b9622..531f187c 100644 --- a/engine/inc/uf/utils/mesh/mesh.h +++ b/engine/inc/uf/utils/mesh/mesh.h @@ -261,6 +261,7 @@ namespace uf { uf::Mesh alias() const; uf::Mesh expand(); void interleave(); + void prune( const uf::stl::vector& keep ); void updateDescriptor(); @@ -389,6 +390,8 @@ namespace uf { srcBuffer.swap( dstBuffer ); + attribute.offset = 0; + attribute.stride = sizeof(To) * attribute.descriptor.components; attribute.pointer = (uint8_t*) ( srcBuffer.data() ); attribute.descriptor.type = toEnum; attribute.descriptor.size = sizeof(To) * attribute.descriptor.components; @@ -417,6 +420,8 @@ namespace uf { } } } + + updateDescriptor(); } }; } diff --git a/engine/src/engine/graph/decode.cpp b/engine/src/engine/graph/decode.cpp index e4b9c655..5e08aade 100644 --- a/engine/src/engine/graph/decode.cpp +++ b/engine/src/engine/graph/decode.cpp @@ -12,7 +12,6 @@ #include #define UF_GRAPH_LOAD_MULTITHREAD 0 -#define UF_GRAPH_EXTENDED 1 #if UF_ENV_DREAMCAST #define UF_DEBUG_TIMER_MULTITRACE_START(...) UF_TIMER_MULTITRACE_START(__VA_ARGS__) @@ -25,12 +24,20 @@ #endif namespace { + struct PendingImage { + uf::stl::string name; + uf::stl::vector buffer; + uf::stl::string extension; + size_t layers; + }; + size_t deduceFormat( const uf::stl::string& format ) { if ( format == "ARGB4444" ) return uf::renderer::enums::Format::R4G4B4A4_UNORM_PACK16; if ( format == "RGB565" ) return uf::renderer::enums::Format::R5G6B5_UNORM_PACK16; return 0; } - uf::Image decodeImage( ext::json::Value& json, pod::Graph& graph, const uf::stl::string& imageName ) { + + uf::Image decodeImage( ext::json::Value& json, pod::Graph& graph, const uf::stl::string& imageName, uf::stl::unordered_map>& scatterMap, uf::stl::vector& pendingImages ) { uf::Image image; uf::stl::string filename = ""; @@ -82,24 +89,27 @@ namespace { auto& storage = uf::graph::getStorage(graph); graph.streams.images[imageName] = { fullPath, offset, length }; } else { - uf::stl::vector buffer; - if ( length > 0 ) { - uf::io::readAsBuffer( buffer, fullPath, offset, length ); - } else { - uf::io::readAsBuffer( buffer, fullPath ); - } + pendingImages.push_back({ imageName, {}, extension, layers }); + auto& pending = pendingImages.back(); - uf::image::open( image, buffer, extension, false ); - uf::image::layers( image, layers ); + size_t readLen = length > 0 ? length : uf::io::size( fullPath ); + if ( readLen > 0 ) { + pending.buffer.resize(readLen); + scatterMap[fullPath].push_back({ + offset, + readLen, + pending.buffer.data() + }); + } } - + image.setFilename( fullPath ); image.setFormat( format ); return image; } - pod::Animation decodeAnimation( ext::json::Value& json, pod::Graph& graph, const uf::stl::string& animName, const uf::stl::vector& megaBuffer ) { + pod::Animation decodeAnimation( ext::json::Value& json, pod::Graph& graph, const uf::stl::string& animName ) { pod::Animation animation = {}; animation.name = json["name"].as(animation.name); animation.start = json["start"].as(0.0f); @@ -113,7 +123,7 @@ namespace { auto& storage = uf::graph::getStorage(graph); auto& animStream = graph.streams.animations[animName]; - ext::json::forEach( json["samplers"], [&]( ext::json::Value& value ){ + ext::json::forEach( json["samplers"], [&]( ext::json::Value& value ){ auto& sampler = animation.samplers.emplace_back(); sampler.interpolator = value["interpolator"].as(sampler.interpolator); @@ -131,13 +141,17 @@ namespace { sStream.outputs = { binPath, outputsOffset, outputsLen }; animStream.samplers.emplace_back(sStream); } else { - if (inputsLen > 0 && !megaBuffer.empty()) { + if ( inputsLen > 0 ) { + uf::stl::vector temp; + uf::io::readAsBuffer(temp, binPath, inputsOffset, inputsLen); sampler.inputs.resize(inputsCount); - memcpy(sampler.inputs.data(), megaBuffer.data() + inputsOffset, inputsLen); + memcpy(sampler.inputs.data(), temp.data(), inputsLen); } - if (outputsLen > 0 && !megaBuffer.empty()) { + if ( outputsLen > 0 ) { + uf::stl::vector temp; + uf::io::readAsBuffer(temp, binPath, outputsOffset, outputsLen); sampler.outputs.resize(outputsCount); - memcpy(sampler.outputs.data(), megaBuffer.data() + outputsOffset, outputsLen); + memcpy(sampler.outputs.data(), temp.data(), outputsLen); } } }); @@ -152,7 +166,7 @@ namespace { return animation; } - pod::Skin decodeSkin( ext::json::Value& json, pod::Graph& graph, const uf::stl::string& skinName, const uf::stl::vector& megaBuffer ) { + pod::Skin decodeSkin( ext::json::Value& json, pod::Graph& graph, const uf::stl::string& skinName ) { pod::Skin skin; skin.name = json["name"].as(skin.name); @@ -162,7 +176,7 @@ namespace { skin.joints.emplace_back( value.as() ); }); - if (json["inverseBindMatrices"].isObject()) { + if ( json["inverseBindMatrices"].isObject() ) { auto& invJson = json["inverseBindMatrices"]; size_t count = invJson["count"].as(); size_t offset = invJson["offset"].as(); @@ -175,9 +189,11 @@ namespace { if ( graph.settings.stream.enabled ) { skinStream.inverseBindMatrices = { binPath, offset, length }; } else { - if (length > 0 && !megaBuffer.empty()) { + if ( length > 0 ) { + uf::stl::vector temp; + uf::io::readAsBuffer(temp, binPath, offset, length); skin.inverseBindMatrices.resize(count); - memcpy(skin.inverseBindMatrices.data(), megaBuffer.data() + offset, length); + memcpy(skin.inverseBindMatrices.data(), temp.data(), length); } } } @@ -185,7 +201,7 @@ namespace { return skin; } - uf::Mesh decodeMesh( ext::json::Value& json, pod::Graph& graph, const uf::stl::string& meshName, const uf::stl::vector& megaBuffer ) { + uf::Mesh decodeMesh( ext::json::Value& json, pod::Graph& graph, const uf::stl::string& meshName, uf::stl::unordered_map>& scatterMap ) { uf::Mesh mesh; #define DESERIALIZE_MESH(N) {\ @@ -220,129 +236,49 @@ namespace { auto& meshStream = graph.streams.meshes[meshName]; mesh.buffers.reserve( json["buffers"].size() ); + bool deferred = graph.settings.stream.enabled; - uf::stl::vector localRegions; - localRegions.reserve( json["buffers"].size() ); - - bool defered = true; // graph.settings.stream.enabled; ext::json::forEach( json["buffers"], [&]( ext::json::Value& value ){ uf::stl::string filename; size_t offset = 0, length = 0; - if (value.isObject()) { + if ( value.isObject() ) { filename = value["filename"].as(); offset = value["offset"].as(); length = value["length"].as(); } else { filename = value.as(); } - + uf::stl::string fullPath = uf::io::directory( graph.name ) + "/" + filename; - pod::StreamRegion region = { fullPath, offset, length }; - if ( defered ) { - mesh.buffers.emplace_back(); - meshStream.buffers.push_back(region); - } else { - uf::stl::vector buf; - if (length > 0 && !megaBuffer.empty()) { - buf.assign(megaBuffer.begin() + offset, megaBuffer.begin() + offset + length); - } else if (length > 0) { - uf::io::readAsBuffer(buf, fullPath, offset, length); - } else { - uf::io::readAsBuffer(buf, fullPath); - } - mesh.buffers.emplace_back(std::move(buf)); - localRegions.push_back(region); - } + mesh.buffers.emplace_back(); + meshStream.buffers.emplace_back(pod::StreamRegion{ fullPath, offset, length }); }); - auto getRegion = [&](size_t bufferIdx) -> pod::StreamRegion { - if ( defered ) return meshStream.buffers[bufferIdx]; - return localRegions[bufferIdx]; + auto queue = [&]( auto& attributes ) { + for ( auto& attr : attributes ) { + if ( !mesh.buffers[attr.buffer].empty() ) continue; + + auto region = meshStream.buffers[attr.buffer]; + if ( region.length == 0 ) continue; + + mesh.buffers[attr.buffer].resize(region.length); + scatterMap[region.filename].push_back({ + region.offset, + region.length, + mesh.buffers[attr.buffer].data() + }); + } }; - for ( size_t i = 0; i < mesh.instance.attributes.size(); ++i ) { - auto& attr = mesh.instance.attributes[i]; - if ( !mesh.buffers[attr.buffer].empty() ) continue; - auto region = getRegion(attr.buffer); - uf::io::readAsBuffer(mesh.buffers[attr.buffer], region.filename, region.offset, region.length); - } - for ( size_t i = 0; i < mesh.indirect.attributes.size(); ++i ) { - auto& attr = mesh.indirect.attributes[i]; - if ( !mesh.buffers[attr.buffer].empty() ) continue; - auto region = getRegion(attr.buffer); - uf::io::readAsBuffer(mesh.buffers[attr.buffer], region.filename, region.offset, region.length); + queue( mesh.instance.attributes ); + queue( mesh.indirect.attributes ); + if ( !deferred ) { + queue( mesh.vertex.attributes ); + queue( mesh.index.attributes ); } - { - uf::stl::vector attributesKept = ext::json::vector(graph.metadata["decode"]["attributes"]); - - uf::stl::vector deadAttributes; - uf::stl::vector deadBuffers; - - for ( size_t i = 0; i < mesh.vertex.attributes.size(); ++i ) { - auto& attribute = mesh.vertex.attributes[i]; - if ( std::find( attributesKept.begin(), attributesKept.end(), attribute.descriptor.name ) != attributesKept.end() ) continue; - - deadAttributes.push_back(i); - deadBuffers.push_back(attribute.buffer); - } - - std::sort(deadAttributes.rbegin(), deadAttributes.rend()); - std::sort(deadBuffers.rbegin(), deadBuffers.rend()); - - for ( auto idx : deadAttributes ) { - mesh.vertex.attributes.erase(mesh.vertex.attributes.begin() + idx); - } - - for ( auto bufID : deadBuffers ) { - mesh.buffers.erase(mesh.buffers.begin() + bufID); - - if ( graph.settings.stream.enabled ) { - meshStream.buffers.erase(meshStream.buffers.begin() + bufID); - } else { - localRegions.erase(localRegions.begin() + bufID); - } - } - - auto remap_input = [&](uf::Mesh::Input& input) { - for (auto& attr : input.attributes) { - int32_t shift = 0; - for (int32_t db : deadBuffers) { - if (attr.buffer > db) shift++; - } - attr.buffer -= shift; - } - }; - - remap_input(mesh.vertex); - remap_input(mesh.index); - remap_input(mesh.instance); - remap_input(mesh.indirect); - } - - // if ( graph.metadata["renderer"]["separate"].as() ) - { - #if UF_ENV_DREAMCAST && GL_QUANTIZED_SHORT - mesh.convert(); - #else - auto conversion = graph.metadata["decode"]["conversion"].as(); - if ( conversion != "" ) { - #if UF_USE_FLOAT16 - if ( conversion == "float16" ) mesh.convert(); - else if ( conversion == "float" ) mesh.convert(); - #endif - #if UF_USE_BFLOAT16 - if ( conversion == "bfloat16" ) mesh.convert(); - else if ( conversion == "float" ) mesh.convert(); - #endif - if ( conversion == "uint16_t" ) mesh.convert(); - else if ( conversion == "float" ) mesh.convert(); - } - #endif - } - mesh.updateDescriptor(); return mesh; } @@ -396,13 +332,15 @@ void uf::graph::load( pod::Graph& graph, const uf::stl::string& filename, const if ( !graph.storage ) graph.storage = new pod::Graph::Storage(); auto& storage = uf::graph::getStorage( graph ); // will just fetch the above +#if 0 if ( !ext::json::isArray(graph.metadata["decode"]["attributes"]) ) { - #if 0 && UF_USE_OPENGL + #if UF_USE_OPENGL graph.metadata["decode"]["attributes"] = uf::stl::vector({ "position", "uv", "st" }); #else graph.metadata["decode"]["attributes"] = uf::stl::vector({ "position", "color", "uv", "st", "normal", "tangent", "joints", "weights" }); #endif } +#endif // failsafes if ( graph.metadata["stream"]["enabled"].is() && graph.metadata["stream"]["enabled"].as() == "auto" ) { @@ -543,7 +481,7 @@ void uf::graph::load( pod::Graph& graph, const uf::stl::string& filename, const uf::stl::vector ioBuf; pod::Primitive* allPrimitives = nullptr; - if (uf::io::readAsBuffer(ioBuf, directory + binName)) { + if ( uf::io::readAsBuffer(ioBuf, directory + binName) ) { allPrimitives = reinterpret_cast(ioBuf.data()); } @@ -557,7 +495,7 @@ void uf::graph::load( pod::Graph& graph, const uf::stl::string& filename, const bool hasOffset = !value["offset"].isNull(); - if (allPrimitives && hasOffset) { + if ( allPrimitives && hasOffset ) { size_t count = value["count"].as(); size_t offsetBytes = value["offset"].as(); size_t startIndex = offsetBytes / sizeof(pod::Primitive); @@ -574,14 +512,34 @@ void uf::graph::load( pod::Graph& graph, const uf::stl::string& filename, const tasks.queue([&]{ UF_DEBUG_TIMER_MULTITRACE("Reading images..."); graph.images.reserve( serializer["images"].size() ); + + uf::stl::vector pendingImages; + pendingImages.reserve( serializer["images"].size() ); + uf::stl::unordered_map> scatterMap; + ext::json::forEach( serializer["images"], [&]( ext::json::Value& value ){ auto name = key + value["name"].as(); + UF_DEBUG_TIMER_MULTITRACE("Reading image={}", name); storage.images[name] = { - .data = decodeImage( value, graph, name ), + .data = decodeImage( value, graph, name, scatterMap, pendingImages ), }; graph.images.emplace_back(name); }); + + for ( auto& [filename, requests] : scatterMap ) { + uf::io::readScatter( filename, requests ); + } + + for ( auto& pending : pendingImages ) { + auto& image = storage.images[pending.name].data; + if ( !pending.buffer.empty() ) { + uf::image::open( image, pending.buffer, pending.extension, false ); + uf::image::layers( image, pending.layers ); + + pending.buffer.clear(); + } + } UF_DEBUG_TIMER_MULTITRACE("Read images"); }); @@ -589,35 +547,38 @@ void uf::graph::load( pod::Graph& graph, const uf::stl::string& filename, const UF_DEBUG_TIMER_MULTITRACE("Reading meshes..."); graph.meshes.reserve( serializer["meshes"].size() ); - uf::stl::vector megaBuffer; - bool bufferAttempted = false; - #if UF_USE_OPENGL - bool preferInterleaved = true; + bool preferMinified = true; #else - bool preferInterleaved = false; + bool preferMinified = false; #endif - if ( graph.settings.stream.enabled ) preferInterleaved = false; + if ( graph.settings.stream.enabled ) preferMinified = false; + + uf::stl::unordered_map> scatterMap; + uf::stl::vector meshesToMinify; ext::json::forEach( serializer["meshes"], [&]( ext::json::Value& value ){ auto name = key + value["name"].as(); - bool hasInterleavedAsset = value["interleaved"].isObject(); - ext::json::Value& json = ( preferInterleaved && hasInterleavedAsset ) ? value["interleaved"] : value; + bool hasMinifiedAsset = value["min"].isObject(); + ext::json::Value& json = ( preferMinified && hasMinifiedAsset ) ? value["min"] : value; - if ( !bufferAttempted && json["buffers"].size() > 0 && json["buffers"][0].isObject() ) { - uf::stl::string binName = json["buffers"][0]["filename"].as(); - uf::io::readAsBuffer( megaBuffer, directory + "/" + binName ); - bufferAttempted = true; - } - - auto& mesh = (storage.meshes[name] = decodeMesh( json, graph, name, megaBuffer )); - if ( preferInterleaved && !hasInterleavedAsset && !graph.settings.stream.enabled ) { - mesh.interleave(); - } + storage.meshes[name] = decodeMesh( json, graph, name, scatterMap ); graph.meshes.emplace_back(name); + + if ( preferMinified && !hasMinifiedAsset && !graph.settings.stream.enabled ) { + meshesToMinify.emplace_back( name ); + } }); + for ( auto& [filename, requests] : scatterMap ) uf::io::readScatter( filename, requests ); + for ( const auto& name : meshesToMinify ) { + auto& mesh = storage.meshes[name]; + mesh.prune( { "position", "uv", "st" } ); + mesh.convert(); + mesh.interleave(); + } + UF_DEBUG_TIMER_MULTITRACE("Read meshes"); }); @@ -625,21 +586,11 @@ void uf::graph::load( pod::Graph& graph, const uf::stl::string& filename, const UF_DEBUG_TIMER_MULTITRACE("Reading animation information..."); auto& animNode = serializer["animations"]; - uf::stl::vector megaBuffer; - bool bufferAttempted = false; - if (animNode.isObject()) { storage.animations.map.reserve( animNode.size() ); ext::json::forEach( animNode, [&]( const uf::stl::string& rawName, ext::json::Value& value ){ auto name = key + rawName; - - if (!bufferAttempted && value["buffer"].is()) { - uf::stl::string binName = value["buffer"].as(); - uf::io::readAsBuffer(megaBuffer, directory + binName); - bufferAttempted = true; - } - - storage.animations[name] = decodeAnimation( value, graph, name, megaBuffer ); + storage.animations[name] = decodeAnimation( value, graph, name ); graph.animations.emplace_back(name); }); } @@ -651,7 +602,7 @@ void uf::graph::load( pod::Graph& graph, const uf::stl::string& filename, const json.readFromFile( path ); auto name = key + json["name"].as(); - storage.animations[name] = decodeAnimation( json, graph, name, megaBuffer ); + storage.animations[name] = decodeAnimation( json, graph, name ); graph.animations.emplace_back(name); }); } @@ -662,19 +613,9 @@ void uf::graph::load( pod::Graph& graph, const uf::stl::string& filename, const UF_DEBUG_TIMER_MULTITRACE("Reading skinning information..."); graph.skins.reserve( serializer["skins"].size() ); - uf::stl::vector megaBuffer; - bool bufferAttempted = false; - ext::json::forEach( serializer["skins"], [&]( ext::json::Value& value ){ auto name = key + value["name"].as(); - - if (!bufferAttempted && value["inverseBindMatrices"].isObject()) { - uf::stl::string binName = value["inverseBindMatrices"]["buffer"].as(); - uf::io::readAsBuffer(megaBuffer, directory + binName); - bufferAttempted = true; - } - - storage.skins[name] = decodeSkin( value, graph, name, megaBuffer ); + storage.skins[name] = decodeSkin( value, graph, name ); graph.skins.emplace_back(name); }); UF_DEBUG_TIMER_MULTITRACE("Read skins"); diff --git a/engine/src/engine/graph/encode.cpp b/engine/src/engine/graph/encode.cpp index 10c64651..d8b2f89f 100644 --- a/engine/src/engine/graph/encode.cpp +++ b/engine/src/engine/graph/encode.cpp @@ -330,10 +330,10 @@ uf::stl::string uf::graph::save( const pod::Graph& graph, const uf::stl::string& ext::json::reserve( serializer["meshes"], graph.meshes.size() ); uf::stl::vector meshesBuffer; - uf::stl::vector interleavedBuffer; + uf::stl::vector minBuffer; uf::stl::string binName = "meshes." + (settings.compression == "none" ? "bin" : settings.compression); - uf::stl::string interleavedBinName = "meshes.interleaved." + (settings.compression == "none" ? "bin" : settings.compression); + uf::stl::string minBinName = "meshes.min." + (settings.compression == "none" ? "bin" : settings.compression); for ( auto& name : graph.meshes ) { auto& mesh = storage.meshes.map.at(name); @@ -341,10 +341,15 @@ uf::stl::string uf::graph::save( const pod::Graph& graph, const uf::stl::string& auto json = encode(mesh, settings, graph, meshesBuffer, binName); json["name"] = name; - uf::Mesh interleavedMesh = mesh.copy(); - interleavedMesh.interleave(); + // to-do: properly flag this + if ( true ) { + uf::Mesh minMesh = mesh.copy(); + minMesh.prune( { "position", "uv", "st" } ); + minMesh.convert(); + minMesh.interleave(); - json["interleaved"] = encode(interleavedMesh, settings, graph, interleavedBuffer, interleavedBinName);; + json["min"] = encode( minMesh, settings, graph, minBuffer, minBinName ); + } serializer["meshes"].emplace_back(json); } @@ -352,8 +357,8 @@ uf::stl::string uf::graph::save( const pod::Graph& graph, const uf::stl::string& if ( !meshesBuffer.empty() ) { uf::io::write(directory + "/" + binName, meshesBuffer); } - if ( !interleavedBuffer.empty() ) { - uf::io::write(directory + "/" + interleavedBinName, interleavedBuffer); + if ( !minBuffer.empty() ) { + uf::io::write(directory + "/" + minBinName, minBuffer); } }); diff --git a/engine/src/engine/graph/graph.cpp b/engine/src/engine/graph/graph.cpp index e3a23834..f11158da 100644 --- a/engine/src/engine/graph/graph.cpp +++ b/engine/src/engine/graph/graph.cpp @@ -30,6 +30,13 @@ // to-do: fix LOD1+ breaking namespace { + struct PendingTexture { + uf::stl::string key; + uf::stl::string formatHint; + uf::stl::vector buffer; + bool needsUpload; + }; + struct TextureDescriptor { bool srgb = false; size_t layers = 1; @@ -220,6 +227,7 @@ namespace { graphic.descriptor.bind.width = graphic.descriptor.inputs.indirect.count; graphic.descriptor.bind.height = 1; graphic.descriptor.bind.depth = 1; + graphic.descriptor.bind.point = VK_PIPELINE_BIND_POINT_COMPUTE; // compute shader auto& shader = graphic.material.getShader("compute", uf::renderer::settings::pipelines::names::culling); @@ -310,10 +318,6 @@ namespace { // to-do: segregate out buffer updating code if ( uf::renderer::settings::pipelines::rt && mesh.vertex.count ) { if ( graphMetadataJson["renderer"]["skinned"].as() ) { - struct PushConstant { - uint32_t jointID; - }; - uf::stl::string compShaderFilename = graphMetadataJson["shaders"]["skinning"]["compute"].as("/graph/skinning/skinning.comp.spv"); { compShaderFilename = entity.resolveURI( compShaderFilename, root ); } @@ -364,13 +368,15 @@ namespace { auto& shader = graphic.material.getShader("compute", "skinning"); - struct SkinningPush { + struct PushConstant { uint32_t jointID; + uint32_t vertexCount; }; - auto& pushConstant = shader.pushConstants.front().get(); + auto& pushConstant = shader.pushConstants.front().get(); pushConstant = { - .jointID = (uint32_t) primitives.front().instance.jointID + .jointID = (uint32_t) primitives.front().instance.jointID, + .vertexCount = (uint32_t) mesh.vertex.count, }; // bind buffers @@ -2113,6 +2119,7 @@ void uf::graph::reload( pod::Graph& graph, pod::Node& node ) { auto& meshStream = graph.streams.meshes[meshName]; auto& primitives = storage.primitives.map[graph.primitives[node.mesh]]; + bool isStreamable = false; float radius = graph.settings.stream.radius; float radiusSquared = radius * radius; @@ -2122,7 +2129,17 @@ void uf::graph::reload( pod::Graph& graph, pod::Node& node ) { } // disable if not tagged for streaming - if ( graph.settings.stream.world != -1 && node.index != graph.settings.stream.world ) { + if ( node.index == graph.settings.stream.world ) { + isStreamable = true; + }/* else if ( ext::json::isObject(tag) && tag.has("stream") ) { + auto& streamTag = tag["stream"]; + if ( streamTag.has("enabled") && streamTag["enabled"].as() ) { + isStreamable = true; + radius = streamTag["radius"].as(radius); + } + }*/ + + if ( !isStreamable ) { radius = 0; } @@ -2163,7 +2180,6 @@ void uf::graph::reload( pod::Graph& graph, pod::Node& node ) { found = true; int8_t lodLevel = 0; - #if 0 // deduce a simple ratio [0.0 to 1.0] of how far we are into the streaming radius float distRatio = distanceSquared / radiusSquared; if ( distRatio > 0.6f ) lodLevel = 3; @@ -2173,15 +2189,14 @@ void uf::graph::reload( pod::Graph& graph, pod::Node& node ) { while ( lodLevel > 0 && primitive.lod.levels[lodLevel].indices == 0 ) { lodLevel--; } - #endif queuedLODs[drawID] = lodLevel; } } // insert closest primitive if all are out of range (because of cringe logic) - if ( !found ) { - queuedLODs[closestDrawID] = 0; + if ( !found /*&& node.index == graph.settings.stream.world*/ ) { + queuedLODs[closestDrawID] = 3; } // bail if no update is detected @@ -2196,110 +2211,112 @@ void uf::graph::reload( pod::Graph& graph, pod::Node& node ) { // read from disk #if UF_GRAPH_SPARSE_READ_MESH - // needs to be dequantized first, naively copying the descriptor settings just doesn't work - { - #if UF_ENV_DREAMCAST && GL_QUANTIZED_SHORT - mesh.convert(); - #else - auto conversion = graphMetadataJson["decode"]["conversion"].as(); - if ( conversion != "" ) { - #if UF_USE_FLOAT16 - if ( conversion == "float16" ) mesh.convert(); - else if ( conversion == "float" ) mesh.convert(); - #endif - #if UF_USE_BFLOAT16 - if ( conversion == "bfloat16" ) mesh.convert(); - else if ( conversion == "float" ) mesh.convert(); - #endif - if ( conversion == "uint16_t" ) mesh.convert(); - else if ( conversion == "float" ) mesh.convert(); - } - #endif - } + uint32_t currentVertexCount = 0; + uint32_t currentIndexCount = 0; - // reset counts - mesh.vertex.count = 0; - mesh.index.count = 0; + struct ActiveDraw { + size_t drawID; + int8_t lodLevel; + uint32_t fileVertexID; + }; - for (size_t drawID = 0; drawID < queuedLODs.size(); ++drawID) { + static thread_local uf::stl::unordered_map> batchedRanges; batchedRanges.clear(); + static thread_local uf::stl::unordered_map bufferSizes; bufferSizes.clear(); + static thread_local uf::stl::unordered_map> scatterMap; scatterMap.clear(); + static thread_local uf::stl::unordered_map bufferWriteOffsets; bufferWriteOffsets.clear(); + static thread_local uf::stl::unordered_map> newBuffers; newBuffers.clear(); + + STATIC_THREAD_LOCAL(uf::stl::vector, activeDraws); + activeDraws.reserve(queuedLODs.size()); + + for ( size_t drawID = 0; drawID < queuedLODs.size(); ++drawID ) { auto lodLevel = queuedLODs[drawID]; - auto& primitive = primitives[drawID]; - auto& drawCommand = drawCommands[drawID]; - - // reset from LOD0 - //primitives[drawID].drawCommand.instances = 1; - primitives[drawID].drawCommand.indices = primitives[drawID].lod.levels[0].indices; - primitives[drawID].drawCommand.indexID = primitives[drawID].lod.levels[0].indexID; - primitives[drawID].drawCommand.vertexID = primitives[drawID].lod.levels[0].vertexID; - primitives[drawID].drawCommand.vertices = primitives[drawID].lod.levels[0].vertices; - - // copy from primitive - drawCommand = primitive.drawCommand; - - // disable draw call - if ( lodLevel < 0 ) { - //drawCommand.instances = 0; + if ( lodLevel >= 0 ) { + auto& lod = primitives[drawID].lod.levels[lodLevel]; + activeDraws.emplace_back(ActiveDraw{drawID, lodLevel, lod.vertexID}); + } else { + auto& drawCommand = drawCommands[drawID]; drawCommand.vertices = 0; drawCommand.indices = 0; drawCommand.vertexID = 0; drawCommand.indexID = 0; - continue; + primitives[drawID].drawCommand = drawCommand; } - - auto& lod = primitive.lod.levels[lodLevel]; - - // queue up ranges to read from disk using LOD bounds - for (auto& attribute : mesh.index.attributes) { - auto stride = attribute.stride > 0 ? attribute.stride : attribute.descriptor.size; - if (ranges[attribute.buffer].empty() || ranges[attribute.buffer].back().start != lod.indexID * stride) { - ranges[attribute.buffer].emplace_back(pod::Range{ - lod.indexID * stride, - lod.indices * stride, - }); - } - } - for (auto& attribute : mesh.vertex.attributes) { - auto stride = attribute.stride > 0 ? attribute.stride : attribute.descriptor.size; - if (ranges[attribute.buffer].empty() || ranges[attribute.buffer].back().start != lod.vertexID * stride) { - ranges[attribute.buffer].emplace_back(pod::Range{ - lod.vertexID * stride, - lod.vertices * stride, - }); - } - } - - // reset draw call and remap to local compacted buffers - drawCommand.vertices = lod.vertices; - drawCommand.indices = lod.indices; - drawCommand.vertexID = mesh.vertex.count; - drawCommand.indexID = mesh.index.count; - - // synchronize primitive - primitives[drawID].drawCommand = drawCommands[drawID]; - - // increment remap indices - mesh.vertex.count += drawCommand.vertices; - mesh.index.count += drawCommand.indices; } - #define STREAM_MESH_DATA( N ) \ - for ( auto& attribute : mesh.N.attributes ) {\ - if ( processedBuffers.count(attribute.buffer) ) continue; \ - processedBuffers.insert(attribute.buffer); \ - auto& region = meshStream.buffers[attribute.buffer];\ - if ( ranges.count(attribute.buffer) <= 0 || region.length == 0 ) { \ - mesh.buffers[attribute.buffer].clear();\ - } else {\ - auto adjustedRanges = ranges[attribute.buffer];\ - for (auto& r : adjustedRanges) {\ - r.start += region.offset;\ - }\ - uf::io::readAsBuffer( mesh.buffers[attribute.buffer], region.filename, adjustedRanges );\ - }\ + std::sort(activeDraws.begin(), activeDraws.end(), [](const ActiveDraw& a, const ActiveDraw& b) { + return a.fileVertexID < b.fileVertexID; + }); + + for ( auto& active : activeDraws ) { + auto& lod = primitives[active.drawID].lod.levels[active.lodLevel]; + for ( auto& attr : mesh.index.attributes ) { + size_t stride = attr.stride > 0 ? attr.stride : attr.descriptor.size; + bufferSizes[attr.buffer] += lod.indices * stride; + } + for ( auto& attr : mesh.vertex.attributes ) { + size_t stride = attr.stride > 0 ? attr.stride : attr.descriptor.size; + bufferSizes[attr.buffer] += lod.vertices * stride; + } + } + + for ( auto& [b, size] : bufferSizes ) { + newBuffers[b].resize(size); + } + + for ( auto& active : activeDraws ) { + auto& primitive = primitives[active.drawID]; + auto& lod = primitive.lod.levels[active.lodLevel]; + auto& drawCommand = drawCommands[active.drawID]; + + drawCommand.vertices = lod.vertices; + drawCommand.indices = lod.indices; + drawCommand.vertexID = currentVertexCount; + drawCommand.indexID = currentIndexCount; + primitive.drawCommand = drawCommand; + + for ( auto& attr : mesh.index.attributes ) { + size_t stride = attr.stride > 0 ? attr.stride : attr.descriptor.size; + auto& region = meshStream.buffers[attr.buffer]; + size_t readBytes = lod.indices * stride; + + scatterMap[region.filename].push_back({ + region.offset + attr.offset + (lod.indexID * stride), + readBytes, + newBuffers[attr.buffer].data() + bufferWriteOffsets[attr.buffer] + }); + bufferWriteOffsets[attr.buffer] += readBytes; } - STREAM_MESH_DATA( index ); - STREAM_MESH_DATA( vertex ); + for ( auto& attr : mesh.vertex.attributes ) { + size_t stride = attr.stride > 0 ? attr.stride : attr.descriptor.size; + auto& region = meshStream.buffers[attr.buffer]; + size_t readBytes = lod.vertices * stride; + + scatterMap[region.filename].push_back({ + region.offset + attr.offset + (lod.vertexID * stride), + readBytes, + newBuffers[attr.buffer].data() + bufferWriteOffsets[attr.buffer] + }); + bufferWriteOffsets[attr.buffer] += readBytes; + } + + currentVertexCount += lod.vertices; + currentIndexCount += lod.indices; + } + + for ( auto& [filename, requests] : scatterMap ) { + uf::io::readScatter(filename, requests); + } + + mesh.vertex.count = currentVertexCount; + mesh.index.count = currentIndexCount; + + for ( auto& [b, buf] : newBuffers ) mesh.buffers[b] = std::move(buf); + + for ( auto& attr : mesh.vertex.attributes ) attr.offset = 0; + for ( auto& attr : mesh.index.attributes ) attr.offset = 0; + // keep the vertex data intact #else // disable remaining draw commands @@ -2369,6 +2386,10 @@ void uf::graph::reload( pod::Graph& graph, pod::Node& node ) { } uf::stl::unordered_map textureDescriptors; + uf::stl::vector pendingTextures; + uf::stl::unordered_map> textureScatterMap; + pendingTextures.reserve( textureDescriptors.size() ); + for ( size_t drawID = 0; drawID < primitives.size(); ++drawID ) { auto& primitive = primitives[drawID]; auto& instance = primitive.instance; @@ -2396,58 +2417,26 @@ void uf::graph::reload( pod::Graph& graph, pod::Node& node ) { if ( visible && (!texture.generated() || texture.aliased) ) { meshUpdated = true; - // load image + pendingTextures.push_back({ key, "", {}, true }); + auto& pending = pendingTextures.back(); + if ( image.getPixels().empty() ) { auto& imgStream = graph.streams.images[key]; - uf::stl::vector buf; - - if (imgStream.buffer.length > 0) { - uf::io::readAsBuffer(buf, imgStream.buffer.filename, imgStream.buffer.offset, imgStream.buffer.length); - } else { - uf::io::readAsBuffer(buf, imgStream.buffer.filename); - } uf::stl::string formatHint = uf::io::extension(image.getFilename()); if (imgStream.buffer.filename.find(".dtex") != uf::stl::string::npos) formatHint = "dtex"; + pending.formatHint = formatHint; - uf::image::open(image, buf, formatHint, false); - - // to-do: check against format instead - if ( key == "lightmap_atlas" ) { - ::convertLightmap( image ); + size_t readLen = imgStream.buffer.length > 0 ? imgStream.buffer.length : uf::io::size(imgStream.buffer.filename); + if (readLen > 0) { + pending.buffer.resize(readLen); + textureScatterMap[imgStream.buffer.filename].push_back({ + imgStream.buffer.offset, + readLen, + pending.buffer.data() + }); } } - - auto filter = uf::renderer::enums::Filter::LINEAR; - auto tag = ext::json::find( key, graphMetadataJson["tags"] ); - if ( !ext::json::isObject( tag ) ) { - tag["renderer"] = graphMetadataJson["renderer"]; - } - if ( tag["renderer"]["filter"].is() ) { - const auto mode = uf::string::lowercase( tag["renderer"]["filter"].as("linear") ); - if ( mode == "linear" ) filter = uf::renderer::enums::Filter::LINEAR; - else if ( mode == "nearest" ) filter = uf::renderer::enums::Filter::NEAREST; - } - - if ( texture.aliased ) { - texture.aliased = false; - #if UF_USE_OPENGL - texture.image = 0; - #else - texture.image = {}; - texture.view = {}; - #endif - } - - texture.sampler.descriptor.filter.min = filter; - texture.sampler.descriptor.filter.mag = filter; - texture.layers = descriptor.layers; - texture.srgb = descriptor.srgb; - - texture.loadFromImage( image ); - #if UF_ENV_DREAMCAST - image.clear(); - #endif } else if ( !visible && (texture.generated() && !texture.aliased) ) { meshUpdated = true; image.clear(); @@ -2455,32 +2444,64 @@ void uf::graph::reload( pod::Graph& graph, pod::Node& node ) { texture.aliasTexture(uf::renderer::Texture2D::empty); } } + + for ( auto& [filename, requests] : textureScatterMap ) { + uf::io::readScatter( filename, requests ); + } + + for ( auto& pending : pendingTextures ) { + if ( !pending.needsUpload ) continue; + + auto& image = storage.images[pending.key].data; + auto& texture = storage.images[pending.key].handle; + auto& descriptor = textureDescriptors[pending.key]; + + if ( !pending.buffer.empty() ) { + uf::image::open( image, pending.buffer, pending.formatHint, false ); + + if ( pending.key == "lightmap_atlas" ) { + ::convertLightmap( image ); + } + pending.buffer.clear(); + } + + auto filter = uf::renderer::enums::Filter::LINEAR; + auto tag = ext::json::find( pending.key, graphMetadataJson["tags"] ); + if ( !ext::json::isObject( tag ) ) { + tag["renderer"] = graphMetadataJson["renderer"]; + } + if ( tag["renderer"]["filter"].is() ) { + const auto mode = uf::string::lowercase( tag["renderer"]["filter"].as("linear") ); + if ( mode == "linear" ) filter = uf::renderer::enums::Filter::LINEAR; + else if ( mode == "nearest" ) filter = uf::renderer::enums::Filter::NEAREST; + } + + if ( texture.aliased ) { + texture.aliased = false; + #if UF_USE_OPENGL + texture.image = 0; + #else + texture.image = {}; + texture.view = {}; + #endif + } + + texture.sampler.descriptor.filter.min = filter; + texture.sampler.descriptor.filter.mag = filter; + texture.layers = descriptor.layers; + texture.srgb = descriptor.srgb; + + texture.loadFromImage( image ); + + #if UF_ENV_DREAMCAST + image.clear(); + #endif + } #undef INCREMENT_TEXTURE_REFCOUNT } if ( !meshUpdated ) return; - // in the event streamed in mesh data from any pathway isn't already converted - { - #if UF_ENV_DREAMCAST && GL_QUANTIZED_SHORT - mesh.convert(); - #else - auto conversion = graphMetadataJson["decode"]["conversion"].as(); - if ( conversion != "" ) { - #if UF_USE_FLOAT16 - if ( conversion == "float16" ) mesh.convert(); - else if ( conversion == "float" ) mesh.convert(); - #endif - #if UF_USE_BFLOAT16 - if ( conversion == "bfloat16" ) mesh.convert(); - else if ( conversion == "float" ) mesh.convert(); - #endif - if ( conversion == "uint16_t" ) mesh.convert(); - else if ( conversion == "float" ) mesh.convert(); - } - #endif - } - mesh.updateDescriptor(); // necessary for OpenGL because recorded descriptors have invalidated pointers diff --git a/engine/src/ext/meshopt/meshopt.cpp b/engine/src/ext/meshopt/meshopt.cpp index 3b9f914d..ab1b905a 100644 --- a/engine/src/ext/meshopt/meshopt.cpp +++ b/engine/src/ext/meshopt/meshopt.cpp @@ -177,6 +177,107 @@ uf::stl::vector ext::meshopt::computeLODs( size_t count, size_t maxLODs, return factors; } +// basically just optimize with a simplify factor anyways +bool ext::meshopt::simplify( uf::Mesh& mesh, float simplifyFactor ) { + if ( simplifyFactor >= 1.0f || simplifyFactor <= 0.0f ) return false; + + mesh.updateDescriptor(); + const auto& views = mesh.buffer_views; + if ( views.empty() ) return false; + + pod::DrawCommand* drawCommands = mesh.indirect.count > 0 ? (pod::DrawCommand*)mesh.getBuffer(mesh.indirect).data() : nullptr; + const uint8_t* srcIndexData = mesh.index.count > 0 ? mesh.getBuffer(mesh.index).data() : nullptr; + + uf::stl::vector outIndices; + uf::stl::vector> outVertices(mesh.vertex.attributes.size()); + + int posAttrIdx = -1; + for ( size_t i = 0; i < mesh.vertex.attributes.size(); ++i ) { + if ( mesh.vertex.attributes[i].descriptor.name == "position" ) { + posAttrIdx = i; break; + } + } + if ( posAttrIdx == -1 ) return false; + + for ( size_t viewIdx = 0; viewIdx < views.size(); ++viewIdx ) { + auto& view = views[viewIdx]; + uint32_t cmdIdx = view.indirectIndex; + + uint32_t srcVertexOffset = view.vertex.first; + uint32_t srcVertexCount = view.vertex.count; + uint32_t srcIndexOffset = view.index.first; + uint32_t srcIndexCount = view.index.count; + + if ( srcIndexCount == 0 ) continue; + + uf::stl::vector localIndices(srcIndexCount); + if ( srcIndexData ) { + for ( size_t i = 0; i < srcIndexCount; ++i ) { + localIndices[i] = readIndex(srcIndexData, srcIndexOffset + i, mesh.index.size); + } + } else { + for ( size_t i = 0; i < srcIndexCount; ++i ) localIndices[i] = i; + } + + auto& posAttr = mesh.vertex.attributes[posAttrIdx]; + const float* srcPositions = (const float*)((const uint8_t*)posAttr.pointer + srcVertexOffset * posAttr.stride); + + uf::stl::vector simplifiedIndices(srcIndexCount); + float targetError = FLT_MAX; + float realError = 0.0f; + + size_t optimizedIndexCount = meshopt_simplify( + simplifiedIndices.data(), localIndices.data(), srcIndexCount, + srcPositions, srcVertexCount, posAttr.stride, + srcIndexCount * simplifyFactor, targetError, meshopt_SimplifyLockBorder, &realError + ); + simplifiedIndices.resize(optimizedIndexCount); + + uf::stl::vector fetchRemap(srcVertexCount); + size_t finalVertexCount = meshopt_optimizeVertexFetchRemap(fetchRemap.data(), simplifiedIndices.data(), optimizedIndexCount, srcVertexCount); + meshopt_remapIndexBuffer(simplifiedIndices.data(), simplifiedIndices.data(), optimizedIndexCount, fetchRemap.data()); + + uint32_t newVertexOffset = outVertices[0].size() / mesh.vertex.attributes[0].stride; + uint32_t newIndexOffset = outIndices.size(); + + for ( uint32_t idx : simplifiedIndices ) { + outIndices.push_back(idx); + } + + for ( size_t a = 0; a < mesh.vertex.attributes.size(); ++a ) { + auto& attr = mesh.vertex.attributes[a]; + const uint8_t* srcBase = (const uint8_t*)attr.pointer + srcVertexOffset * attr.stride; + + uf::stl::vector remapped(finalVertexCount * attr.stride); + meshopt_remapVertexBuffer(remapped.data(), srcBase, srcVertexCount, attr.stride, fetchRemap.data()); + + outVertices[a].insert(outVertices[a].end(), remapped.begin(), remapped.end()); + } + + if ( drawCommands ) { + drawCommands[cmdIdx].indexID = newIndexOffset; + drawCommands[cmdIdx].indices = optimizedIndexCount; + drawCommands[cmdIdx].vertexID = newVertexOffset; + drawCommands[cmdIdx].vertices = finalVertexCount; + } + } + + mesh.index.count = outIndices.size(); + mesh.resizeIndices(mesh.index.count); + uint8_t* dstIdx = mesh.getBuffer(mesh.index).data(); + for ( size_t i = 0; i < outIndices.size(); ++i ) writeIndex(dstIdx, i, mesh.index.size, outIndices[i]); + + mesh.vertex.count = outVertices[0].size() / mesh.vertex.attributes[0].stride; + for ( size_t a = 0; a < mesh.vertex.attributes.size(); ++a ) { + auto& attr = mesh.vertex.attributes[a]; + mesh.buffers[attr.buffer].swap(outVertices[a]); + attr.pointer = mesh.buffers[attr.buffer].data(); + } + + mesh.updateDescriptor(); + return true; +} + uf::stl::vector ext::meshopt::generateLODs( uf::Mesh& mesh, const uf::stl::vector& lodFactors, bool verbose ) { uf::stl::vector lodMetadata; mesh.updateDescriptor(); diff --git a/engine/src/ext/opengl/commands.cpp b/engine/src/ext/opengl/commands.cpp index e1a4b156..74a525ec 100644 --- a/engine/src/ext/opengl/commands.cpp +++ b/engine/src/ext/opengl/commands.cpp @@ -142,6 +142,39 @@ namespace { return true; } + + void remapAttribute( uint8_t*& ptr, size_t& stride, GLenum& type, uf::stl::vector& remapBuffer, size_t compCount, uf::renderer::enums::Type::type_t enumType, size_t vertexCount ) { + remapBuffer.resize( vertexCount * compCount ); + size_t inStride = stride > 0 ? stride : (compCount * sizeof(uint16_t)); + for ( size_t v = 0; v < vertexCount; ++v ) { + uint8_t* basePtr = ptr + (v * inStride); + for ( size_t c = 0; c < compCount; ++c ) { + size_t outIdx = v * compCount + c; + switch ( enumType ) { + case uf::renderer::enums::Type::SHORT: + case uf::renderer::enums::Type::USHORT: + remapBuffer[outIdx] = uf::quant::dequantize( ((uint16_t*)basePtr)[c] ); + break; + #if UF_USE_FLOAT16 + case uf::renderer::enums::Type::HALF: + remapBuffer[outIdx] = ((float16*)basePtr)[c]; + break; + #endif + #if UF_USE_BFLOAT16 + case uf::renderer::enums::Type::BFLOAT: + remapBuffer[outIdx] = ((bfloat16*)basePtr)[c]; + break; + #endif + default: + remapBuffer[outIdx] = 0.0f; + break; + } + } + } + ptr = (uint8_t*) remapBuffer.data(); + type = GL_FLOAT; + stride = 0; + }; } size_t ext::opengl::CommandBuffer::preallocate = 8; void ext::opengl::CommandBuffer::initialize( Device& device ) { @@ -501,37 +534,21 @@ void ext::opengl::CommandBuffer::drawIndexed( const ext::opengl::CommandBuffer:: uint8_t* vertexPtr = drawInfo.attributes.position.pointer ? (static_cast(drawInfo.attributes.position.pointer) + drawInfo.attributes.position.stride * drawInfo.descriptor.inputs.vertex.first) : NULL; auto vertexStride = drawInfo.attributes.position.stride; + auto normalStride = drawInfo.attributes.normal.stride; + auto uvStride = drawInfo.attributes.uv.stride; + auto stStride = drawInfo.attributes.st.stride; + STATIC_THREAD_LOCAL(uf::stl::vector, vertexBufferRemap); + STATIC_THREAD_LOCAL(uf::stl::vector, normalBufferRemap); + STATIC_THREAD_LOCAL(uf::stl::vector, uvBufferRemap); + STATIC_THREAD_LOCAL(uf::stl::vector, stBufferRemap); + // my copy of GLdc is already patched to handle these without needing a preprocessed buffer #if !UF_ENV_DREAMCAST - if ( vertexType != GL_FLOAT ) { - vertexBufferRemap.resize( drawInfo.descriptor.inputs.vertex.count * 3 ); - for ( size_t i = 0; i < drawInfo.descriptor.inputs.vertex.count * 3; ++i ) { - switch ( drawInfo.attributes.position.descriptor.type ) { - case uf::renderer::enums::Type::SHORT: - case uf::renderer::enums::Type::USHORT: - vertexBufferRemap[i] = uf::quant::dequantize( ((uint16_t*) vertexPtr)[i] ); - break; - #if UF_USE_FLOAT16 - case uf::renderer::enums::Type::HALF: - vertexBufferRemap[i] = ((float16*) vertexPtr)[i]; - break; - #endif - #if UF_USE_BFLOAT16 - case uf::renderer::enums::Type::BFLOAT: - vertexBufferRemap[i] = ((bfloat16*) vertexPtr)[i]; - break; - #endif - default: - vertexBufferRemap[i] = vertexPtr[i]; - break; - } - } - - vertexPtr = (uint8_t*) vertexBufferRemap.data(); - vertexType = GL_FLOAT; - vertexStride = 0; - } + if ( vertexPtr && vertexType != GL_FLOAT ) ::remapAttribute( vertexPtr, vertexStride, vertexType, vertexBufferRemap, drawInfo.attributes.position.descriptor.components, drawInfo.attributes.position.descriptor.type, drawInfo.descriptor.inputs.vertex.count ); + if ( normalPtr && normalType != GL_FLOAT ) ::remapAttribute( normalPtr, normalStride, normalType, normalBufferRemap, drawInfo.attributes.normal.descriptor.components, drawInfo.attributes.normal.descriptor.type, drawInfo.descriptor.inputs.vertex.count ); + if ( uvPtr && uvType != GL_FLOAT ) ::remapAttribute( uvPtr, uvStride, uvType, uvBufferRemap, drawInfo.attributes.uv.descriptor.components, drawInfo.attributes.uv.descriptor.type, drawInfo.descriptor.inputs.vertex.count ); + if ( stPtr && stType != GL_FLOAT ) ::remapAttribute( stPtr, stStride, stType, stBufferRemap, drawInfo.attributes.st.descriptor.components, drawInfo.attributes.st.descriptor.type, drawInfo.descriptor.inputs.vertex.count ); #endif if ( drawInfo.attributes.normal.pointer ) { @@ -539,7 +556,7 @@ void ext::opengl::CommandBuffer::drawIndexed( const ext::opengl::CommandBuffer:: GL_ERROR_CHECK(glEnableClientState(GL_NORMAL_ARRAY)); ::shadowState.normalArrayEnabled = true; } - GL_ERROR_CHECK(glNormalPointer(normalType, drawInfo.attributes.normal.stride, normalPtr)); + GL_ERROR_CHECK(glNormalPointer(normalType, normalStride, normalPtr)); } else if ( ::shadowState.normalArrayEnabled ) { GL_ERROR_CHECK(glDisableClientState(GL_NORMAL_ARRAY)); ::shadowState.normalArrayEnabled = false; @@ -579,7 +596,7 @@ void ext::opengl::CommandBuffer::drawIndexed( const ext::opengl::CommandBuffer:: ::shadowState.boundTexture0 = drawInfo.textures.primary.image; } - GL_ERROR_CHECK(glTexCoordPointer(2, uvType, drawInfo.attributes.uv.stride, uvPtr)); + GL_ERROR_CHECK(glTexCoordPointer(2, uvType, uvStride, uvPtr)); GL_ERROR_CHECK(glTexEnvi(GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, drawInfo.attributes.color.pointer ? GL_MODULATE : GL_REPLACE)); } else { if ( ::shadowState.tex0Enabled ) { @@ -618,7 +635,7 @@ void ext::opengl::CommandBuffer::drawIndexed( const ext::opengl::CommandBuffer:: } GL_ERROR_CHECK(glTexEnvi(GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, GL_MODULATE)); - GL_ERROR_CHECK(glTexCoordPointer(2, stType, drawInfo.attributes.st.stride, stPtr)); + GL_ERROR_CHECK(glTexCoordPointer(2, stType, stStride, stPtr)); } else { if ( ::shadowState.tex1Enabled ) { GL_ERROR_CHECK(glClientActiveTexture(GL_TEXTURE1)); diff --git a/engine/src/ext/vulkan/graphic.cpp b/engine/src/ext/vulkan/graphic.cpp index fed111d9..87d442d5 100644 --- a/engine/src/ext/vulkan/graphic.cpp +++ b/engine/src/ext/vulkan/graphic.cpp @@ -420,12 +420,19 @@ void ext::vulkan::Pipeline::record( const Graphic& graphic, VkCommandBuffer comm RenderMode& renderMode = ext::vulkan::getRenderMode(descriptor.renderMode, true); + // to-do: properly dispatch the bind point to the pipeline because for some reason it is not + VkPipelineBindPoint bindPoint = (VkPipelineBindPoint) descriptor.bind.point; + bool bound = false; for ( auto* shader : shaders ) { // compute shaders if ( shader->descriptor.stage == VK_SHADER_STAGE_COMPUTE_BIT ) { - if ( descriptor.bind.point == VK_PIPELINE_BIND_POINT_COMPUTE ) bound = true; - else continue; + if ( descriptor.bind.point == VK_PIPELINE_BIND_POINT_COMPUTE ) { + bound = true; + } else { + bindPoint = VK_PIPELINE_BIND_POINT_COMPUTE; + continue; + } // raytrace shaders } else if ( shader->descriptor.stage == VK_SHADER_STAGE_RAYGEN_BIT_KHR || @@ -434,12 +441,20 @@ void ext::vulkan::Pipeline::record( const Graphic& graphic, VkCommandBuffer comm shader->descriptor.stage == VK_SHADER_STAGE_ANY_HIT_BIT_KHR || shader->descriptor.stage == VK_SHADER_STAGE_INTERSECTION_BIT_KHR ) { - if ( descriptor.bind.point == VK_PIPELINE_BIND_POINT_RAY_TRACING_KHR ) bound = true; - else continue; + if ( descriptor.bind.point == VK_PIPELINE_BIND_POINT_RAY_TRACING_KHR ) { + bound = true; + } else { + bindPoint = VK_PIPELINE_BIND_POINT_RAY_TRACING_KHR; + continue; + } // anything else } else { - if ( descriptor.bind.point == VK_PIPELINE_BIND_POINT_GRAPHICS ) bound = true; - else continue; + if ( descriptor.bind.point == VK_PIPELINE_BIND_POINT_GRAPHICS ) { + bound = true; + } else { + bindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS; + continue; + } } // automatically bind to our default push constants @@ -459,9 +474,8 @@ void ext::vulkan::Pipeline::record( const Graphic& graphic, VkCommandBuffer comm } } } - // Bind the rendering pipeline - // The pipeline (state object) contains all states of the rendering pipeline, binding it will set all the states specified at pipeline creation time - vkCmdBindPipeline(commandBuffer, (VkPipelineBindPoint) descriptor.bind.point, pipeline); + + vkCmdBindPipeline(commandBuffer, bindPoint, pipeline); } void ext::vulkan::Pipeline::destroy() { if ( aliased ) return; diff --git a/engine/src/ext/vulkan/rendermodes/deferred.cpp b/engine/src/ext/vulkan/rendermodes/deferred.cpp index 662b9525..a6bd9e88 100644 --- a/engine/src/ext/vulkan/rendermodes/deferred.cpp +++ b/engine/src/ext/vulkan/rendermodes/deferred.cpp @@ -847,6 +847,20 @@ void ext::vulkan::DeferredRenderMode::createCommandBuffers( const uf::stl::vecto } } + { + VkMemoryBarrier barrier = {}; + barrier.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER; + barrier.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT; + barrier.dstAccessMask = VK_ACCESS_INDIRECT_COMMAND_READ_BIT | VK_ACCESS_SHADER_READ_BIT; + + vkCmdPipelineBarrier( + commandBuffer, + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, + VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT | VK_PIPELINE_STAGE_VERTEX_SHADER_BIT, + 0, 1, &barrier, 0, nullptr, 0, nullptr + ); + } + // pre-renderpass commands VK_COMMAND_BUFFER_CALLBACK( CALLBACK_BEGIN, commandBuffer, frame, { device->UF_CHECKPOINT_MARK( commandBuffer, pod::Checkpoint::GENERIC, "callback[begin]" ); diff --git a/engine/src/ext/zlib/zlib.cpp b/engine/src/ext/zlib/zlib.cpp index 75014646..277b9d4b 100644 --- a/engine/src/ext/zlib/zlib.cpp +++ b/engine/src/ext/zlib/zlib.cpp @@ -26,34 +26,28 @@ bool ext::zlib::decompressFromFile( uf::stl::vector& buffer, const uf:: z_stream strm{}; if (inflateInit2(&strm, 15 + 32) != Z_OK) return false; - size_t offset = 0; uint8_t outBuffer[ext::zlib::bufferSize]; - while (offset < fileSize) { - size_t bytesToRead = std::min(ext::zlib::bufferSize, fileSize - offset); - uf::stl::vector temp; - if (!uf::vfs::readRange(filename, offset, bytesToRead, temp)) break; - - strm.avail_in = (uInt)temp.size(); - strm.next_in = temp.data(); - offset += temp.size(); + bool success = uf::vfs::stream(filename, ext::zlib::bufferSize, [&](const uint8_t* data, size_t size) -> bool { + strm.avail_in = (uInt)size; + strm.next_in = (Bytef*)data; do { strm.avail_out = sizeof(outBuffer); strm.next_out = outBuffer; int ret = inflate(&strm, Z_NO_FLUSH); - if (ret == Z_STREAM_ERROR || ret == Z_DATA_ERROR || ret == Z_MEM_ERROR) { - UF_MSG_ERROR("Zlib: inflate error on file: {}", filename); - inflateEnd(&strm); - return false; - } + if ( ret < 0 && ret != Z_BUF_ERROR ) return false; + size_t have = sizeof(outBuffer) - strm.avail_out; buffer.insert(buffer.end(), outBuffer, outBuffer + have); + } while (strm.avail_out == 0); - } + + return true; + }); inflateEnd(&strm); - return true; + return success; } bool ext::zlib::decompressFromFile( uf::stl::vector& buffer, const uf::stl::string& filename, size_t start, size_t len ) { @@ -63,34 +57,24 @@ bool ext::zlib::decompressFromFile( uf::stl::vector& buffer, const uf:: z_stream strm{}; if (inflateInit2(&strm, 15 + 32) != Z_OK) return false; - size_t offset = 0; size_t uncompressedOffset = 0; uint8_t outBuffer[ext::zlib::bufferSize]; - while (offset < fileSize) { - size_t bytesToRead = std::min(ext::zlib::bufferSize, fileSize - offset); - uf::stl::vector temp; - if (!uf::vfs::readRange(filename, offset, bytesToRead, temp)) break; - - strm.avail_in = (uInt)temp.size(); - strm.next_in = temp.data(); - offset += temp.size(); + bool success = uf::vfs::stream(filename, ext::zlib::bufferSize, [&](const uint8_t* data, size_t size) -> bool { + strm.avail_in = (uInt)size; + strm.next_in = (Bytef*)data; do { strm.avail_out = sizeof(outBuffer); strm.next_out = outBuffer; int ret = inflate(&strm, Z_NO_FLUSH); - if (ret == Z_STREAM_ERROR || ret == Z_DATA_ERROR || ret == Z_MEM_ERROR) { - inflateEnd(&strm); - return false; - } - size_t have = sizeof(outBuffer) - strm.avail_out; + if ( ret < 0 && ret != Z_BUF_ERROR ) return false; - // Calculate if this chunk overlaps with our requested range + size_t have = sizeof(outBuffer) - strm.avail_out; size_t chunkStart = uncompressedOffset; size_t chunkEnd = uncompressedOffset + have; - if (chunkEnd > start && chunkStart < start + len) { + if ( chunkEnd > start && chunkStart < start + len ) { size_t copyStart = (chunkStart < start) ? (start - chunkStart) : 0; size_t copyLen = std::min(have - copyStart, (start + len) - (chunkStart + copyStart)); buffer.insert(buffer.end(), outBuffer + copyStart, outBuffer + copyStart + copyLen); @@ -98,16 +82,18 @@ bool ext::zlib::decompressFromFile( uf::stl::vector& buffer, const uf:: uncompressedOffset += have; - // If we've reached the end of the requested length, we can abort early! - if (uncompressedOffset >= start + len) { - inflateEnd(&strm); - return true; - } + if ( uncompressedOffset >= start + len ) return false; + } while (strm.avail_out == 0); - } + + return true; + }); inflateEnd(&strm); - return true; + + if ( !success && uncompressedOffset >= start + len ) return true; + + return success; } bool ext::zlib::decompressFromFile( uf::stl::vector& buffer, const uf::stl::string& filename, const uf::stl::vector& ranges ) { @@ -122,34 +108,24 @@ bool ext::zlib::decompressFromFile( uf::stl::vector& buffer, const uf:: z_stream strm{}; if (inflateInit2(&strm, 15 + 32) != Z_OK) return false; - size_t offset = 0; size_t uncompressedOffset = 0; - uint8_t outBuffer[ext::zlib::bufferSize]; size_t currentRangeIdx = 0; + uint8_t outBuffer[ext::zlib::bufferSize]; - while (offset < fileSize && currentRangeIdx < sortedRanges.size()) { - size_t bytesToRead = std::min(ext::zlib::bufferSize, fileSize - offset); - uf::stl::vector temp; - if (!uf::vfs::readRange(filename, offset, bytesToRead, temp)) break; - - strm.avail_in = (uInt)temp.size(); - strm.next_in = temp.data(); - offset += temp.size(); + bool success = uf::vfs::stream(filename, ext::zlib::bufferSize, [&](const uint8_t* data, size_t size) -> bool { + strm.avail_in = (uInt)size; + strm.next_in = (Bytef*)data; do { strm.avail_out = sizeof(outBuffer); strm.next_out = outBuffer; int ret = inflate(&strm, Z_NO_FLUSH); - if (ret < 0 && ret != Z_BUF_ERROR) { - inflateEnd(&strm); - return false; - } - size_t have = sizeof(outBuffer) - strm.avail_out; + if ( ret < 0 && ret != Z_BUF_ERROR ) return false; + size_t have = sizeof(outBuffer) - strm.avail_out; size_t chunkStart = uncompressedOffset; size_t chunkEnd = uncompressedOffset + have; - // Check all remaining ranges against this chunk for (size_t i = currentRangeIdx; i < sortedRanges.size(); ++i) { const auto& r = sortedRanges[i]; if (chunkEnd > r.start && chunkStart < r.start + r.len) { @@ -158,15 +134,19 @@ bool ext::zlib::decompressFromFile( uf::stl::vector& buffer, const uf:: buffer.insert(buffer.end(), outBuffer + copyStart, outBuffer + copyStart + copyLen); } if (chunkEnd >= r.start + r.len) { - currentRangeIdx = i + 1; // Move past completed ranges + currentRangeIdx = i + 1; } } uncompressedOffset += have; + if ( currentRangeIdx >= sortedRanges.size() ) return false; + } while (strm.avail_out == 0); - } + + return true; + }); inflateEnd(&strm); - return true; + return success; } bool ext::zlib::decompressFromMemory( uf::stl::vector& dst, const void* src, size_t size, size_t usize ) { @@ -188,6 +168,60 @@ bool ext::zlib::decompressFromMemory( uf::stl::vector& dst, const void* return (ret == Z_STREAM_END || ret == Z_OK); } +bool ext::zlib::decompressScatter( const uf::stl::string& filename, uf::stl::vector& requests ) { + if ( requests.empty() ) return true; + + std::sort(requests.begin(), requests.end(), [](const pod::ScatterRequest& a, const pod::ScatterRequest& b) { + return a.start < b.start; + }); + + z_stream strm{}; + if ( inflateInit2(&strm, 15 + 32) != Z_OK ) return false; + + size_t uncompressedOffset = 0; + size_t currentReqIdx = 0; + uint8_t outBuffer[ext::zlib::bufferSize]; + + bool success = uf::vfs::stream(filename, ext::zlib::bufferSize, [&](const uint8_t* data, size_t size) -> bool { + strm.avail_in = (uInt)size; + strm.next_in = (Bytef*)data; + + do { + strm.avail_out = sizeof(outBuffer); + strm.next_out = outBuffer; + int ret = inflate(&strm, Z_NO_FLUSH); + if ( ret < 0 && ret != Z_BUF_ERROR ) return false; + + size_t have = sizeof(outBuffer) - strm.avail_out; + size_t chunkStart = uncompressedOffset; + size_t chunkEnd = uncompressedOffset + have; + + for ( size_t i = currentReqIdx; i < requests.size(); ++i ) { + auto& req = requests[i]; + if ( chunkEnd > req.start && chunkStart < req.start + req.len ) { + size_t copyStart = (chunkStart < req.start) ? (req.start - chunkStart) : 0; + size_t copyLen = std::min(have - copyStart, (req.start + req.len) - (chunkStart + copyStart)); + + size_t destOffset = (chunkStart + copyStart) - req.start; + std::memcpy(req.dest + destOffset, outBuffer + copyStart, copyLen); + } + if ( chunkEnd >= req.start + req.len && i == currentReqIdx ) { + currentReqIdx++; + } + } + + uncompressedOffset += have; + if ( currentReqIdx >= requests.size() ) return false; + + } while (strm.avail_out == 0); + + return true; + }); + + inflateEnd(&strm); + return success; +} + size_t ext::zlib::compressToFile( const uf::stl::string& filename, const void* data, size_t size ) { z_stream strm{}; // 31 means gzip format diff --git a/engine/src/utils/io/file.cpp b/engine/src/utils/io/file.cpp index 3208cd8a..2471464e 100644 --- a/engine/src/utils/io/file.cpp +++ b/engine/src/utils/io/file.cpp @@ -170,6 +170,40 @@ bool uf::io::readAsBuffer( uf::stl::vector& buffer, const uf::stl::str return true; } +bool uf::io::readScatter( const uf::stl::string& filename, uf::stl::vector& requests ) { +#if UF_ENV_DREAMCAST + const size_t THRESHOLD = 2 * 1024 * 1024; +#else + const size_t THRESHOLD = 16 * 1024 * 1024; +#endif + + uf::stl::string extension = uf::io::extension(filename); + bool isZlib = (extension == "gz"); + size_t fileSize = uf::io::size(filename); + + if ( 0 < fileSize && fileSize <= THRESHOLD ) { + uf::stl::vector fullBuffer; + if ( isZlib ) ext::zlib::decompressFromFile(fullBuffer, filename); + else uf::vfs::read(filename, fullBuffer); + + for ( auto& req : requests ) { + if ( req.start + req.len <= fullBuffer.size() ) { + std::memcpy(req.dest, fullBuffer.data() + req.start, req.len); + } + } + return true; + } + + if ( isZlib ) return ext::zlib::decompressScatter(filename, requests); + + for ( auto& req : requests ) { + uf::stl::vector temp; + uf::vfs::readRange( filename, req.start, req.len, temp ); + std::memcpy(req.dest, temp.data(), temp.size()); + } + return true; +} + size_t uf::io::write( const uf::stl::string& filename, const void* buffer, size_t size ) { uf::stl::string extension = uf::io::extension( filename ); if ( extension == "gz" || extension == "lz4" ) return uf::io::compress( filename, buffer, size ); diff --git a/engine/src/utils/io/vfs.cpp b/engine/src/utils/io/vfs.cpp index ebf27f36..f23e1305 100644 --- a/engine/src/utils/io/vfs.cpp +++ b/engine/src/utils/io/vfs.cpp @@ -94,6 +94,22 @@ namespace { buffer.resize(currentOffset); return true; } + + bool vfs_stream( pod::Mount& mount, const uf::stl::string& file, size_t chunkSize, std::function callback ) { + uf::stl::string path = mount.path + file; + std::ifstream is(path, std::ios::binary); + if ( !is.is_open() ) return false; + + uf::stl::vector buffer(chunkSize); + while ( is.good() ) { + is.read((char*)buffer.data(), chunkSize); + size_t bytesRead = is.gcount(); + if ( bytesRead == 0 ) break; + + if ( !callback(buffer.data(), bytesRead) ) break; + } + return true; + } } pod::Mount uf::vfs::createDiskMount( const uf::stl::string& uri, int priority) { @@ -113,7 +129,8 @@ pod::Mount uf::vfs::createDiskMount( const uf::stl::string& uri, int priority) { .write = ::vfs_write, .mkdir = ::vfs_mkdir, .readRange = ::vfs_readRange, - .readRanges = ::vfs_readRanges + .readRanges = ::vfs_readRanges, + .stream = ::vfs_stream }; } @@ -277,6 +294,7 @@ bool uf::vfs::readRange( const uf::stl::string& path, size_t start, size_t len, uf::stl::vector fullBuffer; if ( !mount.read( mount, relative, fullBuffer ) ) continue; + UF_MSG_DEBUG("hitting fallback: {}", path); if ( start < fullBuffer.size() ) { size_t actualLen = std::min(len, fullBuffer.size() - start); @@ -302,6 +320,7 @@ bool uf::vfs::readRanges( const uf::stl::string& path, const uf::stl::vector fullBuffer; if ( !mount.read( mount, relative, fullBuffer ) ) continue; + UF_MSG_DEBUG("hitting fallback: {}", path); size_t totalBytes = 0; for ( const auto& r : ranges ) { @@ -324,6 +343,35 @@ bool uf::vfs::readRanges( const uf::stl::string& path, const uf::stl::vector callback ) { + uf::stl::string prefix, relative; + uf::io::splitUri(path, prefix, relative); + for ( auto& mount : mounts ) { + if ( prefix.empty() && mount.priority < 0 ) continue; + if ( prefix.empty() || mount.prefix == prefix ) { + if ( !mount.exists( mount, relative ) ) continue; + if ( mount.stream ) return mount.stream( mount, relative, chunkSize, callback ); + + if ( !mount.read ) continue; + UF_MSG_DEBUG("hitting fallback: {}", path); + + uf::stl::vector fullBuffer; + if ( !mount.read( mount, relative, fullBuffer ) ) continue; + + size_t offset = 0; + size_t totalSize = fullBuffer.size(); + + while ( offset < totalSize ) { + size_t currentChunkSize = std::min(chunkSize, totalSize - offset); + if ( !callback(fullBuffer.data() + offset, currentChunkSize) ) break; + offset += currentChunkSize; + } + + return true; + } + } + return false; +} uf::stl::string uf::vfs::resolveBase( const uf::stl::string& path ) { uf::stl::string prefix, relative; diff --git a/engine/src/utils/mesh/mesh.cpp b/engine/src/utils/mesh/mesh.cpp index 1f6c71de..c6c7e546 100644 --- a/engine/src/utils/mesh/mesh.cpp +++ b/engine/src/utils/mesh/mesh.cpp @@ -288,6 +288,48 @@ void uf::Mesh::interleave() { updateDescriptor(); } +void uf::Mesh::prune( const uf::stl::vector& keep ) { + uf::stl::vector deadAttributes; + uf::stl::vector deadBuffers; + + for ( size_t i = 0; i < vertex.attributes.size(); ++i ) { + auto& attribute = vertex.attributes[i]; + if ( std::find( keep.begin(), keep.end(), attribute.descriptor.name ) != keep.end() ) continue; + + deadAttributes.push_back(i); + deadBuffers.push_back(attribute.buffer); + } + + std::sort(deadAttributes.rbegin(), deadAttributes.rend()); + std::sort(deadBuffers.rbegin(), deadBuffers.rend()); + + for ( auto idx : deadAttributes ) { + vertex.attributes.erase(vertex.attributes.begin() + idx); + } + + for ( auto bufID : deadBuffers ) { + buffers.erase(buffers.begin() + bufID); + } + + // + auto remap_input = [&](uf::Mesh::Input& input) { + for (auto& attr : input.attributes) { + int32_t shift = 0; + for (int32_t db : deadBuffers) { + if (attr.buffer > db) shift++; + } + attr.buffer -= shift; + } + }; + + remap_input(vertex); + remap_input(index); + remap_input(instance); + remap_input(indirect); + + updateDescriptor(); +} + void uf::Mesh::clearAttribute( uf::Mesh::Input& input, const uf::Mesh::Attribute& attribute ) { for ( size_t i = 0; i < input.attributes.size(); ++i ) if ( input.attributes[i].descriptor == attribute.descriptor ) return clearAttribute( input, i ); }