ecker/engine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

sanity fixes for gltf conversion (and LOD generation), (mostly) repaired streaming in mesh data (barycentric-deferred rendering would desync because of separated indirection buffers), streamed in mesh data picks the correct LOD (mostly working), squashed hidden bug where meshes were constantly destroying/creating the physics mesh and updating buffers

Browse Source
This commit is contained in:
ecker 2026-05-05 19:48:18 -05:00
parent a145bae065
commit ec2122ab12
11 changed files with 154 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
bin/data/config.json
View File

@ -1,7 +1,7 @@
{
"engine": {
"scenes": {
"start": "SourceEngine",
"start": "StartMenu",
"matrix": { "reverseInfinite": true },
"meshes": { "interleaved": false },
"lights": { "enabled": false,
@ -109,7 +109,7 @@
"invariant": {
"default stage buffers": true,
"default defer buffer destroy": true,
"default command buffer immediate": false,
"default command buffer immediate": true,
"multithreaded recording": true
},
"pipelines": {
@ -118,7 +118,7 @@
"vsync": true, // vsync on vulkan side rather than engine-side
"hdr": true,
"vxgi": false,
"culling": true,
"culling": false,
"bloom": true,
"dof": true,
"rt": false,

3
bin/data/shaders/graph/cull/comp.glsl
View File

@ -192,6 +192,9 @@ void main() {
if ( projectedSize < 0.08 ) lodLevel = 2;
if ( projectedSize < 0.02 ) lodLevel = 3;
lodLevel = min(lodLevel, MAX_LODS - 1);
while ( lodLevel > 0 && lodMetadata[drawCommand.instanceID].levels[lodLevel].indices == 0 ) {
lodLevel--;
}
LOD lod = lodMetadata[drawCommand.instanceID].levels[lodLevel];

2
engine/inc/uf/ext/meshopt/meshopt.h
View File

@ -11,7 +11,7 @@ namespace ext {
namespace meshopt {
bool UF_API optimize( uf::Mesh&, float simplify = 1.0f, size_t = SIZE_MAX, bool verbose = false );
uf::stl::vector<float> computeLODs( size_t count, size_t maxLODs = 4, size_t minIndices = 32 );
uf::stl::vector<float> computeLODs( size_t count, size_t maxLODs = 4, size_t minIndices = 3 );
uf::stl::vector<pod::LODMetadata> UF_API generateLODs( uf::Mesh&, const uf::stl::vector<float>&, bool verbose = false );
template<typename T, typename U = uint32_t>

125
engine/src/engine/graph/graph.cpp
View File

@ -23,8 +23,10 @@
#define UF_DEBUG_TIMER_MULTITRACE_END(...)
#endif
#define UF_GRAPH_SPARSE_READ_MESH 1
#define UF_GRAPH_EXTENDED 1
#define UF_GRAPH_SPARSE_READ_MESH 1
// to-do: fix LOD1+ breaking, fix physics mesh not updating
namespace {
bool newGraphAdded = true;
@ -39,6 +41,14 @@ namespace {
}
return hash;
}
inline uint64_t fnv1aHash(const uf::stl::vector<int8_t>& values) {
uint64_t hash = 1469598103934665603ULL;
for (bool v : values) {
hash ^= static_cast<uint64_t>(static_cast<uint8_t>(v));
hash *= 1099511628211ULL;
}
return hash;
}
size_t allocateObjectID( pod::Graph::Storage& storage ) {
return storage.entities.keys.size();
@ -1280,14 +1290,16 @@ void uf::graph::process( pod::Graph& graph, int32_t index, uf::Object& parent )
pod::Instance::Bounds bounds = {};
size_t baseInstanceID = ::allocateInstanceID( storage, graph.primitives[node.mesh] );
// setup instances
for ( auto i = 0; i < primitives.size(); ++i ) {
auto& primitive = primitives[i];
for ( auto drawID = 0; drawID < primitives.size(); ++drawID ) {
auto& primitive = primitives[drawID];
auto& instance = primitive.instance;
size_t instanceID = baseInstanceID + i;
size_t instanceID = baseInstanceID + drawID;
instance.objectID = node.object;
instance.jointID = graphMetadataJson["renderer"]["skinned"].as<bool>() ? 0 : -1;
primitive.drawCommand.instanceID = instanceID;
bounds.min = uf::vector::min( bounds.min, instance.bounds.min );
bounds.max = uf::vector::max( bounds.max, instance.bounds.max );
@ -1295,7 +1307,7 @@ void uf::graph::process( pod::Graph& graph, int32_t index, uf::Object& parent )
auto& attribute = mesh.indirect.attributes.front();
auto& buffer = mesh.buffers[mesh.isInterleaved(mesh.indirect.interleaved) ? mesh.indirect.interleaved : attribute.buffer];
pod::DrawCommand* drawCommands = (pod::DrawCommand*) buffer.data();
auto& drawCommand = drawCommands[i];
auto& drawCommand = drawCommands[drawID];
drawCommand.instanceID = instanceID;
}
}
@ -1615,6 +1627,7 @@ void uf::graph::reload( pod::Graph& graph, pod::Node& node ) {
}
}
bool meshUpdated = false;
auto model = uf::transform::model( transform );
auto& mesh = storage.meshes.map[graph.meshes[node.mesh]];
auto& primitives = storage.primitives.map[graph.primitives[node.mesh]];
@ -1623,6 +1636,11 @@ void uf::graph::reload( pod::Graph& graph, pod::Node& node ) {
float radius = graph.settings.stream.radius;
float radiusSquared = radius * radius;
// force update if entity isn't already bound to the graphic
if ( !entity.hasComponent<uf::renderer::Graphic>() ) {
meshUpdated = true;
}
// disable if not tagged for streaming
// to-do: check tag
if ( graph.settings.stream.tag != "" && node.name != graph.settings.stream.tag ) {
@ -1640,7 +1658,7 @@ void uf::graph::reload( pod::Graph& graph, pod::Node& node ) {
pod::DrawCommand* drawCommands = (pod::DrawCommand*) buffer.data();
// queues
uf::stl::unordered_map<size_t, uf::stl::vector<pod::Range>> ranges;
uf::stl::vector<bool> queuedDrawIDs( primitives.size(), false ); // this is to maintain draw command order because apparently my code requires draw commands to stay in order
uf::stl::vector<int8_t> queuedLODs( primitives.size(), -1 ); // this is to maintain draw command order because apparently my code requires draw commands to stay in order
// fallbacks for when no draw calls are requested (mainly for the collision mesh)
float closestDistance = std::numeric_limits<float>::max();
size_t closestDrawID = 0;
@ -1661,24 +1679,37 @@ void uf::graph::reload( pod::Graph& graph, pod::Node& node ) {
closestDrawID = drawID;
}
// queue if we're within the radius
if ( (queuedDrawIDs[drawID] = distanceSquared <= radiusSquared) ) {
if ( distanceSquared <= radiusSquared ) {
found = true;
int8_t lodLevel = 0;
// deduce a simple ratio [0.0 to 1.0] of how far we are into the streaming radius
float distRatio = distanceSquared / radiusSquared;
if ( distRatio > 0.6f ) lodLevel = 3;
else if ( distRatio > 0.3f ) lodLevel = 2;
else if ( distRatio > 0.1f ) lodLevel = 1;
while ( lodLevel > 0 && primitive.lod.levels[lodLevel].indices == 0 ) {
lodLevel--;
}
queuedLODs[drawID] = lodLevel;
}
}
// insert closest primitive if all are out of range (because of cringe logic)
if ( !found ) {
queuedDrawIDs[closestDrawID] = true;
queuedLODs[closestDrawID] = 0;
}
// bail if no update is detected
auto drawCommandHash = ::fnv1aHash(queuedDrawIDs);
auto drawCommandHash = ::fnv1aHash(queuedLODs);
graph.settings.stream.lastUpdate = uf::physics::time::current;
if ( drawCommandHash == graph.settings.stream.hash ) {
return;
}
graph.settings.stream.hash = drawCommandHash;
meshUpdated = true;
// read from disk
#if UF_GRAPH_SPARSE_READ_MESH
@ -1707,14 +1738,24 @@ void uf::graph::reload( pod::Graph& graph, pod::Node& node ) {
mesh.vertex.count = 0;
mesh.index.count = 0;
for (size_t drawID = 0; drawID < queuedDrawIDs.size(); ++drawID) {
bool queued = queuedDrawIDs[drawID];
for (size_t drawID = 0; drawID < queuedLODs.size(); ++drawID) {
auto lodLevel = queuedLODs[drawID];
auto& primitive = primitives[drawID];
auto& drawCommand = drawCommands[drawID];
// reset from LOD0
//primitives[drawID].drawCommand.instances = 1;
primitives[drawID].drawCommand.indexID = primitives[drawID].lod.levels[0].indexID;
primitives[drawID].drawCommand.indices = primitives[drawID].lod.levels[0].indices;
primitives[drawID].drawCommand.vertexID = primitives[drawID].lod.levels[0].vertexID;
primitives[drawID].drawCommand.vertices = primitives[drawID].lod.levels[0].vertices;
// copy from primitive
drawCommand = primitive.drawCommand;
// disable draw call
if ( !queued ) {
drawCommand.instances = 0;
if ( lodLevel < 0 ) {
//drawCommand.instances = 0;
drawCommand.vertices = 0;
drawCommand.indices = 0;
drawCommand.vertexID = 0;
@ -1722,26 +1763,33 @@ void uf::graph::reload( pod::Graph& graph, pod::Node& node ) {
continue;
}
// queue up ranges to read from disk
auto& lod = primitive.lod.levels[lodLevel];
// queue up ranges to read from disk using LOD bounds
for (auto& attribute : mesh.index.attributes) {
auto size = attribute.descriptor.size;
ranges[attribute.buffer].emplace_back(pod::Range{
primitive.drawCommand.indexID * size,
primitive.drawCommand.indices * size,
lod.indexID * size,
lod.indices * size,
});
}
for (auto& attribute : mesh.vertex.attributes) {
auto size = attribute.descriptor.size;
ranges[attribute.buffer].emplace_back(pod::Range{
primitive.drawCommand.vertexID * size,
primitive.drawCommand.vertices * size,
lod.vertexID * size,
lod.vertices * size,
});
}
// reset draw call and remap
drawCommand = primitive.drawCommand;
// reset draw call and remap to local compacted buffers
drawCommand.vertices = lod.vertices;
drawCommand.indices = lod.indices;
drawCommand.vertexID = mesh.vertex.count;
drawCommand.indexID = mesh.index.count;
// synchronize primitive
primitives[drawID].drawCommand = drawCommands[drawID];
// increment remap indices
mesh.vertex.count += drawCommand.vertices;
mesh.index.count += drawCommand.indices;
@ -1762,17 +1810,34 @@ void uf::graph::reload( pod::Graph& graph, pod::Node& node ) {
#else
// disable remaining draw commands
for ( auto drawID = 0; drawID < primitives.size(); ++drawID ) {
bool queued = queuedDrawIDs[drawID];
if ( !queued ) {
drawCommands[drawID].instances = 0;
int8_t lodLevel = queuedLODs[drawID];
// reset from LOD0
//primitives[drawID].drawCommand.instances = 1;
primitives[drawID].drawCommand.indexID = primitives[drawID].lod.levels[0].indexID;
primitives[drawID].drawCommand.indices = primitives[drawID].lod.levels[0].indices;
primitives[drawID].drawCommand.vertexID = primitives[drawID].lod.levels[0].vertexID;
primitives[drawID].drawCommand.vertices = primitives[drawID].lod.levels[0].vertices;
// copy from primitive
drawCommands[drawID] = primitives[drawID].drawCommand;
if ( lodLevel < 0 ) {
//drawCommands[drawID].instances = 0;
drawCommands[drawID].vertices = 0;
drawCommands[drawID].indices = 0;
drawCommands[drawID].indexID = 0;
drawCommands[drawID].vertexID = 0;
} else {
drawCommands[drawID] = primitives[drawID].drawCommand;
// to-do: LOD pick here for OpenGL
auto& lod = primitives[drawID].lod.levels[lodLevel];
drawCommands[drawID].indexID = lod.indexID;
drawCommands[drawID].indices = lod.indices;
drawCommands[drawID].vertexID = lod.vertexID;
drawCommands[drawID].vertices = lod.vertices;
}
// synchronize primitive
primitives[drawID].drawCommand = drawCommands[drawID];
}
#define STREAM_MESH_DATA( N ) \
@ -1868,10 +1933,12 @@ void uf::graph::reload( pod::Graph& graph, pod::Node& node ) {
}
}
}
} else { // this shouldn't be reached
} else {
// load mesh if not already loaded
#define LOAD_MESH_DATA( N ) \
for ( auto& attribute : mesh.N.attributes ) {\
if ( !mesh.buffers[attribute.buffer].empty() || mesh.buffer_paths.empty() ) continue;\
meshUpdated = true;\
uf::io::readAsBuffer( mesh.buffers[attribute.buffer], mesh.buffer_paths[attribute.buffer] );\
}
@ -1879,6 +1946,8 @@ void uf::graph::reload( pod::Graph& graph, pod::Node& node ) {
LOAD_MESH_DATA( vertex );
}
if ( !meshUpdated ) return;
// in the event streamed in mesh data from any pathway isn't already converted
{
#if UF_ENV_DREAMCAST && GL_QUANTIZED_SHORT
@ -1908,6 +1977,8 @@ void uf::graph::reload( pod::Graph& graph, pod::Node& node ) {
uf::renderer::states::rebuild = true;
#endif
::newGraphAdded = true; // force rebuffering the draw commands
// update graphic
if ( graphMetadataJson["renderer"]["render"].as<bool>() ) {
bool exists = entity.hasComponent<uf::renderer::Graphic>();

22
engine/src/ext/gltf/gltf.cpp
View File

@ -552,6 +552,8 @@ void ext::gltf::load( pod::Graph& graph, const uf::stl::string& filename, const
}
auto& mesh = storage.meshes[keyName];
auto& primitives = storage.primitives[keyName];
UF_MSG_DEBUG("Optimizing mesh at level {}: {}", level, keyName);
if ( !ext::meshopt::optimize( mesh, simplify, level, print ) ) {
UF_MSG_ERROR("Mesh optimization failed: {}", keyName );
@ -563,9 +565,10 @@ void ext::gltf::load( pod::Graph& graph, const uf::stl::string& filename, const
UF_MSG_ERROR("LOD generation failed: {}", keyName );
} else {
UF_MSG_DEBUG("Generated {} LODs: {}", factors.size() - 1, keyName);
auto& primitives = storage.primitives[keyName];
UF_ASSERT( primitives.size() == lodMetadata.size() );
for ( auto i = 0; i < primitives.size(); ++i ) primitives[i].lod = lodMetadata[i];
for ( auto i = 0; i < primitives.size(); ++i ) {
primitives[i].lod = lodMetadata[i];
}
}
}
}
@ -573,6 +576,21 @@ void ext::gltf::load( pod::Graph& graph, const uf::stl::string& filename, const
UF_MSG_DEBUG( "Optimized mesh" );
}
#endif
{
// update primitive info
for ( auto& keyName : graph.meshes ) {
auto& mesh = storage.meshes[keyName];
auto& primitives = storage.primitives[keyName];
UF_ASSERT( primitives.size() == mesh.indirect.count );
auto& attribute = mesh.indirect.attributes.front();
auto& buffer = mesh.buffers[mesh.isInterleaved(mesh.indirect.interleaved) ? mesh.indirect.interleaved : attribute.buffer];
pod::DrawCommand* drawCommands = (pod::DrawCommand*) buffer.data();
for ( auto drawID = 0; drawID < primitives.size(); ++drawID ) {
primitives[drawID].drawCommand = drawCommands[drawID];
}
}
}
if ( graph.metadata["exporter"]["enabled"].as<bool>() ) {
#if !UF_ENV_DREAMCAST

6
engine/src/ext/gltf/processPrimitives.inl
View File

@ -385,6 +385,12 @@ if ( meshopt.should ) {
meshlet.primitive.drawCommand.vertexID = vertexID;
meshlet.primitive.drawCommand.vertices = meshlet.vertices.size();
// copy to LOD metadata
meshlet.primitive.lod.levels[0].indexID = indexID;
meshlet.primitive.lod.levels[0].indices = meshlet.indices.size();
meshlet.primitive.lod.levels[0].vertexID = vertexID;
meshlet.primitive.lod.levels[0].vertices = meshlet.vertices.size();
drawCommands.emplace_back(meshlet.primitive.drawCommand);
primitives.emplace_back( meshlet.primitive );

16
engine/src/ext/meshopt/meshopt.cpp
View File

@ -90,7 +90,7 @@ bool ext::meshopt::optimize( uf::Mesh& mesh, float simplify, size_t o, bool verb
size_t optimizedIndexCount = srcIndexCount;
if ( 0.0f < simplify && simplify < 1.0f ) {
uf::stl::vector<uint32_t> simplified(srcIndexCount);
float targetError = 1e-2f / simplify;
float targetError = FLT_MAX; // 1e-2f / simplify;
float realError = 0.0f;
optimizedIndexCount = meshopt_simplify(
@ -233,16 +233,18 @@ uf::stl::vector<pod::LODMetadata> ext::meshopt::generateLODs( uf::Mesh& mesh, co
// source from LOD0
auto& cmd0 = lodMetadata[cmdIdx].levels[0];
size_t previousIndicesCount = lodMetadata[cmdIdx].levels[lodIdx - 1].indices;
uf::stl::vector<uint32_t> baseIndices(cmd0.indices);
for ( size_t i = 0; i < cmd0.indices; ++i ) baseIndices[i] = outIndices[cmd0.indexID + i];
// copy from LOD0
lodMetadata[cmdIdx].levels[lodIdx] = lodMetadata[cmdIdx].levels[0];
// generate LOD
if ( 0.0f < simplify && simplify < 1.0f ) {
float targetError = 1e-2f / simplify;
float targetError = FLT_MAX; // 1e-2f / simplify;
float realError = 0.0f;
size_t currentIndicesCount = cmd0.indices;
uf::stl::vector<uint32_t> baseIndices(cmd0.indices);
for ( size_t i = 0; i < cmd0.indices; ++i ) baseIndices[i] = outIndices[cmd0.indexID + i];
uf::stl::vector<uint32_t> lodIndices = baseIndices;
const float* basePositions = (const float*) (outVertices[posAttrIdx].data() + cmd0.vertexID * mesh.vertex.attributes[posAttrIdx].stride);
@ -254,6 +256,7 @@ uf::stl::vector<pod::LODMetadata> ext::meshopt::generateLODs( uf::Mesh& mesh, co
);
// couldn't simplify further, use previous LOD
size_t previousIndicesCount = lodMetadata[cmdIdx].levels[lodIdx - 1].indices;
if ( currentIndicesCount == previousIndicesCount ) {
lodMetadata[cmdIdx].levels[lodIdx] = lodMetadata[cmdIdx].levels[lodIdx - 1];
continue;
@ -288,9 +291,6 @@ uf::stl::vector<pod::LODMetadata> ext::meshopt::generateLODs( uf::Mesh& mesh, co
meshopt_remapVertexBuffer(packed.data(), srcPtr, cmd0.vertices, attr.stride, fetchRemap.data());
outVertices[a].insert(outVertices[a].end(), packed.begin(), packed.end());
}
} else {
// no simplification, just use LOD0 (shouldn't happen)
lodMetadata[cmdIdx].levels[lodIdx] = lodMetadata[cmdIdx].levels[0];
}
}
}

13
engine/src/utils/io/file.cpp
View File

@ -189,13 +189,12 @@ uf::stl::vector<uint8_t>& uf::io::readAsBuffer( uf::stl::vector<uint8_t>& buffer
buffer.resize(totalBytes);
// Read each range
for (const auto& r : ranges) {
is.seekg(r.start, std::ios::beg);
size_t oldSize = buffer.size();
buffer.resize(oldSize + r.len);
is.read(reinterpret_cast<char*>(buffer.data() + oldSize), r.len);
buffer.resize(oldSize + static_cast<size_t>(is.gcount()));
}
size_t currentOffset = 0;
for (const auto& r : ranges) {
is.seekg(r.start, std::ios::beg);
is.read(reinterpret_cast<char*>(buffer.data() + currentOffset), r.len);
currentOffset += static_cast<size_t>(is.gcount());
}
}
uf::stl::string expected;

2
engine/src/utils/math/physics/broadphase/bvh.cpp
View File

@ -232,7 +232,7 @@ void impl::buildMeshBVH( pod::BVH& bvh, const uf::Mesh& mesh, pod::BVH::index_t
auto aabb = impl::computeTriangleAABB( tri );
auto triID = triIndexID + (view.index.first / 3);
if ( triID != bounds.size() ) UF_MSG_DEBUG("triID={}, bounds.size()={}", triID, bounds.size());
// if ( triID != bounds.size() ) UF_MSG_DEBUG("triID={}, bounds.size()={}", triID, bounds.size());
bounds.emplace_back( aabb );
bvh.indices.emplace_back( triID ); // triID => mesh.index.buffer[triID * 3];

3
engine/src/utils/math/physics/impl.cpp
View File

@ -516,6 +516,9 @@ void uf::physics::destroy( pod::PhysicsBody& body ) {
if ( body.collider.type == pod::ShapeType::MESH ) {
if ( body.collider.mesh.bvh ) delete body.collider.mesh.bvh;
}
if ( body.collider.type == pod::ShapeType::CONVEX_HULL ) {
if ( body.collider.convexHull.bvh ) delete body.collider.convexHull.bvh;
}
}
pod::RayQuery uf::physics::rayCast( const pod::Ray& ray, const pod::PhysicsBody& body, float maxDistance ) {

6
engine/src/utils/mesh/mesh.cpp
View File

@ -407,7 +407,11 @@ uf::Mesh::View uf::Mesh::makeView( size_t i, const uf::stl::vector<uf::stl::stri
uf::stl::vector<uf::Mesh::View> uf::Mesh::makeViews( const uf::stl::vector<uf::stl::string>& wanted, size_t lod ) const {
uf::stl::vector<uf::Mesh::View> views;
if ( indirect.count > 0 ) {
for ( auto i = 0; i < indirect.count; i++ ) views.emplace_back(makeView(i, wanted, lod));
for ( auto i = 0; i < indirect.count; i++ ) {
auto view = makeView( i, wanted, lod );
if ( view.index.count == 0 && view.vertex.count == 0 ) continue;
views.emplace_back( view );
}
} else {
views.emplace_back( makeView(wanted, lod) );
}