ecker/engine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fixing issues related to asynchronous asset loading (disabled for now because there's an inconsistent vulkan-related crash that I'd like to fix)

Browse Source
This commit is contained in:
ecker 2026-06-17 22:49:13 -05:00
parent 2944335904
commit 941ad2ead9
20 changed files with 174 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
bin/data/config.json
View File

@ -425,8 +425,8 @@
"auto validate": false
},
"loader": {
"assert": true
//"async": true
"assert": true,
"async": false
},
"hooks": {
"defer lazy calls": true

2
bin/data/scenes/sourceengine/mds_mcdonalds.json
View File

@ -3,6 +3,6 @@
"assets": [
// { "filename": "./maps/mcdonalds-mds.bsp" }
{ "filename": "./maps/mcdonalds-mds/graph.json" },
{ "filename": "ent://burger.json", "delay": 1 }
{ "filename": "ent://burger.json", "delay": 4 }
]
}

4
bin/data/scenes/sourceengine/sourceengine.json
View File

@ -1,7 +1,7 @@
{
// "import": "./rp_downtown_v2.json"
// "import": "./ss2_medsci1.json"
// "import": "./mds_mcdonalds.json"
"import": "./cs_office.json"
"import": "./mds_mcdonalds.json"
// "import": "./cs_office.json"
// "import": "./gm_construct.json"
}

2
engine/inc/uf/engine/asset/payload.h
View File

@ -26,7 +26,7 @@ namespace uf {
uf::stl::string uri = "";
bool initialize = true;
bool monoThreaded = false;
bool async = true;
bool asComponent = false;
uf::Serializer metadata;

5
engine/inc/uf/engine/graph/graph.h
View File

@ -102,8 +102,6 @@ namespace pod {
uf::stl::KeyMap<uf::stl::vector<pod::Matrix4f>> joints;
uf::stl::KeyMap<uf::Entity*> entities;
uf::stl::vector<uf::renderer::TextureCube> cubemaps;
uf::stl::vector<uf::renderer::Texture2D> shadow2Ds;
uf::stl::vector<uf::renderer::TextureCube> shadowCubes;
@ -127,6 +125,8 @@ namespace pod {
bool stale = false;
bool shouldRebind = false;
std::shared_ptr<std::mutex> mutex = std::make_shared<std::mutex>();
}/* storage*/;
pod::Graph::Storage* storage = NULL;
@ -203,6 +203,7 @@ namespace uf {
pod::Graph& UF_API convert( uf::Object&, bool = false ); // converts an object into a graph
void UF_API postprocess( pod::Graph& ); // applies post-processing for format importing
void UF_API import( pod::Graph::Storage& to, pod::Graph::Storage& from, bool move = true ); // moves storage from one to the other
uf::stl::string UF_API save( const pod::Graph&, const uf::stl::string& ); // saves a graph to disk
uf::stl::string UF_API print( const pod::Graph& graph );

22
engine/inc/uf/utils/memory/key_map.h
View File

@ -22,6 +22,8 @@ namespace uf {
uf::stl::vector<T> flatten() const;
uf::stl::vector<T> flattenByIndex() const;
void clear();
void merge( KeyMap<T, Key>&& other );
void import( const KeyMap<T, Key>& other );
};
}
}
@ -92,4 +94,24 @@ uf::stl::vector<T> uf::stl::KeyMap<T,Key>::flattenByIndex() const {
}
return res;
}
template<typename T, typename Key>
void uf::stl::KeyMap<T,Key>::merge( KeyMap<T, Key>&& other ) {
this->reserve( this->keys.size() + other.keys.size() );
for ( auto& key : other.keys ) {
(*this)[key] = std::move( other.map[key] );
}
other.clear();
}
template<typename T, typename Key>
void uf::stl::KeyMap<T,Key>::import( const KeyMap<T, Key>& other ) {
this->reserve( this->keys.size() + other.keys.size() );
for ( auto& key : other.keys ) {
(*this)[key] = other.map.at(key);
}
}

12
engine/inc/uf/utils/thread/thread.h
View File

@ -33,6 +33,12 @@ namespace pod {
typedef uf::stl::vector<pod::Thread::function_t> container_t;
struct UF_API Tasks {
struct UF_API Tracker {
std::atomic<uint32_t> pending{0};
std::mutex mutex;
std::condition_variable cv;
};
uf::stl::string name = uf::thread::workerThreadName;
bool waits = true;
@ -94,9 +100,8 @@ namespace uf {
pod::Thread& UF_API fetchWorker( const uf::stl::string& name = uf::thread::workerThreadName );
pod::Thread::Tasks UF_API schedule( bool multithread, bool waits = true );
pod::Thread::Tasks UF_API schedule( const uf::stl::string& name = uf::thread::workerThreadName, bool waits = true );
uf::stl::vector<pod::Thread*> UF_API execute( pod::Thread::Tasks& tasks );
void UF_API wait( uf::stl::vector<pod::Thread*>& );
void UF_API wait( const uf::stl::vector<pod::Thread*>& );
std::shared_ptr<pod::Thread::Tasks::Tracker> UF_API execute( pod::Thread::Tasks& tasks );
void UF_API wait( std::shared_ptr<pod::Thread::Tasks::Tracker> );
/* Acts on global threads */
typedef uf::stl::unordered_map<uf::stl::string, pod::Thread*> container_t;
@ -145,6 +150,7 @@ namespace uf {
void UF_API wait( pod::Thread& );
uf::stl::string UF_API name( std::thread::id id );
const uf::stl::string& UF_API name( const pod::Thread& );
std::thread::id UF_API id( const pod::Thread& );
pod::Thread::id_t UF_API uid( const pod::Thread& );

9
engine/src/engine/asset/asset.cpp
View File

@ -16,6 +16,7 @@
#include <uf/ext/lua/lua.h>
#include <uf/ext/gltf/gltf.h>
#include <uf/engine/graph/graph.h>
#include <uf/engine/scene/scene.h>
#include <mutex>
@ -56,12 +57,15 @@ uf::stl::unordered_map<uf::stl::string, uf::asset::userdata_t> uf::asset::map;
uf::Serializer uf::asset::metadata;
void uf::asset::processQueue() {
auto tasks = uf::asset::asyncQueue ? uf::thread::schedule(false) : uf::thread::schedule(true, false);
if ( uf::asset::jobs.empty() ) return;
STATIC_THREAD_LOCAL(uf::asset::Job::container_t, jobs);
mutex.lock();
auto jobs = std::move(uf::asset::jobs);
std::swap( jobs, uf::asset::jobs );
mutex.unlock();
bool async = false; // uf::asset::asyncQueue; // a bit buggy
auto tasks = uf::thread::schedule(async ? uf::thread::asyncThreadName : uf::thread::mainThreadName, !true);
for ( auto& job : jobs ) tasks.queue([=]{
auto callback = job.callback;
auto type = job.type;
@ -244,7 +248,6 @@ uf::stl::string uf::asset::load( uf::asset::Payload& payload ) {
}
#endif
// asset = uf::graph::load( filename, payload.metadata );
uf::graph::load( asset, filename, payload.metadata );
uf::graph::process( asset );

5
engine/src/engine/entity/entity.cpp
View File

@ -23,8 +23,9 @@ uf::Entity::~Entity(){
this->destroy();
}
bool uf::Entity::isValid() const {
if ( uf::Entity::memoryPool.size() > 0 && !uf::Entity::memoryPool.exists((void*) this) ) return false;
return /*this != NULL &&*/ (0 < this->m_uid && this->m_uid <= uf::Entity::uids);
// if ( uf::Entity::memoryPool.size() > 0 && !uf::Entity::memoryPool.exists((void*) this) ) return false;
if ( this == 0x0 ) return false;
return (0 < this->m_uid && this->m_uid <= uf::Entity::uids);
}
bool uf::Entity::operator==( const uf::Entity& e ) const {
return this == &e;

2
engine/src/engine/ext/player/model/behavior.cpp
View File

@ -1,5 +1,5 @@
#include "behavior.h"
#include "../behavior.h"
#include <uf/utils/serialize/serializer.h>
#include <uf/utils/hook/hook.h>
#include <uf/utils/time/time.h>

54
engine/src/engine/graph/convert.cpp
View File

@ -331,4 +331,58 @@ void uf::graph::postprocess( pod::Graph& graph ) {
graph.settings.stream.radius = 0;
graph.settings.stream.every = 0;
}
// migrate
if ( graph.storage && uf::graph::storageMode != pod::Graph::Storage::GRAPH ) {
auto* pointer = graph.storage;
graph.storage = NULL;
auto& storage = *pointer;
auto& target = uf::graph::getStorage( graph );
uf::graph::import( target, storage );
delete pointer;
}
}
void uf::graph::import( pod::Graph::Storage& target, pod::Graph::Storage& storage, bool move ) {
std::lock_guard<std::mutex> lock(*target.mutex);
if ( move ) {
target.primitives.merge(std::move(storage.primitives));
target.instances.merge(std::move(storage.instances));
target.meshes.merge(std::move(storage.meshes));
target.images.merge(std::move(storage.images));
target.materials.merge(std::move(storage.materials));
target.textures.merge(std::move(storage.textures));
target.samplers.merge(std::move(storage.samplers));
target.skins.merge(std::move(storage.skins));
target.animations.merge(std::move(storage.animations));
target.atlases.merge(std::move(storage.atlases));
target.objects.merge(std::move(storage.objects));
target.joints.merge(std::move(storage.joints));
target.entities.merge(std::move(storage.entities));
for ( auto& v :storage.lights ) target.lights.emplace_back(std::move(v));
for ( auto& v :storage.shadow2Ds ) target.shadow2Ds.emplace_back(std::move(v));
for ( auto& v :storage.shadowCubes ) target.shadowCubes.emplace_back(std::move(v));
for ( auto& v :storage.flattenedPrimitives ) target.flattenedPrimitives.emplace_back(std::move(v));
} else {
target.primitives.import(storage.primitives);
target.instances.import(storage.instances);
target.meshes.import(storage.meshes);
target.images.import(storage.images);
target.materials.import(storage.materials);
target.textures.import(storage.textures);
target.samplers.import(storage.samplers);
target.skins.import(storage.skins);
target.animations.import(storage.animations);
target.atlases.import(storage.atlases);
target.objects.import(storage.objects);
target.joints.import(storage.joints);
target.entities.import(storage.entities);
for ( auto& v :storage.lights ) target.lights.emplace_back(v);
for ( auto& v :storage.shadow2Ds ) target.shadow2Ds.emplace_back(v);
for ( auto& v :storage.shadowCubes ) target.shadowCubes.emplace_back(v);
for ( auto& v :storage.flattenedPrimitives ) target.flattenedPrimitives.emplace_back(v);
}
}

22
engine/src/engine/graph/decode.cpp
View File

@ -11,13 +11,7 @@
#include <uf/ext/valve/bsp.h>
#include <uf/utils/io/fmt.h>
// it's too unstable right now to do multithreaded loading, perhaps there's a better way
#if UF_USE_OPENGL
#define UF_GRAPH_LOAD_MULTITHREAD 0
#else
#define UF_GRAPH_LOAD_MULTITHREAD 1
#endif
#define UF_GRAPH_LOAD_MULTITHREAD 0
#define UF_GRAPH_EXTENDED 1
#if UF_ENV_DREAMCAST
@ -388,7 +382,8 @@ void uf::graph::load( pod::Graph& graph, const uf::stl::string& filename, const
auto tasks = uf::thread::schedule(false);
#endif
auto& storage = uf::graph::getStorage( graph );
if ( !graph.storage ) graph.storage = new pod::Graph::Storage();
auto& storage = uf::graph::getStorage( graph ); // will just fetch the above
if ( !ext::json::isArray(graph.metadata["decode"]["attributes"]) ) {
#if UF_USE_OPENGL
@ -648,6 +643,17 @@ void uf::graph::load( pod::Graph& graph, const uf::stl::string& filename, const
}
UF_DEBUG_TIMER_MULTITRACE_END("Processing graph...");
// migrate
if ( graph.storage && uf::graph::storageMode != pod::Graph::Storage::GRAPH ) {
auto* pointer = graph.storage;
graph.storage = NULL;
auto& storage = *pointer;
auto& target = uf::graph::getStorage( graph );
uf::graph::import( target, storage );
delete pointer;
}
#if UF_ENV_DREAMCAST
DC_STATS();
#endif

12
engine/src/engine/graph/graph.cpp
View File

@ -615,6 +615,8 @@ UF_VERTEX_INTERPOLATE(uf::graph::mesh::Skinned_u16q, {
})
pod::Graph::Storage& uf::graph::getStorage( pod::Graph& graph ) {
if ( graph.storage ) return *graph.storage; // just fetch it if it already exists
switch ( uf::graph::storageMode ) {
case pod::Graph::Storage::OBJECT: {
if ( !graph.root.entity ) {
@ -771,6 +773,9 @@ void uf::graph::process( pod::Graph& graph ) {
auto& graphMetadataJson = graph.metadata;
auto& graphMetadataValve = graphMetadataJson["valve"];
auto& storage = uf::graph::getStorage( graph );
std::lock_guard<std::mutex> lock(*storage.mutex);
uf::graph::initialize( storage );
// merge light settings with global settings
{
@ -1657,9 +1662,14 @@ void uf::graph::tick() {
}
void uf::graph::tick( uf::Object& object ) {
auto& storage = uf::graph::getStorage( object );
// ! NOTE ! additionally, uncommenting these out also breaks things
// if ( !object.hasComponent<pod::Graph>() ) return;
// auto& graph = object.getComponent<pod::Graph>();
// if ( !graph.root.entity || !graph.root.entity->isValid() ) return;
storage.shouldRebind = uf::graph::tick( storage );
}
bool uf::graph::tick( pod::Graph::Storage& storage ) {
std::lock_guard<std::mutex> lock(*storage.mutex);
bool rebuild = false;
STATIC_THREAD_LOCAL(uf::stl::vector<pod::Instance>, instances);
@ -1955,7 +1965,6 @@ void uf::graph::destroy( pod::Graph::Storage& storage, bool soft ) {
}
for ( auto& t : storage.shadow2Ds ) t.destroy();
for ( auto& t : storage.shadowCubes ) t.destroy();
for ( auto& t : storage.cubemaps ) t.destroy();
for ( auto pair : storage.atlases.map ) pair.second.clear();
for ( auto pair : storage.meshes.map ) pair.second.destroy();
@ -1974,7 +1983,6 @@ void uf::graph::destroy( pod::Graph::Storage& storage, bool soft ) {
storage.entities.clear();
storage.shadow2Ds.clear();
storage.shadowCubes.clear();
storage.cubemaps.clear();
// cleanup storage buffers
if ( !soft ) {

32
engine/src/engine/object/behavior.cpp
View File

@ -65,40 +65,12 @@ void uf::ObjectBehavior::initialize( uf::Object& self ) {
});
this->addHook( "asset:QueueLoad.%UID%", [&](pod::payloads::assetLoad& payload){
auto callback = this->formatHookName("asset:FinishedLoad.%UID%");
/*
switch ( payload.type ) {
case uf::asset::Type::AUDIO:
case uf::asset::Type::IMAGE:
case uf::asset::Type::LUA: {
if ( payload.monoThreaded ) {
if ( uf::asset::cache( payload ) != "" ) this->queueHook( callback, payload );
} else {
uf::asset::cache( callback, payload );
}
} break;
case uf::asset::Type::GRAPH: {
if ( payload.monoThreaded ) {
if ( uf::asset::load( payload ) != "" ) this->queueHook( callback, payload );
} else {
uf::asset::load( callback, payload );
}
} break;
}
*/
payload.object = this->resolvable<>();
if ( payload.monoThreaded ) {
if ( uf::asset::load( payload ) != "" ) this->queueHook( callback, payload );
} else {
if ( payload.async ) {
uf::asset::load( callback, payload );
}
/*
if ( payload.monoThreaded ) {
if ( uf::asset::cache( payload ) != "" ) this->queueHook( callback, payload );
} else {
uf::asset::cache( callback, payload );
if ( uf::asset::load( payload ) != "" ) this->queueHook( callback, payload );
}
*/
});
this->addHook( "asset:FinishedLoad.%UID%", [&](pod::payloads::assetLoad& payload){
this->queueHook("asset:Load.%UID%", payload);

2
engine/src/engine/object/behaviors/graph.cpp
View File

@ -44,6 +44,7 @@ void uf::GraphBehavior::initialize( uf::Object& self ) {
if ( !uf::asset::isExpected( payload, uf::asset::Type::GRAPH ) ) return;
if ( !uf::asset::has( payload ) ) uf::asset::load( payload );
auto& graph = payload.asComponent ? this->getComponent<pod::Graph>() : uf::asset::get<pod::Graph>( payload );
if ( !payload.asComponent ) {
auto userdata = uf::asset::release( payload.filename );
this->moveComponent<pod::Graph>( userdata );
@ -71,6 +72,7 @@ void uf::GraphBehavior::destroy( uf::Object& self ) {}
void uf::GraphBehavior::tick( uf::Object& self ) {
if ( !this->hasComponent<pod::Graph>() ) return;
auto& graph = this->getComponent<pod::Graph>();
if ( !graph.root.entity || !graph.root.entity->isValid() ) return;
if ( !graph.metadata["debug"]["draw"]["armature"].as<bool>(false) ) return;
auto& transform = this->getComponent<pod::Transform<>>();
auto& storage = uf::graph::getStorage( graph );

6
engine/src/engine/object/object.cpp
View File

@ -159,8 +159,8 @@ void uf::Object::loadAssets( const uf::Serializer& _json ){
payload.hash = isObject ? target[i]["hash"].as<uf::stl::string>("") : "";
payload.object = this->resolvable<>();
payload.monoThreaded = isObject ? !target[i]["multithreaded"].as<bool>(true) : !true;
payload.asComponent = isObject ? target[i]["component"].as<bool>(true) : true;
payload.async = isObject ? target[i]["async"].as<bool>(true) : true;
payload.asComponent = isObject ? target[i]["component"].as<bool>(false) : false;
switch ( assetType ) {
case uf::asset::Type::AUDIO: {
@ -172,7 +172,7 @@ void uf::Object::loadAssets( const uf::Serializer& _json ){
if ( bind ) uf::instantiator::bind("LuaBehavior", *this);
} break;
case uf::asset::Type::GRAPH: {
// payload.asComponent = true;
payload.asComponent = true;
if ( bind ) uf::instantiator::bind("GraphBehavior", *this);
auto metadata = json["metadata"]["graph"];

2
engine/src/engine/scene/behavior.cpp
View File

@ -25,7 +25,9 @@ void uf::SceneBehavior::initialize( uf::Object& self ) {
});
uf::physics::initialize( self );
UF_MSG_DEBUG("Initializing graph...");
uf::graph::initialize( self );
UF_MSG_DEBUG("Initialized graph.");
auto& metadata = this->getComponent<uf::SceneBehavior::Metadata>();
}

3
engine/src/ext/gltf/gltf.cpp
View File

@ -156,7 +156,8 @@ void ext::gltf::load( pod::Graph& graph, const uf::stl::string& filename, const
uf::stl::string key = graph.metadata["key"].as<uf::stl::string>("");
if ( key != "" ) key += ":";
auto& storage = uf::graph::getStorage( graph );
if ( !graph.storage ) graph.storage = new pod::Graph::Storage();
auto& storage = uf::graph::getStorage( graph ); // will just fetch the above
// load images
{

1
engine/src/ext/valve/bsp.cpp
View File

@ -679,6 +679,7 @@ void ext::valve::loadBsp( pod::Graph& graph, const uf::stl::string& filename, co
return;
}
if ( !graph.storage ) graph.storage = new pod::Graph::Storage();
auto& storage = uf::graph::getStorage( graph );
graph.name = filename;
graph.metadata = metadata;

48
engine/src/utils/thread/thread.cpp
View File

@ -66,9 +66,11 @@ pod::Thread::Tasks uf::thread::schedule( const uf::stl::string& name, bool wait
return tasks;
}
uf::stl::vector<pod::Thread*> uf::thread::execute( pod::Thread::Tasks& tasks ) {
uf::stl::vector<pod::Thread*> workers;
if ( tasks.container.empty() ) return workers;
std::shared_ptr<pod::Thread::Tasks::Tracker> uf::thread::execute( pod::Thread::Tasks& tasks ) {
auto tracker = std::make_shared<pod::Thread::Tasks::Tracker>();
if ( tasks.container.empty() ) return tracker;
tracker->pending.store( tasks.container.size(), std::memory_order_relaxed );
if ( tasks.name == uf::thread::mainThreadName ) {
#if UF_THREAD_METRICS
@ -78,28 +80,39 @@ uf::stl::vector<pod::Thread*> uf::thread::execute( pod::Thread::Tasks& tasks ) {
task();
++tasksThisFrame;
}
tasks.container.clear();
thread.metrics.tasksProcessed.store(tasksThisFrame, std::memory_order_relaxed);
#else
for ( auto& task : tasks.container ) task();
#endif
tasks.container.clear();
tracker->pending.store(0, std::memory_order_release);
} else {
for ( auto& task : tasks.container ) {
auto& worker = uf::thread::fetchWorker( tasks.name );
uf::thread::queue( worker, task );
workers.emplace_back(&worker);
uf::thread::queue( worker, [task, tracker]() {
struct Decrementer {
std::shared_ptr<pod::Thread::Tasks::Tracker> t;
~Decrementer() {
if ( t->pending.fetch_sub(1, std::memory_order_release) == 1 ) {
std::lock_guard<std::mutex> lock(t->mutex);
t->cv.notify_all();
}
}
} dec{ tracker };
task();
});
}
tasks.container.clear();
if ( tasks.waits ) uf::thread::wait( workers );
if ( tasks.waits ) uf::thread::wait( tracker );
}
return workers;
return tracker;
}
void uf::thread::wait( uf::stl::vector<pod::Thread*>& workers ) {
for ( auto& worker : workers ) uf::thread::wait( *worker );
workers.clear();
}
void uf::thread::wait( const uf::stl::vector<pod::Thread*>& workers ) {
for ( auto& worker : workers ) uf::thread::wait( *worker );
void uf::thread::wait( std::shared_ptr<pod::Thread::Tasks::Tracker> tracker ) {
if ( !tracker ) return;
std::unique_lock<std::mutex> lock(tracker->mutex);
tracker->cv.wait(lock, [&]{ return tracker->pending.load(std::memory_order_acquire) == 0; });
}
void uf::thread::add( pod::Thread& thread, const pod::Thread::function_t& function ) {
@ -225,6 +238,13 @@ void uf::thread::wait( pod::Thread& thread ) {
// while ( thread.pending.load() > 0 ) std::this_thread::yield();
}
uf::stl::string uf::thread::name( std::thread::id id ) {
if ( id == uf::thread::mainThreadId ) return uf::thread::mainThreadName;
for ( auto& [ name, thread ] : uf::thread::threads ) {
if ( thread->thread.get_id() == id ) return name;
}
return "?";
}
const uf::stl::string& uf::thread::name( const pod::Thread& thread ) {
return thread.name;
}