ecker/engine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

re-enabled internal allocator for STL containers (because of lines being swapped), fixed async asset loading (for OpenGL, Vulkan still doesn't like it), grouped OpenGL draw calls by material type for a speedup, some other stuff

Browse Source
This commit is contained in:
ecker 2026-08-09 23:28:11 -05:00
parent 956b0ddb4a
commit f7f6d54c54
20 changed files with 200 additions and 135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
bin/data/config.json
View File

@ -437,7 +437,7 @@
},
"loader": {
"assert": "auto",
"async": "auto" // unironically works on opengl, not on vulkan
"async": "auto"
},
"hooks": {
"defer lazy calls": true

2
bin/data/entities/model.json
View File

@ -79,7 +79,7 @@
"stream": {
"tag": "worldspawn",
"player": "info_player_start",
"enabled": "auto",
"enabled": false, // "auto",
"radius": 50,
"every": 1
}

2
bin/data/entities/player.json
View File

@ -12,7 +12,7 @@
"./scripts/player.lua"
],
"system": {
"noclipped": true,
// "noclipped": true,
"hot reload": {
"enabled": true
}

35
engine/inc/uf/engine/asset/asset.h
View File

@ -13,6 +13,30 @@
namespace uf {
namespace asset {
typedef uf::Hooks::name_t callback_t;
struct Job {
typedef uf::stl::vector<Job> container_t;
uf::asset::callback_t callback = "";
uf::stl::string type = "";
uf::asset::Payload payload = {};
};
struct Read {
typedef uf::stl::unordered_map<uf::stl::string, uf::stl::vector<Read>> container_t;
size_t offset;
size_t length;
uint8_t* dest;
std::function<void()> callback;
std::function<void(uf::stl::vector<uint8_t>&&)> callbackBuffered;
};
struct Stream {
typedef uf::stl::unordered_map<uf::stl::string, uf::stl::vector<Stream>> container_t;
size_t offset;
size_t length;
size_t chunkSize;
std::function<bool(const uint8_t* data, size_t size, size_t fileOffset)> callback;
};
#if UF_COMPONENT_POINTERED_USERDATA
typedef pod::PointeredUserdata userdata_t;
@ -22,6 +46,7 @@ namespace uf {
extern UF_API bool assertionLoad;
extern UF_API bool asyncQueue;
extern UF_API bool deferQueues;
extern UF_API uf::stl::unordered_map<uf::stl::string, uf::asset::userdata_t> map;
@ -33,14 +58,22 @@ namespace uf {
// URL or file path
void UF_API processQueue();
void UF_API processIO();
void UF_API processIO( const uf::asset::Read::container_t& pendingReads, bool async = uf::asset::asyncQueue, bool wait = false );
void UF_API processIO( const uf::asset::Stream::container_t& pendingStreams, bool async = uf::asset::asyncQueue, bool wait = false );
void UF_API processIO( const uf::asset::Read::container_t& pendingReads, const uf::asset::Stream::container_t& pendingStreams, bool async = uf::asset::asyncQueue, bool wait = false );
void UF_API cache( const uf::asset::callback_t&, const uf::asset::Payload& );
void UF_API load( const uf::asset::callback_t&, const uf::asset::Payload& );
void UF_API read( const uf::stl::string& filename, size_t offset, size_t length, uint8_t* dest, std::function<void()> callback = {} );
void UF_API read( const uf::stl::string& filename, size_t offset, size_t length, std::function<void(uf::stl::vector<uint8_t>&&)> callback = {} );
void UF_API read( const uf::stl::string& filename, size_t offset, size_t length, std::function<void(uf::stl::vector<uint8_t>&&)> callback );
void UF_API stream( const uf::stl::string& filename, size_t offset, size_t length, size_t chunkSize, std::function<bool(const uint8_t* data, size_t size, size_t fileOffset)> callback );
void UF_API read( uf::asset::Read::container_t& container, const uf::stl::string& filename, size_t offset, size_t length, uint8_t* dest, std::function<void()> callback = {} );
void UF_API read( uf::asset::Read::container_t& container, const uf::stl::string& filename, size_t offset, size_t length, std::function<void(uf::stl::vector<uint8_t>&&)> callback );
void UF_API stream( uf::asset::Stream::container_t& container, const uf::stl::string& filename, size_t offset, size_t length, size_t chunkSize, std::function<bool(const uint8_t* data, size_t size, size_t fileOffset)> callback );
uf::stl::string UF_API cache( uf::asset::Payload& );
uf::stl::string UF_API load( uf::asset::Payload& );

2
engine/inc/uf/engine/graph/graph.h
View File

@ -81,6 +81,8 @@ namespace pod {
uf::stl::unordered_map<uf::stl::string, pod::MeshStream> meshes;
uf::stl::unordered_map<uf::stl::string, pod::ImageStream> images;
uf::stl::unordered_map<uf::stl::string, pod::BvhStream> bvhs;
uf::asset::Read::container_t queue;
} streams;
// Local storage, used for save/load

4
engine/inc/uf/utils/memory/deque.h
View File

@ -10,9 +10,9 @@ namespace uf {
template<
class T,
#if UF_MEMORYPOOL_USE_ALLOCATOR
class Allocator = std::allocator<T>
#else
class Allocator = uf::Allocator<T>
#else
class Allocator = std::allocator<T>
#endif
>
using deque = std::deque<T, Allocator>;

4
engine/inc/uf/utils/memory/fifo_map.h
View File

@ -12,9 +12,9 @@ namespace uf {
class T,
class Compare = nlohmann::fifo_map_compare<Key>,
#if UF_MEMORYPOOL_USE_ALLOCATOR
class Allocator = std::allocator<std::pair<const Key, T>>
#else
class Allocator = uf::Allocator<std::pair<const Key, T>>
#else
class Allocator = std::allocator<std::pair<const Key, T>>
#endif
>
using fifo_map = nlohmann::fifo_map<Key, T, Compare, Allocator>;

4
engine/inc/uf/utils/memory/map.h
View File

@ -12,9 +12,9 @@ namespace uf {
class T,
class Compare = std::less<Key>,
#if UF_MEMORYPOOL_USE_ALLOCATOR
class Allocator = std::allocator<std::pair<const Key, T>>
#else
class Allocator = uf::Allocator<std::pair<const Key, T>>
#else
class Allocator = std::allocator<std::pair<const Key, T>>
#endif
>
using map = std::map<Key, T, Compare, Allocator>;

4
engine/inc/uf/utils/memory/unordered_map.h
View File

@ -15,9 +15,9 @@ namespace uf {
class Hash = uf::algo::hasher,
class KeyEqual = std::equal_to<Key>,
#if UF_MEMORYPOOL_USE_ALLOCATOR
class Allocator = std::allocator<std::pair<const Key, T>>
#else
class Allocator = uf::Allocator<std::pair<const Key, T>>
#else
class Allocator = std::allocator<std::pair<const Key, T>>
#endif
>
using unordered_map = std::unordered_map<Key, T, Hash, KeyEqual, Allocator>;

4
engine/inc/uf/utils/memory/unordered_set.h
View File

@ -13,9 +13,9 @@ namespace uf {
class Hash = std::hash<Key>,
class KeyEqual = std::equal_to<Key>,
#if UF_MEMORYPOOL_USE_ALLOCATOR
class Allocator = std::allocator<Key>
#else
class Allocator = uf::Allocator<Key>
#else
class Allocator = std::allocator<Key>
#endif
>
using unordered_set = std::unordered_set<Key, Hash, KeyEqual, Allocator>;

4
engine/inc/uf/utils/memory/vector.h
View File

@ -10,9 +10,9 @@ namespace uf {
template<
class T,
#if UF_MEMORYPOOL_USE_ALLOCATOR
class Allocator = std::allocator<T>
#else
class Allocator = uf::Allocator<T>
#else
class Allocator = std::allocator<T>
#endif
>
using vector = std::vector<T, Allocator>;

88
engine/src/engine/asset/asset.cpp
View File

@ -39,70 +39,43 @@ namespace {
}
namespace jobs {
struct Job {
typedef uf::stl::vector<Job> container_t;
uf::asset::callback_t callback = "";
uf::stl::string type = "";
uf::asset::Payload payload = {};
};
std::mutex mutex;
Job::container_t queue;
Job::container_t finished;
uf::asset::Job::container_t queue;
uf::asset::Job::container_t finished;
};
namespace io_read {
struct Job {
typedef uf::stl::unordered_map<uf::stl::string, uf::stl::vector<Job>> container_t;
size_t offset;
size_t length;
uint8_t* dest;
std::function<void()> callback;
std::function<void(uf::stl::vector<uint8_t>&&)> callbackBuffered;
};
std::mutex mutex;
Job::container_t queue;
Job::container_t finished;
uf::asset::Read::container_t queue;
uf::asset::Read::container_t finished;
};
namespace io_stream {
struct Job {
typedef uf::stl::unordered_map<uf::stl::string, uf::stl::vector<Job>> container_t;
size_t offset;
size_t length;
size_t chunkSize;
std::function<bool(const uint8_t* data, size_t size, size_t fileOffset)> callback;
};
std::mutex mutex;
Job::container_t queue;
Job::container_t finished;
uf::asset::Stream::container_t queue;
uf::asset::Stream::container_t finished;
};
}
// uf::asset uf::asset::masterAssetLoader;
bool uf::asset::assertionLoad = true;
bool uf::asset::asyncQueue = true;
bool uf::asset::deferQueues = false;
uf::stl::unordered_map<uf::stl::string, uf::asset::userdata_t> uf::asset::map;
uf::Serializer uf::asset::metadata;
void uf::asset::processQueue() {
if ( ::jobs::queue.empty() && ::jobs::finished.empty() ) return;
STATIC_THREAD_LOCAL(::jobs::Job::container_t, jobs);
::jobs::Job::container_t finishedJobs;
STATIC_THREAD_LOCAL(uf::asset::Job::container_t, jobs);
uf::asset::Job::container_t finishedJobs;
::jobs::mutex.lock();
std::swap( jobs, ::jobs::queue );
std::swap( finishedJobs, ::jobs::finished );
::jobs::mutex.unlock();
if ( jobs.empty() && finishedJobs.empty() ) return;
bool async = uf::asset::asyncQueue; // a bit buggy
auto tasks = uf::thread::schedule(async ? uf::thread::asyncThreadName : uf::thread::mainThreadName, !true);
auto tasks = uf::thread::schedule(uf::asset::asyncQueue ? uf::thread::asyncThreadName : uf::thread::mainThreadName, false);
if ( !finishedJobs.empty() ) {
tasks.queue([jobs = std::move(finishedJobs)]() {
@ -131,8 +104,8 @@ void uf::asset::processQueue() {
}
void uf::asset::processIO() {
STATIC_THREAD_LOCAL(::io_read::Job::container_t, pendingReads);
STATIC_THREAD_LOCAL(::io_stream::Job::container_t, pendingStreams);
STATIC_THREAD_LOCAL(uf::asset::Read::container_t, pendingReads);
STATIC_THREAD_LOCAL(uf::asset::Stream::container_t, pendingStreams);
::io_read::mutex.lock();
std::swap(pendingReads, ::io_read::queue);
@ -143,8 +116,16 @@ void uf::asset::processIO() {
if ( pendingReads.empty() && pendingStreams.empty() ) return;
bool async = uf::asset::asyncQueue;
auto tasks = uf::thread::schedule(async ? uf::thread::asyncThreadName : uf::thread::mainThreadName, false);
uf::asset::processIO( pendingReads, pendingStreams, uf::asset::asyncQueue, false );
}
void uf::asset::processIO( const uf::asset::Read::container_t& pendingReads, bool async, bool wait ) {
return uf::asset::processIO( pendingReads, {}, async, wait );
}
void uf::asset::processIO( const uf::asset::Stream::container_t& pendingStreams, bool async, bool wait ) {
return uf::asset::processIO( {}, pendingStreams, async, wait );
}
void uf::asset::processIO( const uf::asset::Read::container_t& pendingReads, const uf::asset::Stream::container_t& pendingStreams, bool async, bool wait ) {
auto tasks = uf::thread::schedule(async ? uf::thread::asyncThreadName : uf::thread::mainThreadName, wait);
for ( auto& [filename, requests] : pendingReads ) {
tasks.queue([filename = filename, requests = std::move(requests)]() {
@ -215,25 +196,36 @@ void uf::asset::processIO() {
void uf::asset::cache( const uf::asset::callback_t& callback, const uf::asset::Payload& payload ) {
std::lock_guard<std::mutex> lock(::jobs::mutex);
::jobs::queue.emplace_back(::jobs::Job{ callback, "cache", payload });
::jobs::queue.emplace_back(uf::asset::Job{ callback, "cache", payload });
}
void uf::asset::load( const uf::asset::callback_t& callback, const uf::asset::Payload& payload ) {
std::lock_guard<std::mutex> lock(::jobs::mutex);
::jobs::queue.emplace_back(::jobs::Job{ callback, "load", payload });
::jobs::queue.emplace_back(uf::asset::Job{ callback, "load", payload });
}
void uf::asset::read( const uf::stl::string& filename, size_t offset, size_t length, uint8_t* dest, std::function<void()> callback ) {
std::lock_guard<std::mutex> lock(::io_read::mutex);
::io_read::queue[filename].emplace_back(::io_read::Job{ offset, length, dest, callback });
uf::asset::read( ::io_read::queue, filename, offset, length, dest, callback );
}
void uf::asset::read( const uf::stl::string& filename, size_t offset, size_t length, std::function<void(uf::stl::vector<uint8_t>&&)> callback ) {
std::lock_guard<std::mutex> lock(::io_read::mutex);
::io_read::queue[filename].emplace_back(::io_read::Job{ offset, length, nullptr, nullptr, callback });
uf::asset::read( ::io_read::queue, filename, offset, length, callback );
}
void uf::asset::stream( const uf::stl::string& filename, size_t offset, size_t length, size_t chunkSize, std::function<bool(const uint8_t* data, size_t size, size_t fileOffset)> callback ) {
std::lock_guard<std::mutex> lock(::io_stream::mutex);
::io_stream::queue[filename].emplace_back(::io_stream::Job{ offset, length, chunkSize, callback });
uf::asset::stream( ::io_stream::queue, filename, offset, length, chunkSize, callback );
}
void uf::asset::read( uf::asset::Read::container_t& container, const uf::stl::string& filename, size_t offset, size_t length, uint8_t* dest, std::function<void()> callback ) {
container[filename].emplace_back(uf::asset::Read{ offset, length, dest, callback });
}
void uf::asset::read( uf::asset::Read::container_t& container, const uf::stl::string& filename, size_t offset, size_t length, std::function<void(uf::stl::vector<uint8_t>&&)> callback ) {
container[filename].emplace_back(uf::asset::Read{ offset, length, nullptr, nullptr, callback });
}
void uf::asset::stream( uf::asset::Stream::container_t& container, const uf::stl::string& filename, size_t offset, size_t length, size_t chunkSize, std::function<bool(const uint8_t* data, size_t size, size_t fileOffset)> callback ) {
container[filename].emplace_back(uf::asset::Stream{ offset, length, chunkSize, callback });
}
uf::asset::Payload uf::asset::resolveToPayload( const uf::stl::string& uri, const uf::stl::string& mime ) {

2
engine/src/engine/ext/ext.cpp
View File

@ -208,7 +208,7 @@ void UF_API uf::load( ext::json::Value& json ) {
uf::asset::assertionLoad = configEngineDebugJson["loader"]["assert"].as( uf::asset::assertionLoad );
}
if ( configEngineDebugJson["loader"]["async"].as<uf::stl::string>() == "auto" ) {
uf::asset::asyncQueue = false; // to-do: fix
uf::asset::asyncQueue = false; // to-do: find out why vulkan really does not like this
#if UF_USE_OPENGL && !UF_ENV_DREAMCAST
uf::asset::asyncQueue = true;
#endif

10
engine/src/engine/graph/animation.cpp
View File

@ -30,8 +30,8 @@ namespace {
auto& animation = storage.animations.map[name];
auto& animStream = graph.streams.animations[name];
bool needsIO = false;
uf::asset::Read::container_t queue;
for ( size_t i = 0; i < animation.samplers.size(); ++i ) {
auto& sampler = animation.samplers[i];
auto& stream = animStream.samplers[i];
@ -40,18 +40,16 @@ namespace {
if ( stream.inputs.length > 0 ) {
sampler.inputs.resize( stream.inputs.length / sizeof(float) );
uf::asset::read( stream.inputs.filename, stream.inputs.offset, stream.inputs.length, (uint8_t*)(sampler.inputs.data()) );
needsIO = true;
uf::asset::read( queue, stream.inputs.filename, stream.inputs.offset, stream.inputs.length, (uint8_t*)(sampler.inputs.data()) );
}
if ( stream.outputs.length > 0 ) {
sampler.outputs.resize( stream.outputs.length / sizeof(pod::Vector4f) );
uf::asset::read( stream.outputs.filename, stream.outputs.offset, stream.outputs.length, (uint8_t*)(sampler.outputs.data()) );
needsIO = true;
uf::asset::read( queue, stream.outputs.filename, stream.outputs.offset, stream.outputs.length, (uint8_t*)(sampler.outputs.data()) );
}
}
if ( needsIO ) uf::asset::processIO();
uf::asset::processIO( queue );
}
void unloadAnimation( pod::Graph& graph, const uf::stl::string& name ) {

2
engine/src/engine/graph/convert.cpp
View File

@ -361,6 +361,7 @@ void uf::graph::import( pod::Graph::Storage& target, pod::Graph::Storage& storag
target.primitives.merge(std::move(storage.primitives));
target.instances.merge(std::move(storage.instances));
target.meshes.merge(std::move(storage.meshes));
target.bvhs.merge(std::move(storage.bvhs));
target.images.merge(std::move(storage.images));
target.materials.merge(std::move(storage.materials));
target.textures.merge(std::move(storage.textures));
@ -387,6 +388,7 @@ void uf::graph::import( pod::Graph::Storage& target, pod::Graph::Storage& storag
target.primitives.import(storage.primitives);
target.instances.import(storage.instances);
target.meshes.import(storage.meshes);
target.bvhs.import(storage.bvhs);
target.images.import(storage.images);
target.materials.import(storage.materials);
target.textures.import(storage.textures);

104
engine/src/engine/graph/decode.cpp
View File

@ -25,14 +25,10 @@
#define UF_DEBUG_TIMER_MULTITRACE_END(...)
#endif
namespace {
struct PendingImage {
uf::stl::string name;
uf::stl::vector<uint8_t> buffer;
uf::stl::string extension;
size_t layers;
};
// to force deferred loading
#define DEFERRED(...) __VA_ARGS__
namespace {
size_t deduceFormat( const uf::stl::string& format ) {
if ( format == "ARGB4444" ) return uf::renderer::enums::Format::R4G4B4A4_UNORM_PACK16;
if ( format == "RGB565" ) return uf::renderer::enums::Format::R5G6B5_UNORM_PACK16;
@ -86,17 +82,17 @@ namespace {
}
uf::stl::string fullPath = uf::io::directory( graph.name ) + "/" + filename;
auto& storage = uf::graph::getStorage(graph);
graph.streams.images[key] = { fullPath, offset, length };
if ( graph.settings.stream.textures ) {
auto& storage = uf::graph::getStorage(graph);
graph.streams.images[key] = { fullPath, offset, length };
} else {
bool deferred = DEFERRED(graph.settings.stream.textures);
if ( !deferred ) {
size_t readLen = length > 0 ? length : uf::io::size( fullPath );
if ( readLen > 0 ) {
uf::asset::read( fullPath, offset, readLen, [&graph, key, extension, layers]( uf::stl::vector<uint8_t>&& buffer ) {
uf::asset::read( graph.streams.queue, fullPath, offset, readLen, [&graph, key, extension, layers]( uf::stl::vector<uint8_t>&& buffer ) {
auto& storage = uf::graph::getStorage(graph);
auto& image = storage.images[key].data;
uf::image::open( image, buffer, extension, false );
uf::image::layers( image, layers );
} );
@ -123,6 +119,7 @@ namespace {
auto& storage = uf::graph::getStorage(graph);
auto& animStream = graph.streams.animations[animName];
bool deferred = DEFERRED(graph.settings.stream.animations);
ext::json::forEach( json["samplers"], [&]( ext::json::Value& value ){
auto& sampler = animation.samplers.emplace_back();
sampler.interpolator = value["interpolator"].as(sampler.interpolator);
@ -135,19 +132,19 @@ namespace {
size_t outputsOffset = value["outputs"]["offset"].as<size_t>();
size_t outputsLen = value["outputs"]["length"].as<size_t>();
if ( graph.settings.stream.animations ) {
pod::AnimationStream::SamplerStream sStream;
sStream.inputs = { binPath, inputsOffset, inputsLen };
sStream.outputs = { binPath, outputsOffset, outputsLen };
animStream.samplers.emplace_back(sStream);
} else {
pod::AnimationStream::SamplerStream sStream;
sStream.inputs = { binPath, inputsOffset, inputsLen };
sStream.outputs = { binPath, outputsOffset, outputsLen };
animStream.samplers.emplace_back(sStream);
if ( !deferred ) {
if ( inputsLen > 0 ) {
sampler.inputs.resize(inputsCount);
uf::asset::read( binPath, inputsOffset, inputsLen, (uint8_t*)(sampler.inputs.data()) );
uf::asset::read( graph.streams.queue, binPath, inputsOffset, inputsLen, (uint8_t*)(sampler.inputs.data()) );
}
if ( outputsLen > 0 ) {
sampler.outputs.resize(outputsCount);
uf::asset::read( binPath, outputsOffset, outputsLen, (uint8_t*)(sampler.outputs.data()) );
uf::asset::read( graph.streams.queue, binPath, outputsOffset, outputsLen, (uint8_t*)(sampler.outputs.data()) );
}
}
});
@ -172,6 +169,7 @@ namespace {
skin.joints.emplace_back( value.as<int32_t>() );
});
bool deferred = DEFERRED(graph.settings.stream.enabled);
if ( json["inverseBindMatrices"].isObject() ) {
auto& invJson = json["inverseBindMatrices"];
size_t count = invJson["count"].as<size_t>();
@ -181,13 +179,12 @@ namespace {
auto& storage = uf::graph::getStorage(graph);
auto& skinStream = graph.streams.skins[skinName];
skinStream.inverseBindMatrices = { binPath, offset, length };
if ( graph.settings.stream.enabled ) {
skinStream.inverseBindMatrices = { binPath, offset, length };
} else {
if ( !deferred ) {
if ( length > 0 ) {
skin.inverseBindMatrices.resize(count);
uf::asset::read( binPath, offset, length, (uint8_t*)(skin.inverseBindMatrices.data()) );
uf::asset::read( graph.streams.queue, binPath, offset, length, (uint8_t*)(skin.inverseBindMatrices.data()) );
}
}
}
@ -195,9 +192,7 @@ namespace {
return skin;
}
uf::Mesh decodeMesh( ext::json::Value& json, pod::Graph& graph, const uf::stl::string& meshName ) {
uf::Mesh mesh;
void decodeMesh( ext::json::Value& json, pod::Graph& graph, const uf::stl::string& meshName, uf::Mesh& mesh ) {
#define DESERIALIZE_MESH(N) {\
auto& input = json["inputs"][#N];\
mesh.N.attributes.reserve( input["attributes"].size() );\
@ -230,7 +225,7 @@ namespace {
auto& meshStream = graph.streams.meshes[meshName];
mesh.buffers.reserve( json["buffers"].size() );
bool deferred = graph.settings.stream.enabled;
bool deferred = DEFERRED(graph.settings.stream.enabled);
ext::json::forEach( json["buffers"], [&]( ext::json::Value& value ){
uf::stl::string filename;
@ -258,7 +253,7 @@ namespace {
if ( region.length == 0 ) continue;
mesh.buffers[attr.buffer].resize(region.length);
uf::asset::read( region.filename, region.offset, region.length, mesh.buffers[attr.buffer].data() );
uf::asset::read( graph.streams.queue, region.filename, region.offset, region.length, mesh.buffers[attr.buffer].data() );
}
};
@ -270,11 +265,9 @@ namespace {
}
mesh.updateDescriptor();
return mesh;
}
pod::BVH decodeBvh( ext::json::Value& json, pod::Graph& graph, const uf::stl::string& key ) {
pod::BVH bvh;
void decodeBvh( ext::json::Value& json, pod::Graph& graph, const uf::stl::string& key, pod::BVH& bvh ) {
auto& storage = uf::graph::getStorage(graph);
auto& bvhStream = graph.streams.bvhs[key];
@ -285,19 +278,17 @@ namespace {
uf::stl::string fullPath = uf::io::directory( graph.name ) + "/" + filename;
bvhStream.buffer = pod::StreamRegion{ fullPath, offset, length };
bool deferred = graph.settings.stream.enabled;
bool deferred = DEFERRED(graph.settings.stream.enabled);
if ( !deferred ) {
size_t readLen = length > 0 ? length : uf::io::size( fullPath );
if ( readLen > 0 ) {
uf::asset::read( fullPath, offset, readLen, [&graph, key]( uf::stl::vector<uint8_t>&& buffer ) {
uf::asset::read( graph.streams.queue, fullPath, offset, readLen, [&graph, key]( uf::stl::vector<uint8_t>&& buffer ) {
auto& storage = uf::graph::getStorage(graph);
auto& bvh = storage.bvhs[key];
uf::bvh::deserialize( bvh, buffer );
auto res = uf::bvh::deserialize( bvh, buffer );
} );
}
}
return bvh;
}
pod::Node decodeNode( ext::json::Value& json, pod::Graph& graph ) {
@ -405,9 +396,7 @@ void uf::graph::load( pod::Graph& graph, const uf::stl::string& filename, const
uf::stl::string key = graph.metadata["key"].as<uf::stl::string>("");
if ( key != "" ) key += ":";
uf::stl::vector<PendingImage> pendingImages;
uf::stl::vector<uf::stl::string> meshesToMinify;
tasks.queue([&]{
UF_DEBUG_TIMER_MULTITRACE("Reading material information...");
auto& node = serializer["materials"];
@ -420,6 +409,8 @@ void uf::graph::load( pod::Graph& graph, const uf::stl::string& filename, const
auto names = node["names"].as<uf::stl::vector<uf::stl::string>>();
graph.materials.reserve(names.size());
storage.materials.reserve(names.size());
for (size_t i = 0; i < names.size(); ++i) {
auto name = key + names[i];
storage.materials[name] = rawMaterials[i];
@ -442,6 +433,8 @@ void uf::graph::load( pod::Graph& graph, const uf::stl::string& filename, const
auto names = node["names"].as<uf::stl::vector<uf::stl::string>>();
graph.textures.reserve(names.size());
storage.textures.reserve(names.size());
for (size_t i = 0; i < names.size(); ++i) {
auto name = key + names[i];
storage.textures[name] = rawTextures[i];
@ -464,6 +457,8 @@ void uf::graph::load( pod::Graph& graph, const uf::stl::string& filename, const
auto names = node["names"].as<uf::stl::vector<uf::stl::string>>();
graph.samplers.reserve(names.size());
storage.samplers.reserve(names.size());
for (size_t i = 0; i < names.size(); ++i) {
auto name = key + names[i];
storage.samplers[name] = rawSamplers[i];
@ -486,6 +481,8 @@ void uf::graph::load( pod::Graph& graph, const uf::stl::string& filename, const
auto names = node["names"].as<uf::stl::vector<uf::stl::string>>();
graph.lights.reserve(names.size());
storage.lights.reserve(names.size());
for (size_t i = 0; i < names.size(); ++i) {
auto name = key + names[i];
graph.lights[name] = rawLights[i];
@ -505,10 +502,11 @@ void uf::graph::load( pod::Graph& graph, const uf::stl::string& filename, const
allPrimitives = reinterpret_cast<pod::Primitive*>(ioBuf.data());
}
auto& primNode = serializer["primitives"];
graph.primitives.reserve( primNode.size() );
auto count = serializer["primitives"].size();
graph.primitives.reserve( count );
storage.primitives.reserve( count );
ext::json::forEach( primNode, [&]( ext::json::Value& value ){
ext::json::forEach( serializer["primitives"], [&]( ext::json::Value& value ){
auto name = key + value["name"].as<uf::stl::string>();
graph.primitives.emplace_back(name);
@ -531,7 +529,9 @@ void uf::graph::load( pod::Graph& graph, const uf::stl::string& filename, const
tasks.queue([&]{
UF_DEBUG_TIMER_MULTITRACE("Reading images...");
graph.images.reserve( serializer["images"].size() );
auto count = serializer["images"].size();
graph.images.reserve( count );
storage.images.reserve( count );
ext::json::forEach( serializer["images"], [&]( ext::json::Value& value ){
auto name = key + value["name"].as<uf::stl::string>();
@ -549,6 +549,7 @@ void uf::graph::load( pod::Graph& graph, const uf::stl::string& filename, const
tasks.queue([&]{
UF_DEBUG_TIMER_MULTITRACE("Reading meshes...");
graph.meshes.reserve( serializer["meshes"].size() );
storage.meshes.reserve( serializer["meshes"].size() );
#if UF_USE_OPENGL
bool preferMinified = true;
@ -562,7 +563,7 @@ void uf::graph::load( pod::Graph& graph, const uf::stl::string& filename, const
bool hasMinifiedAsset = value["min"].isObject();
auto& json = ( preferMinified && hasMinifiedAsset ) ? value["min"] : value;
storage.meshes[name] = decodeMesh( json, graph, name );
decodeMesh( json, graph, name, storage.meshes[name] );
graph.meshes.emplace_back(name);
if ( preferMinified && !hasMinifiedAsset && !graph.settings.stream.enabled )
@ -574,9 +575,10 @@ void uf::graph::load( pod::Graph& graph, const uf::stl::string& filename, const
tasks.queue([&]{
UF_DEBUG_TIMER_MULTITRACE("Reading BVHs...");
storage.bvhs.reserve( serializer["bvhs"].size() );
ext::json::forEach( serializer["bvhs"], [&]( ext::json::Value& value ){
auto name = key + value["name"].as<uf::stl::string>();
storage.bvhs[name] = decodeBvh( value, graph, name );
decodeBvh( value, graph, name, storage.bvhs[name] );
});
UF_DEBUG_TIMER_MULTITRACE("Read BVHs");
@ -587,7 +589,8 @@ void uf::graph::load( pod::Graph& graph, const uf::stl::string& filename, const
auto& animNode = serializer["animations"];
if ( animNode.isObject() ) {
storage.animations.map.reserve( animNode.size() );
storage.animations.reserve( animNode.size() );
ext::json::forEach( animNode, [&]( const uf::stl::string& rawName, ext::json::Value& value ){
auto name = key + rawName;
storage.animations[name] = decodeAnimation( value, graph, name );
@ -595,7 +598,8 @@ void uf::graph::load( pod::Graph& graph, const uf::stl::string& filename, const
});
}
else if ( animNode.isArray() ) {
storage.animations.map.reserve( animNode.size() );
storage.animations.reserve( animNode.size() );
ext::json::forEach( animNode, [&]( ext::json::Value& value ){
uf::stl::string path = directory + "/" + value.as<uf::stl::string>();
uf::Serializer json;
@ -612,6 +616,7 @@ void uf::graph::load( pod::Graph& graph, const uf::stl::string& filename, const
tasks.queue([&]{
UF_DEBUG_TIMER_MULTITRACE("Reading skinning information...");
graph.skins.reserve( serializer["skins"].size() );
storage.skins.reserve( serializer["skins"].size() );
ext::json::forEach( serializer["skins"], [&]( ext::json::Value& value ){
auto name = key + value["name"].as<uf::stl::string>();
@ -639,8 +644,7 @@ void uf::graph::load( pod::Graph& graph, const uf::stl::string& filename, const
UF_DEBUG_TIMER_MULTITRACE("Executing tasks");
uf::thread::execute( tasks );
UF_DEBUG_TIMER_MULTITRACE("Processing IO");
uf::asset::processIO();
uf::asset::processIO( graph.streams.queue, false, true );
// process meshes that need to be minified because I can't easily tie it to the callback
UF_DEBUG_TIMER_MULTITRACE("Processing meshes for minification");

3
engine/src/engine/graph/graph.cpp
View File

@ -2317,8 +2317,9 @@ void uf::graph::reload( pod::Graph& graph ) {
auto& bvh = storage.bvhs.map[meshName];
auto& bvhStream = graph.streams.bvhs[meshName];
if ( bvh.flatNodes.empty() && bvh.nodes.empty() && bvhStream.buffer.length > 0 ) {
if ( bvh.indices.empty() && bvhStream.buffer.length > 0 ) {
work.needsBvhLoad = true;
UF_MSG_DEBUG("Queued BVH load={}", meshName);
}
if ( radius > 0 && mesh.indirect.count && mesh.indirect.count <= primitives.size() ) {

5
engine/src/ext/lua/lua.cpp
View File

@ -313,6 +313,10 @@ namespace binds {
}
}
namespace {
std::mutex mutex;
}
void ext::lua::initialize() {
if ( !ext::lua::enabled ) return;
@ -477,6 +481,7 @@ pod::LuaScript ext::lua::script( const uf::stl::string& filename ) {
}
void ext::lua::script( const uf::stl::string& filename, pod::LuaScript& script ) {
if ( !ext::lua::enabled ) return;
std::lock_guard<std::mutex> lock(::mutex);
script.file = filename;
script.env = sol::environment( ext::lua::state, sol::create, ext::lua::state.globals() );
}

43
engine/src/ext/opengl/commands.cpp
View File

@ -463,20 +463,36 @@ void ext::opengl::CommandBuffer::drawIndexed( const ext::opengl::CommandBuffer::
::shadowState.projectionDirty = false;
}
if ( drawInfo.blend.modeAlpha > 0 ) {
if ( !::shadowState.alphaTestEnabled ) {
GL_ERROR_CHECK(glEnable(GL_ALPHA_TEST));
::shadowState.alphaTestEnabled = true;
}
if ( ::shadowState.alphaCutoff != drawInfo.blend.alphaCutoff ) {
GL_ERROR_CHECK(glAlphaFunc(GL_GREATER, drawInfo.blend.alphaCutoff));
::shadowState.alphaCutoff = drawInfo.blend.alphaCutoff;
bool needsBlend = drawInfo.blend.modeAlpha > 0;
// this might not be necessary
#if UF_ENV_DREAMCAST
if ( drawInfo.textures.secondary.image && drawInfo.attributes.st.pointer ) needsBlend = true;
#endif
if ( needsBlend ) {
if ( drawInfo.blend.modeAlpha > 0 ) {
if ( !::shadowState.alphaTestEnabled ) {
GL_ERROR_CHECK(glEnable(GL_ALPHA_TEST));
::shadowState.alphaTestEnabled = true;
}
if ( ::shadowState.alphaCutoff != drawInfo.blend.alphaCutoff ) {
GL_ERROR_CHECK(glAlphaFunc(GL_GREATER, drawInfo.blend.alphaCutoff));
::shadowState.alphaCutoff = drawInfo.blend.alphaCutoff;
}
} else {
if ( ::shadowState.alphaTestEnabled ) {
GL_ERROR_CHECK(glDisable(GL_ALPHA_TEST));
::shadowState.alphaTestEnabled = false;
}
}
if ( !::shadowState.blendEnabled ) {
GL_ERROR_CHECK(glEnable(GL_BLEND));
GL_ERROR_CHECK(glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA));
::shadowState.blendEnabled = true;
}
GL_ERROR_CHECK(glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA));
} else {
if ( ::shadowState.alphaTestEnabled ) {
GL_ERROR_CHECK(glDisable(GL_ALPHA_TEST));
@ -543,7 +559,6 @@ void ext::opengl::CommandBuffer::drawIndexed( const ext::opengl::CommandBuffer::
GL_ERROR_CHECK(glColor4f( color[0], color[1], color[2], color[3] ));
::shadowState.color = color;
}
GLenum vertexType = GL_FLOAT;
switch ( drawInfo.attributes.position.descriptor.type ) {
@ -665,6 +680,7 @@ void ext::opengl::CommandBuffer::drawIndexed( const ext::opengl::CommandBuffer::
GL_ERROR_CHECK(glDisableClientState(GL_COLOR_ARRAY));
::shadowState.colorArrayEnabled = false;
}
if ( drawInfo.textures.primary.image && drawInfo.attributes.uv.pointer ) {
GL_ERROR_CHECK(glClientActiveTexture(GL_TEXTURE0));
GL_ERROR_CHECK(glActiveTexture(GL_TEXTURE0));
@ -690,7 +706,6 @@ void ext::opengl::CommandBuffer::drawIndexed( const ext::opengl::CommandBuffer::
GLenum texEnv = (drawInfo.attributes.color.pointer || drawInfo.color.enabled) ? GL_MODULATE : GL_REPLACE;
GL_ERROR_CHECK(glTexCoordPointer(2, uvType, uvStride, uvPtr));
GL_ERROR_CHECK(glTexEnvi(GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, texEnv));
//GL_ERROR_CHECK(glTexEnvi(GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, drawInfo.attributes.color.pointer ? GL_MODULATE : GL_REPLACE));
} else {
if ( ::shadowState.tex0Enabled ) {
GL_ERROR_CHECK(glClientActiveTexture(GL_TEXTURE0));
@ -743,6 +758,12 @@ void ext::opengl::CommandBuffer::drawIndexed( const ext::opengl::CommandBuffer::
}
}
// this probably isn't necessary
#if UF_ENV_DREAMCAST
GL_ERROR_CHECK(glClientActiveTexture(GL_TEXTURE0));
GL_ERROR_CHECK(glActiveTexture(GL_TEXTURE0));
#endif
{
GL_ERROR_CHECK(glEnableClientState(GL_VERTEX_ARRAY));
GL_ERROR_CHECK(glVertexPointer(3, vertexType, vertexStride, vertexPtr));

11
engine/src/ext/opengl/graphic.cpp
View File

@ -486,7 +486,9 @@ void ext::opengl::Graphic::record( CommandBuffer& commandBuffer, const GraphicDe
auto& material = materials[materialID];
auto textureID = material.indexAlbedo;
auto& infos = pool[textureID];
uint32_t drawHash = ( (uint32_t)(textureID & 0xFFFF) ) | ( ((uint32_t)(lightmapID & 0xFFFF)) << 16 );
auto& infos = pool[drawHash];
CommandBuffer::InfoDraw& drawCommandInfo = infos.emplace_back( drawCommandInfoBase );
// CommandBuffer::InfoDraw drawCommandInfo = drawCommandInfoBase;
@ -511,6 +513,7 @@ void ext::opengl::Graphic::record( CommandBuffer& commandBuffer, const GraphicDe
drawCommandInfo.color.enabled = drawCommandInfo.color.value != pod::Vector4f{1.0f, 1.0f, 1.0f, 1.0f};
if ( drawCommandInfo.color.value.w == 0.0f ) {
infos.pop_back();
continue;
}
@ -526,7 +529,11 @@ void ext::opengl::Graphic::record( CommandBuffer& commandBuffer, const GraphicDe
if ( drawCommandInfo.blend.modeAlpha == pod::Material::AlphaMode::BLEND ) {
drawCommandInfo.descriptor.renderTarget = 1;
}
commandBuffer.record( drawCommandInfo );
}
}
for ( auto& [ drawHash, infos ] : pool ) {
for ( auto& info : infos ) {
commandBuffer.record( info );
}
}
} else {