From 98ec4933dac649d92bcc806ddd0f0af411703c4d Mon Sep 17 00:00:00 2001 From: ecker Date: Sat, 6 Jun 2026 00:10:29 -0500 Subject: [PATCH] insanity (added VFS, piped everything through it and made it work, 90% done with valve loaders) --- .gitignore | 4 + bin/data/config.json | 11 +- bin/data/shaders/graph/base/vert.h | 2 +- bin/exe/default/arch | 1 - bin/exe/default/cc | 1 - bin/exe/default/renderer | 1 - debug.sh | 0 engine/inc/uf/config.h | 8 +- engine/inc/uf/engine/graph/graph.h | 1 + engine/inc/uf/engine/scene/behavior.h | 4 + engine/inc/uf/ext/freetype/freetype.h | 3 +- engine/inc/uf/ext/valve/common.h | 3 +- engine/inc/uf/ext/valve/vpk.h | 39 ++ engine/inc/uf/ext/zlib/zlib.h | 44 +-- engine/inc/uf/utils/audio/metadata.h | 2 +- engine/inc/uf/utils/http/http.h | 5 + engine/inc/uf/utils/image/image.h | 2 +- engine/inc/uf/utils/io/file.h | 31 +- engine/inc/uf/utils/io/macros.inl | 6 + engine/inc/uf/utils/io/vfs.h | 54 +++ engine/inc/uf/utils/math/hash.h | 8 +- engine/inc/uf/utils/string/ext.h | 1 + engine/inc/uf/utils/userdata/pointered.inl | 2 +- engine/src/engine/asset/asset.cpp | 27 +- engine/src/engine/ext/ext.cpp | 8 + engine/src/engine/ext/scene/behavior.h | 2 +- engine/src/engine/graph/convert.cpp | 12 +- engine/src/engine/graph/graph.cpp | 118 +++++- engine/src/engine/scene/scene.cpp | 13 + engine/src/ext/audio/vorbis.cpp | 108 +++--- engine/src/ext/audio/wav.cpp | 100 +++-- engine/src/ext/freetype/freetype.cpp | 39 +- engine/src/ext/lua/lua.cpp | 76 +++- engine/src/ext/meshopt/meshopt.cpp | 4 +- engine/src/ext/opengl/shader.cpp | 14 +- engine/src/ext/valve/bsp.cpp | 97 ++--- engine/src/ext/valve/common.cpp | 7 + engine/src/ext/valve/mdl.cpp | 85 +++-- engine/src/ext/valve/vpk.cpp | 311 ++++++++++++++++ engine/src/ext/valve/vtf.cpp | 25 +- engine/src/ext/vulkan/shader.cpp | 12 +- engine/src/ext/zlib/zlib.cpp | 414 +++++++++++++++------ engine/src/utils/http/http.cpp | 145 ++++++-- engine/src/utils/image/image.cpp | 27 +- engine/src/utils/io/file.cpp | 283 +++++++------- engine/src/utils/io/vfs.cpp | 316 ++++++++++++++++ engine/src/utils/string/ext.cpp | 36 +- engine/src/utils/text/glyph.cpp | 4 +- makefiles/default/arch | 1 - makefiles/default/cc | 1 - makefiles/default/renderer | 1 - program.sh | 0 52 files changed, 1897 insertions(+), 622 deletions(-) delete mode 100644 bin/exe/default/arch delete mode 100644 bin/exe/default/cc delete mode 100644 bin/exe/default/renderer mode change 100755 => 100644 debug.sh create mode 100644 engine/inc/uf/ext/valve/vpk.h create mode 100644 engine/inc/uf/utils/io/macros.inl create mode 100644 engine/inc/uf/utils/io/vfs.h create mode 100644 engine/src/ext/valve/vpk.cpp create mode 100644 engine/src/utils/io/vfs.cpp delete mode 100644 makefiles/default/arch delete mode 100644 makefiles/default/cc delete mode 100644 makefiles/default/renderer mode change 100755 => 100644 program.sh diff --git a/.gitignore b/.gitignore index df6f77ab..8117a600 100644 --- a/.gitignore +++ b/.gitignore @@ -58,6 +58,10 @@ default/ *.ttf *.otf *.bin +*.bsp +*.vmt +*.vtf +*.ztmp models/ llm/ diff --git a/bin/data/config.json b/bin/data/config.json index 921cb069..edefcd60 100644 --- a/bin/data/config.json +++ b/bin/data/config.json @@ -332,7 +332,16 @@ "scale": 1.0 }, "ultralight": { "enabled": true, "scale": 1.5 }, - "discord": { "enabled": false } + "discord": { "enabled": false }, + "valve": { + "vpks": [ + "Half-Life 2/hl2/hl2_pak_dir.vpk", + "Half-Life 2/hl2/hl2_textures_dir.vpk", + "Half-Life 2/hl2/hl2_sound_misc_dir.vpk", + "Half-Life 2/hl2/hl2_misc_dir.vpk", + "Counter-Strike Source/cstrike/cstrike_pak_dir.vpk" + ] + } }, "physics": { "solvers": { diff --git a/bin/data/shaders/graph/base/vert.h b/bin/data/shaders/graph/base/vert.h index c6004d5a..3d5af454 100644 --- a/bin/data/shaders/graph/base/vert.h +++ b/bin/data/shaders/graph/base/vert.h @@ -64,7 +64,7 @@ void main() { const uint drawID = gl_DrawIDARB; const uint triangleID = gl_VertexIndex / 3; const DrawCommand drawCommand = drawCommands[drawID]; - const uint instanceID = drawCommand.instanceID; // gl_InstanceIndex; + const uint instanceID = gl_InstanceIndex; // drawCommand.instanceID; // gl_InstanceIndex; const Instance instance = instances[instanceID]; const Object object = objects[instance.objectID]; const uint jointID = instance.jointID; diff --git a/bin/exe/default/arch b/bin/exe/default/arch deleted file mode 100644 index 6ec5a6aa..00000000 --- a/bin/exe/default/arch +++ /dev/null @@ -1 +0,0 @@ -win64 \ No newline at end of file diff --git a/bin/exe/default/cc b/bin/exe/default/cc deleted file mode 100644 index b08d5af5..00000000 --- a/bin/exe/default/cc +++ /dev/null @@ -1 +0,0 @@ -gcc \ No newline at end of file diff --git a/bin/exe/default/renderer b/bin/exe/default/renderer deleted file mode 100644 index 53395cff..00000000 --- a/bin/exe/default/renderer +++ /dev/null @@ -1 +0,0 @@ -vulkan \ No newline at end of file diff --git a/debug.sh b/debug.sh old mode 100755 new mode 100644 diff --git a/engine/inc/uf/config.h b/engine/inc/uf/config.h index f7367bb3..8d9a4e07 100644 --- a/engine/inc/uf/config.h +++ b/engine/inc/uf/config.h @@ -27,28 +27,24 @@ #define WINVER 0x0600 #endif - #define UF_IO_ROOT "./data/" #elif defined(linux) || defined(__linux) // Linux #define UF_ENV "Linux" #define UF_ENV_LINUX 1 #define UF_ENV_HEADER "linux.h" - #define UF_IO_ROOT "./data/" #elif defined(__APPLE__) || defined(MACOSX) || defined(macintosh) || defined(Macintosh) // MacOS #define UF_ENV "OSX" #define UF_ENV_OSX 1 #define UF_ENV_HEADER "osx.h" - #define UF_IO_ROOT "./data/" #elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__) // FreeBSD #define UF_ENV "FreeBSD" #define UF_ENV_FREEBSD 1 #define UF_ENV_HEADER "freebsd.h" - #define UF_IO_ROOT "./data/" #elif defined(__sh__) // Dreamcast #define UF_ENV "Dreamcast" @@ -60,7 +56,7 @@ #define _arch_dreamcast #endif - #define UF_IO_ROOT "/cd/" + // #define UF_IO_ROOT "/cd/" #else // Unsupported system #define UF_ENV "Unknown" @@ -117,6 +113,8 @@ #define ALIGN16 #endif +#define UF_IO_ROOT "data://" + #include "macros.h" #include "simd.h" #include "helpers.inl" \ No newline at end of file diff --git a/engine/inc/uf/engine/graph/graph.h b/engine/inc/uf/engine/graph/graph.h index 8a665389..cd668168 100644 --- a/engine/inc/uf/engine/graph/graph.h +++ b/engine/inc/uf/engine/graph/graph.h @@ -86,6 +86,7 @@ namespace pod { GLOBAL, }; + uf::stl::KeyMap> groupedInstances; uf::stl::KeyMap> instanceAddresses; uf::stl::KeyMap> primitives; uf::stl::KeyMap meshes; diff --git a/engine/inc/uf/engine/scene/behavior.h b/engine/inc/uf/engine/scene/behavior.h index 65313279..c2f52930 100644 --- a/engine/inc/uf/engine/scene/behavior.h +++ b/engine/inc/uf/engine/scene/behavior.h @@ -26,6 +26,10 @@ namespace uf { pod::Thread::Tasks serial; pod::Thread::Tasks parallel; } tasks; + + struct { + size_t hash; + } mount; ); } } \ No newline at end of file diff --git a/engine/inc/uf/ext/freetype/freetype.h b/engine/inc/uf/ext/freetype/freetype.h index 09e65acc..c38ac1e4 100644 --- a/engine/inc/uf/ext/freetype/freetype.h +++ b/engine/inc/uf/ext/freetype/freetype.h @@ -11,6 +11,8 @@ #include FT_FREETYPE_H #include +#include +#include namespace pod { struct FT_Glyph { @@ -24,7 +26,6 @@ namespace ext { bool UF_API initialize(); void UF_API terminate(); - pod::FT_Glyph UF_API initialize( const uf::stl::string& ); bool UF_API initialize( pod::FT_Glyph&, const uf::stl::string& ); void UF_API destroy( pod::FT_Glyph& ); diff --git a/engine/inc/uf/ext/valve/common.h b/engine/inc/uf/ext/valve/common.h index 7cf3234d..7ae33c57 100644 --- a/engine/inc/uf/ext/valve/common.h +++ b/engine/inc/uf/ext/valve/common.h @@ -11,7 +11,7 @@ namespace impl { template T str2vec( uf::stl::string string ) { - string = uf::string::join(uf::string::split(string, " "), ","); // replace spaces with commas + string = uf::string::replace(string, " ", ","); // replace spaces with commas string = ::fmt::format("[{}]", string); // wrap as an array ext::json::Value j; ext::json::decode( j, string ); // parse JSON string return uf::vector::decode( j, T{} ); // parse JSON object @@ -21,5 +21,6 @@ namespace impl { return pod::Vector3f{ -vertex.y, vertex.z, vertex.x } * scale; } + uf::stl::string readString( std::ifstream& file ); bool parseKeyValue( const uf::stl::string& line, uf::stl::string& key, uf::stl::string& value ); } \ No newline at end of file diff --git a/engine/inc/uf/ext/valve/vpk.h b/engine/inc/uf/ext/valve/vpk.h new file mode 100644 index 00000000..56888ab8 --- /dev/null +++ b/engine/inc/uf/ext/valve/vpk.h @@ -0,0 +1,39 @@ +#pragma once + +#include +#include + +namespace pod { +#pragma pack(push, 1) + struct VpkData { + uint32_t crc; + uint16_t preloadBytes; + uint16_t archiveIndex; + uint32_t entryOffset; + uint32_t entryLength; + uint16_t terminator; + }; +#pragma pack(pop) + + struct VpkFile { + VpkData metadata; + uint32_t dirFileOffset; + uf::stl::vector preloadData; + }; + + struct VpkArchive { + uf::stl::string basePath; + uf::stl::unordered_map files; + }; +}; + +namespace ext { + namespace valve { + bool UF_API loadVpk( pod::VpkArchive& vpk, const uf::stl::string& filename ); + bool UF_API readVpk( const pod::VpkArchive& vpk, const uf::stl::string& filename, uf::stl::vector& buffer ); + bool UF_API readVpkRange( const pod::VpkArchive& vpk, const uf::stl::string& path, size_t start, size_t len, uf::stl::vector& buffer ); + + size_t UF_API mountVpk( const uf::stl::string& uri ); + pod::Mount UF_API createVpkMount( const uf::stl::string& uri, int priority = 0 ); + } +} \ No newline at end of file diff --git a/engine/inc/uf/ext/zlib/zlib.h b/engine/inc/uf/ext/zlib/zlib.h index 892e52bb..e931394a 100644 --- a/engine/inc/uf/ext/zlib/zlib.h +++ b/engine/inc/uf/ext/zlib/zlib.h @@ -5,33 +5,32 @@ #include #include +#include #include +#include + +namespace pod { + struct ZipEntry { + size_t offset; + size_t compressedSize; + size_t uncompressedSize; + uint16_t compressionMethod; + }; +} namespace ext { namespace zlib { extern UF_API size_t bufferSize; - /* - uf::stl::vector UF_API compress( const void*, size_t ); - uf::stl::vector UF_API decompress( const void*, size_t ); + + bool UF_API decompressFromFile( uf::stl::vector&, const uf::stl::string& ); + bool UF_API decompressFromFile( uf::stl::vector&, const uf::stl::string& filename, size_t start, size_t len ); + bool UF_API decompressFromFile( uf::stl::vector&, const uf::stl::string& filename, const uf::stl::vector& ranges ); + bool UF_API decompressFromMemory( uf::stl::vector&, const void*, size_t, size_t ); + + size_t UF_API compressToFile( const uf::stl::string&, const void*, size_t ); + bool UF_API directory( const uf::stl::vector& buffer, uf::stl::unordered_map& entries ); + pod::Mount UF_API createZipMount( const uf::stl::string& uri, uf::stl::vector& buffer, int priority = 0 ); - template - uf::stl::vector UF_API compress( const uf::stl::vector& data ) { - auto compressed = ext::zlib::compress( data.data(), data.size() ); - uf::stl::vector vector( compressed.size() / sizeof(T) ); - memcpy( vector.data(), compressed.data(), compressed.size() ); - } - template - uf::stl::vector UF_API decompress( const uf::stl::vector& data ) { - auto compressed = ext::zlib::decompress( data.data(), data.size() ); - uf::stl::vector vector( compressed.size() / sizeof(T) ); - memcpy( vector.data(), compressed.data(), compressed.size() ); - } - */ - - uf::stl::vector& UF_API decompressFromFile( uf::stl::vector&, const uf::stl::string& ); - uf::stl::vector& UF_API decompressFromFile( uf::stl::vector&, const uf::stl::string& filename, size_t start, size_t len ); - uf::stl::vector& UF_API decompressFromFile( uf::stl::vector&, const uf::stl::string& filename, const uf::stl::vector& ranges ); - inline uf::stl::vector decompressFromFile( const uf::stl::string& filename ) { uf::stl::vector buffer; decompressFromFile( buffer, filename ); @@ -47,9 +46,6 @@ namespace ext { decompressFromFile( buffer, filename, ranges ); return buffer; } - - - size_t UF_API compressToFile( const uf::stl::string&, const void*, size_t ); } } diff --git a/engine/inc/uf/utils/audio/metadata.h b/engine/inc/uf/utils/audio/metadata.h index a07339c1..4d34a56b 100644 --- a/engine/inc/uf/utils/audio/metadata.h +++ b/engine/inc/uf/utils/audio/metadata.h @@ -25,7 +25,7 @@ namespace uf { uf::stl::string filename = ""; uf::stl::string extension = ""; struct { - FILE* file = NULL; + void* context = NULL; void* handle = NULL; char* buffer = NULL; diff --git a/engine/inc/uf/utils/http/http.h b/engine/inc/uf/utils/http/http.h index ddbe4fb6..3f5cf4c1 100644 --- a/engine/inc/uf/utils/http/http.h +++ b/engine/inc/uf/utils/http/http.h @@ -7,11 +7,16 @@ namespace uf { struct UF_API Http { uf::stl::string header; uf::stl::string response; + size_t contentLength; + size_t mtime; + char* effective; long code; double elapsed; }; namespace http { uf::Http UF_API get( const uf::stl::string& ); + uf::Http UF_API head( const uf::stl::string& ); + uf::Http UF_API post( const uf::stl::string&, const void* data, size_t len ); } } \ No newline at end of file diff --git a/engine/inc/uf/utils/image/image.h b/engine/inc/uf/utils/image/image.h index b389f0e6..f9990f84 100644 --- a/engine/inc/uf/utils/image/image.h +++ b/engine/inc/uf/utils/image/image.h @@ -12,7 +12,7 @@ namespace pod { uf::stl::string filename; pod::Image::container_t pixels; pod::Vector2ui size = {}; - size_t bpp = 8; + size_t bpp = 8 * 4; size_t channels = 4; size_t format = 0; }; diff --git a/engine/inc/uf/utils/io/file.h b/engine/inc/uf/utils/io/file.h index 8dc1aba3..7e605b95 100644 --- a/engine/inc/uf/utils/io/file.h +++ b/engine/inc/uf/utils/io/file.h @@ -22,14 +22,14 @@ namespace uf { uf::stl::string UF_API extension( const uf::stl::string& ); uf::stl::string UF_API extension( const uf::stl::string&, int32_t ); uf::stl::string UF_API directory( const uf::stl::string& ); - uf::stl::string UF_API sanitize( const uf::stl::string&, const uf::stl::string& = "" ); + uf::stl::string UF_API normalize( const uf::stl::string& ); size_t UF_API size( const uf::stl::string& ); - uf::stl::string& UF_API readAsString( uf::stl::string&, const uf::stl::string&, const uf::stl::string& = "" ); + bool UF_API readAsString( uf::stl::string&, const uf::stl::string&, const uf::stl::string& = "" ); - uf::stl::vector& UF_API readAsBuffer( uf::stl::vector&, const uf::stl::string&, const uf::stl::string& = "" ); - uf::stl::vector& UF_API readAsBuffer( uf::stl::vector&, const uf::stl::string&, size_t start, size_t len, const uf::stl::string& = "" ); - uf::stl::vector& UF_API readAsBuffer( uf::stl::vector&, const uf::stl::string&, const uf::stl::vector& ranges, const uf::stl::string& = "" ); + bool UF_API readAsBuffer( uf::stl::vector&, const uf::stl::string&, const uf::stl::string& = "" ); + bool UF_API readAsBuffer( uf::stl::vector&, const uf::stl::string&, size_t start, size_t len, const uf::stl::string& = "" ); + bool UF_API readAsBuffer( uf::stl::vector&, const uf::stl::string&, const uf::stl::vector& ranges, const uf::stl::string& = "" ); // yuck! // wrapper in case this gets called without feeding into a buffer directly to avoid additional memory allocations @@ -62,9 +62,9 @@ namespace uf { return write( filename, string.c_str(), std::min( string.size(), size ) ); } - uf::stl::vector& UF_API decompress( uf::stl::vector&, const uf::stl::string& ); - uf::stl::vector& UF_API decompress( uf::stl::vector&, const uf::stl::string&, size_t, size_t ); - uf::stl::vector& UF_API decompress( uf::stl::vector&, const uf::stl::string&, const uf::stl::vector& ); + bool UF_API decompress( uf::stl::vector&, const uf::stl::string& ); + bool UF_API decompress( uf::stl::vector&, const uf::stl::string&, size_t, size_t ); + bool UF_API decompress( uf::stl::vector&, const uf::stl::string&, const uf::stl::vector& ); inline uf::stl::vector decompress( const uf::stl::string& filename ) { uf::stl::vector buffer; @@ -94,8 +94,21 @@ namespace uf { bool UF_API exists( const uf::stl::string& ); size_t UF_API mtime( const uf::stl::string& ); bool UF_API mkdir( const uf::stl::string& ); - uf::stl::string UF_API assetType( const uf::stl::string _filename ); + uf::stl::string UF_API assetType( const uf::stl::string& _filename ); + uf::stl::string UF_API assetScheme( const uf::stl::string& _filename ); uf::stl::string UF_API resolveURI( const uf::stl::string&, const uf::stl::string& = "" ); uf::stl::string UF_API preferred( const uf::stl::string& filename ); + + // splits "prefix://path/to/file" into "prefix://" and "path/to/file" + inline void splitUri( const uf::stl::string& uri, uf::stl::string& prefix, uf::stl::string& relativePath ) { + size_t pos = uri.find("://"); + if ( pos != uf::stl::string::npos ) { + prefix = uri.substr(0, pos + 3); + relativePath = uri.substr(pos + 3); + } else { + prefix = ""; + relativePath = uri; + } + } } } \ No newline at end of file diff --git a/engine/inc/uf/utils/io/macros.inl b/engine/inc/uf/utils/io/macros.inl new file mode 100644 index 00000000..20425caa --- /dev/null +++ b/engine/inc/uf/utils/io/macros.inl @@ -0,0 +1,6 @@ +#define UF_VFS_MOUNT_CPP( LAMBDA, URI, PRIORITY ) \ +namespace {\ + static uf::StaticInitialization TOKEN_PASTE(STATIC_INITIALIZATION_, __LINE__)( []{\ + uf::vfs::mount( LAMBDA( URI, PRIORITY ) );\ + });\ +} diff --git a/engine/inc/uf/utils/io/vfs.h b/engine/inc/uf/utils/io/vfs.h new file mode 100644 index 00000000..1896c106 --- /dev/null +++ b/engine/inc/uf/utils/io/vfs.h @@ -0,0 +1,54 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace pod { + struct Range; + + struct Mount { + uf::stl::string prefix = ""; + uf::stl::string path = ""; + int priority = 0; + pod::PointeredUserdata userdata; + + std::function exists; + std::function size; + std::function mtime; + std::function&)> read; + std::function write; + + std::function&)> readRange; + std::function&, uf::stl::vector&)> readRanges; + }; +} + +namespace uf { + namespace vfs { + extern UF_API uf::stl::vector mounts; + + size_t UF_API mount( const pod::Mount& mount ); + bool UF_API unmount( size_t ); + bool UF_API unmount( const uf::stl::string& prefix, const uf::stl::string& base ); + + bool UF_API exists( const uf::stl::string& path ); + size_t UF_API size( const uf::stl::string& path ); + size_t UF_API mtime( const uf::stl::string& path ); + bool UF_API read( const uf::stl::string& path, uf::stl::vector& buffer ); + + size_t UF_API write( const uf::stl::string& path, const void* data, size_t len ); + size_t UF_API write( const uf::stl::string& path, uf::stl::vector& buffer ); + + bool UF_API readRange( const uf::stl::string& path, size_t start, size_t len, uf::stl::vector& buffer ); + bool UF_API readRanges( const uf::stl::string& path, const uf::stl::vector& ranges, uf::stl::vector& buffer ); + + pod::Mount UF_API createDiskMount( const uf::stl::string& uri, int priority = 0 ); + uf::stl::string UF_API resolveBase( const uf::stl::string& path ); + } +} + +#include "macros.inl" \ No newline at end of file diff --git a/engine/inc/uf/utils/math/hash.h b/engine/inc/uf/utils/math/hash.h index efed04ed..06c6c503 100644 --- a/engine/inc/uf/utils/math/hash.h +++ b/engine/inc/uf/utils/math/hash.h @@ -1,11 +1,11 @@ #pragma once namespace uf { - inline void hash(std::size_t& seed) { } + inline void hash( size_t& seed ) { } template - inline void hash(std::size_t& seed, const T& v, Rest... rest) { - seed ^= std::hash()(v) + 0x9e3779b9 + (seed<<6) + (seed>>2); - hash(seed, rest...); + inline void hash( size_t& seed, const T& v, Rest... rest ) { + seed ^= std::hash()(v) + 0x9e3779b9 + (seed<<6) + (seed>>2); + hash(seed, rest...); } } \ No newline at end of file diff --git a/engine/inc/uf/utils/string/ext.h b/engine/inc/uf/utils/string/ext.h index f977110f..2fbedcc1 100644 --- a/engine/inc/uf/utils/string/ext.h +++ b/engine/inc/uf/utils/string/ext.h @@ -17,6 +17,7 @@ namespace uf { bool UF_API matched( const uf::stl::string& str, const uf::stl::string& r ); uf::stl::string UF_API replace( const uf::stl::string&, const uf::stl::string&, const uf::stl::string& ); + uf::stl::string UF_API replace( const uf::stl::string&, const uf::stl::string&, const uf::stl::string&, bool regexp ); uf::stl::string UF_API lowercase( const uf::stl::string& ); uf::stl::string UF_API uppercase( const uf::stl::string& ); uf::stl::string UF_API ltrim( const uf::stl::string& ); diff --git a/engine/inc/uf/utils/userdata/pointered.inl b/engine/inc/uf/utils/userdata/pointered.inl index 19f05404..0a55f365 100644 --- a/engine/inc/uf/utils/userdata/pointered.inl +++ b/engine/inc/uf/utils/userdata/pointered.inl @@ -40,7 +40,7 @@ namespace uf { public: // Easily access POD's type typedef pod::PointeredUserdata pod_t; - bool autoDestruct = uf::userdata::autoDestruct; + bool autoDestruct = uf::userdata::autoDestruct; // shouldn't ever use in containers because of auto-destruct...... protected: // POD storage PointeredUserdata::pod_t m_pod = {}; diff --git a/engine/src/engine/asset/asset.cpp b/engine/src/engine/asset/asset.cpp index a4786273..23398e99 100644 --- a/engine/src/engine/asset/asset.cpp +++ b/engine/src/engine/asset/asset.cpp @@ -152,19 +152,7 @@ bool uf::asset::isExpected( const uf::asset::Payload& payload, uf::asset::Type e uf::stl::string uf::asset::cache( uf::asset::Payload& payload ) { uf::stl::string filename = payload.filename; uf::stl::string extension = uf::io::extension( filename ); - if ( filename.substr(0,5) == "https" ) { - uf::stl::string hash = uf::string::sha256( filename ); - uf::stl::string cached = uf::io::root + "/cache/http/" + hash + "." + extension; - if ( !uf::io::exists( cached ) && !retrieve( filename, cached, hash ) ) { - if ( !uf::asset::assertionLoad ) { - UF_MSG_ERROR("Failed to preload {} ({}): HTTP error", filename, cached); - } else { - UF_EXCEPTION("Failed to preload {} ({}): HTTP error", filename, cached); - } - return ""; - } - filename = cached; - } + if ( !uf::io::exists( filename ) ) { if ( !uf::asset::assertionLoad ) { UF_MSG_ERROR("Failed to preload {}, does not exist", filename); @@ -193,19 +181,6 @@ uf::stl::string uf::asset::load( uf::asset::Payload& payload ) { uf::stl::string extension = uf::string::lowercase(uf::io::extension( payload.filename, -1 )); uf::stl::string basename = uf::string::lowercase( uf::string::replace( uf::io::filename( payload.filename ), "/.(?:gz|lz4?)$/", "" ) ); - if ( payload.filename.substr(0,5) == "https" ) { - uf::stl::string hash = uf::string::sha256( payload.filename ); - uf::stl::string cached = uf::io::root + "/cache/http/" + hash + "." + extension; - if ( !uf::io::exists( cached ) && !retrieve( payload.filename, cached, hash ) ) { - if ( !uf::asset::assertionLoad ) { - UF_MSG_ERROR("Failed to load {} ({}): HTTP error", payload.filename, cached); - } else { - UF_EXCEPTION("Failed to load {} ({}): HTTP error", payload.filename, cached); - } - return ""; - } - filename = cached; - } if ( !uf::io::exists( filename ) ) { if ( !uf::asset::assertionLoad ) { UF_MSG_ERROR("Failed to load {}: does not exist", filename); diff --git a/engine/src/engine/ext/ext.cpp b/engine/src/engine/ext/ext.cpp index a984c148..f6f02a4e 100644 --- a/engine/src/engine/ext/ext.cpp +++ b/engine/src/engine/ext/ext.cpp @@ -46,6 +46,7 @@ #include #include #include +#include bool uf::ready = false; uf::stl::vector uf::arguments; @@ -493,6 +494,13 @@ void UF_API uf::initialize() { uf::console::initialize(); } + /* Load VPKs */ { + auto& vpks = uf::config["engine"]["ext"]["valve"]["vpks"]; + ext::json::forEach( vpks, []( const uf::stl::string& uri ) { + ext::valve::mountVpk( uri ); + }); + } + /* Create initial scene (kludge) */ { uf::Scene& scene = uf::instantiator::instantiate(); //new uf::Scene; uf::scene::scenes.emplace_back(&scene); diff --git a/engine/src/engine/ext/scene/behavior.h b/engine/src/engine/ext/scene/behavior.h index fc67f7ce..fe708885 100644 --- a/engine/src/engine/ext/scene/behavior.h +++ b/engine/src/engine/ext/scene/behavior.h @@ -74,7 +74,7 @@ namespace ext { } fog; struct { struct { - uf::stl::string filename = "%root%/textures/skybox/%d.png"; + uf::stl::string filename = "/textures/skybox/%d.png"; } box; } sky; diff --git a/engine/src/engine/graph/convert.cpp b/engine/src/engine/graph/convert.cpp index c138ba36..dfbb6afe 100644 --- a/engine/src/engine/graph/convert.cpp +++ b/engine/src/engine/graph/convert.cpp @@ -221,15 +221,15 @@ void uf::graph::postprocess( pod::Graph& graph ) { #if UF_USE_XATLAS // generate STs if ( graph.metadata["exporter"]["unwrap"].as(true) || graph.metadata["exporter"]["unwrap"].as() == "tagged" ) { - UF_MSG_DEBUG( "Generating ST's..." ); + //UF_MSG_DEBUG( "Generating ST's..." ); size_t atlases = ext::xatlas::unwrap( graph ); - UF_MSG_DEBUG( "Generated ST's for {} lightmaps", atlases ); + //UF_MSG_DEBUG( "Generated ST's for {} lightmaps", atlases ); } #endif #if UF_USE_MESHOPT // cleanup if blender's exporter is poopy if ( graph.metadata["exporter"]["optimize"].as(false) || graph.metadata["exporter"]["optimize"].as("") == "tagged" || ext::json::isObject( graph.metadata["exporter"]["optimize"] ) ) { - UF_MSG_DEBUG( "Optimizing meshes..." ); + //UF_MSG_DEBUG( "Optimizing meshes..." ); for ( auto& keyName : graph.meshes ) { size_t level = SIZE_MAX; float simplify = 1.0f; @@ -265,7 +265,7 @@ void uf::graph::postprocess( pod::Graph& graph ) { auto& primitives = storage.primitives[keyName]; if ( level ) { - UF_MSG_DEBUG("Optimizing mesh at level {}: {}", level, keyName); + //UF_MSG_DEBUG("Optimizing mesh at level {}: {}", level, keyName); if ( !ext::meshopt::optimize( mesh, simplify, level, print ) ) { UF_MSG_ERROR("Mesh optimization failed: {}", keyName ); } @@ -276,7 +276,7 @@ void uf::graph::postprocess( pod::Graph& graph ) { if ( lodMetadata.empty() ) { UF_MSG_ERROR("LOD generation failed: {}", keyName ); } else { - UF_MSG_DEBUG("Generated {} LODs: {}", factors.size() - 1, keyName); + //UF_MSG_DEBUG("Generated {} LODs: {}", factors.size() - 1, keyName); UF_ASSERT( primitives.size() == lodMetadata.size() ); for ( auto i = 0; i < primitives.size(); ++i ) { primitives[i].lod = lodMetadata[i]; @@ -285,7 +285,7 @@ void uf::graph::postprocess( pod::Graph& graph ) { } } - UF_MSG_DEBUG( "Optimized mesh" ); + //UF_MSG_DEBUG( "Optimized mesh" ); } #endif { diff --git a/engine/src/engine/graph/graph.cpp b/engine/src/engine/graph/graph.cpp index eb61a764..96b5f38c 100644 --- a/engine/src/engine/graph/graph.cpp +++ b/engine/src/engine/graph/graph.cpp @@ -834,7 +834,7 @@ void uf::graph::process( pod::Graph& graph ) { // lightmaps are considered stale if they're older than the graph's source bool stale = false; for ( auto& pair : filenames ) { - uf::stl::string filename = uf::io::sanitize( pair.second, uf::io::directory( graph.name ) ); + uf::stl::string filename = uf::io::resolveURI( pair.second, uf::io::directory( graph.name ) ); auto time = uf::io::mtime(filename); if ( !uf::io::exists( filename ) ) continue; if ( time < mtime ) { @@ -852,7 +852,7 @@ void uf::graph::process( pod::Graph& graph ) { if ( graphMetadataJson["lights"]["lightmap"].as() ) { for ( auto& pair : filenames ) { auto i = pair.first; - auto f = uf::io::sanitize( pair.second, uf::io::directory( graph.name ) ); + auto f = uf::io::resolveURI( pair.second, uf::io::directory( graph.name ) ); if ( !uf::io::exists( f ) ) { graphMetadataJson["lights"]["lightmap"] = false; UF_MSG_ERROR( "lightmap does not exist: {} {}, disabling lightmaps", i, f ) @@ -863,7 +863,7 @@ void uf::graph::process( pod::Graph& graph ) { if ( graphMetadataJson["lights"]["lightmap"].as() ) { for ( auto& pair : filenames ) { auto i = pair.first; - auto f = uf::io::sanitize( pair.second, uf::io::directory( graph.name ) ); + auto f = uf::io::resolveURI( pair.second, uf::io::directory( graph.name ) ); auto textureID = graph.textures.size(); auto imageID = graph.images.size(); @@ -1087,6 +1087,29 @@ void uf::graph::process( pod::Graph& graph ) { } } } + + for ( auto& instance : storage.groupedInstances.map[name] ) { + if ( 0 <= instance.materialID && instance.materialID < graph.materials.size() ) { + auto& keys = storage.materials.keys; + auto& indices = storage.materials.indices; + auto& needle = graph.materials[instance.materialID]; + instance.materialID = indices[needle]; + } + if ( 0 <= instance.lightmapID && instance.lightmapID < graph.textures.size() ) { + auto& keys = storage.textures.keys; + auto& indices = storage.textures.indices; + auto& needle = graph.textures[instance.lightmapID]; + instance.lightmapID = indices[needle]; + } + if ( 0 <= instance.jointID && instance.jointID < graph.skins.size() ) { + auto& skinName = graph.skins[instance.jointID]; + instance.jointID = 0; + for ( auto key : storage.joints.keys ) { + if ( key == skinName ) break; + instance.jointID += storage.joints[key].size(); + } + } + } } /* if ( graphMetadataJson["debug"]["print"]["lights"].as() ) { @@ -1296,6 +1319,41 @@ void uf::graph::process( pod::Graph& graph, int32_t index, uf::Object& parent ) }; } + auto primitiveName = graph.primitives[node.mesh]; + auto& mesh = storage.meshes.map[graph.meshes[node.mesh]]; + auto& primitives = storage.primitives.map[primitiveName]; + auto& grouped = storage.groupedInstances[primitiveName]; + + node.object = ::allocateObjectID( storage ); + auto objectKeyName = std::to_string( node.object ); + + storage.entities[objectKeyName] = &entity; + storage.objects[objectKeyName] = pod::Instance::Object{ + .model = model, + .previous = model, + }; + + pod::Instance::Bounds bounds = {}; + + for ( auto drawID = 0; drawID < primitives.size(); ++drawID ) { + pod::Instance newInstance = primitives[drawID].instance; + newInstance.objectID = node.object; + newInstance.jointID = graphMetadataJson["renderer"]["skinned"].as() ? 0 : -1; + + bounds.min = uf::vector::min( bounds.min, newInstance.bounds.min ); + bounds.max = uf::vector::max( bounds.max, newInstance.bounds.max ); + + grouped.emplace_back(newInstance); + } + + bool isFirstInstance = ( grouped.size() == primitives.size() ); + #if !UF_GRAPH_EXTENDED + if ( graphMetadataJson["renderer"]["render"].as() && isFirstInstance ) { + uf::graph::initializeGraphics( graph, entity, mesh, addresses ); + } + #endif + + #if 0 auto& mesh = storage.meshes.map[graph.meshes[node.mesh]]; auto& primitives = storage.primitives.map[graph.primitives[node.mesh]]; auto& instanceAddresses = storage.instanceAddresses.map[graph.primitives[node.mesh]]; @@ -1328,6 +1386,7 @@ void uf::graph::process( pod::Graph& graph, int32_t index, uf::Object& parent ) if ( graphMetadataJson["renderer"]["render"].as() ) { uf::graph::initializeGraphics( graph, entity, mesh, addresses ); } + #endif #endif { @@ -1490,15 +1549,52 @@ bool uf::graph::tick( pod::Graph::Storage& storage ) { if ( storage.stale ) { for ( auto& key : storage.primitives.keys ) { - for ( auto& primitive : storage.primitives[key] ) { - drawCommands.emplace_back( primitive.drawCommand ); - instances.emplace_back( primitive.instance ); - lodMetadata.emplace_back( primitive.lod ); - } - } + auto& submeshes = storage.primitives.map[key]; + auto& grouped = storage.groupedInstances.map[key]; + auto& submeshAddresses = storage.instanceAddresses.map[key]; - for ( auto& key : storage.instanceAddresses.keys ) { - instanceAddresses.insert( instanceAddresses.end(), storage.instanceAddresses.map[key].begin(), storage.instanceAddresses.map[key].end() ); + auto& mesh = storage.meshes.map[key]; + pod::DrawCommand* meshIndirectCmds = nullptr; + + if (mesh.indirect.count > 0) { + auto& attr = mesh.indirect.attributes.front(); + meshIndirectCmds = (pod::DrawCommand*) mesh.buffers[attr.buffer].data(); + } + + size_t nodesCount = submeshes.empty() ? 0 : grouped.size() / submeshes.size(); + + for ( size_t drawID = 0; drawID < submeshes.size(); ++drawID ) { + auto& primitive = submeshes[drawID]; + + primitive.drawCommand.instanceID = instances.size(); + primitive.drawCommand.instances = nodesCount; + + if (meshIndirectCmds) { + meshIndirectCmds[drawID].instanceID = primitive.drawCommand.instanceID; + meshIndirectCmds[drawID].instances = primitive.drawCommand.instances; + } + + drawCommands.emplace_back( primitive.drawCommand ); + lodMetadata.emplace_back( primitive.lod ); + + bool hasAddresses = (drawID < submeshAddresses.size()); + + for (size_t i = 0; i < nodesCount; ++i) { + size_t strideIndex = (i * submeshes.size()) + drawID; + instances.emplace_back( grouped[strideIndex] ); + instanceAddresses.emplace_back( hasAddresses ? submeshAddresses[drawID] : pod::Instance::Addresses{} ); + } + } + + if (meshIndirectCmds && !grouped.empty()) { + auto hostKeyName = std::to_string(grouped.front().objectID); + if (storage.entities.map.count(hostKeyName) > 0) { + auto* hostEntity = storage.entities.map[hostKeyName]; + if (hostEntity && hostEntity->hasComponent()) { + hostEntity->getComponent().updateMesh(mesh); + } + } + } } for ( auto& key : storage.textures.keys ) textures.emplace_back( storage.textures.map[key] ); diff --git a/engine/src/engine/scene/scene.cpp b/engine/src/engine/scene/scene.cpp index d8bc9860..e664414c 100644 --- a/engine/src/engine/scene/scene.cpp +++ b/engine/src/engine/scene/scene.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -184,8 +185,15 @@ uf::Scene& uf::scene::loadScene( const uf::stl::string& name, const uf::stl::str #endif scene->load(filename); + auto& metadata = scene->getComponent(); + auto& metadataObject = scene->getComponent(); + auto mountUri = ::fmt::format("://{}", uf::vfs::resolveBase( metadataObject.system.root ) ); + metadata.mount.hash = uf::vfs::mount( uf::vfs::createDiskMount( mountUri, 200 ) ); + auto& metadataJson = scene->getComponent(); metadataJson["system"]["scene"] = name; + + #if UF_USE_VULKAN if ( uf::renderer::settings::pipelines::rt ) uf::instantiator::bind( "RayTraceSceneBehavior", *scene ); if ( uf::renderer::settings::pipelines::vxgi ) uf::instantiator::bind( "VoxelizerSceneBehavior", *scene ); @@ -212,6 +220,11 @@ uf::Scene& uf::scene::loadScene( const uf::stl::string& name, const uf::Serializ void uf::scene::unloadScene() { uf::Scene* current = uf::scene::scenes.back(); current->queueDeletion(); + + { + auto& metadataScene = current->getComponent(); + uf::vfs::unmount( metadataScene.mount.hash ); + } // destroy graph if ( current->hasComponent() ) { diff --git a/engine/src/ext/audio/vorbis.cpp b/engine/src/ext/audio/vorbis.cpp index 52df0174..4b5c1c45 100644 --- a/engine/src/ext/audio/vorbis.cpp +++ b/engine/src/ext/audio/vorbis.cpp @@ -7,10 +7,9 @@ #include #include +#include #include -#include #include -#include #include #if UF_USE_TREMOR @@ -22,45 +21,57 @@ namespace { constexpr int endian = 0; // 0 = little endian + struct VorbisVfsContext { + uf::stl::string filename; + size_t currentOffset; + size_t totalSize; + }; + namespace funs { size_t read(void* destination, size_t size, size_t nmemb, void* userdata) { - uf::Audio::Metadata& metadata = *((uf::Audio::Metadata*) userdata); - FILE* file = (FILE*) metadata.stream.file; - size_t read_count = fread(destination, size, nmemb, file); - metadata.stream.consumed += read_count * size; - return read_count; + VorbisVfsContext* ctx = (VorbisVfsContext*) userdata; + size_t bytesToRead = size * nmemb; + size_t bytesLeft = ctx->totalSize - ctx->currentOffset; + if (bytesToRead > bytesLeft) bytesToRead = bytesLeft; + + if (bytesToRead > 0) { + uf::stl::vector tempBuffer; + if (uf::vfs::readRange(ctx->filename, ctx->currentOffset, bytesToRead, tempBuffer)) { + std::memcpy(destination, tempBuffer.data(), tempBuffer.size()); + ctx->currentOffset += tempBuffer.size(); + return tempBuffer.size() / size; + } + } + return 0; } int seek( void* userdata, ogg_int64_t to, int type ) { - uf::Audio::Metadata& metadata = *((uf::Audio::Metadata*) userdata); - FILE* file = (FILE*) metadata.stream.file; + VorbisVfsContext* ctx = (VorbisVfsContext*) userdata; switch ( type ) { - case SEEK_CUR: metadata.stream.consumed += to; break; - case SEEK_END: metadata.stream.consumed = metadata.info.size - to; break; - case SEEK_SET: metadata.stream.consumed = to; break; + case SEEK_CUR: ctx->currentOffset += to; break; + case SEEK_END: ctx->currentOffset = ctx->totalSize + to; break; + case SEEK_SET: ctx->currentOffset = to; break; default: return -1; } - if (metadata.stream.consumed < 0) metadata.stream.consumed = 0; - if (metadata.stream.consumed > metadata.info.size) metadata.stream.consumed = metadata.info.size; - - if (fseek(file, to, type) != 0) return -1; + + if ((int64_t)ctx->currentOffset < 0) ctx->currentOffset = 0; + if (ctx->currentOffset > ctx->totalSize) ctx->currentOffset = ctx->totalSize; + return 0; } int close(void* userdata) { - uf::Audio::Metadata& metadata = *((uf::Audio::Metadata*) userdata); - FILE* file = (FILE*) metadata.stream.file; - if ( file ) { - fclose(file); - metadata.stream.file = NULL; + VorbisVfsContext* ctx = (VorbisVfsContext*) userdata; + if (ctx) { + delete ctx; } return 0; } long tell(void* userdata) { - uf::Audio::Metadata& metadata = *((uf::Audio::Metadata*) userdata); - return metadata.stream.consumed; + VorbisVfsContext* ctx = (VorbisVfsContext*) userdata; + return (long)ctx->currentOffset; } } @@ -78,20 +89,25 @@ namespace { } void ext::vorbis::open(uf::Audio::Metadata& metadata) { - if ( !metadata.stream.file ) metadata.stream.file = fopen(metadata.filename.c_str(), "rb"); + if ( !metadata.stream.context ) { + VorbisVfsContext* ctx = new VorbisVfsContext(); + ctx->filename = metadata.filename; + ctx->currentOffset = 0; + ctx->totalSize = uf::vfs::size(metadata.filename); + + if ( ctx->totalSize == 0 ) { + UF_MSG_ERROR("Vorbis: failed to open file: {}", metadata.filename); + delete ctx; + return; + } + metadata.stream.context = (void*) ctx; + } if ( !metadata.stream.handle ) metadata.stream.handle = (void*) new OggVorbis_File; - FILE* file = (FILE*) metadata.stream.file; + VorbisVfsContext* ctx = (VorbisVfsContext*) metadata.stream.context; OggVorbis_File* vorbisFile = (OggVorbis_File*) metadata.stream.handle; - if ( !file ) { - UF_MSG_ERROR("Vorbis: failed to open {}. File error.", metadata.filename); - return; - } - - fseek(file, 0, SEEK_END); - metadata.info.size = ftell(file); - fseek(file, 0, SEEK_SET); + metadata.info.size = ctx->totalSize; metadata.stream.consumed = 0; ov_callbacks callbacks; @@ -99,10 +115,12 @@ void ext::vorbis::open(uf::Audio::Metadata& metadata) { callbacks.seek_func = funs::seek; callbacks.close_func = funs::close; callbacks.tell_func = funs::tell; - - int error = ov_open_callbacks((void*) &metadata, vorbisFile, NULL, metadata.settings.streamed ? -1 : 0, callbacks); + + int error = ov_open_callbacks((void*) ctx, vorbisFile, NULL, metadata.settings.streamed ? -1 : 0, callbacks); if (error < 0) { UF_MSG_ERROR("Vorbis: failed call to ov_open_callbacks: {}", metadata.filename); + delete ctx; + metadata.stream.context = nullptr; return; } @@ -114,6 +132,7 @@ void ext::vorbis::open(uf::Audio::Metadata& metadata) { if ( !format(metadata, info->channels, 16) ) { ov_clear(vorbisFile); + metadata.stream.context = nullptr; return; } @@ -123,10 +142,9 @@ void ext::vorbis::open(uf::Audio::Metadata& metadata) { void ext::vorbis::load( uf::Audio::Metadata& metadata ) { if ( metadata.settings.streamed ) return ext::vorbis::stream(metadata); - FILE* file = (FILE*) metadata.stream.file; OggVorbis_File* vorbisFile = (OggVorbis_File*) metadata.stream.handle; - uf::stl::vector bytes; + uf::stl::vector bytes; char buffer[uf::audio::bufferSize]; int bitStream = 0; int read = 0; @@ -139,7 +157,7 @@ void ext::vorbis::load( uf::Audio::Metadata& metadata ) { metadata.al.source.set(AL_BUFFER, (ALint) metadata.al.buffer.getIndex()); ov_clear(vorbisFile); - fclose(file); + metadata.stream.context = nullptr; } void ext::vorbis::stream(uf::Audio::Metadata& metadata) { @@ -147,7 +165,7 @@ void ext::vorbis::stream(uf::Audio::Metadata& metadata) { OggVorbis_File* vorbisFile = (OggVorbis_File*) metadata.stream.handle; - // Fill and queue initial buffers + // fill and queue initial buffers char buffer[uf::audio::bufferSize]; uint8_t queuedBuffers = 0; int bitStream = 0; @@ -192,8 +210,8 @@ void ext::vorbis::update(uf::Audio::Metadata& metadata) { ALint state; metadata.al.source.get(AL_SOURCE_STATE, state); if (state != AL_PLAYING) { - if (!metadata.settings.loop && metadata.stream.consumed >= metadata.info.size) { - // Stream finished + VorbisVfsContext* ctx = (VorbisVfsContext*) metadata.stream.context; + if (!metadata.settings.loop && ctx && ctx->currentOffset >= ctx->totalSize) { return; } metadata.al.source.play(); @@ -250,12 +268,8 @@ void ext::vorbis::close(uf::Audio::Metadata& metadata) { ov_clear(file); delete file; metadata.stream.handle = NULL; - } - if ( metadata.stream.file ) { - FILE* file = (FILE*) metadata.stream.file; - fclose(file); - metadata.stream.file = NULL; + metadata.stream.context = NULL; } } -#endif +#endif \ No newline at end of file diff --git a/engine/src/ext/audio/wav.cpp b/engine/src/ext/audio/wav.cpp index d255696b..523e9328 100644 --- a/engine/src/ext/audio/wav.cpp +++ b/engine/src/ext/audio/wav.cpp @@ -7,17 +7,69 @@ #include #include +#include #include #include +#include #define DR_WAV_IMPLEMENTATION #include "dr_wav.h" namespace { + struct DrWavVfsContext { + drwav wav; + uf::stl::string filename; + size_t currentOffset; + size_t totalSize; + }; + + size_t drwav_vfs_read(void* pUserData, void* pBufferOut, size_t bytesToRead) { + DrWavVfsContext* ctx = (DrWavVfsContext*)pUserData; + size_t bytesLeft = ctx->totalSize - ctx->currentOffset; + if (bytesToRead > bytesLeft) bytesToRead = bytesLeft; + + if (bytesToRead > 0) { + uf::stl::vector tempBuffer; + if (uf::vfs::readRange(ctx->filename, ctx->currentOffset, bytesToRead, tempBuffer)) { + std::memcpy(pBufferOut, tempBuffer.data(), tempBuffer.size()); + ctx->currentOffset += tempBuffer.size(); + return tempBuffer.size(); + } + } + return 0; + } + + drwav_bool32 drwav_vfs_seek(void* pUserData, int offset, drwav_seek_origin origin) { + DrWavVfsContext* ctx = (DrWavVfsContext*)pUserData; + + if ((int)origin == 0) { + ctx->currentOffset = (size_t)offset; + } else if ((int)origin == 1) { + if (offset < 0 && (size_t)(-offset) > ctx->currentOffset) { + ctx->currentOffset = 0; + } else { + ctx->currentOffset += offset; + } + } + + if (ctx->currentOffset > ctx->totalSize) ctx->currentOffset = ctx->totalSize; + return 1; + } + + drwav_bool32 drwav_vfs_tell(void* pUserData, long long int* pCursor) { + DrWavVfsContext* ctx = (DrWavVfsContext*)pUserData; + if (pCursor) { + *pCursor = (long long int)ctx->currentOffset; + } + return 1; + } + namespace funs { size_t read(void* destination, size_t size, size_t nmemb, void* userdata) { uf::Audio::Metadata& metadata = *((uf::Audio::Metadata*)userdata); - drwav* wav = (drwav*) metadata.stream.handle; + DrWavVfsContext* ctx = (DrWavVfsContext*) metadata.stream.handle; + drwav* wav = &ctx->wav; + size_t bytesRequested = size * nmemb; size_t frameSize = wav->channels * (wav->bitsPerSample / 8); size_t framesToRead = bytesRequested / frameSize; @@ -30,7 +82,8 @@ namespace { } int seek(void* userdata, int64_t to, int type) { uf::Audio::Metadata& metadata = *((uf::Audio::Metadata*)userdata); - drwav* wav = (drwav*) metadata.stream.handle; + DrWavVfsContext* ctx = (DrWavVfsContext*) metadata.stream.handle; + drwav* wav = &ctx->wav; drwav_uint64 targetFrame = 0; switch (type) { @@ -45,10 +98,10 @@ namespace { } int close(void* userdata) { uf::Audio::Metadata& metadata = *((uf::Audio::Metadata*)userdata); - drwav* wav = (drwav*) metadata.stream.handle; - if (wav) { - drwav_uninit(wav); - delete wav; + DrWavVfsContext* ctx = (DrWavVfsContext*) metadata.stream.handle; + if (ctx) { + drwav_uninit(&ctx->wav); + delete ctx; metadata.stream.handle = nullptr; } return 0; @@ -61,16 +114,21 @@ namespace { } void ext::wav::open(uf::Audio::Metadata& metadata) { - // open file - drwav* wav = new drwav; - if (!drwav_init_file(wav, metadata.filename.c_str(), nullptr)) { + DrWavVfsContext* ctx = new DrWavVfsContext(); + ctx->filename = metadata.filename; + ctx->currentOffset = 0; + ctx->totalSize = uf::vfs::size(metadata.filename); + + if (!drwav_init(&ctx->wav, drwav_vfs_read, drwav_vfs_seek, drwav_vfs_tell, ctx, nullptr)) { UF_MSG_ERROR("Could not open WAV file: {}", metadata.filename); - delete wav; + delete ctx; return; } + drwav* wav = &ctx->wav; + // fill out metadata - metadata.stream.handle = wav; + metadata.stream.handle = ctx; metadata.info.size = wav->totalPCMFrameCount * wav->channels * (wav->bitsPerSample / 8); metadata.stream.consumed = 0; metadata.info.channels = wav->channels; @@ -94,26 +152,21 @@ void ext::wav::open(uf::Audio::Metadata& metadata) { } // choose load or stream - return metadata.settings.streamed ? ext::wav::stream(metadata) : ext::wav::load(metadata); + if (metadata.settings.streamed) ext::wav::stream(metadata); else ext::wav::load(metadata); } void ext::wav::load(uf::Audio::Metadata& metadata) { // if streaming is requested, use streaming function if (metadata.settings.streamed) return ext::wav::stream(metadata); - - drwav* wav = (drwav*) metadata.stream.handle; - // read all PCM data size_t totalBytes = (size_t) metadata.info.size; std::vector bytes(totalBytes); - // Use funs::read instead of drwav_read_pcm_frames size_t bytesRead = funs::read(bytes.data(), 1, totalBytes, &metadata); if (bytesRead < totalBytes) { - bytes.resize(bytesRead); // in case file is truncated + bytes.resize(bytesRead); } - // upload to OpenAL buffer metadata.al.buffer.buffer(metadata.info.format, bytes.data(), (ALsizei) bytes.size(), metadata.info.frequency); metadata.al.source.set(AL_BUFFER, (ALint) metadata.al.buffer.getIndex()); @@ -123,10 +176,8 @@ void ext::wav::load(uf::Audio::Metadata& metadata) { void ext::wav::stream(uf::Audio::Metadata& metadata) { if (!metadata.settings.streamed) return ext::wav::load(metadata); - // Ensure we're at the start funs::seek(&metadata, 0, SEEK_SET); - drwav* wav = (drwav*) metadata.stream.handle; char buffer[uf::audio::bufferSize]; uint8_t queuedBuffers = 0; for (; queuedBuffers < metadata.settings.buffers; ++queuedBuffers) { @@ -167,26 +218,21 @@ void ext::wav::update(uf::Audio::Metadata& metadata) { metadata.al.source.get(AL_SOURCE_STATE, state); if (state != AL_PLAYING) { if (!metadata.settings.loop && metadata.stream.consumed >= metadata.info.size) { - // stream finished - return; + return; // stream finished } // stream stalled, restart it metadata.al.source.play(); } - ALint processed = 0; metadata.al.source.get(AL_BUFFERS_PROCESSED, processed); if (processed <= 0) return; - drwav* wav = (drwav*) metadata.stream.handle; - ALuint index; char buffer[uf::audio::bufferSize]; while (processed--) { memset(buffer, 0, uf::audio::bufferSize); AL_CHECK_RESULT(alSourceUnqueueBuffers(metadata.al.source.getIndex(), 1, &index)); - // Use funs::read instead of drwav_read_pcm_frames size_t bytesRead = funs::read(buffer, 1, uf::audio::bufferSize, &metadata); if (bytesRead == 0) { @@ -198,6 +244,7 @@ void ext::wav::update(uf::Audio::Metadata& metadata) { } bytesRead = funs::read(buffer, 1, uf::audio::bufferSize, &metadata); if (bytesRead == 0) { + // should never actually reach here UF_MSG_ERROR("WAV: failed to read after looping: {}", metadata.filename); break; } @@ -208,7 +255,6 @@ void ext::wav::update(uf::Audio::Metadata& metadata) { AL_CHECK_RESULT(alSourceQueueBuffers(metadata.al.source.getIndex(), 1, &index)); } if (metadata.settings.loop && bytesRead < uf::audio::bufferSize) { - // should never actually reach here UF_MSG_ERROR("WAV: missing data: {}", metadata.filename); } } diff --git a/engine/src/ext/freetype/freetype.cpp b/engine/src/ext/freetype/freetype.cpp index 35ee3abc..b22cb18c 100644 --- a/engine/src/ext/freetype/freetype.cpp +++ b/engine/src/ext/freetype/freetype.cpp @@ -1,9 +1,12 @@ #include #if UF_USE_FREETYPE #include +#include +#include namespace impl { - FT_Library library; + FT_Library ft_library; + uf::stl::unordered_map> ftCache; uf::stl::string error( int error ) { #undef FTERRORS_H_ @@ -62,33 +65,35 @@ pod::FT_Glyph::~FT_Glyph() { } bool ext::freetype::initialize() { - if ( auto error = FT_Init_FreeType( &impl::library ) ) { + if ( auto error = FT_Init_FreeType( &impl::ft_library ) ) { UF_MSG_ERROR("FreeType failed to initialize: {}", impl::error( error )); return false; } return true; } void ext::freetype::terminate() { - FT_Done_FreeType( impl::library ); + FT_Done_FreeType( impl::ft_library ); + impl::ftCache.clear(); } -pod::FT_Glyph ext::freetype::initialize( const uf::stl::string& font ) { - pod::FT_Glyph g; - if ( auto error = FT_New_Face( impl::library, font.c_str(), 0, &g.face ) ) { - UF_MSG_ERROR("FreeType failed to load file '{}': {}", font, impl::error( error )); - } - if ( auto error = FT_Select_Charmap( g.face, FT_ENCODING_UNICODE ) ) { - UF_MSG_ERROR("FreeType failed to load file '{}': {}", font, impl::error( error )); - } - return g; -} -bool ext::freetype::initialize( pod::FT_Glyph& g, const uf::stl::string& font ) { - if ( auto error = FT_New_Face( impl::library, font.c_str(), 0, &g.face ) ) { - UF_MSG_ERROR("FreeType failed to load file '{}': {}", font, impl::error( error )); +bool ext::freetype::initialize( pod::FT_Glyph& g, const uf::stl::string& filename ) { + // yucky yuck + if ( impl::ftCache.find(filename) == impl::ftCache.end() ) { + uf::stl::vector buffer; + if ( !uf::io::readAsBuffer( buffer, filename ) ) { + UF_MSG_ERROR("FreeType failed to read file: {}", filename); + return false; + } + impl::ftCache[filename] = std::move(buffer); + } + + const auto& buffer = impl::ftCache[filename]; + if ( auto error = FT_New_Memory_Face( impl::ft_library, buffer.data(), (FT_Long) buffer.size(), 0, &g.face ) ) { + UF_MSG_ERROR("FreeType failed to load font memory '{}': {}", filename, impl::error( error )); return false; } if ( auto error = FT_Select_Charmap( g.face, FT_ENCODING_UNICODE ) ) { - UF_MSG_ERROR("FreeType failed to load file '{}': {}", font, impl::error( error )); + UF_MSG_ERROR("FreeType failed to load charmap '{}': {}", filename, impl::error( error )); return false; } return true; diff --git a/engine/src/ext/lua/lua.cpp b/engine/src/ext/lua/lua.cpp index d1ada5a9..b66673ca 100644 --- a/engine/src/ext/lua/lua.cpp +++ b/engine/src/ext/lua/lua.cpp @@ -20,6 +20,31 @@ uf::stl::unordered_map ext::lua::modules; #include #include +struct vfs_reader { + uf::stl::vector buffer; + bool read_once; + + vfs_reader(const uf::stl::string& filename) : read_once(false) { + // Use your VFS abstraction to load the entire file into the buffer + uf::io::readAsBuffer(buffer, filename); + } + + static const char* read(lua_State*, void* data, size_t* size) { + vfs_reader* reader = static_cast(data); + + // If we haven't yielded the buffer to Lua yet, do it now. + if (!reader->read_once && !reader->buffer.empty()) { + *size = reader->buffer.size(); + reader->read_once = true; + return reinterpret_cast(reader->buffer.data()); + } + + // Yield 0 to signify EOF + *size = 0; + return nullptr; + } +}; + sol::table ext::lua::createTable() { return sol::table(ext::lua::state, sol::create); } @@ -210,7 +235,13 @@ void ext::lua::initialize() { const uf::stl::string& name = pair.first; const uf::stl::string& script = pair.second; if ( uf::io::extension(script) == "lua" ) { - state.require_file(name, uf::io::resolveURI( script ), true); + uf::stl::string code; + if ( uf::io::readAsString(code, script) ) { + // Pass 'script' as the chunk name so your modules also have proper error tracing! + state.require_script(name, code, true, script); + } else { + UF_MSG_ERROR("Lua: failed to load module via VFS: {}", script); + } } else { state.require_script(name, script, true); } @@ -299,15 +330,29 @@ void ext::lua::initialize() { bool ext::lua::run( const uf::stl::string& s, bool safe ) { // is file if ( uf::io::extension(s) == "lua" ) { + uf::stl::string resolved = uf::io::resolveURI(s); + vfs_reader reader(resolved); + + uf::stl::string chunkname = "@" + resolved; + sol::load_result loaded = state.load(vfs_reader::read, &reader, chunkname.c_str()); + + if ( !loaded.valid() ) { + sol::error err = loaded; + UF_MSG_ERROR("{}", err.what()); + return false; + } + if ( safe ) { - auto result = state.safe_script_file( uf::io::resolveURI( s ), sol::script_pass_on_error ); + sol::protected_function script = loaded; + auto result = script(); if ( !result.valid() ) { sol::error err = result; UF_MSG_ERROR("{}", err.what()); return false; } } else { - state.script_file( uf::io::resolveURI( s ) ); + sol::unsafe_function script = loaded; + script(); } // is string with lua } else { @@ -327,9 +372,7 @@ bool ext::lua::run( const uf::stl::string& s, bool safe ) { pod::LuaScript ext::lua::script( const uf::stl::string& filename ) { pod::LuaScript script; - if ( !ext::lua::enabled ) return script; - script.file = filename; - script.env = sol::environment( ext::lua::state, sol::create, ext::lua::state.globals() ); + ext::lua::script( filename, script ); return script; } void ext::lua::script( const uf::stl::string& filename, pod::LuaScript& script ) { @@ -340,15 +383,32 @@ void ext::lua::script( const uf::stl::string& filename, pod::LuaScript& script ) bool ext::lua::run( const pod::LuaScript& s, bool safe ) { // is file if ( uf::io::extension(s.file) == "lua" ) { + uf::stl::string resolved = uf::io::resolveURI(s.file); + vfs_reader reader(resolved); + + uf::stl::string chunkname = "@" + resolved; + sol::load_result loaded = state.load(vfs_reader::read, &reader, chunkname.c_str()); + + if ( !loaded.valid() ) { + sol::error err = loaded; + UF_MSG_ERROR("{}", err.what()); + return false; + } + + sol::protected_function script = loaded; + + s.env.set_on(script); + if ( safe ) { - auto result = state.safe_script_file( s.file, s.env, sol::script_pass_on_error ); + auto result = script(); if ( !result.valid() ) { sol::error err = result; UF_MSG_ERROR("{}", err.what()); return false; } } else { - state.script_file( s.file, s.env ); + sol::unsafe_function unsafe_script = script; + unsafe_script(); } // is string with lua } else { diff --git a/engine/src/ext/meshopt/meshopt.cpp b/engine/src/ext/meshopt/meshopt.cpp index e5a695d0..3b9f914d 100644 --- a/engine/src/ext/meshopt/meshopt.cpp +++ b/engine/src/ext/meshopt/meshopt.cpp @@ -98,7 +98,7 @@ bool ext::meshopt::optimize( uf::Mesh& mesh, float simplify, size_t o, bool verb srcIndexCount * simplify, targetError, meshopt_SimplifyLockBorder, &realError ); - if ( verbose ) UF_MSG_DEBUG("[View {}] Simplified: {} -> {}", viewIdx, srcIndexCount, optimizedIndexCount); + //if ( verbose ) UF_MSG_DEBUG("[View {}] Simplified: {} -> {}", viewIdx, srcIndexCount, optimizedIndexCount); localIndices.swap(simplified); localIndices.resize(optimizedIndexCount); } @@ -256,7 +256,7 @@ uf::stl::vector ext::meshopt::generateLODs( uf::Mesh& mesh, co continue; } - if ( verbose ) UF_MSG_DEBUG("[View {}] LOD {}: {} -> {}", viewIdx, lodIdx, cmd0.indices, currentIndicesCount); + //if ( verbose ) UF_MSG_DEBUG("[View {}] LOD {}: {} -> {}", viewIdx, lodIdx, cmd0.indices, currentIndicesCount); lodIndices.resize(currentIndicesCount); diff --git a/engine/src/ext/opengl/shader.cpp b/engine/src/ext/opengl/shader.cpp index e556a5fb..7e3e28e8 100644 --- a/engine/src/ext/opengl/shader.cpp +++ b/engine/src/ext/opengl/shader.cpp @@ -46,17 +46,9 @@ void ext::opengl::Shader::initialize( ext::opengl::Device& device, const uf::stl // GPU-based shader execution #if !UF_USE_OPENGL_FIXED_FUNCTION uf::stl::string glsl; - { - std::ifstream is(this->filename = filename, std::ios::binary | std::ios::in | std::ios::ate); - if ( !is.is_open() ) { - GL_VALIDATION_MESSAGE("Error: Could not open shader file \"" << filename << "\""); - return; - } - is.seekg(0, std::ios::end); spirv.reserve(is.tellg()); is.seekg(0, std::ios::beg); - spirv.assign((std::istreambuf_iterator(is)), std::istreambuf_iterator()); - - assert(spirv.size() > 0); - } + uf::io::readAsString( glsl, this->filename = filename ); + if ( glsl.empty() ) UF_EXCEPTION("Error: Could not open shader file: {}", filename); + UF_ASSERT(glsl.size() > 0); { device.activateContext(); module = glCreateShader(stage); diff --git a/engine/src/ext/valve/bsp.cpp b/engine/src/ext/valve/bsp.cpp index feb181a1..4bd8cb18 100644 --- a/engine/src/ext/valve/bsp.cpp +++ b/engine/src/ext/valve/bsp.cpp @@ -1,9 +1,9 @@ #include #include #include +#include #include - -#include // to-do: get rid of this +#include namespace impl { struct RGBE { @@ -186,7 +186,7 @@ namespace impl { uf::stl::vector dispinfos; uf::stl::vector dispverts; // disptris - // pakfile + uf::stl::vector pakfile; // cubemaps // overlay uf::stl::vector lighting; @@ -390,7 +390,7 @@ namespace impl { for ( auto nodeID : graph.root.children ) { auto& node = graph.nodes[nodeID]; auto classname = node.metadata["classname"].as(""); - UF_MSG_INFO("Entity found: {}", classname); + //UF_MSG_INFO("Entity found: {}", classname); node.name = classname; // parse origin @@ -401,6 +401,7 @@ namespace impl { } // parse angles + // to-do: fix oddities auto angles = node.metadata["angles"].as(""); if ( angles != "" ) { auto pyr = impl::str2vec( angles ) * DEG_2_RAD; @@ -418,9 +419,16 @@ namespace impl { if ( 0 <= modelID && modelID < context.modelToMesh.size() ) { node.mesh = context.modelToMesh[modelID]; } - } /* else { - // let the engine handle loading the model - }*/ + } else if ( model.length() > 4 && model.ends_with(".mdl") ) { + auto it = std::find(graph.meshes.begin(), graph.meshes.end(), model); + if ( it == graph.meshes.end() ) { + if ( ext::valve::loadMdl(graph, model) ) { + node.mesh = (int32_t)(graph.meshes.size() - 1); + } + } else { + node.mesh = (int32_t)std::distance(graph.meshes.begin(), it); + } + } // parse lighting info if ( classname.starts_with("light") ) { @@ -441,6 +449,7 @@ namespace impl { light.color /= 255.0f; if (!(stream >> light.intensity)) light.intensity = 200.0f; + light.intensity /= 10.0f; } // to-do: read range @@ -472,24 +481,15 @@ namespace impl { } void ext::valve::loadBsp( pod::Graph& graph, const uf::stl::string& filename, const uf::Serializer& metadata ) { - std::ifstream file(filename, std::ios::binary | std::ios::ate); - if ( !file ) { - UF_MSG_ERROR("failed to open BSP: {}", filename); - return; - } - - std::streamsize size = file.tellg(); - file.seekg(0, std::ios::beg); - - uf::stl::vector buffer(size); - if ( !file.read((char*)(buffer.data()), size) ) { - UF_MSG_ERROR("failed to read BSP data: {}", filename); + uf::stl::vector buffer; + if ( !uf::io::readAsBuffer(buffer, filename) ) { + UF_MSG_ERROR("Failed to read BSP data: {}", filename); return; } const impl::BspHeader* header = (const impl::BspHeader*)(buffer.data()); if ( header->magic != 0x50534256 ) { - UF_MSG_ERROR("invalid VBSP magic number: {}", filename); + UF_MSG_ERROR("Invalid VBSP magic number: {}", filename); return; } @@ -513,11 +513,15 @@ void ext::valve::loadBsp( pod::Graph& graph, const uf::stl::string& filename, co context.gameLumps = impl::extractLump(buffer, header->lumps[impl::BspLump::LUMP_GAME_LUMP]); context.dispinfos = impl::extractLump(buffer, header->lumps[impl::BspLump::LUMP_DISPINFO]); context.dispverts = impl::extractLump(buffer, header->lumps[impl::BspLump::LUMP_DISP_VERTS]); + context.pakfile = impl::extractLump(buffer, header->lumps[impl::BspLump::LUMP_PAKFILE]); context.lighting = impl::extractLump(buffer, header->lumps[impl::BspLump::LUMP_LIGHTING]); context.modelToMesh.assign( context.models.size(), -1 ); context.texdataToMaterial.assign( context.texdatas.size(), -1 ); + // mount pakfile + size_t pakfileMount = uf::vfs::mount( ext::zlib::createZipMount(::fmt::format("pakfile://{}", filename), context.pakfile, 1000 ) ); + // read materials for ( int32_t texDataID = 0; texDataID < context.texdatas.size(); ++texDataID ) { const auto& data = context.texdatas[texDataID]; @@ -525,11 +529,11 @@ void ext::valve::loadBsp( pod::Graph& graph, const uf::stl::string& filename, co // lookup material name if ( data.nameStringTableID >= 0 && data.nameStringTableID < context.stringTable.size() ) { - int32_t offset = context.stringTable[data.nameStringTableID]; - if ( offset >= 0 && offset < context.stringData.size() ) { - matName = context.stringData.c_str() + offset; - } - } + int32_t offset = context.stringTable[data.nameStringTableID]; + if ( offset >= 0 && offset < context.stringData.size() ) { + matName = uf::string::lowercase( context.stringData.c_str() + offset ); + } + } size_t imageID = graph.images.size(); auto imgKeyName = graph.images.emplace_back(matName); @@ -548,24 +552,7 @@ void ext::valve::loadBsp( pod::Graph& graph, const uf::stl::string& filename, co material.indexAlbedo = textureID; material.colorBase = {1.0f, 1.0f, 1.0f, 1.0f}; - UF_MSG_INFO("Material found: {}", matName); - } - - // load materials - uf::stl::vector missing_pixels = { 255, 0, 255, 255, 0, 0, 0, 255, 0, 0, 0, 255, 255, 0, 255, 255 }; - for ( auto matName : graph.materials ) { - uf::Serializer vmt; - auto vmtPath = ::fmt::format("materials/{}.vmt", matName); - auto vtfPath = ::fmt::format("materials/{}.vtf", matName); - auto& image = storage.images[matName]; - - if ( !ext::valve::loadVmt( vmt, vmtPath ) ) goto PEETAH; - if ( !vmt["$basetexture"].is() ) goto PEETAH; - - vtfPath = ::fmt::format("materials/{}.vtf", vmt["$basetexture"].as()); - if ( !ext::valve::loadVtf( image, vtfPath ) ) goto PEETAH; - PEETAH: - image.loadFromBuffer( missing_pixels, { 2, 2 }, 8, 4 ); + //UF_MSG_INFO("Material found: {}", matName); } // read lightmaps @@ -588,7 +575,7 @@ void ext::valve::loadBsp( pod::Graph& graph, const uf::stl::string& filename, co { UF_MSG_DEBUG("Generating new lightmap atlas..."); uf::atlas::generate( context.lightmapAtlas, 0.0f ); - UF_MSG_DEBUG("Generated lightmap atlas."); + //UF_MSG_DEBUG("Generated lightmap atlas."); auto& imageKey = graph.images.emplace_back("lightmap_atlas"); auto& textureKey = graph.textures.emplace_back("lightmap_atlas"); @@ -748,10 +735,30 @@ void ext::valve::loadBsp( pod::Graph& graph, const uf::stl::string& filename, co } } - impl::processNodes( graph, context ); + // load materials + uf::stl::vector missing_pixels = { 255, 0, 255, 255, 0, 0, 0, 255, 0, 0, 0, 255, 255, 0, 255, 255 }; + for ( auto matName : graph.materials ) { + uf::Serializer vmt; + auto vmtPath = ::fmt::format("materials/{}.vmt", matName); + auto vtfPath = ::fmt::format("materials/{}.vtf", matName); + auto& image = storage.images[matName]; + + if ( !ext::valve::loadVmt( vmt, vmtPath ) ) goto PEETAH; + if ( !vmt["$basetexture"].is() ) goto PEETAH; + + vtfPath = ::fmt::format("materials/{}.vtf", vmt["$basetexture"].as()); + if ( !ext::valve::loadVtf( image, vtfPath ) ) goto PEETAH; + continue; + PEETAH: + image.loadFromBuffer( missing_pixels, { 2, 2 }, 8, 4 ); + } + graph.metadata["exporter"]["unwrap"] = false; // not necessary to unwrap uf::graph::postprocess( graph ); + + // unmount pakfile + uf::vfs::unmount( pakfileMount ); } \ No newline at end of file diff --git a/engine/src/ext/valve/common.cpp b/engine/src/ext/valve/common.cpp index 26959aeb..0a737054 100644 --- a/engine/src/ext/valve/common.cpp +++ b/engine/src/ext/valve/common.cpp @@ -9,4 +9,11 @@ bool impl::parseKeyValue( const uf::stl::string& line, uf::stl::string& key, uf: key = line.substr( q1 + 1, q2 - q1 - 1 ); value = line.substr( q3 + 1, q4 - q3 - 1 ); return true; +} + +uf::stl::string impl::readString( std::ifstream& file ) { + uf::stl::string str; + char c; + while ( file.get(c) && c != '\0' ) str += c; + return str; } \ No newline at end of file diff --git a/engine/src/ext/valve/mdl.cpp b/engine/src/ext/valve/mdl.cpp index eca03d20..ed0be4ca 100644 --- a/engine/src/ext/valve/mdl.cpp +++ b/engine/src/ext/valve/mdl.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include namespace impl { @@ -138,16 +139,12 @@ bool ext::valve::loadMdl( pod::Graph& graph, const uf::stl::string& filename ) { auto& storage = uf::graph::getStorage( graph ); uf::stl::vector meshlets; - // read MDL file - std::ifstream mdlFile(filename, std::ios::binary | std::ios::ate); - if ( !mdlFile ) { + // Read MDL file + uf::stl::vector mdlBuffer; + if ( !uf::io::readAsBuffer(mdlBuffer, filename) ) { UF_MSG_ERROR("Failed to find MDL: {}", filename); return false; } - std::streamsize mdlSize = mdlFile.tellg(); - mdlFile.seekg(0, std::ios::beg); - uf::stl::vector mdlBuffer(mdlSize); - mdlFile.read((char*)mdlBuffer.data(), mdlSize); const impl::studiohdr_t* mdlHdr = (const impl::studiohdr_t*)mdlBuffer.data(); if ( mdlHdr->magic != 0x54534449 ) { // "IDST" @@ -155,17 +152,13 @@ bool ext::valve::loadMdl( pod::Graph& graph, const uf::stl::string& filename ) { return false; } - // read VVD file + // Read VVD file uf::stl::string vvdPath = filename.substr(0, filename.find_last_of('.')) + ".vvd"; - std::ifstream vvdFile(vvdPath, std::ios::binary | std::ios::ate); - if ( !vvdFile ) { + uf::stl::vector vvdBuffer; + if ( !uf::io::readAsBuffer(vvdBuffer, vvdPath) ) { UF_MSG_ERROR("Failed to find VVD: {}", vvdPath); return false; } - std::streamsize vvdSize = vvdFile.tellg(); - vvdFile.seekg(0, std::ios::beg); - uf::stl::vector vvdBuffer(vvdSize); - vvdFile.read((char*)vvdBuffer.data(), vvdSize); const impl::vertexFileHeader_t* vvdHdr = (const impl::vertexFileHeader_t*)vvdBuffer.data(); if ( vvdHdr->magic != 0x56534449 || vvdHdr->checksum != mdlHdr->checksum ) { @@ -173,15 +166,35 @@ bool ext::valve::loadMdl( pod::Graph& graph, const uf::stl::string& filename ) { return false; } - // extract material names from MDL - uf::stl::vector materials(mdlHdr->numtextures); - for ( int i = 0; i < mdlHdr->numtextures; ++i ) { - int32_t texStructOffset = mdlHdr->textureindex + (i * 64); - int32_t nameOffset = *(int32_t*)(mdlBuffer.data() + texStructOffset); + // extract material directories (cdtextures) + uf::stl::vector cdmaterials(mdlHdr->numcdtextures); + for ( int i = 0; i < mdlHdr->numcdtextures; ++i ) { + int32_t cdOffset = *(int32_t*)(mdlBuffer.data() + mdlHdr->cdtextureindex + (i * 4)); + uf::stl::string cdPath = (const char*)(mdlBuffer.data() + cdOffset); - materials[i] = (const char*)(mdlBuffer.data() + texStructOffset + nameOffset); - UF_MSG_INFO("Model Material {}: {}", i, materials[i]); - } + std::replace(cdPath.begin(), cdPath.end(), '\\', '/'); + std::transform(cdPath.begin(), cdPath.end(), cdPath.begin(), ::tolower); + cdmaterials[i] = cdPath; + } + + // extract material names from MDL and resolve their full relative paths + uf::stl::vector materials(mdlHdr->numtextures); + for ( int i = 0; i < mdlHdr->numtextures; ++i ) { + int32_t texStructOffset = mdlHdr->textureindex + (i * 64); + int32_t nameOffset = *(int32_t*)(mdlBuffer.data() + texStructOffset); + uf::stl::string baseName = (const char*)(mdlBuffer.data() + texStructOffset + nameOffset); + std::transform(baseName.begin(), baseName.end(), baseName.begin(), ::tolower); + + materials[i] = baseName; + + for ( const auto& cd : cdmaterials ) { + uf::stl::string attempt = cd + baseName; + if ( uf::vfs::exists("materials/" + attempt + ".vmt") ) { + materials[i] = attempt; + break; + } + } + } // extract LOD0 ertices from VVD const impl::mstudiovertex_t* vvdVertices = (const impl::mstudiovertex_t*)(vvdBuffer.data() + vvdHdr->vertexDataStart); @@ -189,13 +202,8 @@ bool ext::valve::loadMdl( pod::Graph& graph, const uf::stl::string& filename ) { // read VTX file uf::stl::string vtxPath = filename.substr(0, filename.find_last_of('.')) + ".dx90.vtx"; - std::ifstream vtxFile(vtxPath, std::ios::binary | std::ios::ate); - if ( !vtxFile ) return false; - - std::streamsize vtxSize = vtxFile.tellg(); - vtxFile.seekg(0, std::ios::beg); - uf::stl::vector vtxBuffer(vtxSize); - vtxFile.read((char*)vtxBuffer.data(), vtxSize); + uf::stl::vector vtxBuffer; + if ( !uf::io::readAsBuffer(vtxBuffer, vtxPath)) return false; const impl::vtxHeader_t* vtxHdr = (const impl::vtxHeader_t*)vtxBuffer.data(); if ( vtxHdr->checksum != mdlHdr->checksum ) { @@ -239,19 +247,19 @@ bool ext::valve::loadMdl( pod::Graph& graph, const uf::stl::string& filename ) { const auto& srcVert = vvdVertices[originalVvdID]; - vert.position = impl::convertPos( srcVert.m_vecPosition, 1.0f ); - vert.normal = uf::vector::normalize( impl::convertPos( srcVert.m_vecNormal ) ); + vert.position = impl::convertPos( srcVert.m_vecPosition ); + vert.normal = uf::vector::normalize( impl::convertPos( srcVert.m_vecNormal, 1.0f ) ); vert.uv = srcVert.m_vecTexCoord; vert.color = {1.0f, 1.0f, 1.0f, 1.0f}; vert.joints.x = srcVert.m_BoneWeights.numbones > 0 ? std::max(0, srcVert.m_BoneWeights.bone[0]) : 0; - vert.joints.y = srcVert.m_BoneWeights.numbones > 1 ? std::max(0, srcVert.m_BoneWeights.bone[1]) : 0; - vert.joints.z = srcVert.m_BoneWeights.numbones > 2 ? std::max(0, srcVert.m_BoneWeights.bone[2]) : 0; - vert.joints.w = 0; + vert.joints.y = srcVert.m_BoneWeights.numbones > 1 ? std::max(0, srcVert.m_BoneWeights.bone[1]) : 0; + vert.joints.z = srcVert.m_BoneWeights.numbones > 2 ? std::max(0, srcVert.m_BoneWeights.bone[2]) : 0; + vert.joints.w = 0; - vert.weights.x = srcVert.m_BoneWeights.numbones > 0 ? srcVert.m_BoneWeights.weight[0] : 1.0f; - vert.weights.y = srcVert.m_BoneWeights.numbones > 1 ? srcVert.m_BoneWeights.weight[1] : 0.0f; - vert.weights.z = srcVert.m_BoneWeights.numbones > 2 ? srcVert.m_BoneWeights.weight[2] : 0.0f; - vert.weights.w = 0.0f; + vert.weights.x = srcVert.m_BoneWeights.numbones > 0 ? srcVert.m_BoneWeights.weight[0] : 1.0f; + vert.weights.y = srcVert.m_BoneWeights.numbones > 1 ? srcVert.m_BoneWeights.weight[1] : 0.0f; + vert.weights.z = srcVert.m_BoneWeights.numbones > 2 ? srcVert.m_BoneWeights.weight[2] : 0.0f; + vert.weights.w = 0.0f; // Bounds calculation auto& bounds = meshlet.primitive.instance.bounds; @@ -275,6 +283,7 @@ bool ext::valve::loadMdl( pod::Graph& graph, const uf::stl::string& filename ) { for ( ; materialID < graph.materials.size(); ++materialID ) { if ( graph.materials[materialID] == matName ) break; } + } else { // does not exist, register size_t imageID = graph.images.size(); diff --git a/engine/src/ext/valve/vpk.cpp b/engine/src/ext/valve/vpk.cpp new file mode 100644 index 00000000..0a825d16 --- /dev/null +++ b/engine/src/ext/valve/vpk.cpp @@ -0,0 +1,311 @@ +#include +#include +#include +#include +#include + +#include +#include +#include + +#if defined(_WIN32) + #include +#else + #include +#endif +#include + +namespace impl { + uf::stl::vector getSteamLibraries() { + uf::stl::vector libraries; + uf::stl::string steamPath = ""; + + #if defined(_WIN32) + HKEY hKey; + if ( RegOpenKeyExA(HKEY_CURRENT_USER, "Software\\Valve\\Steam", 0, KEY_READ, &hKey) == ERROR_SUCCESS ) { + char pathBuf[MAX_PATH]; + DWORD bufferSize = sizeof(pathBuf); + if ( RegQueryValueExA(hKey, "SteamPath", nullptr, nullptr, (LPBYTE)pathBuf, &bufferSize) == ERROR_SUCCESS ) { + steamPath = pathBuf; + } + RegCloseKey(hKey); + } + #else + const char* home = std::getenv("HOME"); + if (home) { + #if defined(__APPLE__) + steamPath = uf::stl::string(home) + "/Library/Application Support/Steam"; + #else + steamPath = uf::stl::string(home) + "/.steam/steam"; // or ~/.local/share/Steam + #endif + } + #endif + + if ( steamPath.empty() ) { + UF_MSG_WARNING("Could not locate base Steam installation."); + return libraries; + } + + libraries.emplace_back(steamPath + "/steamapps/common"); + + uf::stl::string vdfPath = steamPath + "/steamapps/libraryfolders.vdf"; + std::ifstream file(vdfPath); + if ( !file.is_open() ) return libraries; + + uf::stl::string line; + while ( std::getline(file, line) ) { + uf::stl::string key, value; + + if ( !impl::parseKeyValue(line, key, value) ) continue; + if ( key != "path" ) continue; + + std::replace( value.begin(), value.end(), '\\', '/' ); + value = uf::string::replace(value, "//", "/"); + + libraries.emplace_back(value + "/steamapps/common"); + } + + return libraries; + } +} + +pod::Mount ext::valve::createVpkMount( const uf::stl::string& uri, int priority ) { + struct VpkMountState { + uf::stl::string path; + uf::stl::string name; + + bool loaded = false; + pod::VpkArchive* archive = NULL; + + pod::VpkArchive* get() { + if ( !loaded ) { + loaded = true; + + archive = &uf::asset::add(name); + ext::valve::loadVpk( *archive, path ); + } + return archive; + } + }; + + uf::stl::string prefix; + uf::stl::string path; + uf::io::splitUri( uri, prefix, path ); + + pod::Mount mount; + mount.prefix = prefix; + mount.path = path; + mount.priority = priority; + mount.userdata = uf::pointeredUserdata::create(); + auto& state = uf::pointeredUserdata::get( mount.userdata ); + auto* userdata = &state; + + auto libraries = impl::getSteamLibraries(); + for ( const auto& lib : libraries ) { + uf::stl::string fullPath = lib + "/" + path; + if ( !uf::io::exists(fullPath) ) { + continue; + } + state.path = fullPath; + break; + } + + if ( state.path.empty() ) { + UF_MSG_WARNING("Could not resolve VPK path: {}", path); + state.path = path; + } else { + UF_MSG_DEBUG("Mounted VPK: {}", state.path); + } + state.name = ::fmt::format("vpk://{}", path); + + + mount.exists = [userdata](const uf::stl::string& p) { + auto ptr = userdata->get(); + return ptr && ptr->files.count( uf::string::lowercase( p ) ) > 0; + }; + mount.size = [userdata](const uf::stl::string& p) -> size_t { + auto ptr = userdata->get(); + if ( !ptr ) return 0; + auto it = ptr->files.find( uf::string::lowercase( p ) ); + return it != ptr->files.end() ? it->second.metadata.entryLength : 0; + }; + mount.mtime = [](const uf::stl::string&) -> size_t { return 0; }, + mount.read = [userdata](const uf::stl::string& p, uf::stl::vector& buffer) { + auto ptr = userdata->get(); + return ptr && ext::valve::readVpk(*ptr, uf::string::lowercase( p ), buffer); + }; + mount.readRange = [userdata](const uf::stl::string& p, size_t start, size_t len, uf::stl::vector& buffer) { + auto ptr = userdata->get(); + return ptr && ext::valve::readVpkRange(*ptr, uf::string::lowercase( p ), start, len, buffer); + }; + return mount; +} +bool ext::valve::loadVpk( pod::VpkArchive& vpk, const uf::stl::string& path ) { + uf::stl::vector buffer; + if ( !uf::io::readAsBuffer(buffer, path) ) return false; + + size_t dirPos = path.find("_dir.vpk"); + if ( dirPos == uf::stl::string::npos ) return false; + vpk.basePath = path.substr(0, dirPos); + + size_t offset = 0; + + auto readBytes = [&](void* dest, size_t len) -> bool { + if (offset + len > buffer.size()) return false; + std::memcpy(dest, buffer.data() + offset, len); + offset += len; + return true; + }; + + auto readString = [&]() -> uf::stl::string { + uf::stl::string str; + while (offset < buffer.size()) { + char c = (char)buffer[offset++]; + if (c == '\0') break; + str += c; + } + return str; + }; + + uint32_t signature; + if ( !readBytes(&signature, 4) || signature != 0x55aa1234 ) return false; + + uint32_t version, treeSize; + if ( !readBytes(&version, 4) || !readBytes(&treeSize, 4) ) return false; + + // skip for now + if ( version == 2 ) offset += 16; + + uint32_t headerSize = (version == 1) ? 12 : 28; + + while ( true ) { + uf::stl::string ext = readString(); + if ( ext.empty() ) break; + + while ( true ) { + uf::stl::string dir = readString(); + if ( dir.empty() ) break; + + while ( true ) { + uf::stl::string name = readString(); + if ( name.empty() ) break; + + // construct path + uf::stl::string fullPath = dir == " " ? "" : (dir + "/"); + fullPath += name + "." + ext; + std::replace( fullPath.begin(), fullPath.end(), '\\', '/' ); + std::transform( fullPath.begin(), fullPath.end(), fullPath.begin(), ::tolower ); + + // read data + auto& entry = vpk.files[fullPath]; + if ( !readBytes(&entry.metadata, sizeof(pod::VpkData)) ) return false; + + if ( entry.metadata.preloadBytes > 0 ) { + entry.preloadData.resize(entry.metadata.preloadBytes); + if ( !readBytes(entry.preloadData.data(), entry.metadata.preloadBytes) ) return false; + } + + entry.dirFileOffset = headerSize + treeSize; + } + } + } + return true; +} + +bool ext::valve::readVpk( const pod::VpkArchive& vpk, const uf::stl::string& path, uf::stl::vector& buffer ) { + auto it = vpk.files.find( path ); + if ( it == vpk.files.end() ) return false; + + const auto& entry = it->second; + buffer.resize( entry.metadata.preloadBytes + entry.metadata.entryLength ); + + // copy preload + if ( entry.metadata.preloadBytes > 0 ) { + memcpy(buffer.data(), entry.preloadData.data(), entry.metadata.preloadBytes); + } + + // read payload from disk + if ( entry.metadata.entryLength > 0 ) { + uf::stl::string archivePath; + size_t fileOffset = 0; + + if ( entry.metadata.archiveIndex == 0x7FFF ) { + // data is embedded inside _dir.vpk + archivePath = vpk.basePath + "_dir.vpk"; + fileOffset = entry.dirFileOffset + entry.metadata.entryOffset; + } else { + // data is in a chunk file + archivePath = ::fmt::format("{}_{:03d}.vpk", vpk.basePath, entry.metadata.archiveIndex); + fileOffset = entry.metadata.entryOffset; + } + + std::ifstream file(archivePath, std::ios::binary); + if ( file ) { + file.seekg(fileOffset, std::ios::beg); + file.read((char*)(buffer.data() + entry.metadata.preloadBytes), entry.metadata.entryLength); + } else { + buffer.clear(); + UF_MSG_ERROR("Failed to open VPK chunk: {}", archivePath); + return false; + } + } + + return true; +} + +size_t ext::valve::mountVpk( const uf::stl::string& uri ) { + return uf::vfs::mount( ext::valve::createVpkMount( ::fmt::format( "valve://{}", uri ), 10 ) ); +} +bool ext::valve::readVpkRange( const pod::VpkArchive& vpk, const uf::stl::string& path, size_t start, size_t len, uf::stl::vector& buffer ) { + auto it = vpk.files.find( path ); + if ( it == vpk.files.end() ) return false; + + const auto& entry = it->second; + size_t totalSize = entry.metadata.preloadBytes + entry.metadata.entryLength; + + if ( start >= totalSize ) { + buffer.clear(); + return true; + } + + len = std::min(len, totalSize - start); + buffer.resize(len); + + size_t bufferOffset = 0; + + if ( start < entry.metadata.preloadBytes ) { + size_t preloadRead = std::min(len, (size_t)entry.metadata.preloadBytes - start); + memcpy(buffer.data(), entry.preloadData.data() + start, preloadRead); + + bufferOffset += preloadRead; + start += preloadRead; + len -= preloadRead; + } + + if ( len > 0 ) { + size_t diskStart = start - entry.metadata.preloadBytes; + + uf::stl::string archivePath; + size_t fileOffset = 0; + + if ( entry.metadata.archiveIndex == 0x7FFF ) { + archivePath = vpk.basePath + "_dir.vpk"; + fileOffset = entry.dirFileOffset + entry.metadata.entryOffset; + } else { + archivePath = ::fmt::format("{}_{:03d}.vpk", vpk.basePath, entry.metadata.archiveIndex); + fileOffset = entry.metadata.entryOffset; + } + + std::ifstream file(archivePath, std::ios::binary); + if ( file ) { + file.seekg(fileOffset + diskStart, std::ios::beg); + file.read((char*)(buffer.data() + bufferOffset), len); + } else { + buffer.clear(); + UF_MSG_ERROR("Failed to open VPK chunk for ranged read: {}", archivePath); + return false; + } + } + + return true; +} \ No newline at end of file diff --git a/engine/src/ext/valve/vtf.cpp b/engine/src/ext/valve/vtf.cpp index 05f34c86..6ea99f7f 100644 --- a/engine/src/ext/valve/vtf.cpp +++ b/engine/src/ext/valve/vtf.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include namespace impl { @@ -115,10 +116,11 @@ namespace impl { } bool ext::valve::loadVmt( uf::Serializer& dict, const uf::stl::string& filename ) { - std::ifstream file( filename ); - if ( !file ) return false; + uf::stl::string content; + if ( !uf::io::readAsString(content, filename) ) return false; uf::stl::string line; + std::istringstream file(content); while ( std::getline(file, line) ) { uf::stl::string comment = ""; size_t commentPos = line.find("//"); // strip comments @@ -130,23 +132,22 @@ bool ext::valve::loadVmt( uf::Serializer& dict, const uf::stl::string& filename uf::stl::string key, value; if ( impl::parseKeyValue(line, key, value) ) { std::transform(key.begin(), key.end(), key.begin(), ::tolower); + std::transform(value.begin(), value.end(), value.begin(), ::tolower); std::replace(value.begin(), value.end(), '\\', '/'); - dict[key] = value; + if ( key == "include" ) { + ext::valve::loadVmt( dict, value ); + } else { + dict[key] = value; + } } } return true; } bool ext::valve::loadVtf( pod::Image& image, const uf::stl::string& filename ) { - std::ifstream file( filename, std::ios::binary | std::ios::ate ); - if ( !file ) return false; - - std::streamsize size = file.tellg(); - file.seekg(0, std::ios::beg); - - uf::stl::vector buffer(size); - file.read((char*)buffer.data(), size); + uf::stl::vector buffer; + if ( !uf::io::readAsBuffer(buffer, filename) ) return false; const impl::VTFHeader* header = (const impl::VTFHeader*)(buffer.data()); if ( strncmp(header->signature, "VTF", 3) != 0 ) return false; @@ -172,7 +173,7 @@ bool ext::valve::loadVtf( pod::Image& image, const uf::stl::string& filename ) { const uint8_t* data = buffer.data() + offset; image.size = { header->width, header->height }; image.channels = 4; - image.bpp = 8; + image.bpp = 8 * 4; image.pixels.resize( header->width * header->height * 4 ); int dataSize = 0; diff --git a/engine/src/ext/vulkan/shader.cpp b/engine/src/ext/vulkan/shader.cpp index de45f697..dcc21921 100644 --- a/engine/src/ext/vulkan/shader.cpp +++ b/engine/src/ext/vulkan/shader.cpp @@ -293,15 +293,9 @@ void ext::vulkan::Shader::initialize( ext::vulkan::Device& device, const uf::stl ext::vulkan::Buffers::initialize( device ); uf::stl::string spirv; - - { - std::ifstream is(this->filename = filename, std::ios::binary | std::ios::in | std::ios::ate); - if ( !is.is_open() ) UF_EXCEPTION("Error: Could not open shader file: {}", filename); - is.seekg(0, std::ios::end); spirv.reserve(is.tellg()); is.seekg(0, std::ios::beg); - spirv.assign((std::istreambuf_iterator(is)), std::istreambuf_iterator()); - - assert(spirv.size() > 0); - } + uf::io::readAsString( spirv, this->filename = filename ); + if ( spirv.empty() ) UF_EXCEPTION("Error: Could not open shader file: {}", filename); + UF_ASSERT(spirv.size() > 0); { diff --git a/engine/src/ext/zlib/zlib.cpp b/engine/src/ext/zlib/zlib.cpp index df97da83..5caad0de 100644 --- a/engine/src/ext/zlib/zlib.cpp +++ b/engine/src/ext/zlib/zlib.cpp @@ -1,139 +1,333 @@ #include #if UF_USE_ZLIB #include +#include +#include #include +#include +#include + #if UF_ENV_DREAMCAST #include #else #include #endif -size_t ext::zlib::bufferSize = 2048; -/* -uf::stl::vector ext::zlib::compress( const void* data, size_t size ) { - uf::stl::vector buffer; - // zlib struct - z_stream defstream; - defstream.zalloc = Z_NULL; - defstream.zfree = Z_NULL; - defstream.opaque = Z_NULL; - // setup "a" as the input and "b" as the compressed output - defstream.avail_in = (uInt) strlen(a)+1; // size of input, string + terminator - defstream.next_in = (Bytef*) a; // input char array - defstream.avail_out = (uInt) sizeof(b); // size of output - defstream.next_out = (Bytef*) b; // output char array - - // the actual compression work. - deflateInit(&defstream, Z_BEST_COMPRESSION); - deflate(&defstream, Z_FINISH); - deflateEnd(&defstream); +size_t ext::zlib::bufferSize = 16384; - return buffer; -} -uf::stl::vector ext::zlib::decompress( const void* data, size_t size ) { - uf::stl::vector buffer; - - return buffer; -} -*/ -uf::stl::vector& ext::zlib::decompressFromFile( uf::stl::vector& buffer, const uf::stl::string& filename ) { - - gzFile in = gzopen( filename.c_str(), "rb" ); - if ( !in ) { - UF_MSG_ERROR("Zlib: failed to open file for read: {}", filename); - return buffer; +bool ext::zlib::decompressFromFile( uf::stl::vector& buffer, const uf::stl::string& filename ) { + size_t fileSize = uf::vfs::size(filename); + if (fileSize == 0) { + UF_MSG_ERROR("Zlib: file not found or empty: {}", filename); + return false; } - size_t read{}; - uint8_t gzBuffer[ext::zlib::bufferSize]; - while ( (read = gzread( in, gzBuffer, ext::zlib::bufferSize ) ) > 0 ) { - size_t s = buffer.size(); - buffer.resize(s + read); - memcpy( &buffer[s], &gzBuffer[0], read ); + z_stream strm{}; + if (inflateInit2(&strm, 15 + 32) != Z_OK) return false; + + size_t offset = 0; + uint8_t outBuffer[ext::zlib::bufferSize]; + + while (offset < fileSize) { + size_t bytesToRead = std::min(ext::zlib::bufferSize, fileSize - offset); + uf::stl::vector temp; + if (!uf::vfs::readRange(filename, offset, bytesToRead, temp)) break; + + strm.avail_in = (uInt)temp.size(); + strm.next_in = temp.data(); + offset += temp.size(); + + do { + strm.avail_out = sizeof(outBuffer); + strm.next_out = outBuffer; + int ret = inflate(&strm, Z_NO_FLUSH); + if (ret == Z_STREAM_ERROR || ret == Z_DATA_ERROR || ret == Z_MEM_ERROR) { + UF_MSG_ERROR("Zlib: inflate error on file: {}", filename); + inflateEnd(&strm); + return false; + } + size_t have = sizeof(outBuffer) - strm.avail_out; + buffer.insert(buffer.end(), outBuffer, outBuffer + have); + } while (strm.avail_out == 0); } - gzclose( in ); - return buffer; -} -size_t ext::zlib::compressToFile( const uf::stl::string& filename, const void* data, size_t size ) { - gzFile out = gzopen( filename.c_str(), "wb" ); - if ( !out ) { - UF_MSG_ERROR("Zlib: failed to open file for write: {}", filename); - return 0; - } - gzwrite( out, data, size ); - gzclose( out ); - return uf::io::size( filename ); + + inflateEnd(&strm); + return true; } -uf::stl::vector& ext::zlib::decompressFromFile( uf::stl::vector& buffer, const uf::stl::string& filename, size_t start, size_t len ) { - gzFile in = gzopen(filename.c_str(), "rb"); - if ( !in ) { - UF_MSG_ERROR("Zlib: failed to open file for read: {}", filename); - return buffer; +bool ext::zlib::decompressFromFile( uf::stl::vector& buffer, const uf::stl::string& filename, size_t start, size_t len ) { + size_t fileSize = uf::vfs::size(filename); + if (fileSize == 0) return false; + + z_stream strm{}; + if (inflateInit2(&strm, 15 + 32) != Z_OK) return false; + + size_t offset = 0; + size_t uncompressedOffset = 0; + uint8_t outBuffer[ext::zlib::bufferSize]; + + while (offset < fileSize) { + size_t bytesToRead = std::min(ext::zlib::bufferSize, fileSize - offset); + uf::stl::vector temp; + if (!uf::vfs::readRange(filename, offset, bytesToRead, temp)) break; + + strm.avail_in = (uInt)temp.size(); + strm.next_in = temp.data(); + offset += temp.size(); + + do { + strm.avail_out = sizeof(outBuffer); + strm.next_out = outBuffer; + int ret = inflate(&strm, Z_NO_FLUSH); + if (ret == Z_STREAM_ERROR || ret == Z_DATA_ERROR || ret == Z_MEM_ERROR) { + inflateEnd(&strm); + return false; + } + size_t have = sizeof(outBuffer) - strm.avail_out; + + // Calculate if this chunk overlaps with our requested range + size_t chunkStart = uncompressedOffset; + size_t chunkEnd = uncompressedOffset + have; + + if (chunkEnd > start && chunkStart < start + len) { + size_t copyStart = (chunkStart < start) ? (start - chunkStart) : 0; + size_t copyLen = std::min(have - copyStart, (start + len) - (chunkStart + copyStart)); + buffer.insert(buffer.end(), outBuffer + copyStart, outBuffer + copyStart + copyLen); + } + + uncompressedOffset += have; + + // If we've reached the end of the requested length, we can abort early! + if (uncompressedOffset >= start + len) { + inflateEnd(&strm); + return true; + } + } while (strm.avail_out == 0); } - // Seek to the requested uncompressed offset - if (gzseek(in, static_cast(start), SEEK_SET) == -1) { - UF_MSG_ERROR("Zlib: failed to seek to position {} in file {}", start, filename); - gzclose(in); - return buffer; - } - - buffer.resize(len); - int bytesRead = gzread(in, buffer.data(), static_cast(len)); - if (bytesRead < 0) { - int errnum; - const char* errMsg = gzerror(in, &errnum); - UF_MSG_ERROR("Zlib read error: {}", errMsg ? errMsg : "unknown error"); - buffer.clear(); - } else { - buffer.resize(static_cast(bytesRead)); // Adjust size in case EOF occurs early - } - - gzclose(in); - return buffer; + inflateEnd(&strm); + return true; } -uf::stl::vector& ext::zlib::decompressFromFile( uf::stl::vector& buffer, const uf::stl::string& filename, const uf::stl::vector& ranges ) { - if ( ranges.empty() ) { - return buffer; - } +bool ext::zlib::decompressFromFile( uf::stl::vector& buffer, const uf::stl::string& filename, const uf::stl::vector& ranges ) { + if ( ranges.empty() ) return false; - // ensure they're ordered uf::stl::vector sortedRanges = ranges; std::sort( sortedRanges.begin(), sortedRanges.end(), [](const pod::Range& a, const pod::Range& b) { return a.start < b.start; } ); - gzFile in = gzopen(filename.c_str(), "rb"); - if ( !in ) { - UF_MSG_ERROR("Zlib: failed to open file for read: {}", filename); - return buffer; + size_t fileSize = uf::vfs::size(filename); + if (fileSize == 0) return false; + + z_stream strm{}; + if (inflateInit2(&strm, 15 + 32) != Z_OK) return false; + + size_t offset = 0; + size_t uncompressedOffset = 0; + uint8_t outBuffer[ext::zlib::bufferSize]; + size_t currentRangeIdx = 0; + + while (offset < fileSize && currentRangeIdx < sortedRanges.size()) { + size_t bytesToRead = std::min(ext::zlib::bufferSize, fileSize - offset); + uf::stl::vector temp; + if (!uf::vfs::readRange(filename, offset, bytesToRead, temp)) break; + + strm.avail_in = (uInt)temp.size(); + strm.next_in = temp.data(); + offset += temp.size(); + + do { + strm.avail_out = sizeof(outBuffer); + strm.next_out = outBuffer; + int ret = inflate(&strm, Z_NO_FLUSH); + if (ret < 0 && ret != Z_BUF_ERROR) { + inflateEnd(&strm); + return false; + } + size_t have = sizeof(outBuffer) - strm.avail_out; + + size_t chunkStart = uncompressedOffset; + size_t chunkEnd = uncompressedOffset + have; + + // Check all remaining ranges against this chunk + for (size_t i = currentRangeIdx; i < sortedRanges.size(); ++i) { + const auto& r = sortedRanges[i]; + if (chunkEnd > r.start && chunkStart < r.start + r.len) { + size_t copyStart = (chunkStart < r.start) ? (r.start - chunkStart) : 0; + size_t copyLen = std::min(have - copyStart, (r.start + r.len) - (chunkStart + copyStart)); + buffer.insert(buffer.end(), outBuffer + copyStart, outBuffer + copyStart + copyLen); + } + if (chunkEnd >= r.start + r.len) { + currentRangeIdx = i + 1; // Move past completed ranges + } + } + uncompressedOffset += have; + } while (strm.avail_out == 0); } - for ( const auto& r : sortedRanges ) { - if ( gzseek(in, static_cast(r.start), SEEK_SET) == -1 ) { - UF_MSG_ERROR("Zlib: failed to seek to position {} in file {}", r.start, filename); - gzclose(in); - return buffer; - } - - size_t oldSize = buffer.size(); - buffer.resize(oldSize + r.len); - - int bytesRead = gzread(in, buffer.data() + oldSize, static_cast(r.len)); - if ( bytesRead < 0 ) { - int errnum; - const char* errMsg = gzerror(in, &errnum); - UF_MSG_ERROR("Zlib read error: {}", errMsg ? errMsg : "unknown error"); - gzclose(in); - return buffer; - } - - // In case EOF ended early - buffer.resize(oldSize + static_cast(bytesRead)); - } - - gzclose(in); - return buffer; + inflateEnd(&strm); + return true; } + +bool ext::zlib::decompressFromMemory( uf::stl::vector& dst, const void* src, size_t size, size_t usize ) { + if (size == 0) return false; + + z_stream strm{}; + if (inflateInit2(&strm, 15 + 32) != Z_OK) return false; + + strm.avail_in = (uInt)size; + strm.next_in = (Bytef*)src; + + dst.resize(usize); + strm.avail_out = (uInt)usize; + strm.next_out = dst.data(); + + int ret = inflate(&strm, Z_FINISH); + inflateEnd(&strm); + + return (ret == Z_STREAM_END || ret == Z_OK); +} + +size_t ext::zlib::compressToFile( const uf::stl::string& filename, const void* data, size_t size ) { + z_stream strm{}; + // 31 means gzip format + if (deflateInit2(&strm, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 31, 8, Z_DEFAULT_STRATEGY) != Z_OK) return 0; + + strm.avail_in = (uInt)size; + strm.next_in = (Bytef*)data; + + uf::stl::vector compressedData; + uint8_t outBuffer[ext::zlib::bufferSize]; + int ret; + do { + strm.avail_out = sizeof(outBuffer); + strm.next_out = outBuffer; + ret = deflate(&strm, Z_FINISH); + size_t have = sizeof(outBuffer) - strm.avail_out; + + compressedData.insert(compressedData.end(), outBuffer, outBuffer + have); + } while (strm.avail_out == 0); + + deflateEnd(&strm); + + return uf::vfs::write( filename, compressedData.data(), compressedData.size() ); +} + +bool ext::zlib::directory( const uf::stl::vector& buffer, uf::stl::unordered_map& entries ) { + if ( buffer.size() < 22 ) return false; + + int eocdOffset = -1; + for ( int i = (int)buffer.size() - 22; i >= 0; --i ) { + if (buffer[i] == 0x50 && buffer[i+1] == 0x4b && buffer[i+2] == 0x05 && buffer[i+3] == 0x06) { + eocdOffset = i; + break; + } + } + + if ( eocdOffset == -1 ) { + UF_MSG_ERROR("ZIP Parse: Could not find End of Central Directory!"); + return false; + } + + const uint8_t* eocd = buffer.data() + eocdOffset; + uint16_t numEntries = *(const uint16_t*)(eocd + 10); + uint32_t cdOffset = *(const uint32_t*)(eocd + 16); + + if ( cdOffset >= buffer.size() ) return false; + + size_t currentOffset = cdOffset; + for (uint16_t i = 0; i < numEntries; ++i) { + if (currentOffset + 46 > buffer.size()) break; + + const uint8_t* cdHeader = buffer.data() + currentOffset; + + if (*(const uint32_t*)cdHeader != 0x02014b50) break; + + uint16_t compressionMethod = *(const uint16_t*)(cdHeader + 10); + uint32_t compressedSize = *(const uint32_t*)(cdHeader + 20); + uint32_t uncompressedSize = *(const uint32_t*)(cdHeader + 24); + uint16_t nameLen = *(const uint16_t*)(cdHeader + 28); + uint16_t extraLen = *(const uint16_t*)(cdHeader + 30); + uint16_t commentLen = *(const uint16_t*)(cdHeader + 32); + uint32_t localHeaderOffset = *(const uint32_t*)(cdHeader + 42); + + uf::stl::string filename((const char*)(cdHeader + 46), nameLen); + + std::replace(filename.begin(), filename.end(), '\\', '/'); + std::transform(filename.begin(), filename.end(), filename.begin(), ::tolower); + + if ( localHeaderOffset + 30 <= buffer.size() ) { + const uint8_t* localHeader = buffer.data() + localHeaderOffset; + + if (*(const uint32_t*)localHeader == 0x04034b50) { + uint16_t localNameLen = *(const uint16_t*)(localHeader + 26); + uint16_t localExtraLen = *(const uint16_t*)(localHeader + 28); + + size_t dataOffset = localHeaderOffset + 30 + localNameLen + localExtraLen; + + entries[filename] = { + dataOffset, + compressedSize, + uncompressedSize, + compressionMethod + }; + } + } + + currentOffset += 46 + nameLen + extraLen + commentLen; + } + + return true; +} + + +pod::Mount ext::zlib::createZipMount( const uf::stl::string& uri, uf::stl::vector& buffer, int priority ) { + struct ZipMountState { + uf::stl::vector buffer; + uf::stl::unordered_map entries; + }; + + uf::stl::string prefix; + uf::stl::string path; + uf::io::splitUri( uri, prefix, path ); + + pod::Mount mount; + mount.prefix = prefix; + mount.path = path; + mount.priority = priority; + + mount.userdata = uf::pointeredUserdata::create(); + auto& state = uf::pointeredUserdata::get( mount.userdata ); + auto* userdata = &state; + state.buffer = buffer; + + ext::zlib::directory( state.buffer, state.entries ); + + mount.exists = [userdata](const uf::stl::string& file) -> bool { + return userdata->entries.find(file) != userdata->entries.end(); + }; + mount.size = [userdata](const uf::stl::string& file) -> size_t { + auto it = userdata->entries.find(file); + return (it != userdata->entries.end()) ? it->second.uncompressedSize : 0; + }; + mount.read = [userdata](const uf::stl::string& file, uf::stl::vector& buffer) -> bool { + auto it = userdata->entries.find(file); + if (it == userdata->entries.end()) return false; + + const auto& entry = it->second; + const uint8_t* fileData = userdata->buffer.data() + entry.offset; + + if (entry.compressionMethod == 0) { + buffer.assign(fileData, fileData + entry.uncompressedSize); + return true; + } + if (entry.compressionMethod == 8) { + return ext::zlib::decompressFromMemory(buffer, fileData, entry.compressedSize, entry.uncompressedSize); + } + return false; + }; + + return mount; +} + #endif \ No newline at end of file diff --git a/engine/src/utils/http/http.cpp b/engine/src/utils/http/http.cpp index ed6b6bcd..1fd011a5 100644 --- a/engine/src/utils/http/http.cpp +++ b/engine/src/utils/http/http.cpp @@ -1,43 +1,134 @@ #include +#include +#include #if UF_USE_CURL #include #endif #include +namespace impl { + pod::Mount createHttpMount( const uf::stl::string& uri, int priority ) { + uf::stl::string prefix; + uf::stl::string path; + uf::io::splitUri( uri, prefix, path ); + + return pod::Mount{ + .prefix = prefix, + .path = path, + .priority = priority, + .exists = [prefix](const uf::stl::string& p) -> bool { + uf::stl::string url = prefix + p; + uf::Http http = uf::http::head(url); + return ( http.code >= 200 && http.code < 300 ); + }, + .size = [prefix](const uf::stl::string& p) -> size_t { + uf::stl::string url = prefix + p; + uf::Http http = uf::http::head(url); + return http.contentLength; + }, + .mtime = [prefix](const uf::stl::string& p) -> size_t { + uf::stl::string url = prefix + p; + uf::Http http = uf::http::head(url); + return http.mtime; + }, + .read = [prefix](const uf::stl::string& p, uf::stl::vector& buffer) -> bool { + uf::stl::string url = prefix + p; + + uf::Http http = uf::http::get(url); + if ( http.code < 200 || http.code >= 300 ) { + UF_MSG_ERROR("HTTP Error {} on GET {}", http.code, url); + return false; + } + + buffer.assign(http.response.begin(), http.response.end()); + return true; + }, + .write = [prefix](const uf::stl::string& p, const void* buffer, size_t size) -> size_t { + uf::stl::string url = prefix + p; + uf::Http http = uf::http::post(url, buffer, size); + + if ( http.code < 200 || http.code >= 300 ) { + UF_MSG_ERROR("HTTP Error {} on POST {}", http.code, url); + return 0; + } + return size; + }, + }; + } +} + namespace { - size_t writeFunction(void *ptr, size_t size, size_t nmemb, uf::stl::string* data) { + size_t writeFunction( void *ptr, size_t size, size_t nmemb, uf::stl::string* data ) { data->append((char*) ptr, size * nmemb); return size * nmemb; } + + uf::Http cURL( const uf::stl::string& url, const uf::stl::string& method = "GET", const void* data = nullptr, size_t size = 0 ) { + uf::Http http; + #if UF_USE_CURL + auto curl = curl_easy_init(); + if ( !curl ) return http; + + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L); + curl_easy_setopt(curl, CURLOPT_USERAGENT, "curl/7.42.0"); + curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 50L); + curl_easy_setopt(curl, CURLOPT_TCP_KEEPALIVE, 1L); + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L); + curl_easy_setopt(curl, CURLOPT_FILETIME, 1L); + + if ( method == "HEAD" ) { + curl_easy_setopt(curl, CURLOPT_NOBODY, 1L); + } else { + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeFunction); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &http.response); + } + + if ( method == "POST" ) { + curl_easy_setopt(curl, CURLOPT_POST, 1L); + if (data) curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, (long) size); + } + + curl_easy_setopt(curl, CURLOPT_HEADERDATA, &http.header); + + curl_easy_perform(curl); + + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http.code); + curl_easy_getinfo(curl, CURLINFO_TOTAL_TIME, &http.elapsed); + curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &http.effective); + + double cl; + if (curl_easy_getinfo(curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &cl) == CURLE_OK && cl >= 0.0) { + http.contentLength = (size_t)cl; + } else { + http.contentLength = 0; + } + + long filetime = -1; + if (curl_easy_getinfo(curl, CURLINFO_FILETIME, &filetime) == CURLE_OK && filetime >= 0) { + http.mtime = (size_t)filetime; + } else { + http.mtime = 0; + } + + curl_easy_cleanup(curl); + #endif + return http; + } } uf::Http uf::http::get( const uf::stl::string& url ) { - uf::Http http; + return ::cURL( url, "GET" ); +} -#if UF_USE_CURL - auto curl = curl_easy_init(); - if ( !curl ) return http; +uf::Http uf::http::head( const uf::stl::string& url ) { + return ::cURL( url, "HEAD" ); +} - std::cout << curl << " " << url << std::endl; - curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L); -// curl_easy_setopt(curl, CURLOPT_USERPWD, "user:pass"); - curl_easy_setopt(curl, CURLOPT_USERAGENT, "curl/7.42.0"); - curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 50L); - curl_easy_setopt(curl, CURLOPT_TCP_KEEPALIVE, 1L); - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeFunction); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &http.response); - curl_easy_setopt(curl, CURLOPT_HEADERDATA, &http.header); +uf::Http uf::http::post( const uf::stl::string& url, const void* data, size_t size ) { + return ::cURL( url, "POST", data, size ); +} - curl_easy_perform(curl); - - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http.code); - curl_easy_getinfo(curl, CURLINFO_TOTAL_TIME, &http.elapsed); - curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &http.effective); - - curl_easy_cleanup(curl); - curl = NULL; -#endif - return http; -} \ No newline at end of file +UF_VFS_MOUNT_CPP( impl::createHttpMount, "https://", -10 ); +UF_VFS_MOUNT_CPP( impl::createHttpMount, "http://", -10 ); \ No newline at end of file diff --git a/engine/src/utils/image/image.cpp b/engine/src/utils/image/image.cpp index 9f5a195e..378249a9 100644 --- a/engine/src/utils/image/image.cpp +++ b/engine/src/utils/image/image.cpp @@ -88,10 +88,11 @@ namespace impl { } -bool uf::image::open( pod::Image& image, const uf::stl::string& _filename, bool flip ) { - // to-do: use preferred - uf::stl::string filename = uf::io::preferred( _filename ); - if ( !uf::io::exists(filename) ) UF_EXCEPTION("IO error: file does not exist: {}", filename); +bool uf::image::open( pod::Image& image, const uf::stl::string& filename, bool flip ) { + uf::stl::vector buffer; + if ( !uf::io::readAsBuffer( buffer, filename ) ) { + return false; + } #if UF_USE_OPENGL_GLDC auto extension = uf::io::extension( filename ); if ( extension != "dtex" ) UF_MSG_WARNING("non-dtex loading is highly discouraged on this platform: {}", filename); @@ -110,12 +111,14 @@ bool uf::image::open( pod::Image& image, const uf::stl::string& _filename, bool uint32_t size; } header; - FILE* file = NULL; - file = fopen(filename.c_str(), "rb"); - fread(&header, sizeof(header), 1, file); + uf::stl::vector buffer; + if ( !uf::io::readAsBuffer( buffer, filename ) ) UF_EXCEPTION("IO error: could not read DTEX: {}", filename); + if ( buffer.size() < sizeof(header) ) UF_EXCEPTION("IO error: DTEX file is too small to contain a header: {}", filename); + memcpy(&header, buffer.data(), sizeof(header)); + if ( buffer.size() < sizeof(header) + header.size ) UF_EXCEPTION("IO error: DTEX file is truncated or corrupted: {}", filename); + image.pixels.resize(header.size); - fread(image.pixels.data(), header.size, 1, file); - fclose(file); + memcpy(image.pixels.data(), buffer.data() + sizeof(header), header.size); bool twiddled = (header.type & (1 << 26)) < 1; bool compressed = (header.type & (1 << 30)) > 0; @@ -150,12 +153,10 @@ bool uf::image::open( pod::Image& image, const uf::stl::string& _filename, bool #endif { stbi_set_flip_vertically_on_load(flip); - uint8_t* buffer = stbi_load( filename.c_str(), &width, &height, &channelsDud, STBI_rgb_alpha ); + uint8_t* stbi_pixels = stbi_load_from_memory( buffer.data(), buffer.size(), &width, &height, &channelsDud, STBI_rgb_alpha ); size_t len = width * height * channels; image.pixels.resize( len ); - memcpy( &image.pixels[0], buffer, len ); - // image.pixels.insert( image.pixels.end(), (uint8_t*) buffer, buffer + len ); - stbi_image_free(buffer); + memcpy( &image.pixels[0], stbi_pixels, len ); } image.size.x = width; diff --git a/engine/src/utils/io/file.cpp b/engine/src/utils/io/file.cpp index b8d14d37..049c781c 100644 --- a/engine/src/utils/io/file.cpp +++ b/engine/src/utils/io/file.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -59,104 +60,84 @@ uf::stl::string uf::io::directory( const uf::stl::string& str ) { return str.substr( 0, str.find_last_of('/') ) + "/"; } size_t uf::io::size( const uf::stl::string& filename ) { - std::ifstream is(filename, std::ios::binary | std::ios::in | std::ios::ate); - if ( !is.is_open() ) return 0; - is.seekg(0, std::ios::end); - return is.tellg(); + return uf::vfs::size( uf::io::resolveURI(filename) ); } -uf::stl::string uf::io::sanitize( const uf::stl::string& str, const uf::stl::string& _root ) { - // resolve %root% to hard root - uf::stl::string path = str; - uf::stl::string root = _root; - // append root to path - if ( path.find("%root%") == 0 ) { - path = uf::string::replace( path, "%root%", "" ); - if ( root == "" ) root = uf::io::root; +uf::stl::string uf::io::normalize( const uf::stl::string& path ) { + uf::stl::string clean = path; + + std::replace(clean.begin(), clean.end(), '\\', '/'); + + clean = uf::string::replace(clean, "/./", "/", false); // explicitly set as non-regex + + size_t schemePos = clean.find("://"); + uf::stl::string scheme = ""; + if ( schemePos != uf::stl::string::npos ) { + scheme = clean.substr(0, schemePos + 3); + clean = clean.substr(schemePos + 3); } - if ( path.find(root) == uf::stl::string::npos ) { - path = root + "/" + path; - } - // flatten all "/./" - { - uf::stl::string tmp; - while ( path != (tmp = uf::string::replace(path, "/\\/\\.\\//", "/")) ) { - path = tmp; - } - } - // flatten all "//" - { - uf::stl::string tmp; - while ( path != (tmp = uf::string::replace(path, "/\\/\\//", "/")) ) { - path = tmp; - } - } - return path; + + clean = uf::string::replace(clean, "//", "/"); + + return scheme + clean; } // would just use readAsBuffer and convert to string, but that's double the memory cost -uf::stl::string& uf::io::readAsString( uf::stl::string& buffer, const uf::stl::string& _filename, const uf::stl::string& hash ) { +bool uf::io::readAsString( uf::stl::string& buffer, const uf::stl::string& _filename, const uf::stl::string& hash ) { buffer.clear(); - uf::stl::string filename = sanitize(_filename); + uf::stl::string filename = uf::io::normalize( _filename ); uf::stl::string extension = uf::io::extension( filename ); + if ( extension == "gz" ) { auto decompressed = uf::io::decompress( filename ); buffer.resize(decompressed.size()); buffer.assign(decompressed.begin(), decompressed.end()); } else { - std::ifstream is(filename, std::ios::binary | std::ios::in | std::ios::ate); - if ( !is.is_open() ) { + uf::stl::vector tempBuffer; + if ( !uf::vfs::read( filename, tempBuffer ) ) { UF_MSG_ERROR("Error: Could not open file: {}", filename); - return buffer; + return false; } - is.seekg(0, std::ios::end); buffer.resize(is.tellg()); is.seekg(0, std::ios::beg); - buffer.assign((std::istreambuf_iterator(is)), std::istreambuf_iterator()); + buffer.assign(tempBuffer.begin(), tempBuffer.end()); } + uf::stl::string expected = ""; if ( hash != "" && (expected = uf::string::sha256( buffer )) != hash ) { UF_MSG_ERROR("Error: Hash mismatch for file {}; expecting {}, got {}", filename, hash, expected); - // should probably clear } - return buffer; + return true; } -uf::stl::vector& uf::io::readAsBuffer( uf::stl::vector& buffer, const uf::stl::string& _filename, const uf::stl::string& hash ) { +bool uf::io::readAsBuffer( uf::stl::vector& buffer, const uf::stl::string& _filename, const uf::stl::string& hash ) { buffer.clear(); - uf::stl::string filename = sanitize(_filename); + uf::stl::string filename = uf::io::normalize( _filename ); uf::stl::string extension = uf::io::extension( filename ); + if ( extension == "gz" || extension == "lz4" ) { uf::io::decompress( buffer, filename ); } else { - std::ifstream is(filename, std::ios::binary | std::ios::in | std::ios::ate); - if ( !is.is_open() ) { + if ( !uf::vfs::read( filename, buffer ) ) { UF_MSG_ERROR("Error: Could not open file: {}", filename); - return buffer; + return false; } - is.seekg(0, std::ios::end); buffer.resize(is.tellg()); is.seekg(0, std::ios::beg); - buffer.assign((std::istreambuf_iterator(is)), std::istreambuf_iterator()); } + uf::stl::string expected = ""; if ( !hash.empty() && (expected = uf::string::sha256( buffer )) != hash ) { UF_MSG_ERROR("Error: Hash mismatch for file {}; expecting {}, got {}", filename, hash, expected); - // should probably clear } - return buffer; + return true; } -uf::stl::vector& uf::io::readAsBuffer( uf::stl::vector& buffer, const uf::stl::string& _filename, size_t start, size_t len, const uf::stl::string& hash ) { +bool uf::io::readAsBuffer( uf::stl::vector& buffer, const uf::stl::string& _filename, size_t start, size_t len, const uf::stl::string& hash ) { buffer.clear(); - uf::stl::string filename = sanitize(_filename); + uf::stl::string filename = uf::io::normalize(_filename); uf::stl::string extension = uf::io::extension(filename); if ( extension == "gz" || extension == "lz4" ) { uf::io::decompress( buffer, filename, start, len ); } else { - std::ifstream is(filename, std::ios::binary); - if (!is.is_open()) { - UF_MSG_ERROR("Error: Could not open file: {}", filename); - return buffer; + if (!uf::vfs::readRange(filename, start, len, buffer)) { + UF_MSG_ERROR("Error: Could not open file range: {}", filename); + return false; } - is.seekg(start, std::ios::beg); - buffer.resize(len); - is.read(reinterpret_cast(buffer.data()), len); - buffer.resize(static_cast(is.gcount())); // adjust if EOF } uf::stl::string expected; @@ -164,36 +145,20 @@ uf::stl::vector& uf::io::readAsBuffer( uf::stl::vector& buffer UF_MSG_ERROR("Error: Hash mismatch for file {}; expecting {}, got {}", filename, hash, expected); // should probably clear } - return buffer; + return true; } -uf::stl::vector& uf::io::readAsBuffer( uf::stl::vector& buffer, const uf::stl::string& _filename, const uf::stl::vector& ranges, const uf::stl::string& hash ) { +bool uf::io::readAsBuffer( uf::stl::vector& buffer, const uf::stl::string& _filename, const uf::stl::vector& ranges, const uf::stl::string& hash ) { buffer.clear(); - uf::stl::string filename = sanitize(_filename); + uf::stl::string filename = uf::io::normalize(_filename); uf::stl::string extension = uf::io::extension(filename); if ( extension == "gz" || extension == "lz4" ) { uf::io::decompress( buffer, filename, ranges ); } else { - std::ifstream is(filename, std::ios::binary); - if (!is.is_open()) { - UF_MSG_ERROR("Error: Could not open file: {}", filename); - return buffer; - } - - // Precompute total size to reserve memory - size_t totalBytes = 0; - for (const auto& r : ranges) { - totalBytes += r.len; - } - buffer.resize(totalBytes); - - // Read each range - size_t currentOffset = 0; - for (const auto& r : ranges) { - is.seekg(r.start, std::ios::beg); - is.read(reinterpret_cast(buffer.data() + currentOffset), r.len); - currentOffset += static_cast(is.gcount()); + if (!uf::vfs::readRanges(filename, ranges, buffer)) { + UF_MSG_ERROR("Error: Could not open file ranges: {}", filename); + return false; } } @@ -202,41 +167,37 @@ uf::stl::vector& uf::io::readAsBuffer( uf::stl::vector& buffer UF_MSG_ERROR("Error: Hash mismatch for file {}; expecting {}, got {}", filename, hash, expected); // should probably clear } - return buffer; + return true; } size_t uf::io::write( const uf::stl::string& filename, const void* buffer, size_t size ) { uf::stl::string extension = uf::io::extension( filename ); if ( extension == "gz" || extension == "lz4" ) return uf::io::compress( filename, buffer, size ); - std::ofstream output; - output.open( uf::io::sanitize( filename ), std::ios::binary); - output.write( (const char*) buffer, size ); - output.close(); - return size; + return uf::vfs::write( uf::io::resolveURI( filename ), buffer, size ); } // indirection for different compression formats, currently only using zlib's gzFile shit -uf::stl::vector& uf::io::decompress( uf::stl::vector& buffer, const uf::stl::string& filename ) { +bool uf::io::decompress( uf::stl::vector& buffer, const uf::stl::string& filename ) { uf::stl::string extension = uf::io::extension( filename ); if ( extension == "gz" ) return ext::zlib::decompressFromFile( buffer, filename ); // if ( extension == "lz4" ) return ext::lz4::decompressFromFile( buffer, filename ); UF_MSG_ERROR("unsupported compression format requested: {}", extension); - return buffer; + return false; } -uf::stl::vector& uf::io::decompress( uf::stl::vector& buffer, const uf::stl::string& filename, size_t start, size_t len ) { +bool uf::io::decompress( uf::stl::vector& buffer, const uf::stl::string& filename, size_t start, size_t len ) { uf::stl::string extension = uf::io::extension( filename ); if ( extension == "gz" ) return ext::zlib::decompressFromFile( buffer, filename, start, len ); // if ( extension == "lz4" ) return ext::lz4::decompressFromFile( buffer, filename, start, len ); UF_MSG_ERROR("unsupported compression format requested: {}", extension); - return buffer; + return false; } -uf::stl::vector& uf::io::decompress( uf::stl::vector& buffer, const uf::stl::string& filename, const uf::stl::vector& ranges ) { +bool uf::io::decompress( uf::stl::vector& buffer, const uf::stl::string& filename, const uf::stl::vector& ranges ) { uf::stl::string extension = uf::io::extension( filename ); if ( extension == "gz" ) return ext::zlib::decompressFromFile( buffer, filename, ranges ); // if ( extension == "lz4" ) return ext::lz4::decompressFromFile( buffer, filename, ranges ); UF_MSG_ERROR("unsupported compression format requested: {}", extension); - return buffer; + return false; } size_t uf::io::compress( const uf::stl::string& filename, const void* buffer, size_t size ) { uf::stl::string extension = uf::io::extension( filename ); @@ -249,37 +210,23 @@ size_t uf::io::compress( const uf::stl::string& filename, const void* buffer, si uf::stl::string uf::io::hash( const uf::stl::string& filename ) { return uf::string::sha256( uf::io::readAsBuffer( filename ) ); } -bool uf::io::exists( const uf::stl::string& _filename ) { -#if UF_ENV_DREAMCAST - FILE* file = fopen(_filename.c_str(), "r"); - if (file) { - fclose(file); - return true; - } - return false; -#else - uf::stl::string filename = sanitize(_filename); - static struct stat buffer; - return stat(filename.c_str(), &buffer) == 0; -#endif +bool uf::io::exists( const uf::stl::string& filename ) { + return uf::vfs::exists( uf::io::resolveURI(filename) ); } -size_t uf::io::mtime( const uf::stl::string& _filename ) { - uf::stl::string filename = sanitize(_filename); - static struct stat buffer; - if ( stat(filename.c_str(), &buffer) != 0 ) return 0; - return buffer.st_mtime; +size_t uf::io::mtime( const uf::stl::string& filename ) { + return uf::vfs::mtime( uf::io::resolveURI(filename) ); } bool uf::io::mkdir( const uf::stl::string& _filename ) { #if UF_ENV_DREAMCAST || UF_ENV_LINUX return false; #else - uf::stl::string filename = sanitize(_filename); + uf::stl::string filename = uf::io::normalize(_filename); int status = ::mkdir(filename.c_str()); return status != -1; #endif } -uf::stl::string uf::io::assetType( const uf::stl::string _filename ) { +uf::stl::string uf::io::assetType( const uf::stl::string& _filename ) { // remove .gz uf::stl::string filename = uf::string::replace( _filename, ".gz", "" ); @@ -302,36 +249,94 @@ uf::stl::string uf::io::assetType( const uf::stl::string _filename ) { return ""; } -uf::stl::string uf::io::resolveURI( const uf::stl::string& filename, const uf::stl::string& _root ) { - uf::stl::string root = _root; - if ( filename.substr(0,8) == "https://" ) return filename; - const uf::stl::string extension = uf::io::extension(filename); - // just sanitize - if ( filename.find(uf::io::root) == 0 ) { - return uf::io::preferred( uf::io::sanitize( uf::io::filename( filename ), uf::io::directory( filename ) ) ); - } - if ( filename.find("%root%") == 0 ) { - const uf::stl::string f = uf::string::replace( filename, "%root%", uf::io::root ); - return uf::io::preferred( uf::io::sanitize( uf::io::filename( f ), uf::io::directory( f ) ) ); - } - // if the filename contains an absolute path or if no root is provided - if ( filename[0] == '/' || root == "" ) { - const uf::stl::string assetType = uf::io::assetType( filename ); - - if ( filename[0] == '/' && filename[1] == '/' ) root = uf::io::root; - else if ( assetType != "" ) { - if ( assetType == "model" ) root = uf::io::root + "/models/"; - else if ( assetType == "scene" ) root = uf::io::root + "/scenes/"; - else if ( assetType == "entity" ) root = uf::io::root + "/entities/"; - else if ( assetType == "texture" ) root = uf::io::root + "/textures/"; - else if ( assetType == "audio" ) root = uf::io::root + "/audio/"; - else if ( assetType == "shader" ) root = uf::io::root + "/shaders/"; - else if ( assetType == "script" ) root = uf::io::root + "/scripts/"; +// to-do: map to above +uf::stl::string uf::io::assetScheme( const uf::stl::string& _filename ) { + uf::stl::string filename = uf::string::replace( _filename, ".gz", "" ); + uf::stl::string basename = uf::io::filename( filename ); + uf::stl::string extension = uf::io::extension( filename ); - else root = uf::io::root + "/" + assetType + "/"; + if ( basename == "graph.json" ) return "mdl://"; + if ( basename == "scene.json" ) return "scene://"; + if ( extension == "json" ) return "ent://"; + if ( extension == "png" || extension == "dtex" ) return "tex://"; + if ( extension == "glb" || extension == "gltf" || extension == "graph" ) return "mdl://"; + if ( extension == "ogg" || extension == "wav" ) return "snd://"; + if ( extension == "spv" ) return "spv://"; + if ( extension == "lua" ) return "lua://"; + + return "data://"; +} + +uf::stl::string uf::io::resolveURI( const uf::stl::string& _filename, const uf::stl::string& _root ) { + if ( _filename.length() >= 8 && _filename.substr(0,8) == "https://" ) return _filename; + + uf::stl::string f = uf::io::normalize( _filename ); + uf::stl::string root = uf::io::normalize( _root ); + bool schemeResolved = f.find("://") != uf::stl::string::npos; + bool fAlreadyHasScheme = schemeResolved; + + // process macros (only %root%) + if ( !schemeResolved && f.starts_with("%root%") ) { + f = f.substr(6); + root = "data://"; + schemeResolved = true; + } + + // explicit relative path + if ( !schemeResolved && f.length() >= 2 && f.substr(0, 2) == "./" ) { + f = f.substr(2); + if ( root.empty() ) schemeResolved = true; + } + + // explicit absolute + if ( !fAlreadyHasScheme && f.length() > 0 && f[0] == '/' ) { + root = ""; + } + + // deduce scheme and apply + if ( !schemeResolved && f.substr(0, 3) != "../" ) { + if ( root.empty() ) { + uf::stl::string deducedScheme = uf::io::assetScheme( f ); + + if ( deducedScheme != "data://" ) { + root = deducedScheme; + schemeResolved = true; + } else { + root = deducedScheme; + } } } - return uf::io::preferred( uf::io::sanitize( filename, root ) ); + + // absolute system paths + bool isWindowsAbsolute = !fAlreadyHasScheme && f.length() >= 2 && std::isalpha(f[0]) && f[1] == ':'; + bool isUnixAbsolute = !fAlreadyHasScheme && !schemeResolved && f.length() > 0 && f[0] == '/'; + + if ( isWindowsAbsolute || isUnixAbsolute ) { + root = "sys://"; + if ( isUnixAbsolute ) f = f.substr(1); // might not be necessary because of the final normalization? + } + + // apply root + if ( !fAlreadyHasScheme && !root.empty() ) { + if ( f.length() > 0 && f[0] == '/' ) f = f.substr(1); + + // remove cringe like `tex://textures/` + for ( const auto& mount : uf::vfs::mounts ) { + if ( root != mount.prefix ) continue; + + uf::stl::string target = mount.path; + if (target.back() == '/') target.pop_back(); + target = target.substr(target.find_last_of('/') + 1) + "/"; + + if ( f.starts_with(target) ) f = f.substr(target.length()); + break; + } + + if ( root.back() != '/') root += "/"; + f = root + f; + } + + return uf::io::preferred( uf::io::normalize( f ) ); } // attempts to coerce files into a preferred one if it exists diff --git a/engine/src/utils/io/vfs.cpp b/engine/src/utils/io/vfs.cpp new file mode 100644 index 00000000..74d1c8da --- /dev/null +++ b/engine/src/utils/io/vfs.cpp @@ -0,0 +1,316 @@ +#include +#include +#include +#include +#include + +pod::Mount uf::vfs::createDiskMount( const uf::stl::string& uri, int priority) { + uf::stl::string prefix; + uf::stl::string path; + uf::io::splitUri( uri, prefix, path ); + if ( !path.empty() && path.back() != '/' && path.back() != '\\' ) path += '/'; + + return pod::Mount{ + .prefix = prefix, + .path = path, + .priority = priority, + .exists = [path](const uf::stl::string& file) -> bool { + uf::stl::string fullPath = path + file; + #if UF_ENV_DREAMCAST + FILE* file = fopen(fullPath.c_str(), "r"); + if (file) { + fclose(file); + return true; + } + return false; + #else + static struct stat buffer; + return stat(fullPath.c_str(), &buffer) == 0; + #endif + }, + .size = [path](const uf::stl::string& file) -> size_t { + uf::stl::string fullPath = path + file; + std::ifstream is(fullPath, std::ios::binary | std::ios::in | std::ios::ate); + if ( !is.is_open() ) return 0; + is.seekg(0, std::ios::end); + return is.tellg(); + }, + .mtime = [path](const uf::stl::string& file) -> size_t { + uf::stl::string fullPath = path + file; + static struct stat buffer; + if ( stat(fullPath.c_str(), &buffer) != 0 ) return 0; + return buffer.st_mtime; + }, + .read = [path](const uf::stl::string& file, uf::stl::vector& buffer) -> bool { + uf::stl::string fullPath = path + file; + std::ifstream is(fullPath, std::ios::binary | std::ios::ate); + if (!is.is_open()) return false; + + size_t len = is.tellg(); + is.seekg(0, std::ios::beg); + + buffer.resize(len); + is.read((char*)buffer.data(), len); + return true; + }, + .write = [path](const uf::stl::string& file, const void* buffer, size_t size) -> size_t { + uf::stl::string fullPath = path + file; + std::ofstream output(fullPath, std::ios::binary); + if (!output.is_open()) return 0; + output.write((const char*)buffer, size); + output.close(); + return size; + }, + .readRange = [path](const uf::stl::string& file, size_t start, size_t len, uf::stl::vector& buffer) -> bool { + uf::stl::string fullPath = path + file; + std::ifstream is(fullPath, std::ios::binary); + if (!is.is_open()) return false; + + is.seekg(start, std::ios::beg); + buffer.resize(len); + is.read((char*)buffer.data(), len); + buffer.resize(static_cast(is.gcount())); // to-do: adjust if EOF hit early + return true; + }, + .readRanges = [path](const uf::stl::string& file, const uf::stl::vector& ranges, uf::stl::vector& buffer) -> bool { + uf::stl::string fullPath = path + file; + std::ifstream is(fullPath, std::ios::binary); + if (!is.is_open()) return false; + + size_t totalBytes = 0; + for (const auto& r : ranges) totalBytes += r.len; + buffer.resize(totalBytes); + + size_t currentOffset = 0; + for (const auto& r : ranges) { + is.seekg(r.start, std::ios::beg); + is.read((char*)(buffer.data() + currentOffset), r.len); + currentOffset += static_cast(is.gcount()); + } + buffer.resize(currentOffset); + return true; + } + }; +} + +uf::stl::vector uf::vfs::mounts; +size_t uf::vfs::mount( const pod::Mount& mount ) { + // compute hash + size_t hash = {}; + uf::hash( hash, mount.prefix, mount.path ); + + // already exists + for ( auto& m : mounts ) { + size_t hash2 = {}; + uf::hash( hash2, m.prefix, m.path ); + if ( hash == hash2 ) { + return hash; + } + } + + // add mount + mounts.emplace_back(mount); + + // resort + std::sort( mounts.begin(), mounts.end(), [](const pod::Mount& a, const pod::Mount& b) { + return a.priority > b.priority; + }); + + return hash; +} + +bool uf::vfs::unmount( size_t hash ) { + auto it = std::remove_if( mounts.begin(), mounts.end(), [&](const pod::Mount& m) { + size_t hash2 = {}; + uf::hash( hash2, m.prefix, m.path ); + return hash == hash2; + }); + + if ( it != mounts.end() ) { + mounts.erase( it, mounts.end() ); + return true; + } + return false; +} +bool uf::vfs::unmount( const uf::stl::string& prefix, const uf::stl::string& base ) { + uf::stl::string cleanBase = base; + if ( !cleanBase.empty() && cleanBase.back() != '/' && cleanBase.back() != '\\' ) cleanBase += '/'; + + size_t hash = {}; + uf::hash( hash, prefix, cleanBase ); + + // erase by hash first + if ( uf::vfs::unmount( hash ) ) return true; + + auto it = std::remove_if( mounts.begin(), mounts.end(), [&](const pod::Mount& m) { + return m.prefix == prefix && m.path == cleanBase; + }); + + if ( it != mounts.end() ) { + mounts.erase( it, mounts.end() ); + return true; + } + return false; +} + +bool uf::vfs::exists( const uf::stl::string& path ) { + uf::stl::string prefix, relative; + uf::io::splitUri(path, prefix, relative); + + for ( const auto& mount : mounts ) { + if ( prefix.empty() && mount.priority < 0 ) continue; + if ( prefix.empty() || mount.prefix == prefix ) { + if ( mount.exists(relative) ) return true; + } + } + return false; +} + +size_t uf::vfs::size( const uf::stl::string& path ) { + uf::stl::string prefix, relative; + uf::io::splitUri(path, prefix, relative); + + for ( const auto& mount : mounts ) { + if ( prefix.empty() && mount.priority < 0 ) continue; + if ( prefix.empty() || mount.prefix == prefix ) { + if ( mount.exists(relative) ) return mount.size(relative); + } + } + return 0; +} + +size_t uf::vfs::mtime( const uf::stl::string& path ) { + uf::stl::string prefix, relative; + uf::io::splitUri(path, prefix, relative); + + for ( const auto& mount : mounts ) { + if ( prefix.empty() && mount.priority < 0 ) continue; + if ( prefix.empty() || mount.prefix == prefix ) { + if ( mount.exists(relative) ) return mount.mtime(relative); + } + } + return 0; +} + +bool uf::vfs::read( const uf::stl::string& path, uf::stl::vector& buffer ) { + uf::stl::string prefix, relative; + uf::io::splitUri(path, prefix, relative); + + for ( const auto& mount : mounts ) { + if ( prefix.empty() && mount.priority < 0 ) continue; + if ( prefix.empty() || mount.prefix == prefix ) { + bool res = mount.exists( relative ); + if ( mount.exists(relative) ) return mount.read(relative, buffer); + } + } + return false; +} + +size_t uf::vfs::write( const uf::stl::string& path, const void* buffer, size_t size ) { + uf::stl::string prefix, relative; + uf::io::splitUri(path, prefix, relative); + + for ( const auto& mount : mounts ) { + if ( prefix.empty() && mount.priority < 0 ) continue; + if ( prefix.empty() || mount.prefix == prefix ) { + if ( mount.write ) return mount.write(relative, buffer, size); + } + } + return 0; +} + +size_t uf::vfs::write( const uf::stl::string& path, uf::stl::vector& buffer ) { + return uf::vfs::write( path, buffer.data(), buffer.size() ); +} + +bool uf::vfs::readRange( const uf::stl::string& path, size_t start, size_t len, uf::stl::vector& buffer ) { + uf::stl::string prefix, relative; + uf::io::splitUri(path, prefix, relative); + for ( const auto& mount : mounts ) { + if ( prefix.empty() && mount.priority < 0 ) continue; + if ( prefix.empty() || mount.prefix == prefix ) { + if ( !mount.exists(relative) ) continue; + if ( mount.readRange ) return mount.readRange(relative, start, len, buffer); + if ( !mount.read ) continue; + + uf::stl::vector fullBuffer; + if ( !mount.read(relative, fullBuffer) ) continue; + + if ( start < fullBuffer.size() ) { + size_t actualLen = std::min(len, fullBuffer.size() - start); + buffer.assign(fullBuffer.begin() + start, fullBuffer.begin() + start + actualLen); + } else { + buffer.clear(); + } + return true; + } + } + return false; +} + +bool uf::vfs::readRanges( const uf::stl::string& path, const uf::stl::vector& ranges, uf::stl::vector& buffer ) { + uf::stl::string prefix, relative; + uf::io::splitUri(path, prefix, relative); + for ( const auto& mount : mounts ) { + if ( prefix.empty() && mount.priority < 0 ) continue; + if ( prefix.empty() || mount.prefix == prefix ) { + if ( !mount.exists(relative) ) continue; + if ( mount.readRanges ) return mount.readRanges(relative, ranges, buffer); + if ( !mount.read ) continue; + + uf::stl::vector fullBuffer; + if ( !mount.read(relative, fullBuffer) ) continue; + + size_t totalBytes = 0; + for ( const auto& r : ranges ) { + if ( r.start < fullBuffer.size() ) { + totalBytes += std::min(r.len, fullBuffer.size() - r.start); + } + } + + buffer.clear(); + buffer.reserve(totalBytes); + + for ( const auto& r : ranges ) { + if ( r.start < fullBuffer.size() ) { + size_t actualLen = std::min(r.len, fullBuffer.size() - r.start); + buffer.insert(buffer.end(), fullBuffer.begin() + r.start, fullBuffer.begin() + r.start + actualLen); + } + } + return true; + } + } + return false; +} + +uf::stl::string uf::vfs::resolveBase( const uf::stl::string& path ) { + uf::stl::string prefix, relative; + uf::io::splitUri(path, prefix, relative); + + for ( const auto& mount : mounts ) { + if ( prefix.empty() && mount.priority < 0 ) continue; + if ( prefix.empty() || mount.prefix == prefix ) { + uf::stl::string resolved = mount.path + relative; + return resolved; + } + } + + return path; +} + +UF_VFS_MOUNT_CPP( uf::vfs::createDiskMount, "mdl://./data/models", 100 ); +UF_VFS_MOUNT_CPP( uf::vfs::createDiskMount, "scene://./data/scenes", 100 ); +UF_VFS_MOUNT_CPP( uf::vfs::createDiskMount, "ent://./data/entities", 100 ); +UF_VFS_MOUNT_CPP( uf::vfs::createDiskMount, "tex://./data/textures", 100 ); +UF_VFS_MOUNT_CPP( uf::vfs::createDiskMount, "snd://./data/audio", 100 ); +UF_VFS_MOUNT_CPP( uf::vfs::createDiskMount, "spv://./data/shaders", 100 ); +UF_VFS_MOUNT_CPP( uf::vfs::createDiskMount, "lua://./data/scripts", 100 ); + +#if UF_USE_DREAMCAST +UF_VFS_MOUNT_CPP( uf::vfs::createDiskMount, "data:///cd/", 50 ); +#else +UF_VFS_MOUNT_CPP( uf::vfs::createDiskMount, "data://./data", 50 ); +#endif +UF_VFS_MOUNT_CPP( uf::vfs::createDiskMount, "", 0 ); + +UF_VFS_MOUNT_CPP( uf::vfs::createDiskMount, "sys://", 999 ); \ No newline at end of file diff --git a/engine/src/utils/string/ext.cpp b/engine/src/utils/string/ext.cpp index 51cd452d..4c055b11 100644 --- a/engine/src/utils/string/ext.cpp +++ b/engine/src/utils/string/ext.cpp @@ -83,35 +83,41 @@ uf::stl::vector uf::string::cStrings( const uf::stl::vector #if UF_USE_FREETYPE -pod::FT_Glyph uf::glyph::initialize( const uf::stl::string& font ) { - return ext::freetype::initialize( font ); -} + uint8_t* uf::glyph::generate( pod::Glyph& glyph, pod::FT_Glyph& g, uint64_t c, size_t size ) { ext::freetype::setPixelSizes( g, size ); if ( !ext::freetype::load( g, c ) ) return NULL; diff --git a/makefiles/default/arch b/makefiles/default/arch deleted file mode 100644 index 6ec5a6aa..00000000 --- a/makefiles/default/arch +++ /dev/null @@ -1 +0,0 @@ -win64 \ No newline at end of file diff --git a/makefiles/default/cc b/makefiles/default/cc deleted file mode 100644 index b08d5af5..00000000 --- a/makefiles/default/cc +++ /dev/null @@ -1 +0,0 @@ -gcc \ No newline at end of file diff --git a/makefiles/default/renderer b/makefiles/default/renderer deleted file mode 100644 index 53395cff..00000000 --- a/makefiles/default/renderer +++ /dev/null @@ -1 +0,0 @@ -vulkan \ No newline at end of file diff --git a/program.sh b/program.sh old mode 100755 new mode 100644