ecker/engine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

insanity (added VFS, piped everything through it and made it work, 90% done with valve loaders)

Browse Source
This commit is contained in:
ecker 2026-06-06 00:10:29 -05:00
parent 06e88cbf17
commit 98ec4933da
52 changed files with 1897 additions and 622 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@ -58,6 +58,10 @@ default/
*.ttf
*.otf
*.bin
*.bsp
*.vmt
*.vtf
*.ztmp
models/
llm/

11
bin/data/config.json
View File

@ -332,7 +332,16 @@
"scale": 1.0
},
"ultralight": { "enabled": true, "scale": 1.5 },
"discord": { "enabled": false }
"discord": { "enabled": false },
"valve": {
"vpks": [
"Half-Life 2/hl2/hl2_pak_dir.vpk",
"Half-Life 2/hl2/hl2_textures_dir.vpk",
"Half-Life 2/hl2/hl2_sound_misc_dir.vpk",
"Half-Life 2/hl2/hl2_misc_dir.vpk",
"Counter-Strike Source/cstrike/cstrike_pak_dir.vpk"
]
}
},
"physics": {
"solvers": {

2
bin/data/shaders/graph/base/vert.h
View File

@ -64,7 +64,7 @@ void main() {
const uint drawID = gl_DrawIDARB;
const uint triangleID = gl_VertexIndex / 3;
const DrawCommand drawCommand = drawCommands[drawID];
const uint instanceID = drawCommand.instanceID; // gl_InstanceIndex;
const uint instanceID = gl_InstanceIndex; // drawCommand.instanceID; // gl_InstanceIndex;
const Instance instance = instances[instanceID];
const Object object = objects[instance.objectID];
const uint jointID = instance.jointID;

1
bin/exe/default/arch
View File

@ -1 +0,0 @@
win64

1
bin/exe/default/cc
View File

@ -1 +0,0 @@
gcc

1
bin/exe/default/renderer
View File

@ -1 +0,0 @@
vulkan

0
debug.sh Executable file → Normal file
View File

8
engine/inc/uf/config.h
View File

@ -27,28 +27,24 @@
#define WINVER 0x0600
#endif
#define UF_IO_ROOT "./data/"
#elif defined(linux) || defined(__linux)
// Linux
#define UF_ENV "Linux"
#define UF_ENV_LINUX 1
#define UF_ENV_HEADER "linux.h"
#define UF_IO_ROOT "./data/"
#elif defined(__APPLE__) || defined(MACOSX) || defined(macintosh) || defined(Macintosh)
// MacOS
#define UF_ENV "OSX"
#define UF_ENV_OSX 1
#define UF_ENV_HEADER "osx.h"
#define UF_IO_ROOT "./data/"
#elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
// FreeBSD
#define UF_ENV "FreeBSD"
#define UF_ENV_FREEBSD 1
#define UF_ENV_HEADER "freebsd.h"
#define UF_IO_ROOT "./data/"
#elif defined(__sh__)
// Dreamcast
#define UF_ENV "Dreamcast"
@ -60,7 +56,7 @@
#define _arch_dreamcast
#endif
#define UF_IO_ROOT "/cd/"
// #define UF_IO_ROOT "/cd/"
#else
// Unsupported system
#define UF_ENV "Unknown"
@ -117,6 +113,8 @@
#define ALIGN16
#endif
#define UF_IO_ROOT "data://"
#include "macros.h"
#include "simd.h"
#include "helpers.inl"

1
engine/inc/uf/engine/graph/graph.h
View File

@ -86,6 +86,7 @@ namespace pod {
GLOBAL,
};
uf::stl::KeyMap<uf::stl::vector<pod::Instance>> groupedInstances;
uf::stl::KeyMap<uf::stl::vector<pod::Instance::Addresses>> instanceAddresses;
uf::stl::KeyMap<uf::stl::vector<pod::Primitive>> primitives;
uf::stl::KeyMap<uf::Mesh> meshes;

4
engine/inc/uf/engine/scene/behavior.h
View File

@ -26,6 +26,10 @@ namespace uf {
pod::Thread::Tasks serial;
pod::Thread::Tasks parallel;
} tasks;
struct {
size_t hash;
} mount;
);
}
}

3
engine/inc/uf/ext/freetype/freetype.h
View File

@ -11,6 +11,8 @@
#include FT_FREETYPE_H
#include <uf/utils/string/string.h>
#include <uf/utils/memory/vector.h>
#include <memory>
namespace pod {
struct FT_Glyph {
@ -24,7 +26,6 @@ namespace ext {
bool UF_API initialize();
void UF_API terminate();
pod::FT_Glyph UF_API initialize( const uf::stl::string& );
bool UF_API initialize( pod::FT_Glyph&, const uf::stl::string& );
void UF_API destroy( pod::FT_Glyph& );

3
engine/inc/uf/ext/valve/common.h
View File

@ -11,7 +11,7 @@ namespace impl {
template<typename T>
T str2vec( uf::stl::string string ) {
string = uf::string::join(uf::string::split(string, " "), ","); // replace spaces with commas
string = uf::string::replace(string, " ", ","); // replace spaces with commas
string = ::fmt::format("[{}]", string); // wrap as an array
ext::json::Value j; ext::json::decode( j, string ); // parse JSON string
return uf::vector::decode( j, T{} ); // parse JSON object
@ -21,5 +21,6 @@ namespace impl {
return pod::Vector3f{ -vertex.y, vertex.z, vertex.x } * scale;
}
uf::stl::string readString( std::ifstream& file );
bool parseKeyValue( const uf::stl::string& line, uf::stl::string& key, uf::stl::string& value );
}

39
engine/inc/uf/ext/valve/vpk.h Normal file
View File

@ -0,0 +1,39 @@
#pragma once
#include <uf/config.h>
#include <uf/utils/io/vfs.h>
namespace pod {
#pragma pack(push, 1)
struct VpkData {
uint32_t crc;
uint16_t preloadBytes;
uint16_t archiveIndex;
uint32_t entryOffset;
uint32_t entryLength;
uint16_t terminator;
};
#pragma pack(pop)
struct VpkFile {
VpkData metadata;
uint32_t dirFileOffset;
uf::stl::vector<uint8_t> preloadData;
};
struct VpkArchive {
uf::stl::string basePath;
uf::stl::unordered_map<uf::stl::string, VpkFile> files;
};
};
namespace ext {
namespace valve {
bool UF_API loadVpk( pod::VpkArchive& vpk, const uf::stl::string& filename );
bool UF_API readVpk( const pod::VpkArchive& vpk, const uf::stl::string& filename, uf::stl::vector<uint8_t>& buffer );
bool UF_API readVpkRange( const pod::VpkArchive& vpk, const uf::stl::string& path, size_t start, size_t len, uf::stl::vector<uint8_t>& buffer );
size_t UF_API mountVpk( const uf::stl::string& uri );
pod::Mount UF_API createVpkMount( const uf::stl::string& uri, int priority = 0 );
}
}

44
engine/inc/uf/ext/zlib/zlib.h
View File

@ -5,33 +5,32 @@
#include <uf/utils/memory/vector.h>
#include <uf/utils/memory/string.h>
#include <uf/utils/memory/unordered_map.h>
#include <uf/utils/io/file.h>
#include <uf/utils/io/vfs.h>
namespace pod {
struct ZipEntry {
size_t offset;
size_t compressedSize;
size_t uncompressedSize;
uint16_t compressionMethod;
};
}
namespace ext {
namespace zlib {
extern UF_API size_t bufferSize;
/*
uf::stl::vector<uint8_t> UF_API compress( const void*, size_t );
uf::stl::vector<uint8_t> UF_API decompress( const void*, size_t );
bool UF_API decompressFromFile( uf::stl::vector<uint8_t>&, const uf::stl::string& );
bool UF_API decompressFromFile( uf::stl::vector<uint8_t>&, const uf::stl::string& filename, size_t start, size_t len );
bool UF_API decompressFromFile( uf::stl::vector<uint8_t>&, const uf::stl::string& filename, const uf::stl::vector<pod::Range>& ranges );
bool UF_API decompressFromMemory( uf::stl::vector<uint8_t>&, const void*, size_t, size_t );
size_t UF_API compressToFile( const uf::stl::string&, const void*, size_t );
bool UF_API directory( const uf::stl::vector<uint8_t>& buffer, uf::stl::unordered_map<uf::stl::string, pod::ZipEntry>& entries );
pod::Mount UF_API createZipMount( const uf::stl::string& uri, uf::stl::vector<uint8_t>& buffer, int priority = 0 );
template<typename T>
uf::stl::vector<T> UF_API compress( const uf::stl::vector<T>& data ) {
auto compressed = ext::zlib::compress( data.data(), data.size() );
uf::stl::vector<T> vector( compressed.size() / sizeof(T) );
memcpy( vector.data(), compressed.data(), compressed.size() );
}
template<typename T>
uf::stl::vector<T> UF_API decompress( const uf::stl::vector<T>& data ) {
auto compressed = ext::zlib::decompress( data.data(), data.size() );
uf::stl::vector<T> vector( compressed.size() / sizeof(T) );
memcpy( vector.data(), compressed.data(), compressed.size() );
}
*/
uf::stl::vector<uint8_t>& UF_API decompressFromFile( uf::stl::vector<uint8_t>&, const uf::stl::string& );
uf::stl::vector<uint8_t>& UF_API decompressFromFile( uf::stl::vector<uint8_t>&, const uf::stl::string& filename, size_t start, size_t len );
uf::stl::vector<uint8_t>& UF_API decompressFromFile( uf::stl::vector<uint8_t>&, const uf::stl::string& filename, const uf::stl::vector<pod::Range>& ranges );
inline uf::stl::vector<uint8_t> decompressFromFile( const uf::stl::string& filename ) {
uf::stl::vector<uint8_t> buffer;
decompressFromFile( buffer, filename );
@ -47,9 +46,6 @@ namespace ext {
decompressFromFile( buffer, filename, ranges );
return buffer;
}
size_t UF_API compressToFile( const uf::stl::string&, const void*, size_t );
}
}

2
engine/inc/uf/utils/audio/metadata.h
View File

@ -25,7 +25,7 @@ namespace uf {
uf::stl::string filename = "";
uf::stl::string extension = "";
struct {
FILE* file = NULL;
void* context = NULL;
void* handle = NULL;
char* buffer = NULL;

5
engine/inc/uf/utils/http/http.h
View File

@ -7,11 +7,16 @@ namespace uf {
struct UF_API Http {
uf::stl::string header;
uf::stl::string response;
size_t contentLength;
size_t mtime;
char* effective;
long code;
double elapsed;
};
namespace http {
uf::Http UF_API get( const uf::stl::string& );
uf::Http UF_API head( const uf::stl::string& );
uf::Http UF_API post( const uf::stl::string&, const void* data, size_t len );
}
}

2
engine/inc/uf/utils/image/image.h
View File

@ -12,7 +12,7 @@ namespace pod {
uf::stl::string filename;
pod::Image::container_t pixels;
pod::Vector2ui size = {};
size_t bpp = 8;
size_t bpp = 8 * 4;
size_t channels = 4;
size_t format = 0;
};

31
engine/inc/uf/utils/io/file.h
View File

@ -22,14 +22,14 @@ namespace uf {
uf::stl::string UF_API extension( const uf::stl::string& );
uf::stl::string UF_API extension( const uf::stl::string&, int32_t );
uf::stl::string UF_API directory( const uf::stl::string& );
uf::stl::string UF_API sanitize( const uf::stl::string&, const uf::stl::string& = "" );
uf::stl::string UF_API normalize( const uf::stl::string& );
size_t UF_API size( const uf::stl::string& );
uf::stl::string& UF_API readAsString( uf::stl::string&, const uf::stl::string&, const uf::stl::string& = "" );
bool UF_API readAsString( uf::stl::string&, const uf::stl::string&, const uf::stl::string& = "" );
uf::stl::vector<uint8_t>& UF_API readAsBuffer( uf::stl::vector<uint8_t>&, const uf::stl::string&, const uf::stl::string& = "" );
uf::stl::vector<uint8_t>& UF_API readAsBuffer( uf::stl::vector<uint8_t>&, const uf::stl::string&, size_t start, size_t len, const uf::stl::string& = "" );
uf::stl::vector<uint8_t>& UF_API readAsBuffer( uf::stl::vector<uint8_t>&, const uf::stl::string&, const uf::stl::vector<pod::Range>& ranges, const uf::stl::string& = "" );
bool UF_API readAsBuffer( uf::stl::vector<uint8_t>&, const uf::stl::string&, const uf::stl::string& = "" );
bool UF_API readAsBuffer( uf::stl::vector<uint8_t>&, const uf::stl::string&, size_t start, size_t len, const uf::stl::string& = "" );
bool UF_API readAsBuffer( uf::stl::vector<uint8_t>&, const uf::stl::string&, const uf::stl::vector<pod::Range>& ranges, const uf::stl::string& = "" );
// yuck!
// wrapper in case this gets called without feeding into a buffer directly to avoid additional memory allocations
@ -62,9 +62,9 @@ namespace uf {
return write( filename, string.c_str(), std::min( string.size(), size ) );
}
uf::stl::vector<uint8_t>& UF_API decompress( uf::stl::vector<uint8_t>&, const uf::stl::string& );
uf::stl::vector<uint8_t>& UF_API decompress( uf::stl::vector<uint8_t>&, const uf::stl::string&, size_t, size_t );
uf::stl::vector<uint8_t>& UF_API decompress( uf::stl::vector<uint8_t>&, const uf::stl::string&, const uf::stl::vector<pod::Range>& );
bool UF_API decompress( uf::stl::vector<uint8_t>&, const uf::stl::string& );
bool UF_API decompress( uf::stl::vector<uint8_t>&, const uf::stl::string&, size_t, size_t );
bool UF_API decompress( uf::stl::vector<uint8_t>&, const uf::stl::string&, const uf::stl::vector<pod::Range>& );
inline uf::stl::vector<uint8_t> decompress( const uf::stl::string& filename ) {
uf::stl::vector<uint8_t> buffer;
@ -94,8 +94,21 @@ namespace uf {
bool UF_API exists( const uf::stl::string& );
size_t UF_API mtime( const uf::stl::string& );
bool UF_API mkdir( const uf::stl::string& );
uf::stl::string UF_API assetType( const uf::stl::string _filename );
uf::stl::string UF_API assetType( const uf::stl::string& _filename );
uf::stl::string UF_API assetScheme( const uf::stl::string& _filename );
uf::stl::string UF_API resolveURI( const uf::stl::string&, const uf::stl::string& = "" );
uf::stl::string UF_API preferred( const uf::stl::string& filename );
// splits "prefix://path/to/file" into "prefix://" and "path/to/file"
inline void splitUri( const uf::stl::string& uri, uf::stl::string& prefix, uf::stl::string& relativePath ) {
size_t pos = uri.find("://");
if ( pos != uf::stl::string::npos ) {
prefix = uri.substr(0, pos + 3);
relativePath = uri.substr(pos + 3);
} else {
prefix = "";
relativePath = uri;
}
}
}
}

6
engine/inc/uf/utils/io/macros.inl Normal file
View File

@ -0,0 +1,6 @@
#define UF_VFS_MOUNT_CPP( LAMBDA, URI, PRIORITY ) \
namespace {\
static uf::StaticInitialization TOKEN_PASTE(STATIC_INITIALIZATION_, __LINE__)( []{\
uf::vfs::mount( LAMBDA( URI, PRIORITY ) );\
});\
}

54
engine/inc/uf/utils/io/vfs.h Normal file
View File

@ -0,0 +1,54 @@
#pragma once
#include <uf/config.h>
#include <uf/utils/singletons/pre_main.h>
#include <uf/utils/memory/string.h>
#include <uf/utils/memory/vector.h>
#include <uf/utils/userdata/userdata.h>
#include <functional>
namespace pod {
struct Range;
struct Mount {
uf::stl::string prefix = "";
uf::stl::string path = "";
int priority = 0;
pod::PointeredUserdata userdata;
std::function<bool(const uf::stl::string&)> exists;
std::function<size_t(const uf::stl::string&)> size;
std::function<size_t(const uf::stl::string&)> mtime;
std::function<bool(const uf::stl::string&, uf::stl::vector<uint8_t>&)> read;
std::function<size_t(const uf::stl::string&, const void*, size_t)> write;
std::function<bool(const uf::stl::string&, size_t, size_t, uf::stl::vector<uint8_t>&)> readRange;
std::function<bool(const uf::stl::string&, const uf::stl::vector<pod::Range>&, uf::stl::vector<uint8_t>&)> readRanges;
};
}
namespace uf {
namespace vfs {
extern UF_API uf::stl::vector<pod::Mount> mounts;
size_t UF_API mount( const pod::Mount& mount );
bool UF_API unmount( size_t );
bool UF_API unmount( const uf::stl::string& prefix, const uf::stl::string& base );
bool UF_API exists( const uf::stl::string& path );
size_t UF_API size( const uf::stl::string& path );
size_t UF_API mtime( const uf::stl::string& path );
bool UF_API read( const uf::stl::string& path, uf::stl::vector<uint8_t>& buffer );
size_t UF_API write( const uf::stl::string& path, const void* data, size_t len );
size_t UF_API write( const uf::stl::string& path, uf::stl::vector<uint8_t>& buffer );
bool UF_API readRange( const uf::stl::string& path, size_t start, size_t len, uf::stl::vector<uint8_t>& buffer );
bool UF_API readRanges( const uf::stl::string& path, const uf::stl::vector<pod::Range>& ranges, uf::stl::vector<uint8_t>& buffer );
pod::Mount UF_API createDiskMount( const uf::stl::string& uri, int priority = 0 );
uf::stl::string UF_API resolveBase( const uf::stl::string& path );
}
}
#include "macros.inl"

8
engine/inc/uf/utils/math/hash.h
View File

@ -1,11 +1,11 @@
#pragma once
namespace uf {
inline void hash(std::size_t& seed) { }
inline void hash( size_t& seed ) { }
template <typename T, typename... Rest>
inline void hash(std::size_t& seed, const T& v, Rest... rest) {
seed ^= std::hash<T>()(v) + 0x9e3779b9 + (seed<<6) + (seed>>2);
hash(seed, rest...);
inline void hash( size_t& seed, const T& v, Rest... rest ) {
seed ^= std::hash<T>()(v) + 0x9e3779b9 + (seed<<6) + (seed>>2);
hash(seed, rest...);
}
}

1
engine/inc/uf/utils/string/ext.h
View File

@ -17,6 +17,7 @@ namespace uf {
bool UF_API matched( const uf::stl::string& str, const uf::stl::string& r );
uf::stl::string UF_API replace( const uf::stl::string&, const uf::stl::string&, const uf::stl::string& );
uf::stl::string UF_API replace( const uf::stl::string&, const uf::stl::string&, const uf::stl::string&, bool regexp );
uf::stl::string UF_API lowercase( const uf::stl::string& );
uf::stl::string UF_API uppercase( const uf::stl::string& );
uf::stl::string UF_API ltrim( const uf::stl::string& );

2
engine/inc/uf/utils/userdata/pointered.inl
View File

@ -40,7 +40,7 @@ namespace uf {
public:
// Easily access POD's type
typedef pod::PointeredUserdata pod_t;
bool autoDestruct = uf::userdata::autoDestruct;
bool autoDestruct = uf::userdata::autoDestruct; // shouldn't ever use in containers because of auto-destruct......
protected:
// POD storage
PointeredUserdata::pod_t m_pod = {};

27
engine/src/engine/asset/asset.cpp
View File

@ -152,19 +152,7 @@ bool uf::asset::isExpected( const uf::asset::Payload& payload, uf::asset::Type e
uf::stl::string uf::asset::cache( uf::asset::Payload& payload ) {
uf::stl::string filename = payload.filename;
uf::stl::string extension = uf::io::extension( filename );
if ( filename.substr(0,5) == "https" ) {
uf::stl::string hash = uf::string::sha256( filename );
uf::stl::string cached = uf::io::root + "/cache/http/" + hash + "." + extension;
if ( !uf::io::exists( cached ) && !retrieve( filename, cached, hash ) ) {
if ( !uf::asset::assertionLoad ) {
UF_MSG_ERROR("Failed to preload {} ({}): HTTP error", filename, cached);
} else {
UF_EXCEPTION("Failed to preload {} ({}): HTTP error", filename, cached);
}
return "";
}
filename = cached;
}
if ( !uf::io::exists( filename ) ) {
if ( !uf::asset::assertionLoad ) {
UF_MSG_ERROR("Failed to preload {}, does not exist", filename);
@ -193,19 +181,6 @@ uf::stl::string uf::asset::load( uf::asset::Payload& payload ) {
uf::stl::string extension = uf::string::lowercase(uf::io::extension( payload.filename, -1 ));
uf::stl::string basename = uf::string::lowercase( uf::string::replace( uf::io::filename( payload.filename ), "/.(?:gz|lz4?)$/", "" ) );
if ( payload.filename.substr(0,5) == "https" ) {
uf::stl::string hash = uf::string::sha256( payload.filename );
uf::stl::string cached = uf::io::root + "/cache/http/" + hash + "." + extension;
if ( !uf::io::exists( cached ) && !retrieve( payload.filename, cached, hash ) ) {
if ( !uf::asset::assertionLoad ) {
UF_MSG_ERROR("Failed to load {} ({}): HTTP error", payload.filename, cached);
} else {
UF_EXCEPTION("Failed to load {} ({}): HTTP error", payload.filename, cached);
}
return "";
}
filename = cached;
}
if ( !uf::io::exists( filename ) ) {
if ( !uf::asset::assertionLoad ) {
UF_MSG_ERROR("Failed to load {}: does not exist", filename);

8
engine/src/engine/ext/ext.cpp
View File

@ -46,6 +46,7 @@
#include <uf/ext/ffx/fsr.h>
#include <uf/ext/imgui/imgui.h>
#include <uf/ext/vall_e/vall_e.h>
#include <uf/ext/valve/vpk.h>
bool uf::ready = false;
uf::stl::vector<uf::stl::string> uf::arguments;
@ -493,6 +494,13 @@ void UF_API uf::initialize() {
uf::console::initialize();
}
/* Load VPKs */ {
auto& vpks = uf::config["engine"]["ext"]["valve"]["vpks"];
ext::json::forEach( vpks, []( const uf::stl::string& uri ) {
ext::valve::mountVpk( uri );
});
}
/* Create initial scene (kludge) */ {
uf::Scene& scene = uf::instantiator::instantiate<uf::Scene>(); //new uf::Scene;
uf::scene::scenes.emplace_back(&scene);

2
engine/src/engine/ext/scene/behavior.h
View File

@ -74,7 +74,7 @@ namespace ext {
} fog;
struct {
struct {
uf::stl::string filename = "%root%/textures/skybox/%d.png";
uf::stl::string filename = "/textures/skybox/%d.png";
} box;
} sky;

12
engine/src/engine/graph/convert.cpp
View File

@ -221,15 +221,15 @@ void uf::graph::postprocess( pod::Graph& graph ) {
#if UF_USE_XATLAS
// generate STs
if ( graph.metadata["exporter"]["unwrap"].as<bool>(true) || graph.metadata["exporter"]["unwrap"].as<uf::stl::string>() == "tagged" ) {
UF_MSG_DEBUG( "Generating ST's..." );
//UF_MSG_DEBUG( "Generating ST's..." );
size_t atlases = ext::xatlas::unwrap( graph );
UF_MSG_DEBUG( "Generated ST's for {} lightmaps", atlases );
//UF_MSG_DEBUG( "Generated ST's for {} lightmaps", atlases );
}
#endif
#if UF_USE_MESHOPT
// cleanup if blender's exporter is poopy
if ( graph.metadata["exporter"]["optimize"].as<bool>(false) || graph.metadata["exporter"]["optimize"].as<uf::stl::string>("") == "tagged" || ext::json::isObject( graph.metadata["exporter"]["optimize"] ) ) {
UF_MSG_DEBUG( "Optimizing meshes..." );
//UF_MSG_DEBUG( "Optimizing meshes..." );
for ( auto& keyName : graph.meshes ) {
size_t level = SIZE_MAX;
float simplify = 1.0f;
@ -265,7 +265,7 @@ void uf::graph::postprocess( pod::Graph& graph ) {
auto& primitives = storage.primitives[keyName];
if ( level ) {
UF_MSG_DEBUG("Optimizing mesh at level {}: {}", level, keyName);
//UF_MSG_DEBUG("Optimizing mesh at level {}: {}", level, keyName);
if ( !ext::meshopt::optimize( mesh, simplify, level, print ) ) {
UF_MSG_ERROR("Mesh optimization failed: {}", keyName );
}
@ -276,7 +276,7 @@ void uf::graph::postprocess( pod::Graph& graph ) {
if ( lodMetadata.empty() ) {
UF_MSG_ERROR("LOD generation failed: {}", keyName );
} else {
UF_MSG_DEBUG("Generated {} LODs: {}", factors.size() - 1, keyName);
//UF_MSG_DEBUG("Generated {} LODs: {}", factors.size() - 1, keyName);
UF_ASSERT( primitives.size() == lodMetadata.size() );
for ( auto i = 0; i < primitives.size(); ++i ) {
primitives[i].lod = lodMetadata[i];
@ -285,7 +285,7 @@ void uf::graph::postprocess( pod::Graph& graph ) {
}
}
UF_MSG_DEBUG( "Optimized mesh" );
//UF_MSG_DEBUG( "Optimized mesh" );
}
#endif
{

118
engine/src/engine/graph/graph.cpp
View File

@ -834,7 +834,7 @@ void uf::graph::process( pod::Graph& graph ) {
// lightmaps are considered stale if they're older than the graph's source
bool stale = false;
for ( auto& pair : filenames ) {
uf::stl::string filename = uf::io::sanitize( pair.second, uf::io::directory( graph.name ) );
uf::stl::string filename = uf::io::resolveURI( pair.second, uf::io::directory( graph.name ) );
auto time = uf::io::mtime(filename);
if ( !uf::io::exists( filename ) ) continue;
if ( time < mtime ) {
@ -852,7 +852,7 @@ void uf::graph::process( pod::Graph& graph ) {
if ( graphMetadataJson["lights"]["lightmap"].as<bool>() ) {
for ( auto& pair : filenames ) {
auto i = pair.first;
auto f = uf::io::sanitize( pair.second, uf::io::directory( graph.name ) );
auto f = uf::io::resolveURI( pair.second, uf::io::directory( graph.name ) );
if ( !uf::io::exists( f ) ) {
graphMetadataJson["lights"]["lightmap"] = false;
UF_MSG_ERROR( "lightmap does not exist: {} {}, disabling lightmaps", i, f )
@ -863,7 +863,7 @@ void uf::graph::process( pod::Graph& graph ) {
if ( graphMetadataJson["lights"]["lightmap"].as<bool>() ) {
for ( auto& pair : filenames ) {
auto i = pair.first;
auto f = uf::io::sanitize( pair.second, uf::io::directory( graph.name ) );
auto f = uf::io::resolveURI( pair.second, uf::io::directory( graph.name ) );
auto textureID = graph.textures.size();
auto imageID = graph.images.size();
@ -1087,6 +1087,29 @@ void uf::graph::process( pod::Graph& graph ) {
}
}
}
for ( auto& instance : storage.groupedInstances.map[name] ) {
if ( 0 <= instance.materialID && instance.materialID < graph.materials.size() ) {
auto& keys = storage.materials.keys;
auto& indices = storage.materials.indices;
auto& needle = graph.materials[instance.materialID];
instance.materialID = indices[needle];
}
if ( 0 <= instance.lightmapID && instance.lightmapID < graph.textures.size() ) {
auto& keys = storage.textures.keys;
auto& indices = storage.textures.indices;
auto& needle = graph.textures[instance.lightmapID];
instance.lightmapID = indices[needle];
}
if ( 0 <= instance.jointID && instance.jointID < graph.skins.size() ) {
auto& skinName = graph.skins[instance.jointID];
instance.jointID = 0;
for ( auto key : storage.joints.keys ) {
if ( key == skinName ) break;
instance.jointID += storage.joints[key].size();
}
}
}
}
/*
if ( graphMetadataJson["debug"]["print"]["lights"].as<bool>() ) {
@ -1296,6 +1319,41 @@ void uf::graph::process( pod::Graph& graph, int32_t index, uf::Object& parent )
};
}
auto primitiveName = graph.primitives[node.mesh];
auto& mesh = storage.meshes.map[graph.meshes[node.mesh]];
auto& primitives = storage.primitives.map[primitiveName];
auto& grouped = storage.groupedInstances[primitiveName];
node.object = ::allocateObjectID( storage );
auto objectKeyName = std::to_string( node.object );
storage.entities[objectKeyName] = &entity;
storage.objects[objectKeyName] = pod::Instance::Object{
.model = model,
.previous = model,
};
pod::Instance::Bounds bounds = {};
for ( auto drawID = 0; drawID < primitives.size(); ++drawID ) {
pod::Instance newInstance = primitives[drawID].instance;
newInstance.objectID = node.object;
newInstance.jointID = graphMetadataJson["renderer"]["skinned"].as<bool>() ? 0 : -1;
bounds.min = uf::vector::min( bounds.min, newInstance.bounds.min );
bounds.max = uf::vector::max( bounds.max, newInstance.bounds.max );
grouped.emplace_back(newInstance);
}
bool isFirstInstance = ( grouped.size() == primitives.size() );
#if !UF_GRAPH_EXTENDED
if ( graphMetadataJson["renderer"]["render"].as<bool>() && isFirstInstance ) {
uf::graph::initializeGraphics( graph, entity, mesh, addresses );
}
#endif
#if 0
auto& mesh = storage.meshes.map[graph.meshes[node.mesh]];
auto& primitives = storage.primitives.map[graph.primitives[node.mesh]];
auto& instanceAddresses = storage.instanceAddresses.map[graph.primitives[node.mesh]];
@ -1328,6 +1386,7 @@ void uf::graph::process( pod::Graph& graph, int32_t index, uf::Object& parent )
if ( graphMetadataJson["renderer"]["render"].as<bool>() ) {
uf::graph::initializeGraphics( graph, entity, mesh, addresses );
}
#endif
#endif
{
@ -1490,15 +1549,52 @@ bool uf::graph::tick( pod::Graph::Storage& storage ) {
if ( storage.stale ) {
for ( auto& key : storage.primitives.keys ) {
for ( auto& primitive : storage.primitives[key] ) {
drawCommands.emplace_back( primitive.drawCommand );
instances.emplace_back( primitive.instance );
lodMetadata.emplace_back( primitive.lod );
}
}
auto& submeshes = storage.primitives.map[key];
auto& grouped = storage.groupedInstances.map[key];
auto& submeshAddresses = storage.instanceAddresses.map[key];
for ( auto& key : storage.instanceAddresses.keys ) {
instanceAddresses.insert( instanceAddresses.end(), storage.instanceAddresses.map[key].begin(), storage.instanceAddresses.map[key].end() );
auto& mesh = storage.meshes.map[key];
pod::DrawCommand* meshIndirectCmds = nullptr;
if (mesh.indirect.count > 0) {
auto& attr = mesh.indirect.attributes.front();
meshIndirectCmds = (pod::DrawCommand*) mesh.buffers[attr.buffer].data();
}
size_t nodesCount = submeshes.empty() ? 0 : grouped.size() / submeshes.size();
for ( size_t drawID = 0; drawID < submeshes.size(); ++drawID ) {
auto& primitive = submeshes[drawID];
primitive.drawCommand.instanceID = instances.size();
primitive.drawCommand.instances = nodesCount;
if (meshIndirectCmds) {
meshIndirectCmds[drawID].instanceID = primitive.drawCommand.instanceID;
meshIndirectCmds[drawID].instances = primitive.drawCommand.instances;
}
drawCommands.emplace_back( primitive.drawCommand );
lodMetadata.emplace_back( primitive.lod );
bool hasAddresses = (drawID < submeshAddresses.size());
for (size_t i = 0; i < nodesCount; ++i) {
size_t strideIndex = (i * submeshes.size()) + drawID;
instances.emplace_back( grouped[strideIndex] );
instanceAddresses.emplace_back( hasAddresses ? submeshAddresses[drawID] : pod::Instance::Addresses{} );
}
}
if (meshIndirectCmds && !grouped.empty()) {
auto hostKeyName = std::to_string(grouped.front().objectID);
if (storage.entities.map.count(hostKeyName) > 0) {
auto* hostEntity = storage.entities.map[hostKeyName];
if (hostEntity && hostEntity->hasComponent<uf::renderer::Graphic>()) {
hostEntity->getComponent<uf::renderer::Graphic>().updateMesh(mesh);
}
}
}
}
for ( auto& key : storage.textures.keys ) textures.emplace_back( storage.textures.map[key] );

13
engine/src/engine/scene/scene.cpp
View File

@ -7,6 +7,7 @@
#include <uf/utils/renderer/renderer.h>
#include <uf/utils/debug/draw.h>
#include <uf/utils/io/fmt.h>
#include <uf/utils/io/vfs.h>
#include <uf/engine/ext.h>
#include <regex>
@ -184,8 +185,15 @@ uf::Scene& uf::scene::loadScene( const uf::stl::string& name, const uf::stl::str
#endif
scene->load(filename);
auto& metadata = scene->getComponent<uf::SceneBehavior::Metadata>();
auto& metadataObject = scene->getComponent<uf::ObjectBehavior::Metadata>();
auto mountUri = ::fmt::format("://{}", uf::vfs::resolveBase( metadataObject.system.root ) );
metadata.mount.hash = uf::vfs::mount( uf::vfs::createDiskMount( mountUri, 200 ) );
auto& metadataJson = scene->getComponent<uf::Serializer>();
metadataJson["system"]["scene"] = name;
#if UF_USE_VULKAN
if ( uf::renderer::settings::pipelines::rt ) uf::instantiator::bind( "RayTraceSceneBehavior", *scene );
if ( uf::renderer::settings::pipelines::vxgi ) uf::instantiator::bind( "VoxelizerSceneBehavior", *scene );
@ -212,6 +220,11 @@ uf::Scene& uf::scene::loadScene( const uf::stl::string& name, const uf::Serializ
void uf::scene::unloadScene() {
uf::Scene* current = uf::scene::scenes.back();
current->queueDeletion();
{
auto& metadataScene = current->getComponent<uf::SceneBehavior::Metadata>();
uf::vfs::unmount( metadataScene.mount.hash );
}
// destroy graph
if ( current->hasComponent<pod::Graph::Storage>() ) {

108
engine/src/ext/audio/vorbis.cpp
View File

@ -7,10 +7,9 @@
#include <uf/ext/audio/vorbis.h>
#include <uf/utils/memory/pool.h>
#include <uf/utils/io/vfs.h>
#include <iostream>
#include <fstream>
#include <vector>
#include <cstdio>
#include <cstring>
#if UF_USE_TREMOR
@ -22,45 +21,57 @@
namespace {
constexpr int endian = 0; // 0 = little endian
struct VorbisVfsContext {
uf::stl::string filename;
size_t currentOffset;
size_t totalSize;
};
namespace funs {
size_t read(void* destination, size_t size, size_t nmemb, void* userdata) {
uf::Audio::Metadata& metadata = *((uf::Audio::Metadata*) userdata);
FILE* file = (FILE*) metadata.stream.file;
size_t read_count = fread(destination, size, nmemb, file);
metadata.stream.consumed += read_count * size;
return read_count;
VorbisVfsContext* ctx = (VorbisVfsContext*) userdata;
size_t bytesToRead = size * nmemb;
size_t bytesLeft = ctx->totalSize - ctx->currentOffset;
if (bytesToRead > bytesLeft) bytesToRead = bytesLeft;
if (bytesToRead > 0) {
uf::stl::vector<uint8_t> tempBuffer;
if (uf::vfs::readRange(ctx->filename, ctx->currentOffset, bytesToRead, tempBuffer)) {
std::memcpy(destination, tempBuffer.data(), tempBuffer.size());
ctx->currentOffset += tempBuffer.size();
return tempBuffer.size() / size;
}
}
return 0;
}
int seek( void* userdata, ogg_int64_t to, int type ) {
uf::Audio::Metadata& metadata = *((uf::Audio::Metadata*) userdata);
FILE* file = (FILE*) metadata.stream.file;
VorbisVfsContext* ctx = (VorbisVfsContext*) userdata;
switch ( type ) {
case SEEK_CUR: metadata.stream.consumed += to; break;
case SEEK_END: metadata.stream.consumed = metadata.info.size - to; break;
case SEEK_SET: metadata.stream.consumed = to; break;
case SEEK_CUR: ctx->currentOffset += to; break;
case SEEK_END: ctx->currentOffset = ctx->totalSize + to; break;
case SEEK_SET: ctx->currentOffset = to; break;
default: return -1;
}
if (metadata.stream.consumed < 0) metadata.stream.consumed = 0;
if (metadata.stream.consumed > metadata.info.size) metadata.stream.consumed = metadata.info.size;
if (fseek(file, to, type) != 0) return -1;
if ((int64_t)ctx->currentOffset < 0) ctx->currentOffset = 0;
if (ctx->currentOffset > ctx->totalSize) ctx->currentOffset = ctx->totalSize;
return 0;
}
int close(void* userdata) {
uf::Audio::Metadata& metadata = *((uf::Audio::Metadata*) userdata);
FILE* file = (FILE*) metadata.stream.file;
if ( file ) {
fclose(file);
metadata.stream.file = NULL;
VorbisVfsContext* ctx = (VorbisVfsContext*) userdata;
if (ctx) {
delete ctx;
}
return 0;
}
long tell(void* userdata) {
uf::Audio::Metadata& metadata = *((uf::Audio::Metadata*) userdata);
return metadata.stream.consumed;
VorbisVfsContext* ctx = (VorbisVfsContext*) userdata;
return (long)ctx->currentOffset;
}
}
@ -78,20 +89,25 @@ namespace {
}
void ext::vorbis::open(uf::Audio::Metadata& metadata) {
if ( !metadata.stream.file ) metadata.stream.file = fopen(metadata.filename.c_str(), "rb");
if ( !metadata.stream.context ) {
VorbisVfsContext* ctx = new VorbisVfsContext();
ctx->filename = metadata.filename;
ctx->currentOffset = 0;
ctx->totalSize = uf::vfs::size(metadata.filename);
if ( ctx->totalSize == 0 ) {
UF_MSG_ERROR("Vorbis: failed to open file: {}", metadata.filename);
delete ctx;
return;
}
metadata.stream.context = (void*) ctx;
}
if ( !metadata.stream.handle ) metadata.stream.handle = (void*) new OggVorbis_File;
FILE* file = (FILE*) metadata.stream.file;
VorbisVfsContext* ctx = (VorbisVfsContext*) metadata.stream.context;
OggVorbis_File* vorbisFile = (OggVorbis_File*) metadata.stream.handle;
if ( !file ) {
UF_MSG_ERROR("Vorbis: failed to open {}. File error.", metadata.filename);
return;
}
fseek(file, 0, SEEK_END);
metadata.info.size = ftell(file);
fseek(file, 0, SEEK_SET);
metadata.info.size = ctx->totalSize;
metadata.stream.consumed = 0;
ov_callbacks callbacks;
@ -99,10 +115,12 @@ void ext::vorbis::open(uf::Audio::Metadata& metadata) {
callbacks.seek_func = funs::seek;
callbacks.close_func = funs::close;
callbacks.tell_func = funs::tell;
int error = ov_open_callbacks((void*) &metadata, vorbisFile, NULL, metadata.settings.streamed ? -1 : 0, callbacks);
int error = ov_open_callbacks((void*) ctx, vorbisFile, NULL, metadata.settings.streamed ? -1 : 0, callbacks);
if (error < 0) {
UF_MSG_ERROR("Vorbis: failed call to ov_open_callbacks: {}", metadata.filename);
delete ctx;
metadata.stream.context = nullptr;
return;
}
@ -114,6 +132,7 @@ void ext::vorbis::open(uf::Audio::Metadata& metadata) {
if ( !format(metadata, info->channels, 16) ) {
ov_clear(vorbisFile);
metadata.stream.context = nullptr;
return;
}
@ -123,10 +142,9 @@ void ext::vorbis::open(uf::Audio::Metadata& metadata) {
void ext::vorbis::load( uf::Audio::Metadata& metadata ) {
if ( metadata.settings.streamed ) return ext::vorbis::stream(metadata);
FILE* file = (FILE*) metadata.stream.file;
OggVorbis_File* vorbisFile = (OggVorbis_File*) metadata.stream.handle;
uf::stl::vector<char> bytes;
uf::stl::vector<uint8_t> bytes;
char buffer[uf::audio::bufferSize];
int bitStream = 0;
int read = 0;
@ -139,7 +157,7 @@ void ext::vorbis::load( uf::Audio::Metadata& metadata ) {
metadata.al.source.set(AL_BUFFER, (ALint) metadata.al.buffer.getIndex());
ov_clear(vorbisFile);
fclose(file);
metadata.stream.context = nullptr;
}
void ext::vorbis::stream(uf::Audio::Metadata& metadata) {
@ -147,7 +165,7 @@ void ext::vorbis::stream(uf::Audio::Metadata& metadata) {
OggVorbis_File* vorbisFile = (OggVorbis_File*) metadata.stream.handle;
// Fill and queue initial buffers
// fill and queue initial buffers
char buffer[uf::audio::bufferSize];
uint8_t queuedBuffers = 0;
int bitStream = 0;
@ -192,8 +210,8 @@ void ext::vorbis::update(uf::Audio::Metadata& metadata) {
ALint state;
metadata.al.source.get(AL_SOURCE_STATE, state);
if (state != AL_PLAYING) {
if (!metadata.settings.loop && metadata.stream.consumed >= metadata.info.size) {
// Stream finished
VorbisVfsContext* ctx = (VorbisVfsContext*) metadata.stream.context;
if (!metadata.settings.loop && ctx && ctx->currentOffset >= ctx->totalSize) {
return;
}
metadata.al.source.play();
@ -250,12 +268,8 @@ void ext::vorbis::close(uf::Audio::Metadata& metadata) {
ov_clear(file);
delete file;
metadata.stream.handle = NULL;
}
if ( metadata.stream.file ) {
FILE* file = (FILE*) metadata.stream.file;
fclose(file);
metadata.stream.file = NULL;
metadata.stream.context = NULL;
}
}
#endif
#endif

100
engine/src/ext/audio/wav.cpp
View File

@ -7,17 +7,69 @@
#include <uf/ext/audio/wav.h>
#include <uf/utils/memory/pool.h>
#include <uf/utils/io/vfs.h>
#include <iostream>
#include <cstdio>
#include <cstring>
#define DR_WAV_IMPLEMENTATION
#include "dr_wav.h"
namespace {
struct DrWavVfsContext {
drwav wav;
uf::stl::string filename;
size_t currentOffset;
size_t totalSize;
};
size_t drwav_vfs_read(void* pUserData, void* pBufferOut, size_t bytesToRead) {
DrWavVfsContext* ctx = (DrWavVfsContext*)pUserData;
size_t bytesLeft = ctx->totalSize - ctx->currentOffset;
if (bytesToRead > bytesLeft) bytesToRead = bytesLeft;
if (bytesToRead > 0) {
uf::stl::vector<uint8_t> tempBuffer;
if (uf::vfs::readRange(ctx->filename, ctx->currentOffset, bytesToRead, tempBuffer)) {
std::memcpy(pBufferOut, tempBuffer.data(), tempBuffer.size());
ctx->currentOffset += tempBuffer.size();
return tempBuffer.size();
}
}
return 0;
}
drwav_bool32 drwav_vfs_seek(void* pUserData, int offset, drwav_seek_origin origin) {
DrWavVfsContext* ctx = (DrWavVfsContext*)pUserData;
if ((int)origin == 0) {
ctx->currentOffset = (size_t)offset;
} else if ((int)origin == 1) {
if (offset < 0 && (size_t)(-offset) > ctx->currentOffset) {
ctx->currentOffset = 0;
} else {
ctx->currentOffset += offset;
}
}
if (ctx->currentOffset > ctx->totalSize) ctx->currentOffset = ctx->totalSize;
return 1;
}
drwav_bool32 drwav_vfs_tell(void* pUserData, long long int* pCursor) {
DrWavVfsContext* ctx = (DrWavVfsContext*)pUserData;
if (pCursor) {
*pCursor = (long long int)ctx->currentOffset;
}
return 1;
}
namespace funs {
size_t read(void* destination, size_t size, size_t nmemb, void* userdata) {
uf::Audio::Metadata& metadata = *((uf::Audio::Metadata*)userdata);
drwav* wav = (drwav*) metadata.stream.handle;
DrWavVfsContext* ctx = (DrWavVfsContext*) metadata.stream.handle;
drwav* wav = &ctx->wav;
size_t bytesRequested = size * nmemb;
size_t frameSize = wav->channels * (wav->bitsPerSample / 8);
size_t framesToRead = bytesRequested / frameSize;
@ -30,7 +82,8 @@ namespace {
}
int seek(void* userdata, int64_t to, int type) {
uf::Audio::Metadata& metadata = *((uf::Audio::Metadata*)userdata);
drwav* wav = (drwav*) metadata.stream.handle;
DrWavVfsContext* ctx = (DrWavVfsContext*) metadata.stream.handle;
drwav* wav = &ctx->wav;
drwav_uint64 targetFrame = 0;
switch (type) {
@ -45,10 +98,10 @@ namespace {
}
int close(void* userdata) {
uf::Audio::Metadata& metadata = *((uf::Audio::Metadata*)userdata);
drwav* wav = (drwav*) metadata.stream.handle;
if (wav) {
drwav_uninit(wav);
delete wav;
DrWavVfsContext* ctx = (DrWavVfsContext*) metadata.stream.handle;
if (ctx) {
drwav_uninit(&ctx->wav);
delete ctx;
metadata.stream.handle = nullptr;
}
return 0;
@ -61,16 +114,21 @@ namespace {
}
void ext::wav::open(uf::Audio::Metadata& metadata) {
// open file
drwav* wav = new drwav;
if (!drwav_init_file(wav, metadata.filename.c_str(), nullptr)) {
DrWavVfsContext* ctx = new DrWavVfsContext();
ctx->filename = metadata.filename;
ctx->currentOffset = 0;
ctx->totalSize = uf::vfs::size(metadata.filename);
if (!drwav_init(&ctx->wav, drwav_vfs_read, drwav_vfs_seek, drwav_vfs_tell, ctx, nullptr)) {
UF_MSG_ERROR("Could not open WAV file: {}", metadata.filename);
delete wav;
delete ctx;
return;
}
drwav* wav = &ctx->wav;
// fill out metadata
metadata.stream.handle = wav;
metadata.stream.handle = ctx;
metadata.info.size = wav->totalPCMFrameCount * wav->channels * (wav->bitsPerSample / 8);
metadata.stream.consumed = 0;
metadata.info.channels = wav->channels;
@ -94,26 +152,21 @@ void ext::wav::open(uf::Audio::Metadata& metadata) {
}
// choose load or stream
return metadata.settings.streamed ? ext::wav::stream(metadata) : ext::wav::load(metadata);
if (metadata.settings.streamed) ext::wav::stream(metadata); else ext::wav::load(metadata);
}
void ext::wav::load(uf::Audio::Metadata& metadata) {
// if streaming is requested, use streaming function
if (metadata.settings.streamed) return ext::wav::stream(metadata);
drwav* wav = (drwav*) metadata.stream.handle;
// read all PCM data
size_t totalBytes = (size_t) metadata.info.size;
std::vector<uint8_t> bytes(totalBytes);
// Use funs::read instead of drwav_read_pcm_frames
size_t bytesRead = funs::read(bytes.data(), 1, totalBytes, &metadata);
if (bytesRead < totalBytes) {
bytes.resize(bytesRead); // in case file is truncated
bytes.resize(bytesRead);
}
// upload to OpenAL buffer
metadata.al.buffer.buffer(metadata.info.format, bytes.data(), (ALsizei) bytes.size(), metadata.info.frequency);
metadata.al.source.set(AL_BUFFER, (ALint) metadata.al.buffer.getIndex());
@ -123,10 +176,8 @@ void ext::wav::load(uf::Audio::Metadata& metadata) {
void ext::wav::stream(uf::Audio::Metadata& metadata) {
if (!metadata.settings.streamed) return ext::wav::load(metadata);
// Ensure we're at the start
funs::seek(&metadata, 0, SEEK_SET);
drwav* wav = (drwav*) metadata.stream.handle;
char buffer[uf::audio::bufferSize];
uint8_t queuedBuffers = 0;
for (; queuedBuffers < metadata.settings.buffers; ++queuedBuffers) {
@ -167,26 +218,21 @@ void ext::wav::update(uf::Audio::Metadata& metadata) {
metadata.al.source.get(AL_SOURCE_STATE, state);
if (state != AL_PLAYING) {
if (!metadata.settings.loop && metadata.stream.consumed >= metadata.info.size) {
// stream finished
return;
return; // stream finished
}
// stream stalled, restart it
metadata.al.source.play();
}
ALint processed = 0;
metadata.al.source.get(AL_BUFFERS_PROCESSED, processed);
if (processed <= 0) return;
drwav* wav = (drwav*) metadata.stream.handle;
ALuint index;
char buffer[uf::audio::bufferSize];
while (processed--) {
memset(buffer, 0, uf::audio::bufferSize);
AL_CHECK_RESULT(alSourceUnqueueBuffers(metadata.al.source.getIndex(), 1, &index));
// Use funs::read instead of drwav_read_pcm_frames
size_t bytesRead = funs::read(buffer, 1, uf::audio::bufferSize, &metadata);
if (bytesRead == 0) {
@ -198,6 +244,7 @@ void ext::wav::update(uf::Audio::Metadata& metadata) {
}
bytesRead = funs::read(buffer, 1, uf::audio::bufferSize, &metadata);
if (bytesRead == 0) {
// should never actually reach here
UF_MSG_ERROR("WAV: failed to read after looping: {}", metadata.filename);
break;
}
@ -208,7 +255,6 @@ void ext::wav::update(uf::Audio::Metadata& metadata) {
AL_CHECK_RESULT(alSourceQueueBuffers(metadata.al.source.getIndex(), 1, &index));
}
if (metadata.settings.loop && bytesRead < uf::audio::bufferSize) {
// should never actually reach here
UF_MSG_ERROR("WAV: missing data: {}", metadata.filename);
}
}

39
engine/src/ext/freetype/freetype.cpp
View File

@ -1,9 +1,12 @@
#include <iostream>
#if UF_USE_FREETYPE
#include <uf/ext/freetype/freetype.h>
#include <uf/utils/io/file.h>
#include <uf/utils/memory/unordered_map.h>
namespace impl {
FT_Library library;
FT_Library ft_library;
uf::stl::unordered_map<uf::stl::string, uf::stl::vector<uint8_t>> ftCache;
uf::stl::string error( int error ) {
#undef FTERRORS_H_
@ -62,33 +65,35 @@ pod::FT_Glyph::~FT_Glyph() {
}
bool ext::freetype::initialize() {
if ( auto error = FT_Init_FreeType( &impl::library ) ) {
if ( auto error = FT_Init_FreeType( &impl::ft_library ) ) {
UF_MSG_ERROR("FreeType failed to initialize: {}", impl::error( error ));
return false;
}
return true;
}
void ext::freetype::terminate() {
FT_Done_FreeType( impl::library );
FT_Done_FreeType( impl::ft_library );
impl::ftCache.clear();
}
pod::FT_Glyph ext::freetype::initialize( const uf::stl::string& font ) {
pod::FT_Glyph g;
if ( auto error = FT_New_Face( impl::library, font.c_str(), 0, &g.face ) ) {
UF_MSG_ERROR("FreeType failed to load file '{}': {}", font, impl::error( error ));
}
if ( auto error = FT_Select_Charmap( g.face, FT_ENCODING_UNICODE ) ) {
UF_MSG_ERROR("FreeType failed to load file '{}': {}", font, impl::error( error ));
}
return g;
}
bool ext::freetype::initialize( pod::FT_Glyph& g, const uf::stl::string& font ) {
if ( auto error = FT_New_Face( impl::library, font.c_str(), 0, &g.face ) ) {
UF_MSG_ERROR("FreeType failed to load file '{}': {}", font, impl::error( error ));
bool ext::freetype::initialize( pod::FT_Glyph& g, const uf::stl::string& filename ) {
// yucky yuck
if ( impl::ftCache.find(filename) == impl::ftCache.end() ) {
uf::stl::vector<uint8_t> buffer;
if ( !uf::io::readAsBuffer( buffer, filename ) ) {
UF_MSG_ERROR("FreeType failed to read file: {}", filename);
return false;
}
impl::ftCache[filename] = std::move(buffer);
}
const auto& buffer = impl::ftCache[filename];
if ( auto error = FT_New_Memory_Face( impl::ft_library, buffer.data(), (FT_Long) buffer.size(), 0, &g.face ) ) {
UF_MSG_ERROR("FreeType failed to load font memory '{}': {}", filename, impl::error( error ));
return false;
}
if ( auto error = FT_Select_Charmap( g.face, FT_ENCODING_UNICODE ) ) {
UF_MSG_ERROR("FreeType failed to load file '{}': {}", font, impl::error( error ));
UF_MSG_ERROR("FreeType failed to load charmap '{}': {}", filename, impl::error( error ));
return false;
}
return true;

76
engine/src/ext/lua/lua.cpp
View File

@ -20,6 +20,31 @@ uf::stl::unordered_map<uf::stl::string, uf::stl::string> ext::lua::modules;
#include <uf/utils/io/inputs.h>
#include <uf/utils/io/fmt.h>
struct vfs_reader {
uf::stl::vector<uint8_t> buffer;
bool read_once;
vfs_reader(const uf::stl::string& filename) : read_once(false) {
// Use your VFS abstraction to load the entire file into the buffer
uf::io::readAsBuffer(buffer, filename);
}
static const char* read(lua_State*, void* data, size_t* size) {
vfs_reader* reader = static_cast<vfs_reader*>(data);
// If we haven't yielded the buffer to Lua yet, do it now.
if (!reader->read_once && !reader->buffer.empty()) {
*size = reader->buffer.size();
reader->read_once = true;
return reinterpret_cast<const char*>(reader->buffer.data());
}
// Yield 0 to signify EOF
*size = 0;
return nullptr;
}
};
sol::table ext::lua::createTable() {
return sol::table(ext::lua::state, sol::create);
}
@ -210,7 +235,13 @@ void ext::lua::initialize() {
const uf::stl::string& name = pair.first;
const uf::stl::string& script = pair.second;
if ( uf::io::extension(script) == "lua" ) {
state.require_file(name, uf::io::resolveURI( script ), true);
uf::stl::string code;
if ( uf::io::readAsString(code, script) ) {
// Pass 'script' as the chunk name so your modules also have proper error tracing!
state.require_script(name, code, true, script);
} else {
UF_MSG_ERROR("Lua: failed to load module via VFS: {}", script);
}
} else {
state.require_script(name, script, true);
}
@ -299,15 +330,29 @@ void ext::lua::initialize() {
bool ext::lua::run( const uf::stl::string& s, bool safe ) {
// is file
if ( uf::io::extension(s) == "lua" ) {
uf::stl::string resolved = uf::io::resolveURI(s);
vfs_reader reader(resolved);
uf::stl::string chunkname = "@" + resolved;
sol::load_result loaded = state.load(vfs_reader::read, &reader, chunkname.c_str());
if ( !loaded.valid() ) {
sol::error err = loaded;
UF_MSG_ERROR("{}", err.what());
return false;
}
if ( safe ) {
auto result = state.safe_script_file( uf::io::resolveURI( s ), sol::script_pass_on_error );
sol::protected_function script = loaded;
auto result = script();
if ( !result.valid() ) {
sol::error err = result;
UF_MSG_ERROR("{}", err.what());
return false;
}
} else {
state.script_file( uf::io::resolveURI( s ) );
sol::unsafe_function script = loaded;
script();
}
// is string with lua
} else {
@ -327,9 +372,7 @@ bool ext::lua::run( const uf::stl::string& s, bool safe ) {
pod::LuaScript ext::lua::script( const uf::stl::string& filename ) {
pod::LuaScript script;
if ( !ext::lua::enabled ) return script;
script.file = filename;
script.env = sol::environment( ext::lua::state, sol::create, ext::lua::state.globals() );
ext::lua::script( filename, script );
return script;
}
void ext::lua::script( const uf::stl::string& filename, pod::LuaScript& script ) {
@ -340,15 +383,32 @@ void ext::lua::script( const uf::stl::string& filename, pod::LuaScript& script )
bool ext::lua::run( const pod::LuaScript& s, bool safe ) {
// is file
if ( uf::io::extension(s.file) == "lua" ) {
uf::stl::string resolved = uf::io::resolveURI(s.file);
vfs_reader reader(resolved);
uf::stl::string chunkname = "@" + resolved;
sol::load_result loaded = state.load(vfs_reader::read, &reader, chunkname.c_str());
if ( !loaded.valid() ) {
sol::error err = loaded;
UF_MSG_ERROR("{}", err.what());
return false;
}
sol::protected_function script = loaded;
s.env.set_on(script);
if ( safe ) {
auto result = state.safe_script_file( s.file, s.env, sol::script_pass_on_error );
auto result = script();
if ( !result.valid() ) {
sol::error err = result;
UF_MSG_ERROR("{}", err.what());
return false;
}
} else {
state.script_file( s.file, s.env );
sol::unsafe_function unsafe_script = script;
unsafe_script();
}
// is string with lua
} else {

4
engine/src/ext/meshopt/meshopt.cpp
View File

@ -98,7 +98,7 @@ bool ext::meshopt::optimize( uf::Mesh& mesh, float simplify, size_t o, bool verb
srcIndexCount * simplify, targetError, meshopt_SimplifyLockBorder, &realError
);
if ( verbose ) UF_MSG_DEBUG("[View {}] Simplified: {} -> {}", viewIdx, srcIndexCount, optimizedIndexCount);
//if ( verbose ) UF_MSG_DEBUG("[View {}] Simplified: {} -> {}", viewIdx, srcIndexCount, optimizedIndexCount);
localIndices.swap(simplified);
localIndices.resize(optimizedIndexCount);
}
@ -256,7 +256,7 @@ uf::stl::vector<pod::LODMetadata> ext::meshopt::generateLODs( uf::Mesh& mesh, co
continue;
}
if ( verbose ) UF_MSG_DEBUG("[View {}] LOD {}: {} -> {}", viewIdx, lodIdx, cmd0.indices, currentIndicesCount);
//if ( verbose ) UF_MSG_DEBUG("[View {}] LOD {}: {} -> {}", viewIdx, lodIdx, cmd0.indices, currentIndicesCount);
lodIndices.resize(currentIndicesCount);

14
engine/src/ext/opengl/shader.cpp
View File

@ -46,17 +46,9 @@ void ext::opengl::Shader::initialize( ext::opengl::Device& device, const uf::stl
// GPU-based shader execution
#if !UF_USE_OPENGL_FIXED_FUNCTION
uf::stl::string glsl;
{
std::ifstream is(this->filename = filename, std::ios::binary | std::ios::in | std::ios::ate);
if ( !is.is_open() ) {
GL_VALIDATION_MESSAGE("Error: Could not open shader file \"" << filename << "\"");
return;
}
is.seekg(0, std::ios::end); spirv.reserve(is.tellg()); is.seekg(0, std::ios::beg);
spirv.assign((std::istreambuf_iterator<char>(is)), std::istreambuf_iterator<char>());
assert(spirv.size() > 0);
}
uf::io::readAsString( glsl, this->filename = filename );
if ( glsl.empty() ) UF_EXCEPTION("Error: Could not open shader file: {}", filename);
UF_ASSERT(glsl.size() > 0);
{
device.activateContext();
module = glCreateShader(stage);

97
engine/src/ext/valve/bsp.cpp
View File

@ -1,9 +1,9 @@
#include <uf/ext/valve/bsp.h>
#include <uf/ext/valve/mdl.h>
#include <uf/ext/valve/vtf.h>
#include <uf/ext/valve/vpk.h>
#include <uf/ext/valve/common.h>
#include <sstream> // to-do: get rid of this
#include <uf/ext/zlib/zlib.h>
namespace impl {
struct RGBE {
@ -186,7 +186,7 @@ namespace impl {
uf::stl::vector<impl::BspDispInfo> dispinfos;
uf::stl::vector<impl::BspDispVert> dispverts;
// disptris
// pakfile
uf::stl::vector<uint8_t> pakfile;
// cubemaps
// overlay
uf::stl::vector<uint8_t> lighting;
@ -390,7 +390,7 @@ namespace impl {
for ( auto nodeID : graph.root.children ) {
auto& node = graph.nodes[nodeID];
auto classname = node.metadata["classname"].as<uf::stl::string>("");
UF_MSG_INFO("Entity found: {}", classname);
//UF_MSG_INFO("Entity found: {}", classname);
node.name = classname;
// parse origin
@ -401,6 +401,7 @@ namespace impl {
}
// parse angles
// to-do: fix oddities
auto angles = node.metadata["angles"].as<uf::stl::string>("");
if ( angles != "" ) {
auto pyr = impl::str2vec<pod::Vector3f>( angles ) * DEG_2_RAD;
@ -418,9 +419,16 @@ namespace impl {
if ( 0 <= modelID && modelID < context.modelToMesh.size() ) {
node.mesh = context.modelToMesh[modelID];
}
} /* else {
// let the engine handle loading the model
}*/
} else if ( model.length() > 4 && model.ends_with(".mdl") ) {
auto it = std::find(graph.meshes.begin(), graph.meshes.end(), model);
if ( it == graph.meshes.end() ) {
if ( ext::valve::loadMdl(graph, model) ) {
node.mesh = (int32_t)(graph.meshes.size() - 1);
}
} else {
node.mesh = (int32_t)std::distance(graph.meshes.begin(), it);
}
}
// parse lighting info
if ( classname.starts_with("light") ) {
@ -441,6 +449,7 @@ namespace impl {
light.color /= 255.0f;
if (!(stream >> light.intensity)) light.intensity = 200.0f;
light.intensity /= 10.0f;
}
// to-do: read range
@ -472,24 +481,15 @@ namespace impl {
}
void ext::valve::loadBsp( pod::Graph& graph, const uf::stl::string& filename, const uf::Serializer& metadata ) {
std::ifstream file(filename, std::ios::binary | std::ios::ate);
if ( !file ) {
UF_MSG_ERROR("failed to open BSP: {}", filename);
return;
}
std::streamsize size = file.tellg();
file.seekg(0, std::ios::beg);
uf::stl::vector<uint8_t> buffer(size);
if ( !file.read((char*)(buffer.data()), size) ) {
UF_MSG_ERROR("failed to read BSP data: {}", filename);
uf::stl::vector<uint8_t> buffer;
if ( !uf::io::readAsBuffer(buffer, filename) ) {
UF_MSG_ERROR("Failed to read BSP data: {}", filename);
return;
}
const impl::BspHeader* header = (const impl::BspHeader*)(buffer.data());
if ( header->magic != 0x50534256 ) {
UF_MSG_ERROR("invalid VBSP magic number: {}", filename);
UF_MSG_ERROR("Invalid VBSP magic number: {}", filename);
return;
}
@ -513,11 +513,15 @@ void ext::valve::loadBsp( pod::Graph& graph, const uf::stl::string& filename, co
context.gameLumps = impl::extractLump<impl::BspGameLump>(buffer, header->lumps[impl::BspLump::LUMP_GAME_LUMP]);
context.dispinfos = impl::extractLump<impl::BspDispInfo>(buffer, header->lumps[impl::BspLump::LUMP_DISPINFO]);
context.dispverts = impl::extractLump<impl::BspDispVert>(buffer, header->lumps[impl::BspLump::LUMP_DISP_VERTS]);
context.pakfile = impl::extractLump<uint8_t>(buffer, header->lumps[impl::BspLump::LUMP_PAKFILE]);
context.lighting = impl::extractLump<uint8_t>(buffer, header->lumps[impl::BspLump::LUMP_LIGHTING]);
context.modelToMesh.assign( context.models.size(), -1 );
context.texdataToMaterial.assign( context.texdatas.size(), -1 );
// mount pakfile
size_t pakfileMount = uf::vfs::mount( ext::zlib::createZipMount(::fmt::format("pakfile://{}", filename), context.pakfile, 1000 ) );
// read materials
for ( int32_t texDataID = 0; texDataID < context.texdatas.size(); ++texDataID ) {
const auto& data = context.texdatas[texDataID];
@ -525,11 +529,11 @@ void ext::valve::loadBsp( pod::Graph& graph, const uf::stl::string& filename, co
// lookup material name
if ( data.nameStringTableID >= 0 && data.nameStringTableID < context.stringTable.size() ) {
int32_t offset = context.stringTable[data.nameStringTableID];
if ( offset >= 0 && offset < context.stringData.size() ) {
matName = context.stringData.c_str() + offset;
}
}
int32_t offset = context.stringTable[data.nameStringTableID];
if ( offset >= 0 && offset < context.stringData.size() ) {
matName = uf::string::lowercase( context.stringData.c_str() + offset );
}
}
size_t imageID = graph.images.size();
auto imgKeyName = graph.images.emplace_back(matName);
@ -548,24 +552,7 @@ void ext::valve::loadBsp( pod::Graph& graph, const uf::stl::string& filename, co
material.indexAlbedo = textureID;
material.colorBase = {1.0f, 1.0f, 1.0f, 1.0f};
UF_MSG_INFO("Material found: {}", matName);
}
// load materials
uf::stl::vector<uint8_t> missing_pixels = { 255, 0, 255, 255, 0, 0, 0, 255, 0, 0, 0, 255, 255, 0, 255, 255 };
for ( auto matName : graph.materials ) {
uf::Serializer vmt;
auto vmtPath = ::fmt::format("materials/{}.vmt", matName);
auto vtfPath = ::fmt::format("materials/{}.vtf", matName);
auto& image = storage.images[matName];
if ( !ext::valve::loadVmt( vmt, vmtPath ) ) goto PEETAH;
if ( !vmt["$basetexture"].is<uf::stl::string>() ) goto PEETAH;
vtfPath = ::fmt::format("materials/{}.vtf", vmt["$basetexture"].as<uf::stl::string>());
if ( !ext::valve::loadVtf( image, vtfPath ) ) goto PEETAH;
PEETAH:
image.loadFromBuffer( missing_pixels, { 2, 2 }, 8, 4 );
//UF_MSG_INFO("Material found: {}", matName);
}
// read lightmaps
@ -588,7 +575,7 @@ void ext::valve::loadBsp( pod::Graph& graph, const uf::stl::string& filename, co
{
UF_MSG_DEBUG("Generating new lightmap atlas...");
uf::atlas::generate( context.lightmapAtlas, 0.0f );
UF_MSG_DEBUG("Generated lightmap atlas.");
//UF_MSG_DEBUG("Generated lightmap atlas.");
auto& imageKey = graph.images.emplace_back("lightmap_atlas");
auto& textureKey = graph.textures.emplace_back("lightmap_atlas");
@ -748,10 +735,30 @@ void ext::valve::loadBsp( pod::Graph& graph, const uf::stl::string& filename, co
}
}
impl::processNodes( graph, context );
// load materials
uf::stl::vector<uint8_t> missing_pixels = { 255, 0, 255, 255, 0, 0, 0, 255, 0, 0, 0, 255, 255, 0, 255, 255 };
for ( auto matName : graph.materials ) {
uf::Serializer vmt;
auto vmtPath = ::fmt::format("materials/{}.vmt", matName);
auto vtfPath = ::fmt::format("materials/{}.vtf", matName);
auto& image = storage.images[matName];
if ( !ext::valve::loadVmt( vmt, vmtPath ) ) goto PEETAH;
if ( !vmt["$basetexture"].is<uf::stl::string>() ) goto PEETAH;
vtfPath = ::fmt::format("materials/{}.vtf", vmt["$basetexture"].as<uf::stl::string>());
if ( !ext::valve::loadVtf( image, vtfPath ) ) goto PEETAH;
continue;
PEETAH:
image.loadFromBuffer( missing_pixels, { 2, 2 }, 8, 4 );
}
graph.metadata["exporter"]["unwrap"] = false; // not necessary to unwrap
uf::graph::postprocess( graph );
// unmount pakfile
uf::vfs::unmount( pakfileMount );
}

7
engine/src/ext/valve/common.cpp
View File

@ -9,4 +9,11 @@ bool impl::parseKeyValue( const uf::stl::string& line, uf::stl::string& key, uf:
key = line.substr( q1 + 1, q2 - q1 - 1 );
value = line.substr( q3 + 1, q4 - q3 - 1 );
return true;
}
uf::stl::string impl::readString( std::ifstream& file ) {
uf::stl::string str;
char c;
while ( file.get(c) && c != '\0' ) str += c;
return str;
}

85
engine/src/ext/valve/mdl.cpp
View File

@ -1,6 +1,7 @@
#include <uf/ext/valve/bsp.h>
#include <uf/ext/valve/mdl.h>
#include <uf/ext/valve/vtf.h>
#include <uf/ext/valve/vpk.h>
#include <uf/ext/valve/common.h>
namespace impl {
@ -138,16 +139,12 @@ bool ext::valve::loadMdl( pod::Graph& graph, const uf::stl::string& filename ) {
auto& storage = uf::graph::getStorage( graph );
uf::stl::vector<impl::Meshlet> meshlets;
// read MDL file
std::ifstream mdlFile(filename, std::ios::binary | std::ios::ate);
if ( !mdlFile ) {
// Read MDL file
uf::stl::vector<uint8_t> mdlBuffer;
if ( !uf::io::readAsBuffer(mdlBuffer, filename) ) {
UF_MSG_ERROR("Failed to find MDL: {}", filename);
return false;
}
std::streamsize mdlSize = mdlFile.tellg();
mdlFile.seekg(0, std::ios::beg);
uf::stl::vector<uint8_t> mdlBuffer(mdlSize);
mdlFile.read((char*)mdlBuffer.data(), mdlSize);
const impl::studiohdr_t* mdlHdr = (const impl::studiohdr_t*)mdlBuffer.data();
if ( mdlHdr->magic != 0x54534449 ) { // "IDST"
@ -155,17 +152,13 @@ bool ext::valve::loadMdl( pod::Graph& graph, const uf::stl::string& filename ) {
return false;
}
// read VVD file
// Read VVD file
uf::stl::string vvdPath = filename.substr(0, filename.find_last_of('.')) + ".vvd";
std::ifstream vvdFile(vvdPath, std::ios::binary | std::ios::ate);
if ( !vvdFile ) {
uf::stl::vector<uint8_t> vvdBuffer;
if ( !uf::io::readAsBuffer(vvdBuffer, vvdPath) ) {
UF_MSG_ERROR("Failed to find VVD: {}", vvdPath);
return false;
}
std::streamsize vvdSize = vvdFile.tellg();
vvdFile.seekg(0, std::ios::beg);
uf::stl::vector<uint8_t> vvdBuffer(vvdSize);
vvdFile.read((char*)vvdBuffer.data(), vvdSize);
const impl::vertexFileHeader_t* vvdHdr = (const impl::vertexFileHeader_t*)vvdBuffer.data();
if ( vvdHdr->magic != 0x56534449 || vvdHdr->checksum != mdlHdr->checksum ) {
@ -173,15 +166,35 @@ bool ext::valve::loadMdl( pod::Graph& graph, const uf::stl::string& filename ) {
return false;
}
// extract material names from MDL
uf::stl::vector<uf::stl::string> materials(mdlHdr->numtextures);
for ( int i = 0; i < mdlHdr->numtextures; ++i ) {
int32_t texStructOffset = mdlHdr->textureindex + (i * 64);
int32_t nameOffset = *(int32_t*)(mdlBuffer.data() + texStructOffset);
// extract material directories (cdtextures)
uf::stl::vector<uf::stl::string> cdmaterials(mdlHdr->numcdtextures);
for ( int i = 0; i < mdlHdr->numcdtextures; ++i ) {
int32_t cdOffset = *(int32_t*)(mdlBuffer.data() + mdlHdr->cdtextureindex + (i * 4));
uf::stl::string cdPath = (const char*)(mdlBuffer.data() + cdOffset);
materials[i] = (const char*)(mdlBuffer.data() + texStructOffset + nameOffset);
UF_MSG_INFO("Model Material {}: {}", i, materials[i]);
}
std::replace(cdPath.begin(), cdPath.end(), '\\', '/');
std::transform(cdPath.begin(), cdPath.end(), cdPath.begin(), ::tolower);
cdmaterials[i] = cdPath;
}
// extract material names from MDL and resolve their full relative paths
uf::stl::vector<uf::stl::string> materials(mdlHdr->numtextures);
for ( int i = 0; i < mdlHdr->numtextures; ++i ) {
int32_t texStructOffset = mdlHdr->textureindex + (i * 64);
int32_t nameOffset = *(int32_t*)(mdlBuffer.data() + texStructOffset);
uf::stl::string baseName = (const char*)(mdlBuffer.data() + texStructOffset + nameOffset);
std::transform(baseName.begin(), baseName.end(), baseName.begin(), ::tolower);
materials[i] = baseName;
for ( const auto& cd : cdmaterials ) {
uf::stl::string attempt = cd + baseName;
if ( uf::vfs::exists("materials/" + attempt + ".vmt") ) {
materials[i] = attempt;
break;
}
}
}
// extract LOD0 ertices from VVD
const impl::mstudiovertex_t* vvdVertices = (const impl::mstudiovertex_t*)(vvdBuffer.data() + vvdHdr->vertexDataStart);
@ -189,13 +202,8 @@ bool ext::valve::loadMdl( pod::Graph& graph, const uf::stl::string& filename ) {
// read VTX file
uf::stl::string vtxPath = filename.substr(0, filename.find_last_of('.')) + ".dx90.vtx";
std::ifstream vtxFile(vtxPath, std::ios::binary | std::ios::ate);
if ( !vtxFile ) return false;
std::streamsize vtxSize = vtxFile.tellg();
vtxFile.seekg(0, std::ios::beg);
uf::stl::vector<uint8_t> vtxBuffer(vtxSize);
vtxFile.read((char*)vtxBuffer.data(), vtxSize);
uf::stl::vector<uint8_t> vtxBuffer;
if ( !uf::io::readAsBuffer(vtxBuffer, vtxPath)) return false;
const impl::vtxHeader_t* vtxHdr = (const impl::vtxHeader_t*)vtxBuffer.data();
if ( vtxHdr->checksum != mdlHdr->checksum ) {
@ -239,19 +247,19 @@ bool ext::valve::loadMdl( pod::Graph& graph, const uf::stl::string& filename ) {
const auto& srcVert = vvdVertices[originalVvdID];
vert.position = impl::convertPos( srcVert.m_vecPosition, 1.0f );
vert.normal = uf::vector::normalize( impl::convertPos( srcVert.m_vecNormal ) );
vert.position = impl::convertPos( srcVert.m_vecPosition );
vert.normal = uf::vector::normalize( impl::convertPos( srcVert.m_vecNormal, 1.0f ) );
vert.uv = srcVert.m_vecTexCoord;
vert.color = {1.0f, 1.0f, 1.0f, 1.0f};
vert.joints.x = srcVert.m_BoneWeights.numbones > 0 ? std::max<int8_t>(0, srcVert.m_BoneWeights.bone[0]) : 0;
vert.joints.y = srcVert.m_BoneWeights.numbones > 1 ? std::max<int8_t>(0, srcVert.m_BoneWeights.bone[1]) : 0;
vert.joints.z = srcVert.m_BoneWeights.numbones > 2 ? std::max<int8_t>(0, srcVert.m_BoneWeights.bone[2]) : 0;
vert.joints.w = 0;
vert.joints.y = srcVert.m_BoneWeights.numbones > 1 ? std::max<int8_t>(0, srcVert.m_BoneWeights.bone[1]) : 0;
vert.joints.z = srcVert.m_BoneWeights.numbones > 2 ? std::max<int8_t>(0, srcVert.m_BoneWeights.bone[2]) : 0;
vert.joints.w = 0;
vert.weights.x = srcVert.m_BoneWeights.numbones > 0 ? srcVert.m_BoneWeights.weight[0] : 1.0f;
vert.weights.y = srcVert.m_BoneWeights.numbones > 1 ? srcVert.m_BoneWeights.weight[1] : 0.0f;
vert.weights.z = srcVert.m_BoneWeights.numbones > 2 ? srcVert.m_BoneWeights.weight[2] : 0.0f;
vert.weights.w = 0.0f;
vert.weights.x = srcVert.m_BoneWeights.numbones > 0 ? srcVert.m_BoneWeights.weight[0] : 1.0f;
vert.weights.y = srcVert.m_BoneWeights.numbones > 1 ? srcVert.m_BoneWeights.weight[1] : 0.0f;
vert.weights.z = srcVert.m_BoneWeights.numbones > 2 ? srcVert.m_BoneWeights.weight[2] : 0.0f;
vert.weights.w = 0.0f;
// Bounds calculation
auto& bounds = meshlet.primitive.instance.bounds;
@ -275,6 +283,7 @@ bool ext::valve::loadMdl( pod::Graph& graph, const uf::stl::string& filename ) {
for ( ; materialID < graph.materials.size(); ++materialID ) {
if ( graph.materials[materialID] == matName ) break;
}
} else {
// does not exist, register
size_t imageID = graph.images.size();

311
engine/src/ext/valve/vpk.cpp Normal file
View File

@ -0,0 +1,311 @@
#include <uf/ext/valve/bsp.h>
#include <uf/ext/valve/mdl.h>
#include <uf/ext/valve/vtf.h>
#include <uf/ext/valve/vpk.h>
#include <uf/ext/valve/common.h>
#include <uf/utils/io/vfs.h>
#include <uf/utils/userdata/userdata.h>
#include <uf/engine/asset/asset.h>
#if defined(_WIN32)
#include <windows.h>
#else
#include <cstdlib>
#endif
#include <fstream>
namespace impl {
uf::stl::vector<uf::stl::string> getSteamLibraries() {
uf::stl::vector<uf::stl::string> libraries;
uf::stl::string steamPath = "";
#if defined(_WIN32)
HKEY hKey;
if ( RegOpenKeyExA(HKEY_CURRENT_USER, "Software\\Valve\\Steam", 0, KEY_READ, &hKey) == ERROR_SUCCESS ) {
char pathBuf[MAX_PATH];
DWORD bufferSize = sizeof(pathBuf);
if ( RegQueryValueExA(hKey, "SteamPath", nullptr, nullptr, (LPBYTE)pathBuf, &bufferSize) == ERROR_SUCCESS ) {
steamPath = pathBuf;
}
RegCloseKey(hKey);
}
#else
const char* home = std::getenv("HOME");
if (home) {
#if defined(__APPLE__)
steamPath = uf::stl::string(home) + "/Library/Application Support/Steam";
#else
steamPath = uf::stl::string(home) + "/.steam/steam"; // or ~/.local/share/Steam
#endif
}
#endif
if ( steamPath.empty() ) {
UF_MSG_WARNING("Could not locate base Steam installation.");
return libraries;
}
libraries.emplace_back(steamPath + "/steamapps/common");
uf::stl::string vdfPath = steamPath + "/steamapps/libraryfolders.vdf";
std::ifstream file(vdfPath);
if ( !file.is_open() ) return libraries;
uf::stl::string line;
while ( std::getline(file, line) ) {
uf::stl::string key, value;
if ( !impl::parseKeyValue(line, key, value) ) continue;
if ( key != "path" ) continue;
std::replace( value.begin(), value.end(), '\\', '/' );
value = uf::string::replace(value, "//", "/");
libraries.emplace_back(value + "/steamapps/common");
}
return libraries;
}
}
pod::Mount ext::valve::createVpkMount( const uf::stl::string& uri, int priority ) {
struct VpkMountState {
uf::stl::string path;
uf::stl::string name;
bool loaded = false;
pod::VpkArchive* archive = NULL;
pod::VpkArchive* get() {
if ( !loaded ) {
loaded = true;
archive = &uf::asset::add<pod::VpkArchive>(name);
ext::valve::loadVpk( *archive, path );
}
return archive;
}
};
uf::stl::string prefix;
uf::stl::string path;
uf::io::splitUri( uri, prefix, path );
pod::Mount mount;
mount.prefix = prefix;
mount.path = path;
mount.priority = priority;
mount.userdata = uf::pointeredUserdata::create<VpkMountState>();
auto& state = uf::pointeredUserdata::get<VpkMountState>( mount.userdata );
auto* userdata = &state;
auto libraries = impl::getSteamLibraries();
for ( const auto& lib : libraries ) {
uf::stl::string fullPath = lib + "/" + path;
if ( !uf::io::exists(fullPath) ) {
continue;
}
state.path = fullPath;
break;
}
if ( state.path.empty() ) {
UF_MSG_WARNING("Could not resolve VPK path: {}", path);
state.path = path;
} else {
UF_MSG_DEBUG("Mounted VPK: {}", state.path);
}
state.name = ::fmt::format("vpk://{}", path);
mount.exists = [userdata](const uf::stl::string& p) {
auto ptr = userdata->get();
return ptr && ptr->files.count( uf::string::lowercase( p ) ) > 0;
};
mount.size = [userdata](const uf::stl::string& p) -> size_t {
auto ptr = userdata->get();
if ( !ptr ) return 0;
auto it = ptr->files.find( uf::string::lowercase( p ) );
return it != ptr->files.end() ? it->second.metadata.entryLength : 0;
};
mount.mtime = [](const uf::stl::string&) -> size_t { return 0; },
mount.read = [userdata](const uf::stl::string& p, uf::stl::vector<uint8_t>& buffer) {
auto ptr = userdata->get();
return ptr && ext::valve::readVpk(*ptr, uf::string::lowercase( p ), buffer);
};
mount.readRange = [userdata](const uf::stl::string& p, size_t start, size_t len, uf::stl::vector<uint8_t>& buffer) {
auto ptr = userdata->get();
return ptr && ext::valve::readVpkRange(*ptr, uf::string::lowercase( p ), start, len, buffer);
};
return mount;
}
bool ext::valve::loadVpk( pod::VpkArchive& vpk, const uf::stl::string& path ) {
uf::stl::vector<uint8_t> buffer;
if ( !uf::io::readAsBuffer(buffer, path) ) return false;
size_t dirPos = path.find("_dir.vpk");
if ( dirPos == uf::stl::string::npos ) return false;
vpk.basePath = path.substr(0, dirPos);
size_t offset = 0;
auto readBytes = [&](void* dest, size_t len) -> bool {
if (offset + len > buffer.size()) return false;
std::memcpy(dest, buffer.data() + offset, len);
offset += len;
return true;
};
auto readString = [&]() -> uf::stl::string {
uf::stl::string str;
while (offset < buffer.size()) {
char c = (char)buffer[offset++];
if (c == '\0') break;
str += c;
}
return str;
};
uint32_t signature;
if ( !readBytes(&signature, 4) || signature != 0x55aa1234 ) return false;
uint32_t version, treeSize;
if ( !readBytes(&version, 4) || !readBytes(&treeSize, 4) ) return false;
// skip for now
if ( version == 2 ) offset += 16;
uint32_t headerSize = (version == 1) ? 12 : 28;
while ( true ) {
uf::stl::string ext = readString();
if ( ext.empty() ) break;
while ( true ) {
uf::stl::string dir = readString();
if ( dir.empty() ) break;
while ( true ) {
uf::stl::string name = readString();
if ( name.empty() ) break;
// construct path
uf::stl::string fullPath = dir == " " ? "" : (dir + "/");
fullPath += name + "." + ext;
std::replace( fullPath.begin(), fullPath.end(), '\\', '/' );
std::transform( fullPath.begin(), fullPath.end(), fullPath.begin(), ::tolower );
// read data
auto& entry = vpk.files[fullPath];
if ( !readBytes(&entry.metadata, sizeof(pod::VpkData)) ) return false;
if ( entry.metadata.preloadBytes > 0 ) {
entry.preloadData.resize(entry.metadata.preloadBytes);
if ( !readBytes(entry.preloadData.data(), entry.metadata.preloadBytes) ) return false;
}
entry.dirFileOffset = headerSize + treeSize;
}
}
}
return true;
}
bool ext::valve::readVpk( const pod::VpkArchive& vpk, const uf::stl::string& path, uf::stl::vector<uint8_t>& buffer ) {
auto it = vpk.files.find( path );
if ( it == vpk.files.end() ) return false;
const auto& entry = it->second;
buffer.resize( entry.metadata.preloadBytes + entry.metadata.entryLength );
// copy preload
if ( entry.metadata.preloadBytes > 0 ) {
memcpy(buffer.data(), entry.preloadData.data(), entry.metadata.preloadBytes);
}
// read payload from disk
if ( entry.metadata.entryLength > 0 ) {
uf::stl::string archivePath;
size_t fileOffset = 0;
if ( entry.metadata.archiveIndex == 0x7FFF ) {
// data is embedded inside _dir.vpk
archivePath = vpk.basePath + "_dir.vpk";
fileOffset = entry.dirFileOffset + entry.metadata.entryOffset;
} else {
// data is in a chunk file
archivePath = ::fmt::format("{}_{:03d}.vpk", vpk.basePath, entry.metadata.archiveIndex);
fileOffset = entry.metadata.entryOffset;
}
std::ifstream file(archivePath, std::ios::binary);
if ( file ) {
file.seekg(fileOffset, std::ios::beg);
file.read((char*)(buffer.data() + entry.metadata.preloadBytes), entry.metadata.entryLength);
} else {
buffer.clear();
UF_MSG_ERROR("Failed to open VPK chunk: {}", archivePath);
return false;
}
}
return true;
}
size_t ext::valve::mountVpk( const uf::stl::string& uri ) {
return uf::vfs::mount( ext::valve::createVpkMount( ::fmt::format( "valve://{}", uri ), 10 ) );
}
bool ext::valve::readVpkRange( const pod::VpkArchive& vpk, const uf::stl::string& path, size_t start, size_t len, uf::stl::vector<uint8_t>& buffer ) {
auto it = vpk.files.find( path );
if ( it == vpk.files.end() ) return false;
const auto& entry = it->second;
size_t totalSize = entry.metadata.preloadBytes + entry.metadata.entryLength;
if ( start >= totalSize ) {
buffer.clear();
return true;
}
len = std::min(len, totalSize - start);
buffer.resize(len);
size_t bufferOffset = 0;
if ( start < entry.metadata.preloadBytes ) {
size_t preloadRead = std::min(len, (size_t)entry.metadata.preloadBytes - start);
memcpy(buffer.data(), entry.preloadData.data() + start, preloadRead);
bufferOffset += preloadRead;
start += preloadRead;
len -= preloadRead;
}
if ( len > 0 ) {
size_t diskStart = start - entry.metadata.preloadBytes;
uf::stl::string archivePath;
size_t fileOffset = 0;
if ( entry.metadata.archiveIndex == 0x7FFF ) {
archivePath = vpk.basePath + "_dir.vpk";
fileOffset = entry.dirFileOffset + entry.metadata.entryOffset;
} else {
archivePath = ::fmt::format("{}_{:03d}.vpk", vpk.basePath, entry.metadata.archiveIndex);
fileOffset = entry.metadata.entryOffset;
}
std::ifstream file(archivePath, std::ios::binary);
if ( file ) {
file.seekg(fileOffset + diskStart, std::ios::beg);
file.read((char*)(buffer.data() + bufferOffset), len);
} else {
buffer.clear();
UF_MSG_ERROR("Failed to open VPK chunk for ranged read: {}", archivePath);
return false;
}
}
return true;
}

25
engine/src/ext/valve/vtf.cpp
View File

@ -1,6 +1,7 @@
#include <uf/ext/valve/bsp.h>
#include <uf/ext/valve/mdl.h>
#include <uf/ext/valve/vtf.h>
#include <uf/ext/valve/vpk.h>
#include <uf/ext/valve/common.h>
namespace impl {
@ -115,10 +116,11 @@ namespace impl {
}
bool ext::valve::loadVmt( uf::Serializer& dict, const uf::stl::string& filename ) {
std::ifstream file( filename );
if ( !file ) return false;
uf::stl::string content;
if ( !uf::io::readAsString(content, filename) ) return false;
uf::stl::string line;
std::istringstream file(content);
while ( std::getline(file, line) ) {
uf::stl::string comment = "";
size_t commentPos = line.find("//"); // strip comments
@ -130,23 +132,22 @@ bool ext::valve::loadVmt( uf::Serializer& dict, const uf::stl::string& filename
uf::stl::string key, value;
if ( impl::parseKeyValue(line, key, value) ) {
std::transform(key.begin(), key.end(), key.begin(), ::tolower);
std::transform(value.begin(), value.end(), value.begin(), ::tolower);
std::replace(value.begin(), value.end(), '\\', '/');
dict[key] = value;
if ( key == "include" ) {
ext::valve::loadVmt( dict, value );
} else {
dict[key] = value;
}
}
}
return true;
}
bool ext::valve::loadVtf( pod::Image& image, const uf::stl::string& filename ) {
std::ifstream file( filename, std::ios::binary | std::ios::ate );
if ( !file ) return false;
std::streamsize size = file.tellg();
file.seekg(0, std::ios::beg);
uf::stl::vector<uint8_t> buffer(size);
file.read((char*)buffer.data(), size);
uf::stl::vector<uint8_t> buffer;
if ( !uf::io::readAsBuffer(buffer, filename) ) return false;
const impl::VTFHeader* header = (const impl::VTFHeader*)(buffer.data());
if ( strncmp(header->signature, "VTF", 3) != 0 ) return false;
@ -172,7 +173,7 @@ bool ext::valve::loadVtf( pod::Image& image, const uf::stl::string& filename ) {
const uint8_t* data = buffer.data() + offset;
image.size = { header->width, header->height };
image.channels = 4;
image.bpp = 8;
image.bpp = 8 * 4;
image.pixels.resize( header->width * header->height * 4 );
int dataSize = 0;

12
engine/src/ext/vulkan/shader.cpp
View File

@ -293,15 +293,9 @@ void ext::vulkan::Shader::initialize( ext::vulkan::Device& device, const uf::stl
ext::vulkan::Buffers::initialize( device );
uf::stl::string spirv;
{
std::ifstream is(this->filename = filename, std::ios::binary | std::ios::in | std::ios::ate);
if ( !is.is_open() ) UF_EXCEPTION("Error: Could not open shader file: {}", filename);
is.seekg(0, std::ios::end); spirv.reserve(is.tellg()); is.seekg(0, std::ios::beg);
spirv.assign((std::istreambuf_iterator<char>(is)), std::istreambuf_iterator<char>());
assert(spirv.size() > 0);
}
uf::io::readAsString( spirv, this->filename = filename );
if ( spirv.empty() ) UF_EXCEPTION("Error: Could not open shader file: {}", filename);
UF_ASSERT(spirv.size() > 0);
{

414
engine/src/ext/zlib/zlib.cpp
View File

@ -1,139 +1,333 @@
#include <uf/ext/zlib/zlib.h>
#if UF_USE_ZLIB
#include <uf/utils/io/file.h>
#include <uf/utils/io/vfs.h>
#include <uf/utils/userdata/userdata.h>
#include <cstring>
#include <algorithm>
#include <fstream>
#if UF_ENV_DREAMCAST
#include <zlib/zlib.h>
#else
#include <zlib.h>
#endif
size_t ext::zlib::bufferSize = 2048;
/*
uf::stl::vector<uint8_t> ext::zlib::compress( const void* data, size_t size ) {
uf::stl::vector<uint8_t> buffer;
// zlib struct
z_stream defstream;
defstream.zalloc = Z_NULL;
defstream.zfree = Z_NULL;
defstream.opaque = Z_NULL;
// setup "a" as the input and "b" as the compressed output
defstream.avail_in = (uInt) strlen(a)+1; // size of input, string + terminator
defstream.next_in = (Bytef*) a; // input char array
defstream.avail_out = (uInt) sizeof(b); // size of output
defstream.next_out = (Bytef*) b; // output char array
// the actual compression work.
deflateInit(&defstream, Z_BEST_COMPRESSION);
deflate(&defstream, Z_FINISH);
deflateEnd(&defstream);
size_t ext::zlib::bufferSize = 16384;
return buffer;
}
uf::stl::vector<uint8_t> ext::zlib::decompress( const void* data, size_t size ) {
uf::stl::vector<uint8_t> buffer;
return buffer;
}
*/
uf::stl::vector<uint8_t>& ext::zlib::decompressFromFile( uf::stl::vector<uint8_t>& buffer, const uf::stl::string& filename ) {
gzFile in = gzopen( filename.c_str(), "rb" );
if ( !in ) {
UF_MSG_ERROR("Zlib: failed to open file for read: {}", filename);
return buffer;
bool ext::zlib::decompressFromFile( uf::stl::vector<uint8_t>& buffer, const uf::stl::string& filename ) {
size_t fileSize = uf::vfs::size(filename);
if (fileSize == 0) {
UF_MSG_ERROR("Zlib: file not found or empty: {}", filename);
return false;
}
size_t read{};
uint8_t gzBuffer[ext::zlib::bufferSize];
while ( (read = gzread( in, gzBuffer, ext::zlib::bufferSize ) ) > 0 ) {
size_t s = buffer.size();
buffer.resize(s + read);
memcpy( &buffer[s], &gzBuffer[0], read );
z_stream strm{};
if (inflateInit2(&strm, 15 + 32) != Z_OK) return false;
size_t offset = 0;
uint8_t outBuffer[ext::zlib::bufferSize];
while (offset < fileSize) {
size_t bytesToRead = std::min(ext::zlib::bufferSize, fileSize - offset);
uf::stl::vector<uint8_t> temp;
if (!uf::vfs::readRange(filename, offset, bytesToRead, temp)) break;
strm.avail_in = (uInt)temp.size();
strm.next_in = temp.data();
offset += temp.size();
do {
strm.avail_out = sizeof(outBuffer);
strm.next_out = outBuffer;
int ret = inflate(&strm, Z_NO_FLUSH);
if (ret == Z_STREAM_ERROR || ret == Z_DATA_ERROR || ret == Z_MEM_ERROR) {
UF_MSG_ERROR("Zlib: inflate error on file: {}", filename);
inflateEnd(&strm);
return false;
}
size_t have = sizeof(outBuffer) - strm.avail_out;
buffer.insert(buffer.end(), outBuffer, outBuffer + have);
} while (strm.avail_out == 0);
}
gzclose( in );
return buffer;
}
size_t ext::zlib::compressToFile( const uf::stl::string& filename, const void* data, size_t size ) {
gzFile out = gzopen( filename.c_str(), "wb" );
if ( !out ) {
UF_MSG_ERROR("Zlib: failed to open file for write: {}", filename);
return 0;
}
gzwrite( out, data, size );
gzclose( out );
return uf::io::size( filename );
inflateEnd(&strm);
return true;
}
uf::stl::vector<uint8_t>& ext::zlib::decompressFromFile( uf::stl::vector<uint8_t>& buffer, const uf::stl::string& filename, size_t start, size_t len ) {
gzFile in = gzopen(filename.c_str(), "rb");
if ( !in ) {
UF_MSG_ERROR("Zlib: failed to open file for read: {}", filename);
return buffer;
bool ext::zlib::decompressFromFile( uf::stl::vector<uint8_t>& buffer, const uf::stl::string& filename, size_t start, size_t len ) {
size_t fileSize = uf::vfs::size(filename);
if (fileSize == 0) return false;
z_stream strm{};
if (inflateInit2(&strm, 15 + 32) != Z_OK) return false;
size_t offset = 0;
size_t uncompressedOffset = 0;
uint8_t outBuffer[ext::zlib::bufferSize];
while (offset < fileSize) {
size_t bytesToRead = std::min(ext::zlib::bufferSize, fileSize - offset);
uf::stl::vector<uint8_t> temp;
if (!uf::vfs::readRange(filename, offset, bytesToRead, temp)) break;
strm.avail_in = (uInt)temp.size();
strm.next_in = temp.data();
offset += temp.size();
do {
strm.avail_out = sizeof(outBuffer);
strm.next_out = outBuffer;
int ret = inflate(&strm, Z_NO_FLUSH);
if (ret == Z_STREAM_ERROR || ret == Z_DATA_ERROR || ret == Z_MEM_ERROR) {
inflateEnd(&strm);
return false;
}
size_t have = sizeof(outBuffer) - strm.avail_out;
// Calculate if this chunk overlaps with our requested range
size_t chunkStart = uncompressedOffset;
size_t chunkEnd = uncompressedOffset + have;
if (chunkEnd > start && chunkStart < start + len) {
size_t copyStart = (chunkStart < start) ? (start - chunkStart) : 0;
size_t copyLen = std::min(have - copyStart, (start + len) - (chunkStart + copyStart));
buffer.insert(buffer.end(), outBuffer + copyStart, outBuffer + copyStart + copyLen);
}
uncompressedOffset += have;
// If we've reached the end of the requested length, we can abort early!
if (uncompressedOffset >= start + len) {
inflateEnd(&strm);
return true;
}
} while (strm.avail_out == 0);
}
// Seek to the requested uncompressed offset
if (gzseek(in, static_cast<z_off_t>(start), SEEK_SET) == -1) {
UF_MSG_ERROR("Zlib: failed to seek to position {} in file {}", start, filename);
gzclose(in);
return buffer;
}
buffer.resize(len);
int bytesRead = gzread(in, buffer.data(), static_cast<unsigned int>(len));
if (bytesRead < 0) {
int errnum;
const char* errMsg = gzerror(in, &errnum);
UF_MSG_ERROR("Zlib read error: {}", errMsg ? errMsg : "unknown error");
buffer.clear();
} else {
buffer.resize(static_cast<size_t>(bytesRead)); // Adjust size in case EOF occurs early
}
gzclose(in);
return buffer;
inflateEnd(&strm);
return true;
}
uf::stl::vector<uint8_t>& ext::zlib::decompressFromFile( uf::stl::vector<uint8_t>& buffer, const uf::stl::string& filename, const uf::stl::vector<pod::Range>& ranges ) {
if ( ranges.empty() ) {
return buffer;
}
bool ext::zlib::decompressFromFile( uf::stl::vector<uint8_t>& buffer, const uf::stl::string& filename, const uf::stl::vector<pod::Range>& ranges ) {
if ( ranges.empty() ) return false;
// ensure they're ordered
uf::stl::vector<pod::Range> sortedRanges = ranges;
std::sort( sortedRanges.begin(), sortedRanges.end(), [](const pod::Range& a, const pod::Range& b) { return a.start < b.start; } );
gzFile in = gzopen(filename.c_str(), "rb");
if ( !in ) {
UF_MSG_ERROR("Zlib: failed to open file for read: {}", filename);
return buffer;
size_t fileSize = uf::vfs::size(filename);
if (fileSize == 0) return false;
z_stream strm{};
if (inflateInit2(&strm, 15 + 32) != Z_OK) return false;
size_t offset = 0;
size_t uncompressedOffset = 0;
uint8_t outBuffer[ext::zlib::bufferSize];
size_t currentRangeIdx = 0;
while (offset < fileSize && currentRangeIdx < sortedRanges.size()) {
size_t bytesToRead = std::min(ext::zlib::bufferSize, fileSize - offset);
uf::stl::vector<uint8_t> temp;
if (!uf::vfs::readRange(filename, offset, bytesToRead, temp)) break;
strm.avail_in = (uInt)temp.size();
strm.next_in = temp.data();
offset += temp.size();
do {
strm.avail_out = sizeof(outBuffer);
strm.next_out = outBuffer;
int ret = inflate(&strm, Z_NO_FLUSH);
if (ret < 0 && ret != Z_BUF_ERROR) {
inflateEnd(&strm);
return false;
}
size_t have = sizeof(outBuffer) - strm.avail_out;
size_t chunkStart = uncompressedOffset;
size_t chunkEnd = uncompressedOffset + have;
// Check all remaining ranges against this chunk
for (size_t i = currentRangeIdx; i < sortedRanges.size(); ++i) {
const auto& r = sortedRanges[i];
if (chunkEnd > r.start && chunkStart < r.start + r.len) {
size_t copyStart = (chunkStart < r.start) ? (r.start - chunkStart) : 0;
size_t copyLen = std::min(have - copyStart, (r.start + r.len) - (chunkStart + copyStart));
buffer.insert(buffer.end(), outBuffer + copyStart, outBuffer + copyStart + copyLen);
}
if (chunkEnd >= r.start + r.len) {
currentRangeIdx = i + 1; // Move past completed ranges
}
}
uncompressedOffset += have;
} while (strm.avail_out == 0);
}
for ( const auto& r : sortedRanges ) {
if ( gzseek(in, static_cast<z_off_t>(r.start), SEEK_SET) == -1 ) {
UF_MSG_ERROR("Zlib: failed to seek to position {} in file {}", r.start, filename);
gzclose(in);
return buffer;
}
size_t oldSize = buffer.size();
buffer.resize(oldSize + r.len);
int bytesRead = gzread(in, buffer.data() + oldSize, static_cast<unsigned int>(r.len));
if ( bytesRead < 0 ) {
int errnum;
const char* errMsg = gzerror(in, &errnum);
UF_MSG_ERROR("Zlib read error: {}", errMsg ? errMsg : "unknown error");
gzclose(in);
return buffer;
}
// In case EOF ended early
buffer.resize(oldSize + static_cast<size_t>(bytesRead));
}
gzclose(in);
return buffer;
inflateEnd(&strm);
return true;
}
bool ext::zlib::decompressFromMemory( uf::stl::vector<uint8_t>& dst, const void* src, size_t size, size_t usize ) {
if (size == 0) return false;
z_stream strm{};
if (inflateInit2(&strm, 15 + 32) != Z_OK) return false;
strm.avail_in = (uInt)size;
strm.next_in = (Bytef*)src;
dst.resize(usize);
strm.avail_out = (uInt)usize;
strm.next_out = dst.data();
int ret = inflate(&strm, Z_FINISH);
inflateEnd(&strm);
return (ret == Z_STREAM_END || ret == Z_OK);
}
size_t ext::zlib::compressToFile( const uf::stl::string& filename, const void* data, size_t size ) {
z_stream strm{};
// 31 means gzip format
if (deflateInit2(&strm, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 31, 8, Z_DEFAULT_STRATEGY) != Z_OK) return 0;
strm.avail_in = (uInt)size;
strm.next_in = (Bytef*)data;
uf::stl::vector<uint8_t> compressedData;
uint8_t outBuffer[ext::zlib::bufferSize];
int ret;
do {
strm.avail_out = sizeof(outBuffer);
strm.next_out = outBuffer;
ret = deflate(&strm, Z_FINISH);
size_t have = sizeof(outBuffer) - strm.avail_out;
compressedData.insert(compressedData.end(), outBuffer, outBuffer + have);
} while (strm.avail_out == 0);
deflateEnd(&strm);
return uf::vfs::write( filename, compressedData.data(), compressedData.size() );
}
bool ext::zlib::directory( const uf::stl::vector<uint8_t>& buffer, uf::stl::unordered_map<uf::stl::string, pod::ZipEntry>& entries ) {
if ( buffer.size() < 22 ) return false;
int eocdOffset = -1;
for ( int i = (int)buffer.size() - 22; i >= 0; --i ) {
if (buffer[i] == 0x50 && buffer[i+1] == 0x4b && buffer[i+2] == 0x05 && buffer[i+3] == 0x06) {
eocdOffset = i;
break;
}
}
if ( eocdOffset == -1 ) {
UF_MSG_ERROR("ZIP Parse: Could not find End of Central Directory!");
return false;
}
const uint8_t* eocd = buffer.data() + eocdOffset;
uint16_t numEntries = *(const uint16_t*)(eocd + 10);
uint32_t cdOffset = *(const uint32_t*)(eocd + 16);
if ( cdOffset >= buffer.size() ) return false;
size_t currentOffset = cdOffset;
for (uint16_t i = 0; i < numEntries; ++i) {
if (currentOffset + 46 > buffer.size()) break;
const uint8_t* cdHeader = buffer.data() + currentOffset;
if (*(const uint32_t*)cdHeader != 0x02014b50) break;
uint16_t compressionMethod = *(const uint16_t*)(cdHeader + 10);
uint32_t compressedSize = *(const uint32_t*)(cdHeader + 20);
uint32_t uncompressedSize = *(const uint32_t*)(cdHeader + 24);
uint16_t nameLen = *(const uint16_t*)(cdHeader + 28);
uint16_t extraLen = *(const uint16_t*)(cdHeader + 30);
uint16_t commentLen = *(const uint16_t*)(cdHeader + 32);
uint32_t localHeaderOffset = *(const uint32_t*)(cdHeader + 42);
uf::stl::string filename((const char*)(cdHeader + 46), nameLen);
std::replace(filename.begin(), filename.end(), '\\', '/');
std::transform(filename.begin(), filename.end(), filename.begin(), ::tolower);
if ( localHeaderOffset + 30 <= buffer.size() ) {
const uint8_t* localHeader = buffer.data() + localHeaderOffset;
if (*(const uint32_t*)localHeader == 0x04034b50) {
uint16_t localNameLen = *(const uint16_t*)(localHeader + 26);
uint16_t localExtraLen = *(const uint16_t*)(localHeader + 28);
size_t dataOffset = localHeaderOffset + 30 + localNameLen + localExtraLen;
entries[filename] = {
dataOffset,
compressedSize,
uncompressedSize,
compressionMethod
};
}
}
currentOffset += 46 + nameLen + extraLen + commentLen;
}
return true;
}
pod::Mount ext::zlib::createZipMount( const uf::stl::string& uri, uf::stl::vector<uint8_t>& buffer, int priority ) {
struct ZipMountState {
uf::stl::vector<uint8_t> buffer;
uf::stl::unordered_map<uf::stl::string, pod::ZipEntry> entries;
};
uf::stl::string prefix;
uf::stl::string path;
uf::io::splitUri( uri, prefix, path );
pod::Mount mount;
mount.prefix = prefix;
mount.path = path;
mount.priority = priority;
mount.userdata = uf::pointeredUserdata::create<ZipMountState>();
auto& state = uf::pointeredUserdata::get<ZipMountState>( mount.userdata );
auto* userdata = &state;
state.buffer = buffer;
ext::zlib::directory( state.buffer, state.entries );
mount.exists = [userdata](const uf::stl::string& file) -> bool {
return userdata->entries.find(file) != userdata->entries.end();
};
mount.size = [userdata](const uf::stl::string& file) -> size_t {
auto it = userdata->entries.find(file);
return (it != userdata->entries.end()) ? it->second.uncompressedSize : 0;
};
mount.read = [userdata](const uf::stl::string& file, uf::stl::vector<uint8_t>& buffer) -> bool {
auto it = userdata->entries.find(file);
if (it == userdata->entries.end()) return false;
const auto& entry = it->second;
const uint8_t* fileData = userdata->buffer.data() + entry.offset;
if (entry.compressionMethod == 0) {
buffer.assign(fileData, fileData + entry.uncompressedSize);
return true;
}
if (entry.compressionMethod == 8) {
return ext::zlib::decompressFromMemory(buffer, fileData, entry.compressedSize, entry.uncompressedSize);
}
return false;
};
return mount;
}
#endif

145
engine/src/utils/http/http.cpp
View File

@ -1,43 +1,134 @@
#include <uf/utils/http/http.h>
#include <uf/utils/io/vfs.h>
#include <uf/utils/io/file.h>
#if UF_USE_CURL
#include <curl/curl.h>
#endif
#include <iostream>
namespace impl {
pod::Mount createHttpMount( const uf::stl::string& uri, int priority ) {
uf::stl::string prefix;
uf::stl::string path;
uf::io::splitUri( uri, prefix, path );
return pod::Mount{
.prefix = prefix,
.path = path,
.priority = priority,
.exists = [prefix](const uf::stl::string& p) -> bool {
uf::stl::string url = prefix + p;
uf::Http http = uf::http::head(url);
return ( http.code >= 200 && http.code < 300 );
},
.size = [prefix](const uf::stl::string& p) -> size_t {
uf::stl::string url = prefix + p;
uf::Http http = uf::http::head(url);
return http.contentLength;
},
.mtime = [prefix](const uf::stl::string& p) -> size_t {
uf::stl::string url = prefix + p;
uf::Http http = uf::http::head(url);
return http.mtime;
},
.read = [prefix](const uf::stl::string& p, uf::stl::vector<uint8_t>& buffer) -> bool {
uf::stl::string url = prefix + p;
uf::Http http = uf::http::get(url);
if ( http.code < 200 || http.code >= 300 ) {
UF_MSG_ERROR("HTTP Error {} on GET {}", http.code, url);
return false;
}
buffer.assign(http.response.begin(), http.response.end());
return true;
},
.write = [prefix](const uf::stl::string& p, const void* buffer, size_t size) -> size_t {
uf::stl::string url = prefix + p;
uf::Http http = uf::http::post(url, buffer, size);
if ( http.code < 200 || http.code >= 300 ) {
UF_MSG_ERROR("HTTP Error {} on POST {}", http.code, url);
return 0;
}
return size;
},
};
}
}
namespace {
size_t writeFunction(void *ptr, size_t size, size_t nmemb, uf::stl::string* data) {
size_t writeFunction( void *ptr, size_t size, size_t nmemb, uf::stl::string* data ) {
data->append((char*) ptr, size * nmemb);
return size * nmemb;
}
uf::Http cURL( const uf::stl::string& url, const uf::stl::string& method = "GET", const void* data = nullptr, size_t size = 0 ) {
uf::Http http;
#if UF_USE_CURL
auto curl = curl_easy_init();
if ( !curl ) return http;
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L);
curl_easy_setopt(curl, CURLOPT_USERAGENT, "curl/7.42.0");
curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 50L);
curl_easy_setopt(curl, CURLOPT_TCP_KEEPALIVE, 1L);
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
curl_easy_setopt(curl, CURLOPT_FILETIME, 1L);
if ( method == "HEAD" ) {
curl_easy_setopt(curl, CURLOPT_NOBODY, 1L);
} else {
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeFunction);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &http.response);
}
if ( method == "POST" ) {
curl_easy_setopt(curl, CURLOPT_POST, 1L);
if (data) curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data);
curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, (long) size);
}
curl_easy_setopt(curl, CURLOPT_HEADERDATA, &http.header);
curl_easy_perform(curl);
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http.code);
curl_easy_getinfo(curl, CURLINFO_TOTAL_TIME, &http.elapsed);
curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &http.effective);
double cl;
if (curl_easy_getinfo(curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &cl) == CURLE_OK && cl >= 0.0) {
http.contentLength = (size_t)cl;
} else {
http.contentLength = 0;
}
long filetime = -1;
if (curl_easy_getinfo(curl, CURLINFO_FILETIME, &filetime) == CURLE_OK && filetime >= 0) {
http.mtime = (size_t)filetime;
} else {
http.mtime = 0;
}
curl_easy_cleanup(curl);
#endif
return http;
}
}
uf::Http uf::http::get( const uf::stl::string& url ) {
uf::Http http;
return ::cURL( url, "GET" );
}
#if UF_USE_CURL
auto curl = curl_easy_init();
if ( !curl ) return http;
uf::Http uf::http::head( const uf::stl::string& url ) {
return ::cURL( url, "HEAD" );
}
std::cout << curl << " " << url << std::endl;
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L);
// curl_easy_setopt(curl, CURLOPT_USERPWD, "user:pass");
curl_easy_setopt(curl, CURLOPT_USERAGENT, "curl/7.42.0");
curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 50L);
curl_easy_setopt(curl, CURLOPT_TCP_KEEPALIVE, 1L);
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeFunction);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &http.response);
curl_easy_setopt(curl, CURLOPT_HEADERDATA, &http.header);
uf::Http uf::http::post( const uf::stl::string& url, const void* data, size_t size ) {
return ::cURL( url, "POST", data, size );
}
curl_easy_perform(curl);
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http.code);
curl_easy_getinfo(curl, CURLINFO_TOTAL_TIME, &http.elapsed);
curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &http.effective);
curl_easy_cleanup(curl);
curl = NULL;
#endif
return http;
}
UF_VFS_MOUNT_CPP( impl::createHttpMount, "https://", -10 );
UF_VFS_MOUNT_CPP( impl::createHttpMount, "http://", -10 );

27
engine/src/utils/image/image.cpp
View File

@ -88,10 +88,11 @@ namespace impl {
}
bool uf::image::open( pod::Image& image, const uf::stl::string& _filename, bool flip ) {
// to-do: use preferred
uf::stl::string filename = uf::io::preferred( _filename );
if ( !uf::io::exists(filename) ) UF_EXCEPTION("IO error: file does not exist: {}", filename);
bool uf::image::open( pod::Image& image, const uf::stl::string& filename, bool flip ) {
uf::stl::vector<uint8_t> buffer;
if ( !uf::io::readAsBuffer( buffer, filename ) ) {
return false;
}
#if UF_USE_OPENGL_GLDC
auto extension = uf::io::extension( filename );
if ( extension != "dtex" ) UF_MSG_WARNING("non-dtex loading is highly discouraged on this platform: {}", filename);
@ -110,12 +111,14 @@ bool uf::image::open( pod::Image& image, const uf::stl::string& _filename, bool
uint32_t size;
} header;
FILE* file = NULL;
file = fopen(filename.c_str(), "rb");
fread(&header, sizeof(header), 1, file);
uf::stl::vector<uint8_t> buffer;
if ( !uf::io::readAsBuffer( buffer, filename ) ) UF_EXCEPTION("IO error: could not read DTEX: {}", filename);
if ( buffer.size() < sizeof(header) ) UF_EXCEPTION("IO error: DTEX file is too small to contain a header: {}", filename);
memcpy(&header, buffer.data(), sizeof(header));
if ( buffer.size() < sizeof(header) + header.size ) UF_EXCEPTION("IO error: DTEX file is truncated or corrupted: {}", filename);
image.pixels.resize(header.size);
fread(image.pixels.data(), header.size, 1, file);
fclose(file);
memcpy(image.pixels.data(), buffer.data() + sizeof(header), header.size);
bool twiddled = (header.type & (1 << 26)) < 1;
bool compressed = (header.type & (1 << 30)) > 0;
@ -150,12 +153,10 @@ bool uf::image::open( pod::Image& image, const uf::stl::string& _filename, bool
#endif
{
stbi_set_flip_vertically_on_load(flip);
uint8_t* buffer = stbi_load( filename.c_str(), &width, &height, &channelsDud, STBI_rgb_alpha );
uint8_t* stbi_pixels = stbi_load_from_memory( buffer.data(), buffer.size(), &width, &height, &channelsDud, STBI_rgb_alpha );
size_t len = width * height * channels;
image.pixels.resize( len );
memcpy( &image.pixels[0], buffer, len );
// image.pixels.insert( image.pixels.end(), (uint8_t*) buffer, buffer + len );
stbi_image_free(buffer);
memcpy( &image.pixels[0], stbi_pixels, len );
}
image.size.x = width;

283
engine/src/utils/io/file.cpp
View File

@ -1,4 +1,5 @@
#include <uf/utils/io/file.h>
#include <uf/utils/io/vfs.h>
#include <uf/utils/string/ext.h>
#include <uf/utils/string/hash.h>
@ -59,104 +60,84 @@ uf::stl::string uf::io::directory( const uf::stl::string& str ) {
return str.substr( 0, str.find_last_of('/') ) + "/";
}
size_t uf::io::size( const uf::stl::string& filename ) {
std::ifstream is(filename, std::ios::binary | std::ios::in | std::ios::ate);
if ( !is.is_open() ) return 0;
is.seekg(0, std::ios::end);
return is.tellg();
return uf::vfs::size( uf::io::resolveURI(filename) );
}
uf::stl::string uf::io::sanitize( const uf::stl::string& str, const uf::stl::string& _root ) {
// resolve %root% to hard root
uf::stl::string path = str;
uf::stl::string root = _root;
// append root to path
if ( path.find("%root%") == 0 ) {
path = uf::string::replace( path, "%root%", "" );
if ( root == "" ) root = uf::io::root;
uf::stl::string uf::io::normalize( const uf::stl::string& path ) {
uf::stl::string clean = path;
std::replace(clean.begin(), clean.end(), '\\', '/');
clean = uf::string::replace(clean, "/./", "/", false); // explicitly set as non-regex
size_t schemePos = clean.find("://");
uf::stl::string scheme = "";
if ( schemePos != uf::stl::string::npos ) {
scheme = clean.substr(0, schemePos + 3);
clean = clean.substr(schemePos + 3);
}
if ( path.find(root) == uf::stl::string::npos ) {
path = root + "/" + path;
}
// flatten all "/./"
{
uf::stl::string tmp;
while ( path != (tmp = uf::string::replace(path, "/\\/\\.\\//", "/")) ) {
path = tmp;
}
}
// flatten all "//"
{
uf::stl::string tmp;
while ( path != (tmp = uf::string::replace(path, "/\\/\\//", "/")) ) {
path = tmp;
}
}
return path;
clean = uf::string::replace(clean, "//", "/");
return scheme + clean;
}
// would just use readAsBuffer and convert to string, but that's double the memory cost
uf::stl::string& uf::io::readAsString( uf::stl::string& buffer, const uf::stl::string& _filename, const uf::stl::string& hash ) {
bool uf::io::readAsString( uf::stl::string& buffer, const uf::stl::string& _filename, const uf::stl::string& hash ) {
buffer.clear();
uf::stl::string filename = sanitize(_filename);
uf::stl::string filename = uf::io::normalize( _filename );
uf::stl::string extension = uf::io::extension( filename );
if ( extension == "gz" ) {
auto decompressed = uf::io::decompress( filename );
buffer.resize(decompressed.size());
buffer.assign(decompressed.begin(), decompressed.end());
} else {
std::ifstream is(filename, std::ios::binary | std::ios::in | std::ios::ate);
if ( !is.is_open() ) {
uf::stl::vector<uint8_t> tempBuffer;
if ( !uf::vfs::read( filename, tempBuffer ) ) {
UF_MSG_ERROR("Error: Could not open file: {}", filename);
return buffer;
return false;
}
is.seekg(0, std::ios::end); buffer.resize(is.tellg()); is.seekg(0, std::ios::beg);
buffer.assign((std::istreambuf_iterator<char>(is)), std::istreambuf_iterator<char>());
buffer.assign(tempBuffer.begin(), tempBuffer.end());
}
uf::stl::string expected = "";
if ( hash != "" && (expected = uf::string::sha256( buffer )) != hash ) {
UF_MSG_ERROR("Error: Hash mismatch for file {}; expecting {}, got {}", filename, hash, expected);
// should probably clear
}
return buffer;
return true;
}
uf::stl::vector<uint8_t>& uf::io::readAsBuffer( uf::stl::vector<uint8_t>& buffer, const uf::stl::string& _filename, const uf::stl::string& hash ) {
bool uf::io::readAsBuffer( uf::stl::vector<uint8_t>& buffer, const uf::stl::string& _filename, const uf::stl::string& hash ) {
buffer.clear();
uf::stl::string filename = sanitize(_filename);
uf::stl::string filename = uf::io::normalize( _filename );
uf::stl::string extension = uf::io::extension( filename );
if ( extension == "gz" || extension == "lz4" ) {
uf::io::decompress( buffer, filename );
} else {
std::ifstream is(filename, std::ios::binary | std::ios::in | std::ios::ate);
if ( !is.is_open() ) {
if ( !uf::vfs::read( filename, buffer ) ) {
UF_MSG_ERROR("Error: Could not open file: {}", filename);
return buffer;
return false;
}
is.seekg(0, std::ios::end); buffer.resize(is.tellg()); is.seekg(0, std::ios::beg);
buffer.assign((std::istreambuf_iterator<char>(is)), std::istreambuf_iterator<char>());
}
uf::stl::string expected = "";
if ( !hash.empty() && (expected = uf::string::sha256( buffer )) != hash ) {
UF_MSG_ERROR("Error: Hash mismatch for file {}; expecting {}, got {}", filename, hash, expected);
// should probably clear
}
return buffer;
return true;
}
uf::stl::vector<uint8_t>& uf::io::readAsBuffer( uf::stl::vector<uint8_t>& buffer, const uf::stl::string& _filename, size_t start, size_t len, const uf::stl::string& hash ) {
bool uf::io::readAsBuffer( uf::stl::vector<uint8_t>& buffer, const uf::stl::string& _filename, size_t start, size_t len, const uf::stl::string& hash ) {
buffer.clear();
uf::stl::string filename = sanitize(_filename);
uf::stl::string filename = uf::io::normalize(_filename);
uf::stl::string extension = uf::io::extension(filename);
if ( extension == "gz" || extension == "lz4" ) {
uf::io::decompress( buffer, filename, start, len );
} else {
std::ifstream is(filename, std::ios::binary);
if (!is.is_open()) {
UF_MSG_ERROR("Error: Could not open file: {}", filename);
return buffer;
if (!uf::vfs::readRange(filename, start, len, buffer)) {
UF_MSG_ERROR("Error: Could not open file range: {}", filename);
return false;
}
is.seekg(start, std::ios::beg);
buffer.resize(len);
is.read(reinterpret_cast<char*>(buffer.data()), len);
buffer.resize(static_cast<size_t>(is.gcount())); // adjust if EOF
}
uf::stl::string expected;
@ -164,36 +145,20 @@ uf::stl::vector<uint8_t>& uf::io::readAsBuffer( uf::stl::vector<uint8_t>& buffer
UF_MSG_ERROR("Error: Hash mismatch for file {}; expecting {}, got {}", filename, hash, expected);
// should probably clear
}
return buffer;
return true;
}
uf::stl::vector<uint8_t>& uf::io::readAsBuffer( uf::stl::vector<uint8_t>& buffer, const uf::stl::string& _filename, const uf::stl::vector<pod::Range>& ranges, const uf::stl::string& hash ) {
bool uf::io::readAsBuffer( uf::stl::vector<uint8_t>& buffer, const uf::stl::string& _filename, const uf::stl::vector<pod::Range>& ranges, const uf::stl::string& hash ) {
buffer.clear();
uf::stl::string filename = sanitize(_filename);
uf::stl::string filename = uf::io::normalize(_filename);
uf::stl::string extension = uf::io::extension(filename);
if ( extension == "gz" || extension == "lz4" ) {
uf::io::decompress( buffer, filename, ranges );
} else {
std::ifstream is(filename, std::ios::binary);
if (!is.is_open()) {
UF_MSG_ERROR("Error: Could not open file: {}", filename);
return buffer;
}
// Precompute total size to reserve memory
size_t totalBytes = 0;
for (const auto& r : ranges) {
totalBytes += r.len;
}
buffer.resize(totalBytes);
// Read each range
size_t currentOffset = 0;
for (const auto& r : ranges) {
is.seekg(r.start, std::ios::beg);
is.read(reinterpret_cast<char*>(buffer.data() + currentOffset), r.len);
currentOffset += static_cast<size_t>(is.gcount());
if (!uf::vfs::readRanges(filename, ranges, buffer)) {
UF_MSG_ERROR("Error: Could not open file ranges: {}", filename);
return false;
}
}
@ -202,41 +167,37 @@ uf::stl::vector<uint8_t>& uf::io::readAsBuffer( uf::stl::vector<uint8_t>& buffer
UF_MSG_ERROR("Error: Hash mismatch for file {}; expecting {}, got {}", filename, hash, expected);
// should probably clear
}
return buffer;
return true;
}
size_t uf::io::write( const uf::stl::string& filename, const void* buffer, size_t size ) {
uf::stl::string extension = uf::io::extension( filename );
if ( extension == "gz" || extension == "lz4" ) return uf::io::compress( filename, buffer, size );
std::ofstream output;
output.open( uf::io::sanitize( filename ), std::ios::binary);
output.write( (const char*) buffer, size );
output.close();
return size;
return uf::vfs::write( uf::io::resolveURI( filename ), buffer, size );
}
// indirection for different compression formats, currently only using zlib's gzFile shit
uf::stl::vector<uint8_t>& uf::io::decompress( uf::stl::vector<uint8_t>& buffer, const uf::stl::string& filename ) {
bool uf::io::decompress( uf::stl::vector<uint8_t>& buffer, const uf::stl::string& filename ) {
uf::stl::string extension = uf::io::extension( filename );
if ( extension == "gz" ) return ext::zlib::decompressFromFile( buffer, filename );
// if ( extension == "lz4" ) return ext::lz4::decompressFromFile( buffer, filename );
UF_MSG_ERROR("unsupported compression format requested: {}", extension);
return buffer;
return false;
}
uf::stl::vector<uint8_t>& uf::io::decompress( uf::stl::vector<uint8_t>& buffer, const uf::stl::string& filename, size_t start, size_t len ) {
bool uf::io::decompress( uf::stl::vector<uint8_t>& buffer, const uf::stl::string& filename, size_t start, size_t len ) {
uf::stl::string extension = uf::io::extension( filename );
if ( extension == "gz" ) return ext::zlib::decompressFromFile( buffer, filename, start, len );
// if ( extension == "lz4" ) return ext::lz4::decompressFromFile( buffer, filename, start, len );
UF_MSG_ERROR("unsupported compression format requested: {}", extension);
return buffer;
return false;
}
uf::stl::vector<uint8_t>& uf::io::decompress( uf::stl::vector<uint8_t>& buffer, const uf::stl::string& filename, const uf::stl::vector<pod::Range>& ranges ) {
bool uf::io::decompress( uf::stl::vector<uint8_t>& buffer, const uf::stl::string& filename, const uf::stl::vector<pod::Range>& ranges ) {
uf::stl::string extension = uf::io::extension( filename );
if ( extension == "gz" ) return ext::zlib::decompressFromFile( buffer, filename, ranges );
// if ( extension == "lz4" ) return ext::lz4::decompressFromFile( buffer, filename, ranges );
UF_MSG_ERROR("unsupported compression format requested: {}", extension);
return buffer;
return false;
}
size_t uf::io::compress( const uf::stl::string& filename, const void* buffer, size_t size ) {
uf::stl::string extension = uf::io::extension( filename );
@ -249,37 +210,23 @@ size_t uf::io::compress( const uf::stl::string& filename, const void* buffer, si
uf::stl::string uf::io::hash( const uf::stl::string& filename ) {
return uf::string::sha256( uf::io::readAsBuffer( filename ) );
}
bool uf::io::exists( const uf::stl::string& _filename ) {
#if UF_ENV_DREAMCAST
FILE* file = fopen(_filename.c_str(), "r");
if (file) {
fclose(file);
return true;
}
return false;
#else
uf::stl::string filename = sanitize(_filename);
static struct stat buffer;
return stat(filename.c_str(), &buffer) == 0;
#endif
bool uf::io::exists( const uf::stl::string& filename ) {
return uf::vfs::exists( uf::io::resolveURI(filename) );
}
size_t uf::io::mtime( const uf::stl::string& _filename ) {
uf::stl::string filename = sanitize(_filename);
static struct stat buffer;
if ( stat(filename.c_str(), &buffer) != 0 ) return 0;
return buffer.st_mtime;
size_t uf::io::mtime( const uf::stl::string& filename ) {
return uf::vfs::mtime( uf::io::resolveURI(filename) );
}
bool uf::io::mkdir( const uf::stl::string& _filename ) {
#if UF_ENV_DREAMCAST || UF_ENV_LINUX
return false;
#else
uf::stl::string filename = sanitize(_filename);
uf::stl::string filename = uf::io::normalize(_filename);
int status = ::mkdir(filename.c_str());
return status != -1;
#endif
}
uf::stl::string uf::io::assetType( const uf::stl::string _filename ) {
uf::stl::string uf::io::assetType( const uf::stl::string& _filename ) {
// remove .gz
uf::stl::string filename = uf::string::replace( _filename, ".gz", "" );
@ -302,36 +249,94 @@ uf::stl::string uf::io::assetType( const uf::stl::string _filename ) {
return "";
}
uf::stl::string uf::io::resolveURI( const uf::stl::string& filename, const uf::stl::string& _root ) {
uf::stl::string root = _root;
if ( filename.substr(0,8) == "https://" ) return filename;
const uf::stl::string extension = uf::io::extension(filename);
// just sanitize
if ( filename.find(uf::io::root) == 0 ) {
return uf::io::preferred( uf::io::sanitize( uf::io::filename( filename ), uf::io::directory( filename ) ) );
}
if ( filename.find("%root%") == 0 ) {
const uf::stl::string f = uf::string::replace( filename, "%root%", uf::io::root );
return uf::io::preferred( uf::io::sanitize( uf::io::filename( f ), uf::io::directory( f ) ) );
}
// if the filename contains an absolute path or if no root is provided
if ( filename[0] == '/' || root == "" ) {
const uf::stl::string assetType = uf::io::assetType( filename );
if ( filename[0] == '/' && filename[1] == '/' ) root = uf::io::root;
else if ( assetType != "" ) {
if ( assetType == "model" ) root = uf::io::root + "/models/";
else if ( assetType == "scene" ) root = uf::io::root + "/scenes/";
else if ( assetType == "entity" ) root = uf::io::root + "/entities/";
else if ( assetType == "texture" ) root = uf::io::root + "/textures/";
else if ( assetType == "audio" ) root = uf::io::root + "/audio/";
else if ( assetType == "shader" ) root = uf::io::root + "/shaders/";
else if ( assetType == "script" ) root = uf::io::root + "/scripts/";
// to-do: map to above
uf::stl::string uf::io::assetScheme( const uf::stl::string& _filename ) {
uf::stl::string filename = uf::string::replace( _filename, ".gz", "" );
uf::stl::string basename = uf::io::filename( filename );
uf::stl::string extension = uf::io::extension( filename );
else root = uf::io::root + "/" + assetType + "/";
if ( basename == "graph.json" ) return "mdl://";
if ( basename == "scene.json" ) return "scene://";
if ( extension == "json" ) return "ent://";
if ( extension == "png" || extension == "dtex" ) return "tex://";
if ( extension == "glb" || extension == "gltf" || extension == "graph" ) return "mdl://";
if ( extension == "ogg" || extension == "wav" ) return "snd://";
if ( extension == "spv" ) return "spv://";
if ( extension == "lua" ) return "lua://";
return "data://";
}
uf::stl::string uf::io::resolveURI( const uf::stl::string& _filename, const uf::stl::string& _root ) {
if ( _filename.length() >= 8 && _filename.substr(0,8) == "https://" ) return _filename;
uf::stl::string f = uf::io::normalize( _filename );
uf::stl::string root = uf::io::normalize( _root );
bool schemeResolved = f.find("://") != uf::stl::string::npos;
bool fAlreadyHasScheme = schemeResolved;
// process macros (only %root%)
if ( !schemeResolved && f.starts_with("%root%") ) {
f = f.substr(6);
root = "data://";
schemeResolved = true;
}
// explicit relative path
if ( !schemeResolved && f.length() >= 2 && f.substr(0, 2) == "./" ) {
f = f.substr(2);
if ( root.empty() ) schemeResolved = true;
}
// explicit absolute
if ( !fAlreadyHasScheme && f.length() > 0 && f[0] == '/' ) {
root = "";
}
// deduce scheme and apply
if ( !schemeResolved && f.substr(0, 3) != "../" ) {
if ( root.empty() ) {
uf::stl::string deducedScheme = uf::io::assetScheme( f );
if ( deducedScheme != "data://" ) {
root = deducedScheme;
schemeResolved = true;
} else {
root = deducedScheme;
}
}
}
return uf::io::preferred( uf::io::sanitize( filename, root ) );
// absolute system paths
bool isWindowsAbsolute = !fAlreadyHasScheme && f.length() >= 2 && std::isalpha(f[0]) && f[1] == ':';
bool isUnixAbsolute = !fAlreadyHasScheme && !schemeResolved && f.length() > 0 && f[0] == '/';
if ( isWindowsAbsolute || isUnixAbsolute ) {
root = "sys://";
if ( isUnixAbsolute ) f = f.substr(1); // might not be necessary because of the final normalization?
}
// apply root
if ( !fAlreadyHasScheme && !root.empty() ) {
if ( f.length() > 0 && f[0] == '/' ) f = f.substr(1);
// remove cringe like `tex://textures/`
for ( const auto& mount : uf::vfs::mounts ) {
if ( root != mount.prefix ) continue;
uf::stl::string target = mount.path;
if (target.back() == '/') target.pop_back();
target = target.substr(target.find_last_of('/') + 1) + "/";
if ( f.starts_with(target) ) f = f.substr(target.length());
break;
}
if ( root.back() != '/') root += "/";
f = root + f;
}
return uf::io::preferred( uf::io::normalize( f ) );
}
// attempts to coerce files into a preferred one if it exists

316
engine/src/utils/io/vfs.cpp Normal file
View File

@ -0,0 +1,316 @@
#include <uf/utils/io/vfs.h>
#include <uf/utils/io/file.h>
#include <uf/utils/math/hash.h>
#include <algorithm>
#include <sys/stat.h>
pod::Mount uf::vfs::createDiskMount( const uf::stl::string& uri, int priority) {
uf::stl::string prefix;
uf::stl::string path;
uf::io::splitUri( uri, prefix, path );
if ( !path.empty() && path.back() != '/' && path.back() != '\\' ) path += '/';
return pod::Mount{
.prefix = prefix,
.path = path,
.priority = priority,
.exists = [path](const uf::stl::string& file) -> bool {
uf::stl::string fullPath = path + file;
#if UF_ENV_DREAMCAST
FILE* file = fopen(fullPath.c_str(), "r");
if (file) {
fclose(file);
return true;
}
return false;
#else
static struct stat buffer;
return stat(fullPath.c_str(), &buffer) == 0;
#endif
},
.size = [path](const uf::stl::string& file) -> size_t {
uf::stl::string fullPath = path + file;
std::ifstream is(fullPath, std::ios::binary | std::ios::in | std::ios::ate);
if ( !is.is_open() ) return 0;
is.seekg(0, std::ios::end);
return is.tellg();
},
.mtime = [path](const uf::stl::string& file) -> size_t {
uf::stl::string fullPath = path + file;
static struct stat buffer;
if ( stat(fullPath.c_str(), &buffer) != 0 ) return 0;
return buffer.st_mtime;
},
.read = [path](const uf::stl::string& file, uf::stl::vector<uint8_t>& buffer) -> bool {
uf::stl::string fullPath = path + file;
std::ifstream is(fullPath, std::ios::binary | std::ios::ate);
if (!is.is_open()) return false;
size_t len = is.tellg();
is.seekg(0, std::ios::beg);
buffer.resize(len);
is.read((char*)buffer.data(), len);
return true;
},
.write = [path](const uf::stl::string& file, const void* buffer, size_t size) -> size_t {
uf::stl::string fullPath = path + file;
std::ofstream output(fullPath, std::ios::binary);
if (!output.is_open()) return 0;
output.write((const char*)buffer, size);
output.close();
return size;
},
.readRange = [path](const uf::stl::string& file, size_t start, size_t len, uf::stl::vector<uint8_t>& buffer) -> bool {
uf::stl::string fullPath = path + file;
std::ifstream is(fullPath, std::ios::binary);
if (!is.is_open()) return false;
is.seekg(start, std::ios::beg);
buffer.resize(len);
is.read((char*)buffer.data(), len);
buffer.resize(static_cast<size_t>(is.gcount())); // to-do: adjust if EOF hit early
return true;
},
.readRanges = [path](const uf::stl::string& file, const uf::stl::vector<pod::Range>& ranges, uf::stl::vector<uint8_t>& buffer) -> bool {
uf::stl::string fullPath = path + file;
std::ifstream is(fullPath, std::ios::binary);
if (!is.is_open()) return false;
size_t totalBytes = 0;
for (const auto& r : ranges) totalBytes += r.len;
buffer.resize(totalBytes);
size_t currentOffset = 0;
for (const auto& r : ranges) {
is.seekg(r.start, std::ios::beg);
is.read((char*)(buffer.data() + currentOffset), r.len);
currentOffset += static_cast<size_t>(is.gcount());
}
buffer.resize(currentOffset);
return true;
}
};
}
uf::stl::vector<pod::Mount> uf::vfs::mounts;
size_t uf::vfs::mount( const pod::Mount& mount ) {
// compute hash
size_t hash = {};
uf::hash( hash, mount.prefix, mount.path );
// already exists
for ( auto& m : mounts ) {
size_t hash2 = {};
uf::hash( hash2, m.prefix, m.path );
if ( hash == hash2 ) {
return hash;
}
}
// add mount
mounts.emplace_back(mount);
// resort
std::sort( mounts.begin(), mounts.end(), [](const pod::Mount& a, const pod::Mount& b) {
return a.priority > b.priority;
});
return hash;
}
bool uf::vfs::unmount( size_t hash ) {
auto it = std::remove_if( mounts.begin(), mounts.end(), [&](const pod::Mount& m) {
size_t hash2 = {};
uf::hash( hash2, m.prefix, m.path );
return hash == hash2;
});
if ( it != mounts.end() ) {
mounts.erase( it, mounts.end() );
return true;
}
return false;
}
bool uf::vfs::unmount( const uf::stl::string& prefix, const uf::stl::string& base ) {
uf::stl::string cleanBase = base;
if ( !cleanBase.empty() && cleanBase.back() != '/' && cleanBase.back() != '\\' ) cleanBase += '/';
size_t hash = {};
uf::hash( hash, prefix, cleanBase );
// erase by hash first
if ( uf::vfs::unmount( hash ) ) return true;
auto it = std::remove_if( mounts.begin(), mounts.end(), [&](const pod::Mount& m) {
return m.prefix == prefix && m.path == cleanBase;
});
if ( it != mounts.end() ) {
mounts.erase( it, mounts.end() );
return true;
}
return false;
}
bool uf::vfs::exists( const uf::stl::string& path ) {
uf::stl::string prefix, relative;
uf::io::splitUri(path, prefix, relative);
for ( const auto& mount : mounts ) {
if ( prefix.empty() && mount.priority < 0 ) continue;
if ( prefix.empty() || mount.prefix == prefix ) {
if ( mount.exists(relative) ) return true;
}
}
return false;
}
size_t uf::vfs::size( const uf::stl::string& path ) {
uf::stl::string prefix, relative;
uf::io::splitUri(path, prefix, relative);
for ( const auto& mount : mounts ) {
if ( prefix.empty() && mount.priority < 0 ) continue;
if ( prefix.empty() || mount.prefix == prefix ) {
if ( mount.exists(relative) ) return mount.size(relative);
}
}
return 0;
}
size_t uf::vfs::mtime( const uf::stl::string& path ) {
uf::stl::string prefix, relative;
uf::io::splitUri(path, prefix, relative);
for ( const auto& mount : mounts ) {
if ( prefix.empty() && mount.priority < 0 ) continue;
if ( prefix.empty() || mount.prefix == prefix ) {
if ( mount.exists(relative) ) return mount.mtime(relative);
}
}
return 0;
}
bool uf::vfs::read( const uf::stl::string& path, uf::stl::vector<uint8_t>& buffer ) {
uf::stl::string prefix, relative;
uf::io::splitUri(path, prefix, relative);
for ( const auto& mount : mounts ) {
if ( prefix.empty() && mount.priority < 0 ) continue;
if ( prefix.empty() || mount.prefix == prefix ) {
bool res = mount.exists( relative );
if ( mount.exists(relative) ) return mount.read(relative, buffer);
}
}
return false;
}
size_t uf::vfs::write( const uf::stl::string& path, const void* buffer, size_t size ) {
uf::stl::string prefix, relative;
uf::io::splitUri(path, prefix, relative);
for ( const auto& mount : mounts ) {
if ( prefix.empty() && mount.priority < 0 ) continue;
if ( prefix.empty() || mount.prefix == prefix ) {
if ( mount.write ) return mount.write(relative, buffer, size);
}
}
return 0;
}
size_t uf::vfs::write( const uf::stl::string& path, uf::stl::vector<uint8_t>& buffer ) {
return uf::vfs::write( path, buffer.data(), buffer.size() );
}
bool uf::vfs::readRange( const uf::stl::string& path, size_t start, size_t len, uf::stl::vector<uint8_t>& buffer ) {
uf::stl::string prefix, relative;
uf::io::splitUri(path, prefix, relative);
for ( const auto& mount : mounts ) {
if ( prefix.empty() && mount.priority < 0 ) continue;
if ( prefix.empty() || mount.prefix == prefix ) {
if ( !mount.exists(relative) ) continue;
if ( mount.readRange ) return mount.readRange(relative, start, len, buffer);
if ( !mount.read ) continue;
uf::stl::vector<uint8_t> fullBuffer;
if ( !mount.read(relative, fullBuffer) ) continue;
if ( start < fullBuffer.size() ) {
size_t actualLen = std::min(len, fullBuffer.size() - start);
buffer.assign(fullBuffer.begin() + start, fullBuffer.begin() + start + actualLen);
} else {
buffer.clear();
}
return true;
}
}
return false;
}
bool uf::vfs::readRanges( const uf::stl::string& path, const uf::stl::vector<pod::Range>& ranges, uf::stl::vector<uint8_t>& buffer ) {
uf::stl::string prefix, relative;
uf::io::splitUri(path, prefix, relative);
for ( const auto& mount : mounts ) {
if ( prefix.empty() && mount.priority < 0 ) continue;
if ( prefix.empty() || mount.prefix == prefix ) {
if ( !mount.exists(relative) ) continue;
if ( mount.readRanges ) return mount.readRanges(relative, ranges, buffer);
if ( !mount.read ) continue;
uf::stl::vector<uint8_t> fullBuffer;
if ( !mount.read(relative, fullBuffer) ) continue;
size_t totalBytes = 0;
for ( const auto& r : ranges ) {
if ( r.start < fullBuffer.size() ) {
totalBytes += std::min(r.len, fullBuffer.size() - r.start);
}
}
buffer.clear();
buffer.reserve(totalBytes);
for ( const auto& r : ranges ) {
if ( r.start < fullBuffer.size() ) {
size_t actualLen = std::min(r.len, fullBuffer.size() - r.start);
buffer.insert(buffer.end(), fullBuffer.begin() + r.start, fullBuffer.begin() + r.start + actualLen);
}
}
return true;
}
}
return false;
}
uf::stl::string uf::vfs::resolveBase( const uf::stl::string& path ) {
uf::stl::string prefix, relative;
uf::io::splitUri(path, prefix, relative);
for ( const auto& mount : mounts ) {
if ( prefix.empty() && mount.priority < 0 ) continue;
if ( prefix.empty() || mount.prefix == prefix ) {
uf::stl::string resolved = mount.path + relative;
return resolved;
}
}
return path;
}
UF_VFS_MOUNT_CPP( uf::vfs::createDiskMount, "mdl://./data/models", 100 );
UF_VFS_MOUNT_CPP( uf::vfs::createDiskMount, "scene://./data/scenes", 100 );
UF_VFS_MOUNT_CPP( uf::vfs::createDiskMount, "ent://./data/entities", 100 );
UF_VFS_MOUNT_CPP( uf::vfs::createDiskMount, "tex://./data/textures", 100 );
UF_VFS_MOUNT_CPP( uf::vfs::createDiskMount, "snd://./data/audio", 100 );
UF_VFS_MOUNT_CPP( uf::vfs::createDiskMount, "spv://./data/shaders", 100 );
UF_VFS_MOUNT_CPP( uf::vfs::createDiskMount, "lua://./data/scripts", 100 );
#if UF_USE_DREAMCAST
UF_VFS_MOUNT_CPP( uf::vfs::createDiskMount, "data:///cd/", 50 );
#else
UF_VFS_MOUNT_CPP( uf::vfs::createDiskMount, "data://./data", 50 );
#endif
UF_VFS_MOUNT_CPP( uf::vfs::createDiskMount, "", 0 );
UF_VFS_MOUNT_CPP( uf::vfs::createDiskMount, "sys://", 999 );

36
engine/src/utils/string/ext.cpp
View File

@ -83,35 +83,41 @@ uf::stl::vector<const char*> uf::string::cStrings( const uf::stl::vector<uf::stl
uf::stl::string uf::string::ltrim( const uf::stl::string& str ) {
auto s = str;
s.erase(s.begin(), std::find_if(s.begin(), s.end(), [](unsigned char ch) {
return !std::isspace(ch);
}));
return s;
s.erase(s.begin(), std::find_if(s.begin(), s.end(), [](unsigned char ch) {
return !std::isspace(ch);
}));
return s;
}
uf::stl::string uf::string::rtrim( const uf::stl::string& str ) {
auto s = str;
s.erase(std::find_if(s.rbegin(), s.rend(), [](unsigned char ch) {
return !std::isspace(ch);
}).base(), s.end());
return s;
s.erase(std::find_if(s.rbegin(), s.rend(), [](unsigned char ch) {
return !std::isspace(ch);
}).base(), s.end());
return s;
}
uf::stl::string uf::string::trim( const uf::stl::string& str ) {
return uf::string::rtrim( uf::string::ltrim( str ) );
}
uf::stl::string uf::string::replace( const uf::stl::string& string, const uf::stl::string& search, const uf::stl::string& replace ) {
return uf::string::replace( string, search, replace, uf::string::isRegex( search ) );
}
uf::stl::string uf::string::replace( const uf::stl::string& string, const uf::stl::string& search, const uf::stl::string& replace, bool regexp ) {
uf::stl::string result = string;
if ( uf::string::isRegex(search) ) {
if ( regexp ) {
std::regex regex(search.substr(1,search.length()-2));
result = std::regex_replace( string, regex, replace );
} else {
// to-do: replace all
size_t start_pos = string.find(search);
if( start_pos == uf::stl::string::npos ) return result;
result.replace(start_pos, search.length(), replace);
return std::regex_replace( string, regex, replace );
}
if ( search.empty() ) return result;
size_t start_pos = 0;
while ( (start_pos = result.find(search, start_pos)) != uf::stl::string::npos ) {
result.replace(start_pos, search.length(), replace);
start_pos += replace.length();
}
return result;
}
bool uf::string::contains( const uf::stl::string& string, const uf::stl::string& search ) {

4
engine/src/utils/text/glyph.cpp
View File

@ -2,9 +2,7 @@
#include <iostream>
#if UF_USE_FREETYPE
pod::FT_Glyph uf::glyph::initialize( const uf::stl::string& font ) {
return ext::freetype::initialize( font );
}
uint8_t* uf::glyph::generate( pod::Glyph& glyph, pod::FT_Glyph& g, uint64_t c, size_t size ) {
ext::freetype::setPixelSizes( g, size );
if ( !ext::freetype::load( g, c ) ) return NULL;

1
makefiles/default/arch
View File

@ -1 +0,0 @@
win64

1
makefiles/default/cc
View File

@ -1 +0,0 @@
gcc

1
makefiles/default/renderer
View File

@ -1 +0,0 @@
vulkan

0
program.sh Executable file → Normal file
View File