diff --git a/bin/data/config.json b/bin/data/config.json index cb9081ae..f7ace55c 100644 --- a/bin/data/config.json +++ b/bin/data/config.json @@ -1,9 +1,9 @@ { "engine": { "scenes": { - "start": "SourceEngine", + "start": "Dark", "lights": { "enabled": true, - "lightmaps": false, + "lightmaps": true, "max": 32, "shadows": { "enabled": false, @@ -307,7 +307,7 @@ }, "json": { "encoding": "msgpack", - "compression": "gz" + "compression": "lz4" }, "vall_e": { "enabled": true @@ -335,6 +335,9 @@ "Half-Life 2/hl2/hl2_sound_misc_dir.vpk", "Half-Life 2/hl2/hl2_misc_dir.vpk", "Counter-Strike Source/cstrike/cstrike_pak_dir.vpk" + ], + "games": [ + "SS2/" ] } }, diff --git a/bin/data/entities/model.json b/bin/data/entities/model.json index bf1c2934..9a14bfae 100644 --- a/bin/data/entities/model.json +++ b/bin/data/entities/model.json @@ -78,7 +78,7 @@ "stream": { "tag": "worldspawn", "player": "info_player_start", - "enabled": true, // "auto", + "enabled": false, // "auto", "radius": 64, "every": 1 } diff --git a/bin/data/scenes/dark/base_dark.json b/bin/data/scenes/dark/base_dark.json new file mode 100644 index 00000000..02edea48 --- /dev/null +++ b/bin/data/scenes/dark/base_dark.json @@ -0,0 +1,19 @@ +{ + "import": "/model.json", + "metadata": { + "graph": { + "renderer": { "separate": false }, + "exporter": { + "optimize": { "simplify": 0, "lods": true, "print": true } + }, + "baking": { "enabled": true }, + "tags": { + "worldspawn": { + "physics": { "type": "mesh", "static": true, "mass": 0 }, + "grid": { "size": [8,1,8], "epsilon": 0.001, "cleanup": true, "print": true, "clip": true }, + "unwrap mesh": true + } + } + } + } +} \ No newline at end of file diff --git a/bin/data/scenes/dark/dark.json b/bin/data/scenes/dark/dark.json new file mode 100644 index 00000000..a6f6fa6f --- /dev/null +++ b/bin/data/scenes/dark/dark.json @@ -0,0 +1,3 @@ +{ + "import": "./medsci1.json" +} \ No newline at end of file diff --git a/bin/data/scenes/dark/medsci1.json b/bin/data/scenes/dark/medsci1.json new file mode 100644 index 00000000..d373db89 --- /dev/null +++ b/bin/data/scenes/dark/medsci1.json @@ -0,0 +1,8 @@ +{ + "import": "./base_dark.json", + "assets": [ + { "filename": "game://Data/MEDSCI1.MIS" } + // { "filename": "./maps/medsci1.mis" } + // { "filename": "./maps/medsci1/graph.json" } + ] +} \ No newline at end of file diff --git a/bin/data/scenes/dark/player.json b/bin/data/scenes/dark/player.json new file mode 100644 index 00000000..bd1b7f5d --- /dev/null +++ b/bin/data/scenes/dark/player.json @@ -0,0 +1,10 @@ +{ + "import": "/player.json", + "assets": [ +// { "filename": "/gui/hud/hud.json", "delay": 0 } + ], + "transform": { + // "orientation": [ 0, 1, 0, 0 ] + } +// "metadata": { "physics": { "gravity": [ 0, 0, 0 ] } } +} \ No newline at end of file diff --git a/bin/data/scenes/dark/scene.json b/bin/data/scenes/dark/scene.json new file mode 100644 index 00000000..11a4158c --- /dev/null +++ b/bin/data/scenes/dark/scene.json @@ -0,0 +1,29 @@ +{ + "imports": [ + "/scene.json", + "./dark.json" + ], + "assets": [ + // { "filename": "./loading.json", "delay": 0.5 } + ], + "metadata": { + "light": { + "fog-": { + // "color": [ 0.1, 0.1, 0.1 ], + // "color": [ 0.2, 0.2, 0.2 ], + "color": [ 0.3, 0.3, 0.3 ], + "range": [ 64, 256 ], + "step scale": 4, + "absorbtion": 0.125, + "density": { + "threshold": 0.35, + "multiplier": 1.0, + "scale": 25.0, + "offset": [0.2, 0, 1], + "timescale": 32 + } + }, + "ambient": [ 0, 0, 0 ] + } + } +} \ No newline at end of file diff --git a/bin/data/shaders/display/deferred/comp/comp.h b/bin/data/shaders/display/deferred/comp/comp.h index 2499a0bb..3b8c8d37 100644 --- a/bin/data/shaders/display/deferred/comp/comp.h +++ b/bin/data/shaders/display/deferred/comp/comp.h @@ -180,6 +180,18 @@ void postProcess() { } } +/* + { + const Material material = materials[surface.instance.materialID >= materials.length() ? 0 : surface.instance.materialID]; + + if ( ( bool(ubo.settings.lighting.useLightmaps)) && validTextureIndex( surface.instance.lightmapID ) ) { + vec4 light = sampleTexture( surface.instance.lightmapID, surface.st.xy, 0.0 ); + outFragColor.rgb = light.rgb; + } else { + outFragColor.rgb = vec3(1,0,1); + } + } +*/ /* { const Material material = materials[surface.instance.materialID >= materials.length() ? 0 : surface.instance.materialID]; diff --git a/engine/inc/uf/engine/asset/asset.h b/engine/inc/uf/engine/asset/asset.h index 697e7c0f..780dd7c7 100644 --- a/engine/inc/uf/engine/asset/asset.h +++ b/engine/inc/uf/engine/asset/asset.h @@ -32,7 +32,7 @@ namespace uf { // URL or file path void UF_API processQueue(); - void UF_API processIO(); + void UF_API processIO(); void UF_API cache( const uf::asset::callback_t&, const uf::asset::Payload& ); void UF_API load( const uf::asset::callback_t&, const uf::asset::Payload& ); diff --git a/engine/inc/uf/engine/graph/graph.h b/engine/inc/uf/engine/graph/graph.h index 0ca0a461..2b2f5d59 100644 --- a/engine/inc/uf/engine/graph/graph.h +++ b/engine/inc/uf/engine/graph/graph.h @@ -209,6 +209,7 @@ namespace uf { } pod::Graph& UF_API convert( uf::Object&, bool = false ); // converts an object into a graph + void UF_API preprocess( pod::Graph&, const uf::Serializer& = ext::json::null(), const uf::stl::string& = "" ); // applies pre-processing for format importing void UF_API postprocess( pod::Graph& ); // applies post-processing for format importing void UF_API import( pod::Graph::Storage& to, pod::Graph::Storage& from, bool move = true ); // moves storage from one to the other uf::stl::string UF_API save( const pod::Graph&, const uf::stl::string& ); // saves a graph to disk diff --git a/engine/inc/uf/ext/ttlg/bin.h b/engine/inc/uf/ext/ttlg/bin.h new file mode 100644 index 00000000..4723735e --- /dev/null +++ b/engine/inc/uf/ext/ttlg/bin.h @@ -0,0 +1,10 @@ +#pragma once + +#include +#include + +namespace ext { + namespace ttlg { + bool UF_API loadBin( pod::Graph& graph, const uf::stl::string& filename ); + } +} \ No newline at end of file diff --git a/engine/inc/uf/ext/ttlg/common.h b/engine/inc/uf/ext/ttlg/common.h new file mode 100644 index 00000000..d38f3164 --- /dev/null +++ b/engine/inc/uf/ext/ttlg/common.h @@ -0,0 +1,68 @@ +#pragma once + +#include +#include +#include + +namespace impl { + const float darkToMeters = 0.75f; + typedef uf::Meshlet_T Meshlet; + + template + inline bool readStruct(const uf::stl::vector& buffer, uint32_t& offset, T& outValue) { + if (offset + sizeof(T) > buffer.size()) return false; + std::memcpy(&outValue, buffer.data() + offset, sizeof(T)); + offset += sizeof(T); + return true; + } + + template + inline bool readArray(const uf::stl::vector& buffer, uint32_t& offset, size_t count, uf::stl::vector& outArray) { + size_t bytes = count * sizeof(T); + if (offset + bytes > buffer.size()) return false; + outArray.resize(count); + std::memcpy(outArray.data(), buffer.data() + offset, bytes); + offset += bytes; + return true; + } + + inline float findWrap( float x ) { + return -64.0f * std::floor(x / 64.0f); + }; + + inline void encodeRGBE(const pod::Vector3f& color, uint8_t* out) { + float maxColor = std::max({ color.x, color.y, color.z }); + + if ( maxColor < 1e-6f ) { + out[0] = 0; + out[1] = 0; + out[2] = 0; + out[3] = 0; + return; + } + + int exponent; + float mantissa = std::frexp(maxColor, &exponent); + + float scale = std::exp2(-(float)(exponent)); + pod::Vector3f rgb = color * scale; + + out[0] = (uint8_t)(std::clamp(rgb.x * 255.f, 0.f, 255.f)); + out[1] = (uint8_t)(std::clamp(rgb.y * 255.f, 0.f, 255.f)); + out[2] = (uint8_t)(std::clamp(rgb.z * 255.f, 0.f, 255.f)); + out[3] = (uint8_t)(std::clamp(exponent + 128, 0, 255)); + } + + inline pod::Vector3f convertPos_NewDark( const pod::Vector3f& v, float scale = impl::darkToMeters ) { + return pod::Vector3f{ v.x, v.z, v.y } * scale; + } + + inline bool getBit(const uint8_t* bitmap, int32_t bitIndex, int32_t minID) { + int32_t relativeBit = bitIndex - minID; + int32_t byteIndex = relativeBit >> 3; + int32_t bitOffset = relativeBit & 0x07; + return (bitmap[byteIndex] & (1 << bitOffset)) != 0; + } + + uf::stl::string sanitizeString(const char* raw, size_t maxLength = 16); +} \ No newline at end of file diff --git a/engine/inc/uf/ext/ttlg/mis.h b/engine/inc/uf/ext/ttlg/mis.h new file mode 100644 index 00000000..b05b60fc --- /dev/null +++ b/engine/inc/uf/ext/ttlg/mis.h @@ -0,0 +1,10 @@ +#pragma once + +#include +#include + +namespace ext { + namespace ttlg { + void UF_API loadMis( pod::Graph& graph, const uf::stl::string& filename, const uf::Serializer& metadata ); + } +} \ No newline at end of file diff --git a/engine/inc/uf/ext/ttlg/pcx.h b/engine/inc/uf/ext/ttlg/pcx.h new file mode 100644 index 00000000..32601abb --- /dev/null +++ b/engine/inc/uf/ext/ttlg/pcx.h @@ -0,0 +1,12 @@ +#pragma once + +#include +#include + +namespace ext { + namespace ttlg { + bool UF_API loadPalette( const uf::stl::string& family, uf::stl::vector& palette ); + bool UF_API loadPcx( pod::Image& image, const uf::stl::vector& buffer, const uint8_t* paletteData = NULL ); + } +} + diff --git a/engine/inc/uf/ext/valve/vpk.h b/engine/inc/uf/ext/valve/vpk.h index 56888ab8..e8d5f7a9 100644 --- a/engine/inc/uf/ext/valve/vpk.h +++ b/engine/inc/uf/ext/valve/vpk.h @@ -33,7 +33,8 @@ namespace ext { bool UF_API readVpk( const pod::VpkArchive& vpk, const uf::stl::string& filename, uf::stl::vector& buffer ); bool UF_API readVpkRange( const pod::VpkArchive& vpk, const uf::stl::string& path, size_t start, size_t len, uf::stl::vector& buffer ); - size_t UF_API mountVpk( const uf::stl::string& uri ); - pod::Mount UF_API createVpkMount( const uf::stl::string& uri, int priority = 0 ); + uf::vfs::Mount UF_API mountVpk( const uf::stl::string& uri, bool temp = false ); + uf::vfs::Mount UF_API mountGame( const uf::stl::string& uri, bool temp = false ); + pod::Mount UF_API createVpkMount( const uf::stl::string& uri, int priority = 0 ); // exposed here in the weird twisted event non-Steam VPK files are shipped } } \ No newline at end of file diff --git a/engine/inc/uf/ext/zlib/zlib.h b/engine/inc/uf/ext/zlib/zlib.h index 51d8f899..9512381d 100644 --- a/engine/inc/uf/ext/zlib/zlib.h +++ b/engine/inc/uf/ext/zlib/zlib.h @@ -22,15 +22,18 @@ namespace ext { namespace zlib { extern UF_API size_t bufferSize; - bool UF_API decompressFromFile( uf::stl::vector&, const uf::stl::string& ); - bool UF_API decompressFromFile( uf::stl::vector&, const uf::stl::string& filename, size_t start, size_t len ); - bool UF_API decompressFromFile( uf::stl::vector&, const uf::stl::string& filename, const uf::stl::vector& ranges ); - bool UF_API decompressFromMemory( uf::stl::vector&, const void*, size_t, size_t ); - bool UF_API decompressScatter( const uf::stl::string& filename, uf::stl::vector& requests ); + bool UF_API decompressFromFile( uf::stl::vector&, const uf::stl::string&, int = 15 + 32 ); + bool UF_API decompressFromFile( uf::stl::vector&, const uf::stl::string& filename, size_t start, size_t len, int = 15 + 32 ); + bool UF_API decompressFromFile( uf::stl::vector&, const uf::stl::string& filename, const uf::stl::vector& ranges, int = 15 + 32 ); + bool UF_API decompressFromMemory( uf::stl::vector&, const void*, size_t, size_t, int = 15 + 32 ); + bool UF_API decompressScatter( const uf::stl::string& filename, uf::stl::vector& requests, int = 15 + 32 ); size_t UF_API compressToFile( const uf::stl::string&, const void*, size_t ); bool UF_API directory( const uf::stl::vector& buffer, uf::stl::unordered_map& entries ); + pod::Mount UF_API createZipMount( const uf::stl::string& uri, uf::stl::vector& buffer, int priority = 0 ); + pod::Mount UF_API createZipMount( const uf::stl::string& uri, uf::stl::vector&& buffer, int priority = 0 ); + pod::Mount UF_API createZipMount( const uf::stl::string& uri, const uf::stl::string& filename, int priority = 0 ); inline uf::stl::vector decompressFromFile( const uf::stl::string& filename ) { uf::stl::vector buffer; diff --git a/engine/inc/uf/utils/io/vfs.h b/engine/inc/uf/utils/io/vfs.h index e71385f2..f0f408d6 100644 --- a/engine/inc/uf/utils/io/vfs.h +++ b/engine/inc/uf/utils/io/vfs.h @@ -33,9 +33,15 @@ namespace pod { namespace uf { namespace vfs { extern UF_API uf::stl::vector mounts; + struct UF_API Mount { + size_t hash = {}; + bool temp = false; + ~Mount(); + }; - size_t UF_API mount( const pod::Mount& mount ); + uf::vfs::Mount UF_API mount( const pod::Mount& mount, bool = false ); bool UF_API unmount( size_t ); + bool UF_API unmount( const uf::vfs::Mount& ); bool UF_API unmount( const uf::stl::string& prefix, const uf::stl::string& base ); bool UF_API exists( const uf::stl::string& path ); diff --git a/engine/inc/uf/utils/mesh/mesh.h b/engine/inc/uf/utils/mesh/mesh.h index 531f187c..a2c6e193 100644 --- a/engine/inc/uf/utils/mesh/mesh.h +++ b/engine/inc/uf/utils/mesh/mesh.h @@ -622,6 +622,7 @@ namespace uf { const pod::DrawCommand& UF_API fetchDrawCommand( const uf::Mesh& mesh, size_t triID ); const uf::Mesh::View* UF_API fetchView( const uf::Mesh& mesh, size_t& triID ); + template pod::Instance::Bounds bounds( uf::stl::vector& vertices ); template size_t windingOrder( uf::stl::vector& vertices ); template size_t windingOrder( uf::stl::vector& vertices, uf::stl::vector& indices ); template void normals( uf::stl::vector& vertices ); @@ -734,6 +735,22 @@ T uf::mesh::fetchVertexAttribute( const uf::Mesh::View& view, const uf::Mesh::At } } +template +pod::Instance::Bounds uf::mesh::bounds( uf::stl::vector& vertices ) { + pod::Instance::Bounds bounds; + + if ( !vertices.empty() ) { + bounds.min = bounds.max = vertices[0].position; + for ( const auto& v : vertices ) { + bounds.min = uf::vector::min(bounds.min, v.position); + bounds.max = uf::vector::max(bounds.max, v.position); + } + } + + bounds.center = (bounds.max + bounds.min) * 0.5f; + bounds.extent = uf::vector::abs(bounds.max - bounds.min) * 0.5f; + return bounds; +} template size_t uf::mesh::windingOrder( uf::stl::vector& vertices ) { if constexpr ( !uf::mesh::has_normal::value ) return 0; diff --git a/engine/src/engine/asset/asset.cpp b/engine/src/engine/asset/asset.cpp index c09cfe39..ed3c728c 100644 --- a/engine/src/engine/asset/asset.cpp +++ b/engine/src/engine/asset/asset.cpp @@ -267,6 +267,7 @@ uf::asset::Payload uf::asset::resolveToPayload( const uf::stl::string& uri, cons { "mdl", uf::asset::Type::GRAPH }, #endif { "bsp", uf::asset::Type::GRAPH }, + { "mis", uf::asset::Type::GRAPH }, }; payload.filename = uri; diff --git a/engine/src/engine/ext/ext.cpp b/engine/src/engine/ext/ext.cpp index 9eb53935..8db7aa3f 100644 --- a/engine/src/engine/ext/ext.cpp +++ b/engine/src/engine/ext/ext.cpp @@ -523,6 +523,12 @@ void UF_API uf::initialize() { } #if UF_USE_VALVE + /* Mount games */ { + auto& games = uf::config["engine"]["ext"]["valve"]["games"]; + ext::json::forEach( games, []( const uf::stl::string& uri ) { + ext::valve::mountGame( uri ); + }); + } /* Load VPKs */ { auto& vpks = uf::config["engine"]["ext"]["valve"]["vpks"]; ext::json::forEach( vpks, []( const uf::stl::string& uri ) { diff --git a/engine/src/engine/graph/animation.cpp b/engine/src/engine/graph/animation.cpp index 3d33e821..80bb18f9 100644 --- a/engine/src/engine/graph/animation.cpp +++ b/engine/src/engine/graph/animation.cpp @@ -30,28 +30,28 @@ namespace { auto& animation = storage.animations.map[name]; auto& animStream = graph.streams.animations[name]; + bool needsIO = false; + for ( size_t i = 0; i < animation.samplers.size(); ++i ) { auto& sampler = animation.samplers[i]; auto& stream = animStream.samplers[i]; if ( !sampler.inputs.empty() ) continue; - uf::stl::vector ioBuf; - if ( stream.inputs.length > 0 ) { - if ( uf::io::readAsBuffer(ioBuf, stream.inputs.filename, stream.inputs.offset, stream.inputs.length) ) { - sampler.inputs.resize( stream.inputs.length / sizeof(float) ); - memcpy( sampler.inputs.data(), ioBuf.data(), stream.inputs.length ); - } + sampler.inputs.resize( stream.inputs.length / sizeof(float) ); + uf::asset::read( stream.inputs.filename, stream.inputs.offset, stream.inputs.length, (uint8_t*)(sampler.inputs.data()) ); + needsIO = true; } if ( stream.outputs.length > 0 ) { - if ( uf::io::readAsBuffer(ioBuf, stream.outputs.filename, stream.outputs.offset, stream.outputs.length) ) { - sampler.outputs.resize( stream.outputs.length / sizeof(pod::Vector4f) ); - memcpy( sampler.outputs.data(), ioBuf.data(), stream.outputs.length ); - } + sampler.outputs.resize( stream.outputs.length / sizeof(pod::Vector4f) ); + uf::asset::read( stream.outputs.filename, stream.outputs.offset, stream.outputs.length, (uint8_t*)(sampler.outputs.data()) ); + needsIO = true; } } + + if ( needsIO ) uf::asset::processIO(); } void unloadAnimation( pod::Graph& graph, const uf::stl::string& name ) { diff --git a/engine/src/engine/graph/convert.cpp b/engine/src/engine/graph/convert.cpp index 0f98f43a..2689b8f5 100644 --- a/engine/src/engine/graph/convert.cpp +++ b/engine/src/engine/graph/convert.cpp @@ -222,6 +222,14 @@ pod::Graph& uf::graph::convert( uf::Object& object, bool process ) { return graph; } +void uf::graph::preprocess( pod::Graph& graph, const uf::Serializer& metadata, const uf::stl::string& filename ) { + if ( !graph.storage ) graph.storage = new pod::Graph::Storage(); + if ( !filename.empty() ) graph.name = filename; + if ( !metadata.isNull() ) graph.metadata = metadata; + graph.root.name = "%ROOT%"; + graph.root.index = -1; +} + void uf::graph::postprocess( pod::Graph& graph ) { auto& storage = uf::graph::getStorage( graph ); // post-processing diff --git a/engine/src/engine/graph/decode.cpp b/engine/src/engine/graph/decode.cpp index 39633658..3c124d35 100644 --- a/engine/src/engine/graph/decode.cpp +++ b/engine/src/engine/graph/decode.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #define UF_GRAPH_LOAD_MULTITHREAD 0 @@ -94,8 +95,8 @@ namespace { size_t readLen = length > 0 ? length : uf::io::size( fullPath ); if ( readLen > 0 ) { - pending.buffer.resize(readLen); - uf::asset::read( fullPath, offset, readLen, pending.buffer.data()/*, [&graph, &pending]() { + pending.buffer.resize(readLen); + uf::asset::read( fullPath, offset, readLen, pending.buffer.data()/*, [&graph, &pending]() { auto& storage = uf::graph::getStorage(graph); auto& image = storage.images[pending.name].data; @@ -103,8 +104,8 @@ namespace { uf::image::layers( image, pending.layers ); pending.buffer.clear(); - }*/ ); - } + }*/ ); + } } image.setFilename( fullPath ); @@ -146,16 +147,12 @@ namespace { animStream.samplers.emplace_back(sStream); } else { if ( inputsLen > 0 ) { - uf::stl::vector temp; - uf::io::readAsBuffer(temp, binPath, inputsOffset, inputsLen); sampler.inputs.resize(inputsCount); - memcpy(sampler.inputs.data(), temp.data(), inputsLen); + uf::asset::read( binPath, inputsOffset, inputsLen, (uint8_t*)(sampler.inputs.data()) ); } if ( outputsLen > 0 ) { - uf::stl::vector temp; - uf::io::readAsBuffer(temp, binPath, outputsOffset, outputsLen); sampler.outputs.resize(outputsCount); - memcpy(sampler.outputs.data(), temp.data(), outputsLen); + uf::asset::read( binPath, outputsOffset, outputsLen, (uint8_t*)(sampler.outputs.data()) ); } } }); @@ -194,10 +191,8 @@ namespace { skinStream.inverseBindMatrices = { binPath, offset, length }; } else { if ( length > 0 ) { - uf::stl::vector temp; - uf::io::readAsBuffer(temp, binPath, offset, length); skin.inverseBindMatrices.resize(count); - memcpy(skin.inverseBindMatrices.data(), temp.data(), length); + uf::asset::read( binPath, offset, length, (uint8_t*)(skin.inverseBindMatrices.data()) ); } } } @@ -304,17 +299,17 @@ namespace { } void uf::graph::load( pod::Graph& graph, const uf::stl::string& filename, const uf::Serializer& metadata ) { - const uf::stl::string extension = uf::io::extension( filename ); + const uf::stl::string extension = uf::string::lowercase( uf::io::extension( filename ) ); #if UF_USE_GLTF - if ( extension == "glb" || extension == "gltf" ) { - return ext::gltf::load( graph, filename, metadata ); - } + if ( extension == "glb" || extension == "gltf" ) return ext::gltf::load( graph, filename, metadata ); #endif #if UF_USE_VALVE - if ( extension == "bsp" ) { - return ext::valve::loadBsp( graph, filename, metadata ); - } + if ( extension == "bsp" ) return ext::valve::loadBsp( graph, filename, metadata ); #endif +#if UF_USE_TTLG + if ( extension == "mis" ) return ext::ttlg::loadMis( graph, filename, metadata ); +#endif + const uf::stl::string directory = uf::io::directory( filename ) + "/"; uf::Serializer serializer; UF_DEBUG_TIMER_MULTITRACE_START("Reading {}", filename); diff --git a/engine/src/engine/graph/graph.cpp b/engine/src/engine/graph/graph.cpp index 4876c9b7..fac041b3 100644 --- a/engine/src/engine/graph/graph.cpp +++ b/engine/src/engine/graph/graph.cpp @@ -1058,16 +1058,15 @@ void uf::graph::process( pod::Graph& graph ) { } } if ( spawnID == -1 && !spawns.empty() ) spawnID = uf::stl::random( spawns ); - if ( spawnID != -1 ) { - auto& node = graph.nodes[spawnID]; - auto& child = /*graph.root.entity->*/node.entity->loadChild( "./player.json", false ); // to-do: do not hardcode this - auto& childTransform = child.getComponent>(); + + auto& node = spawnID == -1 ? graph.root : graph.nodes[spawnID]; + auto& child = /*graph.root.entity->*/node.entity->loadChild( "./player.json", false ); // to-do: do not hardcode this + auto& childTransform = child.getComponent>(); - auto flatten = uf::transform::flatten( node.transform ); - childTransform = flatten; + auto flatten = uf::transform::flatten( node.transform ); + childTransform = flatten; - graph.settings.stream.player = spawnID; - } + graph.settings.stream.player = spawnID; } // patch materials/textures @@ -2169,9 +2168,9 @@ void uf::graph::reload( pod::Graph& graph ) { for ( auto& node : graph.nodes ) { if ( !(0 <= node.mesh && node.mesh < graph.meshes.size()) ) continue; - if ( !node.entity ) continue; + if ( !node.entity ) continue; - bool isStreamable = false; + bool isStreamable = false; float radius = graph.settings.stream.radius; float radiusSquared = radius * radius; @@ -2472,7 +2471,7 @@ void uf::graph::reload( pod::Graph& graph ) { if ( readLen > 0 ) { pending.buffer.resize( readLen ); uf::asset::read( imgStream.buffer.filename, imgStream.buffer.offset, readLen, pending.buffer.data() ); - } + } } } else if ( !visible && (texture.generated() && !texture.aliased) ) { image.clear(); diff --git a/engine/src/engine/scene/scene.cpp b/engine/src/engine/scene/scene.cpp index 388085c6..d27e3dd9 100644 --- a/engine/src/engine/scene/scene.cpp +++ b/engine/src/engine/scene/scene.cpp @@ -200,7 +200,8 @@ uf::Scene& uf::scene::loadScene( const uf::stl::string& name, const uf::stl::str auto& metadata = scene->getComponent(); auto& metadataObject = scene->getComponent(); auto mountUri = ::fmt::format("://{}", uf::vfs::resolveBase( metadataObject.system.root ) ); - metadata.mount.hash = uf::vfs::mount( uf::vfs::createDiskMount( mountUri, 200 ) ); + auto mount = uf::vfs::mount( uf::vfs::createDiskMount( mountUri, 200 ) ); + metadata.mount.hash = mount.hash; auto& metadataJson = scene->getComponent(); metadataJson["system"]["scene"] = name; diff --git a/engine/src/ext/gltf/gltf.cpp b/engine/src/ext/gltf/gltf.cpp index fd5819ff..493d0f31 100644 --- a/engine/src/ext/gltf/gltf.cpp +++ b/engine/src/ext/gltf/gltf.cpp @@ -150,15 +150,12 @@ void ext::gltf::load( pod::Graph& graph, const uf::stl::string& filename, const return; } - graph.name = filename; - graph.metadata = metadata; - + uf::graph::preprocess( graph, metadata, filename ); + auto& storage = uf::graph::getStorage( graph ); + uf::stl::string key = graph.metadata["key"].as(""); if ( key != "" ) key += ":"; - if ( !graph.storage ) graph.storage = new pod::Graph::Storage(); - auto& storage = uf::graph::getStorage( graph ); // will just fetch the above - // load images { graph.images.reserve(model.images.size()); @@ -458,9 +455,6 @@ void ext::gltf::load( pod::Graph& graph, const uf::stl::string& filename, const { const auto& scene = model.scenes[model.defaultScene > -1 ? model.defaultScene : 0]; graph.nodes.resize( model.nodes.size() ); - - graph.root.name = "%ROOT%"; - graph.root.index = -1; graph.root.children.reserve( scene.nodes.size() ); for ( auto i : scene.nodes ) { size_t childIndex = loadNode( model, graph, i, -1 ); diff --git a/engine/src/ext/ttlg/bin.cpp b/engine/src/ext/ttlg/bin.cpp new file mode 100644 index 00000000..ab583e5a --- /dev/null +++ b/engine/src/ext/ttlg/bin.cpp @@ -0,0 +1,393 @@ +#include +#include +#include +#include +#include + +namespace impl { +#pragma pack(push, 1) + struct BinMainHeader { + char magic[4]; + uint32_t version; + }; + + struct BinHeader { + char name[8]; + float sphere_rad; + float max_poly_rad; + + pod::Vector3f bmax; + pod::Vector3f bmin; + pod::Vector3f parent_cen; + + uint16_t num_polys; + uint16_t num_verts; + uint16_t num_parms; + + uint8_t num_mats; + uint8_t num_vcalls; + uint8_t num_vhots; + uint8_t num_objs; + + uint32_t offset_objs; + uint32_t offset_mats; + uint32_t offset_uv; + uint32_t offset_vhots; + uint32_t offset_verts; + uint32_t offset_light; + uint32_t offset_norms; + uint32_t offset_poly_list; + uint32_t offset_nodes; + uint32_t model_size; + + // version 4 has extra properties + }; + + struct BinMaterial { + char name[16]; + uint8_t type; + uint8_t slot_num; + uint32_t handle_or_color; + float uvscale_or_ipal; + }; + + struct BinVertex { + pod::Vector3f position; + }; + + struct BinUV { + pod::Vector2f uv; + }; + + struct BinPolyHeader { + uint16_t index; + int16_t data; + uint8_t type; + uint8_t num_verts; + uint16_t norm_index; + float d; + }; + + struct SubObjTransform { + int32_t parent; + float min_range; + float max_range; + float rot[9]; + pod::Vector3f axle_point; + }; + + struct SubObjectHeader { + char name[8]; + uint8_t movement; + SubObjTransform trans; + int16_t child_sub_obj; + int16_t next_sub_obj; + int16_t vhot_start; + int16_t sub_num_vhots; + int16_t point_start; + int16_t sub_num_points; + int16_t light_start; + int16_t sub_num_lights; + int16_t norm_start; + int16_t sub_num_norms; + int16_t node_start; + int16_t sub_num_nodes; + }; +#pragma pack(pop) +} + +namespace impl { + void computeTransforms( uf::stl::vector& subObjects, uf::stl::vector& transforms, uf::stl::vector& offsets, int16_t nodeIdx = 0, int16_t parentIdx = -1 ) { + if ( nodeIdx < 0 || nodeIdx >= subObjects.size() ) return; + + const auto& subObj = subObjects[nodeIdx]; + pod::Vector3f offset = subObj.trans.axle_point; + pod::Matrix4f rot = uf::matrix::identity(); + + bool hasRot = false; + for ( auto i = 0; i < 9; ++i) if ( std::abs(subObj.trans.rot[i]) > 0.0001f ) { hasRot = true; break; } + if ( hasRot ) { + rot(0,0) = subObj.trans.rot[0]; rot(1,0) = subObj.trans.rot[1]; rot(2,0) = subObj.trans.rot[2]; + rot(0,1) = subObj.trans.rot[3]; rot(1,1) = subObj.trans.rot[4]; rot(2,1) = subObj.trans.rot[5]; + rot(0,2) = subObj.trans.rot[6]; rot(1,2) = subObj.trans.rot[7]; rot(2,2) = subObj.trans.rot[8]; + } + + if ( parentIdx >= 0 ) { + transforms[nodeIdx] = transforms[parentIdx] * rot; + offsets[nodeIdx] = uf::matrix::multiply( transforms[parentIdx], offset, 1.0f ) + offsets[parentIdx]; + } else { + transforms[nodeIdx] = rot; + offsets[nodeIdx] = offset; + } + + if ( subObj.child_sub_obj >= 0 ) computeTransforms( subObjects, transforms, offsets, subObj.child_sub_obj, nodeIdx ); + if ( subObj.next_sub_obj >= 0 ) computeTransforms( subObjects, transforms, offsets, subObj.next_sub_obj, parentIdx ); + }; +} + +bool ext::ttlg::loadBin( pod::Graph& graph, const uf::stl::string& filename ) { + uf::stl::vector buffer; + if ( !uf::io::exists( filename ) ) { + UF_MSG_ERROR("BIN does not exist: {}", filename); + return false; + } + if ( !uf::io::readAsBuffer( buffer, filename ) ) { + UF_MSG_ERROR("Failed to read BIN data: {}", filename); + return false; + } + + uint32_t offset = 0; + + impl::BinMainHeader mainHeader; + if ( !impl::readStruct( buffer, offset, mainHeader ) ) { + UF_MSG_ERROR("Failed to read BIN header: {}", filename); + return false; + } + + if ( strncmp(mainHeader.magic, "LGMM", 4) == 0 ) { + UF_MSG_ERROR("Attempting to read LGMM file as an LMGD: {}", filename); + return false; + } + + if ( strncmp(mainHeader.magic, "LGMD", 4) != 0 ) { + UF_MSG_ERROR("Invalid BIN file magic (Expected LGMD): {}", filename); + return false; + } + + impl::BinHeader header; + if ( !impl::readStruct(buffer, offset, header ) ) { + UF_MSG_ERROR("Failed to parse BIN header: {}", filename); + return false; + } + + // skip additional information (to-do: parse) + if ( mainHeader.version == 4 ) offset += 12; + + if ( header.offset_poly_list == 0 || header.offset_poly_list >= buffer.size() ) { + UF_MSG_ERROR("Invalid BIN file (poly list offset out of bounds): {}", filename); + return false; + } + + auto& storage = uf::graph::getStorage( graph ); + uf::stl::unordered_map meshlets; + uf::stl::vector textureToMaterialId( header.num_mats, -1 ); + + uf::stl::vector vertices; + uf::stl::vector uvs; + uf::stl::vector normals; + uf::stl::vector subObjects; + uf::stl::vector> subObjectPolys; + + size_t numUVs = 0; + size_t numNormals = 0; + + // read materials + if ( header.num_mats > 0 && header.offset_mats > 0 && header.offset_mats < buffer.size() ) { + uint32_t matOffset = header.offset_mats; + uf::stl::vector binMats; + + if ( impl::readArray(buffer, matOffset, header.num_mats, binMats ) ) { + for ( uint32_t i = 0; i < header.num_mats; ++i ) { + uf::stl::string matName = impl::sanitizeString(binMats[i].name, 16); + if ( matName.empty() ) { + matName = "missing_texture"; + } else { + std::transform(matName.begin(), matName.end(), matName.begin(), ::tolower); + std::replace(matName.begin(), matName.end(), '\\', '/'); + + size_t dotPos = matName.find_last_of('.'); + if ( dotPos != uf::stl::string::npos ) matName = matName.substr(0, dotPos); + } + + + auto it = std::find(graph.materials.begin(), graph.materials.end(), matName); + if ( it != graph.materials.end() ) { + textureToMaterialId[i] = (int32_t)(std::distance(graph.materials.begin(), it)); + } else { + textureToMaterialId[i] = graph.materials.size(); + graph.materials.emplace_back(matName); + storage.materials[matName].indexAlbedo = -1; + } + } + } + } + + // read vertices + if ( header.num_verts > 0 && header.offset_verts > 0 && header.offset_verts < buffer.size() ) { + uint32_t offset = header.offset_verts; + impl::readArray( buffer, offset, header.num_verts, vertices ); + } + // read UVs + if ( header.offset_uv > 0 && header.offset_uv < buffer.size() ) { + numUVs = (buffer.size() - header.offset_uv) / sizeof(impl::BinUV); + } + if ( numUVs > 0 ) { + uint32_t offset = header.offset_uv; + impl::readArray( buffer, offset, numUVs, uvs ); + } + // read normals + if ( header.offset_norms > 0 && header.offset_norms < buffer.size() ) { + numNormals = (buffer.size() - header.offset_norms) / sizeof(pod::Vector3f); + } + if ( numNormals > 0 ) { + uint32_t offset = header.offset_norms; + impl::readArray( buffer, offset, numNormals, normals ); + } + // read subobject information + if ( header.num_objs > 0 && header.offset_objs > 0 && header.offset_objs < buffer.size() ) { + uint32_t offset = header.offset_objs; + impl::readArray( buffer, offset, header.num_objs, subObjects ); + } + + uf::stl::vector transforms( subObjects.size(), uf::matrix::identity() ); + uf::stl::vector offsets( subObjects.size() ); + + // read transforms + if ( !subObjects.empty() ) impl::computeTransforms( subObjects, transforms, offsets ); + + // read subobject faces + if ( header.offset_nodes > 0 && header.offset_nodes < buffer.size() ) { + uint32_t offset = header.offset_nodes; + uf::stl::vector faces; + while ( offset < buffer.size() ) { + faces.clear(); + uint8_t nodeType = buffer[offset]; + + if ( nodeType == 4 ) { + subObjectPolys.emplace_back(); + offset += 3; + } else if ( nodeType == 3 ) { + offset += 19; + } else if ( nodeType == 2 ) { + uint16_t nf1{}, nf2{}; + impl::readStruct( buffer, offset += 17, nf1 ); + impl::readStruct( buffer, offset += 4, nf2 ); + + if ( impl::readArray( buffer, offset, nf1 + nf2, faces ) && !subObjectPolys.empty() ) { + subObjectPolys.back().insert( subObjectPolys.back().end(), faces.begin(), faces.end() ); + } + } else if (nodeType == 1) { + uint16_t nf1{}, nf2{}; + impl::readStruct( buffer, offset += 17, nf1 ); + impl::readStruct( buffer, offset += 12, nf2 ); + + if ( impl::readArray( buffer, offset, nf1 + nf2, faces ) && !subObjectPolys.empty() ) { + subObjectPolys.back().insert( subObjectPolys.back().end(), faces.begin(), faces.end() ); + } + } else if (nodeType == 0) { + uint16_t nf{}; + impl::readStruct( buffer, offset += 17, nf ); + if ( impl::readArray(buffer, offset, nf, faces) && !subObjectPolys.empty() ) { + subObjectPolys.back().insert(subObjectPolys.back().end(), faces.begin(), faces.end()); + } + } else { + break; + } + } + } + + // iterate subobjects + size_t numParsedSubs = std::min(subObjects.size(), subObjectPolys.size()); + for ( size_t objIdx = 0; objIdx < numParsedSubs; ++objIdx ) { + const auto& subObj = subObjects[objIdx]; + const auto& polyOffsets = subObjectPolys[objIdx]; + + pod::Matrix4f transform = transforms[objIdx]; + pod::Vector3f offsetPos = offsets[objIdx]; + + for ( uint16_t polyOffset : polyOffsets ) { + uint32_t absolutePolyOffset = header.offset_poly_list + polyOffset; + if (absolutePolyOffset >= buffer.size()) continue; + + impl::BinPolyHeader polyHeader; + uint32_t readOffset = absolutePolyOffset; + if (!impl::readStruct(buffer, readOffset, polyHeader)) continue; + + uint32_t dataOffset = absolutePolyOffset + 12; + + uf::stl::vector vertIndices; + impl::readArray(buffer, dataOffset, polyHeader.num_verts, vertIndices); + + uf::stl::vector normIndices; + impl::readArray(buffer, dataOffset, polyHeader.num_verts, normIndices); + + uf::stl::vector uvIndices; + bool hasUVs = ((polyHeader.type & 3) == 3); + if (hasUVs) impl::readArray(buffer, dataOffset, polyHeader.num_verts, uvIndices); + + int32_t graphMatID = -1; + int32_t localMatID = polyHeader.data > 0 ? polyHeader.data - 1 : 0; + if (localMatID >= 0 && localMatID < textureToMaterialId.size()) { + graphMatID = textureToMaterialId[localMatID]; + } + + auto& meshlet = meshlets[graphMatID]; + meshlet.primitive.instance.materialID = graphMatID; + + uint32_t startVertIdx = meshlet.vertices.size(); + pod::Vector3f polyNormal = {0.f, 1.f, 0.f}; + if (polyHeader.norm_index < normals.size()) polyNormal = normals[polyHeader.norm_index]; + polyNormal = impl::convertPos_NewDark(uf::matrix::multiply(transforms[objIdx], polyNormal, 0.0f)); + + for (uint8_t v = 0; v < polyHeader.num_verts; ++v) { + auto& vert = meshlet.vertices.emplace_back(); + uint16_t vIdx = vertIndices[v]; + + if ( vIdx < vertices.size() ) { + vert.position = impl::convertPos_NewDark( uf::matrix::multiply(transforms[objIdx], vertices[vIdx].position, 1.0f) + offsets[objIdx] ); + } + vert.normal = impl::convertPos_NewDark(polyNormal, 1.0f); + vert.color = { 255, 255, 255, 255 }; + + if ( hasUVs && v < uvIndices.size() ) { + uint16_t uvIdx = uvIndices[v]; + if ( uvIdx < uvs.size() ) vert.uv = uvs[uvIdx].uv; + } + } + + // triangle fan => triangles + for ( uint8_t v = 1; v < polyHeader.num_verts - 1; ++v ) { + meshlet.indices.emplace_back(startVertIdx); + meshlet.indices.emplace_back(startVertIdx + v); + meshlet.indices.emplace_back(startVertIdx + v + 1); + } + } + } + + if ( meshlets.empty() ) { + if ( header.num_vhots > 0 || header.num_objs > 0 || header.num_polys == 0 ) { + UF_MSG_DEBUG("BIN file acts as a dummy node (no polygons, but has VHOTs/Objs): {}", filename); + /* + uf::stl::string meshName = filename; + graph.meshes.emplace_back(meshName); + graph.primitives.emplace_back(meshName); + return true; + */ + } else { + UF_MSG_WARNING("BIN file contained no valid polygons: {}", filename); + } + return false; + } + + uf::stl::string meshName = filename; + graph.meshes.emplace_back(meshName); + graph.primitives.emplace_back(meshName); + + auto& mesh = storage.meshes[meshName]; + auto& primitives = storage.primitives[meshName]; + + size_t primitiveID = 0; + for ( auto& [matID, meshlet] : meshlets ) { + meshlet.primitive.drawCommand.indices = meshlet.indices.size(); + meshlet.primitive.drawCommand.vertices = meshlet.vertices.size(); + meshlet.primitive.instance.materialID = matID; + meshlet.primitive.instance.primitiveID = primitiveID++; + meshlet.primitive.instance.bounds = uf::mesh::bounds( meshlet.vertices ); + + uf::mesh::tangents( meshlet.vertices, meshlet.indices ); + } + + mesh.compile( meshlets, primitives ); + + return true; +} \ No newline at end of file diff --git a/engine/src/ext/ttlg/common.cpp b/engine/src/ext/ttlg/common.cpp new file mode 100644 index 00000000..56662614 --- /dev/null +++ b/engine/src/ext/ttlg/common.cpp @@ -0,0 +1,22 @@ +#include + +uf::stl::string impl::sanitizeString( const char* raw, size_t maxLength ) { + if (!raw || maxLength == 0) return ""; + + size_t len = 0; + while ( len < maxLength && raw[len] != '\0' ) ++len; + + uf::stl::string clean; + clean.reserve(len); + for ( size_t i = 0; i < len; ++i ) { + unsigned char c = (unsigned char)(raw[i]); + if ( c >= 32 && c <= 126 ) { + clean += c; + } else { + if (c == '\t') clean += "\\t"; + else if (c == '\n') clean += "\\n"; + else if (c == '\r') clean += "\\r"; + } + } + return clean; +} \ No newline at end of file diff --git a/engine/src/ext/ttlg/mis.cpp b/engine/src/ext/ttlg/mis.cpp new file mode 100644 index 00000000..6edee889 --- /dev/null +++ b/engine/src/ext/ttlg/mis.cpp @@ -0,0 +1,1025 @@ +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace impl { +#pragma pack(push, 1) + struct DarkDBHeader { + uint32_t inv_offset; + uint32_t zero; + uint32_t one; + uint8_t zeros[256]; + uint32_t dead_beef; // 0xEFBEADDE + }; + + struct DarkDBInvItem { + char name[12]; + uint32_t offset; + uint32_t length; + }; + + struct DarkDBChunkHeader { + char name[12]; + uint32_t version_high; + uint32_t version_low; + uint32_t zero; + }; + + struct WRHeader { + uint32_t unk; + uint32_t numCells; + }; + + struct WRCellHeader { + uint8_t numVertices; + uint8_t numPolygons; + uint8_t numTextured; + uint8_t numPortals; + uint8_t numPlanes; + uint8_t mediaType; + uint8_t cellFlags; + uint32_t nxn; + uint16_t polymapSize; + uint8_t numAnimLights; + uint8_t flowGroup; + pod::Vector3f center; + float radius; + }; + + struct WRPolygon { + uint8_t flags; + uint8_t count; + uint8_t plane; + uint8_t unk; + uint16_t tgtCell; + uint8_t unk1; + uint8_t unk2; + }; + + struct WRPolygonTexturing { + pod::Vector3f axisU; + pod::Vector3f axisV; + int16_t u; + int16_t v; + uint8_t txt; + uint8_t originVertex; + uint16_t unk; + float scale; + pod::Vector3f center; + }; + + struct WRLightInfo { + int16_t u; + int16_t v; + uint16_t lx; + uint8_t ly; + uint8_t lx8; + uint32_t staticLmapPtr; + uint32_t dynamicLmapPtr; + uint32_t animflags; + }; + + struct WRPlane { + pod::Vector3f normal; + float d; + }; + + struct DarkDBChunkTXLIST { + uint32_t length; + uint32_t txt_count; + uint32_t fam_count; + }; + + struct DarkDBTXLIST_fam { + char name[16]; + }; + + struct DarkDBTXLIST_texture { + uint8_t one; + uint8_t fam; + uint16_t zero; + char name[16]; + }; + + struct PropertyChunkHeader { + uint32_t numEntries; + }; + + struct PropertyEntry { + int32_t objectId; + uint32_t size; + }; + + struct LinkData { + int32_t sourceId; + int32_t destId; + uint16_t flavor; + }; + + struct DarkPartitionedLink { + uint32_t id; + int32_t src; + int32_t dest; + uint16_t flavor; + }; + + struct sPositionProp { + pod::Vector3f position; + uint32_t cell; + uint16_t headingRaw; + uint16_t pitchRaw; + uint16_t rollRaw; + }; + + struct sLightProp { + pod::Vector3f color; + float intensity; + float range; + uint8_t type; + uint8_t active; + uint8_t castShadows; + }; + + struct sAmbientSoundProp { + char schemaName[16]; + uint32_t flags; + float volume; + float radius; + }; + + struct sAnimLightProp { + uint32_t mode; + float millis; + pod::Vector3f color; + float brightness; + float radius; + }; +#pragma pack(pop) + + // do not load these + uf::stl::unordered_set modelBlacklist = { + "fx_particle.bin", + "spark_.bin" + }; + + struct DarkContext { + uf::stl::unordered_map inventory; + uf::stl::vector families; + + uf::stl::unordered_map archetypes; + uf::stl::unordered_map parentMap; + uf::stl::unordered_map modelNames; + uf::stl::unordered_map customNames; + uf::stl::unordered_map lightProps; + uf::stl::unordered_map ambientSounds; + uf::stl::unordered_map> scripts; + + uf::stl::vector links; + uf::stl::unordered_map linkFlavorNames; + + uf::stl::unordered_map> palettes; + uf::stl::vector textureToMaterialId; + pod::Atlas lightmapAtlas; + + template + bool findInheritedProperty(int32_t objId, const uf::stl::unordered_map& propMap, T& outValue) const { + auto it = propMap.find(objId); + if ( it != propMap.end() ) { + outValue = it->second; + return true; + } + auto pit = parentMap.find(objId); + if ( pit != parentMap.end() && pit->second != 0 ) { + return findInheritedProperty(pit->second, propMap, outValue); + } + return false; + } + }; + + pod::Atlas::hash_t darkFaceHash(size_t cellIdx, size_t polyIdx) { + return ::fmt::format("c_{}_p_{}", cellIdx, polyIdx); + } + + uf::stl::string getStringFromOffsetSafe(const uf::stl::vector& buffer, uint32_t offset, size_t maxScan = 128) { + if ( offset >= buffer.size()) return ""; + size_t max_len = std::min(maxScan, buffer.size() - offset); + const char* ptr = (const char*)(buffer.data() + offset); + return uf::stl::string(ptr, strnlen( ptr, max_len )); + } + + void parseRelations(impl::DarkContext& ctx, const uf::stl::vector& buffer, const impl::DarkDBInvItem& item) { + uint32_t offset = item.offset + sizeof(impl::DarkDBChunkHeader); + uint32_t endOffset = item.offset + item.length; + + uint16_t flavorId = 1; + while ( offset + 32 <= endOffset ) { + uf::stl::string relName = impl::sanitizeString((const char*)(buffer.data() + offset), 32); + if ( !relName.empty() ) ctx.linkFlavorNames[flavorId] = relName; + flavorId++; + offset += 32; + } + } + + void parsePartitionedLinks(impl::DarkContext& ctx, const uf::stl::vector& buffer, const impl::DarkDBInvItem& item) { + uint32_t offset = item.offset + sizeof(impl::DarkDBChunkHeader); + uint32_t endOffset = item.offset + item.length; + + size_t count = (endOffset - offset) / sizeof(impl::DarkPartitionedLink); + + uf::stl::vector chunkLinks; + if ( !impl::readArray( buffer, offset, count, chunkLinks ) ) return; + for ( const auto& l : chunkLinks ) { + ctx.links.push_back({l.src, l.dest, l.flavor}); // to-do: emplace_back + } + } + + void parseProperty_Strings(impl::DarkContext& ctx, const uf::stl::vector& buffer, const impl::DarkDBInvItem& item, const uf::stl::string& propType) { + uint32_t offset = item.offset + sizeof(impl::DarkDBChunkHeader); + uint32_t endOffset = item.offset + item.length; + + while ( offset < endOffset ) { + PropertyEntry entry; + + if ( !impl::readStruct( buffer, offset, entry ) ) break; + + uint32_t dataStart = offset; + + if ( propType == "SymName" ) { + if ( entry.size > 4 ) { + auto symName = impl::getStringFromOffsetSafe( buffer, dataStart + 4, entry.size - 4 ); + if ( !symName.empty() ) ctx.archetypes[entry.objectId] = symName; + } + } + else if ( propType == "ModelName" ) { + auto modelName = impl::getStringFromOffsetSafe(buffer, dataStart, entry.size); + if ( !modelName.empty() ) ctx.modelNames[entry.objectId] = modelName; + } + + offset = dataStart + entry.size; + } + } + + void parseGenericProperties(impl::DarkContext& ctx, const uf::stl::vector& buffer, const uf::stl::string& propName, const impl::DarkDBInvItem& item ) { + uint32_t offset = item.offset + sizeof(impl::DarkDBChunkHeader); + uint32_t endOffset = item.offset + item.length; + + while ( offset < endOffset ) { + PropertyEntry entry; + if ( !impl::readStruct( buffer, offset, entry ) ) break; + + uint32_t dataStart = offset; + + if ( propName == "Light" && entry.size >= sizeof(sLightProp) ) { + sLightProp light; + if ( impl::readStruct( buffer, dataStart, light ) ) ctx.lightProps[entry.objectId] = light; + /* + if ( std::isnan(light.color.x) || std::isinf(light.color.x) ) light.color = {1.f, 1.f, 1.f}; + if ( std::isnan(light.intensity) || std::isinf(light.intensity) ) light.intensity = 100.f; + */ + } + else if ( propName == "Ambient" && entry.size >= sizeof(sAmbientSoundProp) ) { + sAmbientSoundProp sound; + if ( impl::readStruct( buffer, dataStart, sound ) ) { + auto schemaName = sanitizeString( sound.schemaName, 16 ); + memset(sound.schemaName, 0, 16); + memcpy(sound.schemaName, schemaName.c_str(), std::min(15, schemaName.size())); + ctx.ambientSounds[entry.objectId] = sound; + } + } + else if ( propName == "Scripts" ) { + const char* scriptCursor = (const char*)(buffer.data() + dataStart); + size_t remainingBytes = entry.size; + for ( auto s = 0; s < 4; ++s ) { + if ( remainingBytes <= 0 || *scriptCursor == '\0' ) break; + + auto script = sanitizeString( scriptCursor, remainingBytes ); + if ( script.empty() ) break; + + ctx.scripts[entry.objectId].emplace_back(script); + size_t step = strnlen(scriptCursor, remainingBytes) + 1; + if ( step > remainingBytes ) break; + scriptCursor += step; + remainingBytes -= step; + } + } + + offset = dataStart + entry.size; + } + } + + void parseTextures( pod::Graph& graph, impl::DarkContext& ctx, const uf::stl::vector& buffer, const impl::DarkDBInvItem& item ) { + uint32_t offset = item.offset + sizeof(impl::DarkDBChunkHeader); + uint32_t size = item.length; + DarkDBChunkTXLIST header; + if ( !impl::readStruct( buffer, offset, header ) ) return; + + uf::stl::vector fams; + if ( impl::readArray( buffer, offset, header.fam_count, fams ) ) { + ctx.families.resize( header.fam_count ); + for ( auto i = 0; i < header.fam_count; ++i ) { + ctx.families[i] = uf::stl::string(fams[i].name, strnlen(fams[i].name, 16)); + std::transform(ctx.families[i].begin(), ctx.families[i].end(), ctx.families[i].begin(), ::tolower); + std::replace(ctx.families[i].begin(), ctx.families[i].end(), '\\', '/'); + } + } + + auto& storage = uf::graph::getStorage(graph); + ctx.textureToMaterialId.resize( header.txt_count, -1 ); + + uf::stl::vector texs; + if ( impl::readArray( buffer, offset, header.txt_count, texs ) ) { + for (uint32_t i = 0; i < header.txt_count; ++i) { + const auto& tex = texs[i]; + uf::stl::string texName(tex.name, strnlen(tex.name, 16)); + if (texName.empty() || texName == "null") continue; + + std::transform(texName.begin(), texName.end(), texName.begin(), ::tolower); + std::replace(texName.begin(), texName.end(), '\\', '/'); + + uf::stl::string matName = texName; + if (tex.fam > 0 && tex.fam < ctx.families.size() && !ctx.families[tex.fam].empty()) { + matName = ctx.families[tex.fam] + "/" + texName; + } + + // could instead just check against storage.materials.map.count( matName ) + auto it = std::find( graph.materials.begin(), graph.materials.end(), matName ); + if ( it == graph.materials.end() ) { + ctx.textureToMaterialId[i] = graph.materials.size(); + graph.materials.emplace_back(matName); + storage.materials[matName].indexAlbedo = -1; + } else { + ctx.textureToMaterialId[i] = std::distance(graph.materials.begin(), it); + } + } + } + } + + void loadMaterials( pod::Graph& graph, impl::DarkContext& ctx ) { + auto& storage = uf::graph::getStorage(graph); + uf::stl::vector extensions = { ".png", ".dds", ".tga", ".pcx", ".gif", ".bmp" }; + uf::stl::vector missing_pixels = { 255, 0, 255, 255, 0, 0, 0, 255, 0, 0, 0, 255, 255, 0, 255, 255 }; + + uf::stl::vector buffer; + for ( const auto& matName : graph.materials ) { + auto& image = storage.images[matName].data; + if ( !image.getPixels().empty() ) continue; // already loaded + + bool loaded = false; + + // sanitize name and family + uf::stl::string family = ""; + uf::stl::string texName = matName; + uf::stl::vector searchDirs; + size_t slashPos = matName.find('/'); + if ( slashPos != uf::stl::string::npos ) { + family = matName.substr(0, slashPos); + texName = matName.substr(slashPos + 1); + } + + // family deduced, add it to search queue + if ( !family.empty() ) { + searchDirs.emplace_back("fam://" + family + "/"); + searchDirs.emplace_back("fam://" + family + "/anim/"); + for ( const auto& fam : ctx.families ) { + if ( fam != family && !fam.empty() ) searchDirs.emplace_back("fam://" + fam + "/"); + if ( fam != family && !fam.empty() ) searchDirs.emplace_back("fam://" + fam + "/anim/"); + } + // no family deduced, most likely an object + } else { + searchDirs.emplace_back("obj://txt16/"); + searchDirs.emplace_back("obj://txt/"); + searchDirs.emplace_back("obj://"); + for ( const auto& fam : ctx.families ) { + if ( !fam.empty() ) { + searchDirs.emplace_back("fam://" + fam + "/"); + searchDirs.emplace_back("fam://" + fam + "/anim/"); + } + } + } + + // search for target + for ( const auto& dir : searchDirs ) { + if ( loaded ) break; + + if ( dir.starts_with("fam://") ) { + size_t fStart = 6; + size_t fEnd = dir.find('/', fStart); + if ( fEnd != uf::stl::string::npos ) family = dir.substr(fStart, fEnd - fStart); + } + + for ( const auto& ext : extensions ) { + uf::stl::string target = dir + texName + ext; + // target doesn't exist + if ( !uf::io::exists( target ) ) continue; + // failed to read target + if ( !uf::io::readAsBuffer( buffer, target ) ) continue; + + // is a PCX + if ( ext == ".pcx" ) { + if ( !family.empty() ) ext::ttlg::loadPalette( family, ctx.palettes[family] ); + const uint8_t* palette = (!family.empty() && !ctx.palettes[family].empty()) ? ctx.palettes[family].data() : nullptr; + if ( !ext::ttlg::loadPcx( image, buffer, palette ) ) { + UF_MSG_ERROR("Failed to load PCX: {}", target); + continue; + } + // not a PCX, load it + } else if ( !uf::image::open(image, buffer, target ) ) { + continue; + } + + // animated + if ( dir.find("/anim/") != uf::stl::string::npos ) { + UF_MSG_DEBUG("Animated texture found: {}", target); + } + + loaded = true; + break; + } + } + + // did not load, fallback to missing_texture + if ( !loaded ) { + UF_MSG_DEBUG("Could not load material: {}", texName); + image.loadFromBuffer( missing_pixels, { 2, 2 }, 8, 4 ); + } + + size_t imageID = graph.images.size(); + size_t textureID = graph.textures.size(); + + graph.images.emplace_back( matName ); + graph.textures.emplace_back( matName ); + storage.textures[matName].index = imageID; + + auto& material = storage.materials[matName]; + material.colorBase = { 1.0f, 1.0f, 1.0f, 1.0f }; + material.factorRoughness = 1.0f; + material.factorMetallic = 0.0f; + material.factorOcclusion = 1.0f; + material.factorAlphaCutoff = 0.1f; + material.indexAlbedo = textureID; + material.indexNormal = -1; + material.indexEmissive = -1; + material.indexMetallicRoughness = -1; + material.indexOcclusion = -1; + material.indexCubemap = -1; + + // to-do: fill out + if ( matName.find("glass") != uf::stl::string::npos ) { + material.modeAlpha = pod::Material::AlphaMode::BLEND; + } + } + } + + void parseWorld( pod::Graph& graph, impl::DarkContext& ctx, const uf::stl::vector& buffer, const impl::DarkDBInvItem& item ) { + uint32_t offset = item.offset + sizeof(impl::DarkDBChunkHeader); + impl::WRHeader header; + if ( !impl::readStruct( buffer, offset, header ) ) return; + + auto& storage = uf::graph::getStorage(graph); + uf::stl::string meshName = "worldspawn"; + graph.meshes.emplace_back(meshName); + graph.primitives.emplace_back(meshName); + + auto& mesh = storage.meshes[meshName]; + auto& primitives = storage.primitives[meshName]; + uf::stl::unordered_map meshlets; + + // cell information + struct ParsedCell { + uint32_t cellIdx; + impl::WRCellHeader header; + uf::stl::vector vertices; + uf::stl::vector polys; + uf::stl::vector texInfo; + uf::stl::vector> polyIndices; + uf::stl::vector planes; + uf::stl::vector lmInfos; + }; + uf::stl::vector cells( header.numCells ); + + // fill cell information + for ( uint32_t c = 0; c < header.numCells; ++c ) { + auto& cell = cells[c]; + cell.cellIdx = c; + + if ( !impl::readStruct( buffer, offset, cell.header ) ) break; + + // read cell information + impl::readArray( buffer, offset, cell.header.numVertices, cell.vertices ); + impl::readArray( buffer, offset, cell.header.numPolygons, cell.polys ); + impl::readArray( buffer, offset, cell.header.numTextured, cell.texInfo ); + + // read index count + uint32_t numIndices; + if ( !impl::readStruct( buffer, offset, numIndices ) ) break; + if ( offset + numIndices > buffer.size() ) break; + + // fill indices + cell.polyIndices.resize( cell.header.numPolygons ); + for ( uint8_t i = 0; i < cell.header.numPolygons; ++i ) { + cell.polyIndices[i].resize( cell.polys[i].count ); + for ( uint8_t j = 0; j < cell.polys[i].count; ++j ) { + cell.polyIndices[i][j] = buffer[offset++]; + } + } + + // read planes + impl::readArray( buffer, offset, cell.header.numPlanes, cell.planes ); + + // read animated lights + offset += cell.header.numAnimLights * sizeof(int16_t); + + // to-do: read lightmaps information + impl::readArray(buffer, offset, cell.header.numTextured, cell.lmInfos); + + // read lightmap + uint32_t lightPixelSize = 2; // to-do: deduce? + for ( uint8_t i = 0; i < cell.header.numTextured; ++i ) { + int lmCount = 1; + uint32_t flags = cell.lmInfos[i].animflags; + while ( flags ) { + if ( flags & 1 ) lmCount++; + flags >>= 1; + } + + uint32_t w = cell.lmInfos[i].lx; + uint32_t h = cell.lmInfos[i].ly; + uint32_t lmSizeBytes = w * h * lightPixelSize; + + if ( lmSizeBytes > 0 && ( offset + lmSizeBytes * lmCount ) <= buffer.size()) { + pod::Image image; + image.size = { w, h }; + image.channels = 4; + image.bpp = 32; + image.pixels.resize( w * h * 4 ); + + // read lightmap + uf::stl::vector samples; + impl::readArray( buffer, offset, w * h * lmCount, samples ); + + // read pixels + for ( auto y = 0; y < h; ++y ) { + for ( auto x = 0; x < w; ++x ) { + uint16_t sample = samples[y * w + x]; + auto color = pod::Vector3f{ + (float)((sample >> 10) & 0x1F), + (float)((sample >> 5) & 0x1F), + (float)((sample ) & 0x1F), + } / 31.0f; + impl::encodeRGBE( color, &image.pixels[(y * w + x) * 4] ); + } + } + uf::atlas::add( ctx.lightmapAtlas, image, impl::darkFaceHash(c, i) ); + } else { + offset += ( lmSizeBytes * lmCount ); + } + } + + // to-do: read light information + uint32_t lightCount; + if ( impl::readStruct( buffer, offset, lightCount ) ) { + offset += ( lightCount * sizeof(uint16_t) ); // skip + } + } + + // combine lightmap into atlas + auto atlasImageID = graph.images.size(); + auto atlasTextureID = graph.textures.size(); + + if ( !ctx.lightmapAtlas.tiles.empty() ) { + uf::atlas::generate(ctx.lightmapAtlas, 1); + auto& imageKey = graph.images.emplace_back("lightmap_atlas"); + auto& textureKey = graph.textures.emplace_back("lightmap_atlas"); + + storage.images[imageKey].data = uf::atlas::get( ctx.lightmapAtlas ); + storage.textures[textureKey].index = atlasImageID; + } + + // build mesh from cells + uf::stl::vector indices; + uf::stl::vector uvs; + uf::stl::vector sts; + for ( const auto& cell : cells ) { + uint8_t solidPolys = cell.header.numPolygons - cell.header.numPortals; + for ( uint8_t p = 0; p < solidPolys; ++p ) { + const auto& poly = cell.polys[p]; + if ( p >= cell.header.numTextured || poly.plane >= cell.header.numPlanes) continue; + + const auto& lm = cell.lmInfos[p]; + const auto& texInfo = cell.texInfo[p]; + const auto& plane = cell.planes[poly.plane]; + + // clear buffer + indices.clear(); indices.resize(poly.count); + uvs.clear(); uvs.resize(poly.count); + sts.clear(); sts.resize(poly.count); + + // bind material information + int32_t materialID = -1; + uf::stl::string matName = ""; + if ( texInfo.txt < ctx.textureToMaterialId.size() ) { + materialID = ctx.textureToMaterialId[texInfo.txt]; + if ( materialID >= 0 && materialID < graph.materials.size() ) { + matName = graph.materials[materialID]; + } + } + + // prepare meshlet + auto& meshlet = meshlets[materialID]; + meshlet.primitive.instance.materialID = materialID; + meshlet.primitive.instance.lightmapID = atlasTextureID; + + pod::Vector3f origin = cell.vertices[cell.polyIndices[p][texInfo.originVertex]]; + + // project UV + pod::Vector2f mag2 = { uf::vector::dot(texInfo.axisU, texInfo.axisU), uf::vector::dot(texInfo.axisV, texInfo.axisV) }; + float dotp = uf::vector::dot( texInfo.axisU, texInfo.axisV ); + + pod::Vector2f sh = pod::Vector2f{ (float)texInfo.u, (float)texInfo.v } / 4096.0f; + + pod::Vector2f texSize = { 64.0f, 64.0f }; + if ( !matName.empty() && storage.images.map.count(matName) ) { + const auto& img = storage.images[matName].data; + if ( img.size.x > 0 && img.size.y > 0 ) { + texSize = { (float)img.size.x, (float)img.size.y }; + } + } + + pod::Vector2f rs = texSize / 64.0f; + + pod::Vector2f lmBase = { (float)lm.u, (float)lm.v }; + pod::Vector2f texBase = { (float)texInfo.u, (float)texInfo.v }; + pod::Vector2f lsh = (pod::Vector2f{0.5f, 0.5f} - lmBase) + (texBase / 1024.0f); + + float corr = 0.0f; + float det = ( mag2.x * mag2.y - dotp * dotp ); + if ( std::abs(det) > EPS ) corr = 1.0f / det; + + pod::Vector2f c = { corr * mag2.y, corr * mag2.x }; + float cross = corr * dotp; + + pod::Vector2f minSt = { std::numeric_limits::max(), std::numeric_limits::max() }; + + for ( uint8_t v = 0; v < poly.count; ++v ) { + pod::Vector3f pos = cell.vertices[cell.polyIndices[p][v]]; + pod::Vector3f vrel = pos - origin; + + pod::Vector2f pr = { uf::vector::dot(texInfo.axisU, vrel), uf::vector::dot(texInfo.axisV, vrel) }; + pod::Vector2f proj; + + if ( dotp == 0.0f ) { + proj = pr / mag2; + } else { + proj.x = pr.x * c.x - pr.y * cross; + proj.y = pr.y * c.y - pr.x * cross; + } + + uvs[v] = (proj + sh) / rs; + sts[v] = (proj * 4.0f) + lsh; + + minSt = uf::vector::min(minSt, sts[v]); + } + + pod::Vector2f lmSh = { + impl::findWrap(minSt.x), + impl::findWrap(minSt.y) + }; + + pod::Vector2f lmSize = { + lm.lx > 0 ? (float)(lm.lx) : 0.5f, + lm.ly > 0 ? (float)(lm.ly) : 0.5f + }; + + // write vertices + for ( uint8_t v = 0; v < poly.count; ++v ) { + indices[v] = meshlet.vertices.size(); + + auto& vert = meshlet.vertices.emplace_back(); + vert.position = impl::convertPos_NewDark( cell.vertices[cell.polyIndices[p][v]] ); + vert.normal = impl::convertPos_NewDark( plane.normal, 1.0f ); + vert.color = { 255, 255, 255, 255 }; + vert.uv = uvs[v]; + vert.st = uf::atlas::mapUv(ctx.lightmapAtlas, (sts[v] + lmSh) / lmSize, impl::darkFaceHash(cell.cellIdx, p)); + } + + // triangle strip => triangle + for ( uint8_t t = 1; t < poly.count - 1; ++t ) { + meshlet.indices.emplace_back(indices[0]); + meshlet.indices.emplace_back(indices[t]); + meshlet.indices.emplace_back(indices[t + 1]); + } + } + } + + if ( meshlets.empty()) return; + + size_t primitiveID = 0; + for ( auto& [ matID, meshlet ] : meshlets ) { + meshlet.primitive.drawCommand.indices = meshlet.indices.size(); + meshlet.primitive.drawCommand.vertices = meshlet.vertices.size(); + meshlet.primitive.instance.materialID = matID; + meshlet.primitive.instance.primitiveID = primitiveID++; + meshlet.primitive.instance.bounds = uf::mesh::bounds( meshlet.vertices ); + + uf::mesh::tangents( meshlet.vertices, meshlet.indices ); + } + + if ( false ) { + uf::meshgrid::Grid grid; + grid.divisions = {8, 1, 8}; + auto mlets = uf::stl::values( meshlets ); + auto partitioned = uf::meshgrid::partition( grid, mlets, EPS, true, true ); + mesh.compile( partitioned, primitives ); + } else { + mesh.compile( meshlets, primitives ); + } + + + auto nodeID = graph.nodes.size(); + auto& node = graph.nodes.emplace_back(); + node.name = "worldspawn"; + node.mesh = (int32_t)(graph.meshes.size() - 1); + graph.root.children.emplace_back(nodeID); + } + + void parseObjects( pod::Graph& graph, const impl::DarkContext& ctx, const uf::stl::vector& buffer, const impl::DarkDBInvItem& objVecItem, const impl::DarkDBInvItem& posItem ) { + // read ObjVec + if ( objVecItem.length < 8 ) return; + + const uint8_t* bitmap = buffer.data(); + int32_t minID, maxID; + { + uint32_t offset = objVecItem.offset + sizeof(impl::DarkDBChunkHeader); + if ( !impl::readStruct( buffer, offset, minID ) ) return; + if ( !impl::readStruct( buffer, offset, maxID ) ) return; + bitmap += offset; + } + + // iterate through P$Position + uint32_t offset = posItem.offset + sizeof(impl::DarkDBChunkHeader); + uint32_t end = offset + posItem.length; + while ( offset < end ) { + PropertyEntry entry; + if ( !impl::readStruct( buffer, offset, entry ) ) break; + uint32_t start = offset; // to reset the offset later + + bool isActive = ( entry.objectId >= minID && entry.objectId < maxID && getBit( bitmap, entry.objectId, minID ) ); + if ( !isActive || entry.objectId < 0 ) { + offset = start + entry.size; + continue; + } + + // read position (and orientation) property + sPositionProp prop; + if ( !impl::readStruct( buffer, offset, prop ) ) break; + + auto nodeID = graph.nodes.size(); + graph.root.children.emplace_back(nodeID); + auto& node = graph.nodes.emplace_back(); + auto& metadata = node.metadata["dark"]; + + auto facing = pod::Vector3f{ prop.pitchRaw, prop.headingRaw, -prop.rollRaw } * M_PI / 32768.0f; + + node.transform.position = impl::convertPos_NewDark(prop.position); + node.transform.orientation = uf::quaternion::euler( impl::convertPos_NewDark( facing, 1.0f ) ); + + // deduce name + if ( ctx.customNames.count(entry.objectId) ) { + node.name = impl::sanitizeString(ctx.customNames.at(entry.objectId).c_str(), 128); + } else { + uf::stl::string symName; + if ( ctx.findInheritedProperty(entry.objectId, ctx.archetypes, symName) ) { + node.name = impl::sanitizeString(symName.c_str(), 128); + } + if ( node.name.empty()) { + node.name = ::fmt::format("object_{}", entry.objectId); + } else { + node.name = ::fmt::format("{}_{}", node.name, entry.objectId); + } + } + + // deduce model + node.mesh = -1; + uf::stl::string modelName; + if ( ctx.findInheritedProperty( entry.objectId, ctx.modelNames, modelName ) && !modelName.empty() ) { + uf::stl::string model = sanitizeString(modelName.c_str(), 64); + std::transform( model.begin(), model.end(), model.begin(), ::tolower ); + + if ( !model.ends_with(".bin") ) model += ".bin"; + + uf::stl::string path = "obj://" + model; + auto it = std::find( graph.meshes.begin(), graph.meshes.end(), path ); + if ( it != graph.meshes.end() ) { + node.mesh = (int32_t)std::distance(graph.meshes.begin(), it); + } else if ( !impl::modelBlacklist.count( model ) ) { + size_t meshID = graph.meshes.size(); + if ( ext::ttlg::loadBin( graph, path ) ) { + node.mesh = (int32_t)(meshID); + } else { + node.mesh = -1; + } + } + } + + // deduce light + sLightProp light; + if ( ctx.findInheritedProperty( entry.objectId, ctx.lightProps, light ) && light.active ) { + auto lightKey = ::fmt::format("light_{}", entry.objectId); + auto& graphLight = graph.lights[lightKey]; + graphLight.color = light.color; + graphLight.intensity = light.intensity; + graphLight.range = light.range; + + metadata["light_source"] = lightKey; + } + + // deduce sound + sAmbientSoundProp ambient; + if (ctx.findInheritedProperty(entry.objectId, ctx.ambientSounds, ambient)) { + metadata["sound"]["schema"] = uf::stl::string(ambient.schemaName); + metadata["sound"]["volume"] = ambient.volume; + metadata["sound"]["radius"] = ambient.radius; + metadata["sound"]["flags"] = ambient.flags; + } + + // deduce + uf::stl::vector scripts; + if ( ctx.findInheritedProperty( entry.objectId, ctx.scripts, scripts ) ) { + for ( const auto& s : scripts ) metadata["scripts"].emplace_back(s); + } + + metadata["id"] = entry.objectId; + metadata["cell"] = prop.cell; + + + offset = start + entry.size; + } + } + + void processLinks( pod::Graph& graph, const impl::DarkContext& ctx ) { + uf::stl::unordered_map objectToNode; + for ( size_t i = 0; i < graph.nodes.size(); ++i ) { + auto& node = graph.nodes[i]; + if ( node.metadata["dark"].isObject() && node.metadata["dark"]["id"].is() ) { + int32_t objId = node.metadata["dark"]["id"].as(); + objectToNode[objId] = i; + } + } + + for ( const auto& link : ctx.links ) { + if ( !objectToNode.count(link.sourceId) || !objectToNode.count(link.destId) ) continue; + size_t sourceIdx = objectToNode[link.sourceId]; + size_t destIdx = objectToNode[link.destId]; + + auto& srcNode = graph.nodes[sourceIdx]; + auto& connection = srcNode.metadata["dark"]["connections"].emplace_back(); + + connection["target_node"] = graph.nodes[destIdx].name; + connection["target_id"] = link.destId; + connection["flavor"] = link.flavor; + } + } + + void readInventory( impl::DarkContext& ctx, uf::stl::unordered_map& inventory, uf::stl::vector& buffer, const impl::DarkDBHeader& header ) { + uint32_t chunkCount; + uint32_t offset = header.inv_offset; + if ( !impl::readStruct( buffer, offset, chunkCount ) ) return; + for ( uint32_t i = 0; i < chunkCount; ++i ) { + impl::DarkDBInvItem item; + if ( !impl::readStruct(buffer, offset, item) ) break; + uf::stl::string name = impl::sanitizeString( item.name ); + inventory[name] = item; + } + + // parse strings + if ( inventory.count("P$SymName") > 0 ) { + impl::parseProperty_Strings( ctx, buffer, inventory["P$SymName"], "SymName" ); + } + if ( inventory.count("P$ModelName") > 0 ) { + impl::parseProperty_Strings( ctx, buffer, inventory["P$ModelName"], "ModelName" ); + } + // parse properties + if ( inventory.count("P$Light") > 0 ) { + impl::parseGenericProperties( ctx, buffer, "Light", inventory["P$Light"] ); + } + if ( inventory.count("P$Ambient") > 0 ) { + impl::parseGenericProperties( ctx, buffer, "Ambient", inventory["P$Ambient"] ); + } + if ( inventory.count("P$Scripts") > 0 ) { + impl::parseGenericProperties( ctx, buffer, "Scripts", inventory["P$Scripts"] ); + } + // parse links + if ( inventory.count("Relations") > 0 ) { + impl::parseRelations( ctx, buffer, inventory["Relations"] ); + } + for ( const auto& [ name, item ] : inventory ) { + if ( name.starts_with("L$") && !name.starts_with("LD$")) { + impl::parsePartitionedLinks( ctx, buffer, item ); + } + } + + // bind hierarchy + { + uf::stl::unordered_map flavorCounts; + int16_t bestFlavor = -1; + size_t maxCount = 0; + + for ( const auto& link : ctx.links ) { + if ( link.destId >= 0 ) continue; + flavorCounts[link.flavor]++; + if ( flavorCounts[link.flavor] <= maxCount ) continue; + maxCount = flavorCounts[link.flavor]; + bestFlavor = link.flavor; + } + + if ( bestFlavor != -1 ) { + for ( const auto& link : ctx.links ) { + if ( link.flavor != bestFlavor ) continue; + if ( ctx.parentMap.find(link.sourceId) != ctx.parentMap.end() ) continue; + ctx.parentMap[link.sourceId] = link.destId; + } + } + } + } + + void loadGam( impl::DarkContext& ctx, const uf::stl::string& filename ) { + uf::stl::vector buffer; + if ( !uf::io::readAsBuffer( buffer, filename ) ) { + UF_MSG_WARNING("Failed to read Gamesys data: {}. Archetypes will be missing!", filename); + return; + } + + uint32_t offset = 0; + impl::DarkDBHeader header; + if ( !impl::readStruct( buffer, offset, header ) ) return; + + if ( header.dead_beef != 0xEFBEADDE ) { + UF_MSG_ERROR("Invalid DB in GAM: DEADBEEF not found."); + return; + } + + uf::stl::unordered_map inventory; // temporary inventory, mainly parsing to load in archetypes + impl::readInventory( ctx, inventory, buffer, header ); + } +} + +void ext::ttlg::loadMis( pod::Graph& graph, const uf::stl::string& filename, const uf::Serializer& metadata ) { + uf::stl::vector buffer; + if ( !uf::io::readAsBuffer( buffer, filename ) ) { + UF_MSG_ERROR("Failed to read MIS data: {}", filename); + return; + } + + uint32_t offset = 0; + impl::DarkDBHeader header; + if ( !impl::readStruct( buffer, offset, header ) ) { + UF_MSG_ERROR("Failed to read DB header: {}", filename); + return; + } + if ( header.dead_beef != 0xEFBEADDE ) { + UF_MSG_ERROR("Invalid DB: DEADBEEF not found: {}", filename); + return; + } + if ( header.inv_offset >= buffer.size() ) { + UF_MSG_ERROR("Inventory offset is past EOF: {}", filename); + return; + } + + uf::graph::preprocess( graph, metadata, filename ); + auto& storage = uf::graph::getStorage( graph ); + + // mount files + auto famMount = uf::vfs::mount(ext::zlib::createZipMount("fam://", "game://Data/res/FAM.CRF", 1000), true); + auto mshMount = uf::vfs::mount(ext::zlib::createZipMount("obj://", "game://Data/res/MESH.CRF", 1000), true); + auto bmpMount = uf::vfs::mount(ext::zlib::createZipMount("obj://", "game://Data/res/BITMAP.CRF", 1000), true); + auto objMount = uf::vfs::mount(ext::zlib::createZipMount("obj://", "game://Data/res/OBJ.CRF", 1000), true); + // load initial data + impl::DarkContext ctx; + impl::loadGam( ctx, /*metadata["game"].as*/("game://Data/SHOCK2.GAM") ); // to-do: deduce path from metadata + impl::readInventory( ctx, ctx.inventory, buffer, header ); + // mission parsing + if ( ctx.inventory.count("TXLIST") > 0 ) { + impl::parseTextures( graph, ctx, buffer, ctx.inventory["TXLIST"] ); + impl::loadMaterials( graph, ctx ); + } + if ( ctx.inventory.count("WRRGB") > 0 ) { + impl::parseWorld( graph, ctx, buffer, ctx.inventory["WRRGB"] ); + } + if ( ctx.inventory.count("ObjVec") > 0 && ctx.inventory.count("P$Position") > 0 ) { + impl::parseObjects( graph, ctx, buffer, ctx.inventory["ObjVec"], ctx.inventory["P$Position"] ); + impl::loadMaterials( graph, ctx ); + } + impl::processLinks( graph, ctx ); + // disable postprocessing flags + if ( filename.starts_with("game://") ) graph.metadata["exporter"]["enabled"] = false; // disable exporting if loaded from a VPK + graph.metadata["exporter"]["unwrap"] = false; // do not unwrap UVs for baking (we already have those) + graph.metadata["baking"]["enabled"] = false; // disable lightmap baking (we already have those) + + uf::graph::postprocess(graph); +} \ No newline at end of file diff --git a/engine/src/ext/ttlg/pcx.cpp b/engine/src/ext/ttlg/pcx.cpp new file mode 100644 index 00000000..207932a7 --- /dev/null +++ b/engine/src/ext/ttlg/pcx.cpp @@ -0,0 +1,96 @@ +#include +#include + +bool ext::ttlg::loadPalette( const uf::stl::string& family, uf::stl::vector& palette ) { + if ( family.empty() ) return false; + if ( !palette.empty() ) return true; + + uf::stl::vector buffer; + if ( uf::io::readAsBuffer(buffer, "fam://" + family + "/full.pcx")) { + if ( buffer.size() >= 769 && buffer[buffer.size() - 769] == 0x0C ) { + palette.assign(buffer.end() - 768, buffer.end()); + return true; + } + buffer.clear(); + } + + if ( !uf::io::readAsBuffer(buffer, "fam://" + family + "/full.gif") ) + return false; + if ( buffer.size() < 13 || buffer[0] != 'G' || buffer[1] != 'I' || buffer[2] != 'F') + return false; + + uint8_t packed = buffer[10]; + if ( !(packed & 0x80) ) return false; + + int gctSize = 2 << (packed & 0x07); + int gctBytes = gctSize * 3; + + if ( buffer.size() < 13 + gctBytes) return false; + + palette.assign(buffer.begin() + 13, buffer.begin() + 13 + gctBytes); + while ( palette.size() < 768 ) palette.emplace_back(0); + return true; +} + +bool ext::ttlg::loadPcx( pod::Image& image, const uf::stl::vector& buffer, const uint8_t* paletteData ) { + if ( buffer.size() < 128 ) return false; + if ( buffer[0] != 10 || buffer[2] != 1 ) return false; + + uint16_t xmin = buffer[4] | (buffer[5] << 8); + uint16_t ymin = buffer[6] | (buffer[7] << 8); + uint16_t xmax = buffer[8] | (buffer[9] << 8); + uint16_t ymax = buffer[10] | (buffer[11] << 8); + + int width = xmax - xmin + 1; + int height = ymax - ymin + 1; + uint16_t bytesPerLine = buffer[66] | (buffer[67] << 8); + + uf::stl::vector indices(bytesPerLine * height); + size_t offset = 128; + size_t dest = 0; + + while ( dest < indices.size() && offset < buffer.size() ) { + uint8_t data = buffer[offset++]; + if ( (data & 0xC0) == 0xC0 ) { + uint8_t runLength = data & 0x3F; + uint8_t colorIndex = buffer[offset++]; + for ( int i = 0; i < runLength && dest < indices.size(); ++i ) { + indices[dest++] = colorIndex; + } + } else { + indices[dest++] = data; + } + } + + if ( buffer.size() >= offset + 769 && buffer[buffer.size() - 769] == 0x0C ) { + paletteData = &buffer[buffer.size() - 768]; + } + if ( !paletteData ) { + return false; + } + + image.pixels.resize(width * height * 4); + for ( int y = 0; y < height; ++y ) { + for ( int x = 0; x < width; ++x ) { + uint8_t index = indices[y * bytesPerLine + x]; + int outIdx = (y * width + x) * 4; + + if ( index == 0 ) { + image.pixels[outIdx + 0] = 0; + image.pixels[outIdx + 1] = 0; + image.pixels[outIdx + 2] = 0; + image.pixels[outIdx + 3] = 0; + } else { + image.pixels[outIdx + 0] = paletteData[index * 3 + 0]; + image.pixels[outIdx + 1] = paletteData[index * 3 + 1]; + image.pixels[outIdx + 2] = paletteData[index * 3 + 2]; + image.pixels[outIdx + 3] = 255; + } + } + } + + image.size = { width, height }; + image.channels = 4; + image.bpp = 32; + return true; +} \ No newline at end of file diff --git a/engine/src/ext/valve/bsp.cpp b/engine/src/ext/valve/bsp.cpp index c6a567e2..727df8c9 100644 --- a/engine/src/ext/valve/bsp.cpp +++ b/engine/src/ext/valve/bsp.cpp @@ -603,168 +603,6 @@ namespace impl { impl::extractLumpString( buffer, lump, str ); return str; } - - void processNodes( pod::Graph& graph, const impl::BspContext& context, float scale = impl::sourceToMeters ) { - size_t lights = 0; - - int32_t spawnID = -1; - uf::stl::vector spawns; - uf::stl::unordered_map targets; - - for ( auto nodeID : graph.root.children ) { - auto& node = graph.nodes[nodeID]; - auto& metadata = node.metadata["valve"]; - - if ( !ext::json::isObject( metadata ) ) continue; - - auto classname = metadata["classname"].as(""); - //UF_MSG_INFO("Entity found: {}", classname); - node.name = classname; - node.mesh = -1; - - // parse origin - auto origin = metadata["origin"].as(""); - if ( origin != "" ) { - auto position = impl::str2vec( origin ); - node.transform.position = impl::convertPos( position, scale ); - } - - // parse angles - auto angles = metadata["angles"].as(""); - if ( angles != "" ) { - auto pyr = impl::str2vec( angles ) * -DEG_2_RAD; - node.transform.orientation = uf::quaternion::euler( pyr ); - } - - // parse model - auto model = metadata["model"].as(); - if ( classname == "worldspawn" ) { - node.mesh = context.modelToMesh[0]; // implicitly bind to model 0 - } else if ( model.starts_with("*") ) { - int modelID = std::stoi( model.substr(1) ); - if ( 0 <= modelID && modelID < context.modelToMesh.size() ) { - node.mesh = context.modelToMesh[modelID]; - } - } else if ( model.length() > 4 && model.ends_with(".mdl") ) { - auto it = std::find(graph.meshes.begin(), graph.meshes.end(), model); - if ( it == graph.meshes.end() ) { - auto meshID = graph.meshes.size(); - if ( ext::valve::loadMdl(graph, model) ) { - node.mesh = meshID; - } else { - uf::stl::string model = "models/error.mdl"; - auto it = std::find(graph.meshes.begin(), graph.meshes.end(), model); - - if ( it != graph.meshes.end() ) { - node.mesh = (int32_t)std::distance(graph.meshes.begin(), it); - } else if ( ext::valve::loadMdl( graph, model ) ) { - node.mesh = (int32_t)(graph.meshes.size() - 1); - } else { - node.mesh = -1; - } - } - } else { - node.mesh = (int32_t)std::distance(graph.meshes.begin(), it); - } - } - - // parse lighting info - if ( classname.starts_with("light") ) { - auto lightKeyName = ::fmt::format( "{}_{}", classname, nodeID ); - auto& light = graph.lights[lightKeyName]; - light.color = { 1.0f, 1.0f, 1.0f }; - light.intensity = 200.0f; - light.range = 0.0f; - - // read color and intensity - auto _light = metadata["_light"].as(""); - if ( _light != "" ) { - // to-do: do not use stringstream - std::istringstream stream(_light); - light.color = { 255.0f, 255.0f, 255.0f }; - - stream >> light.color.x >> light.color.y >> light.color.z; - light.color /= 255.0f; - - if (!(stream >> light.intensity)) light.intensity = 200.0f; - } - // to-do: read range - light.intensity *= 0.2f; // scale down - } - - // parse door - if ( classname.starts_with("func_door") ) { - auto& metadataDoor = metadata["door"]; - - float scale = impl::sourceToMeters; - if ( classname == "func_door" ) { - auto movedirStr = metadata["movedir"].as(""); - if ( movedirStr == "" && metadata["angle"].as() ) { - float ang = metadata["angle"].as(); - if ( ang == -1 ) movedirStr = "-90 0 0"; - else if ( ang == -2 ) movedirStr = "90 0 0"; - else movedirStr = ::fmt::format("0 {} 0", ang); - } - - if ( movedirStr != "" ) { - auto pyr = impl::str2vec( movedirStr ); - - float pitch = pyr.x * DEG_2_RAD; - float yaw = pyr.y * DEG_2_RAD; - - pod::Vector3f slideDir; - slideDir.x = cos(yaw) * cos(pitch); - slideDir.z = -(sin(yaw) * cos(pitch)); - slideDir.y = -sin(pitch); - - metadataDoor["direction"] = uf::vector::encode( uf::vector::normalize(slideDir) ); - } - } else if ( classname == "func_door_rotating" ) { - scale = 1.0f; - metadataDoor["distance"] = metadata["distance"].as(90.0f); - - int flags = metadataDoor["spawnflags"].as(); - pod::Vector3f axis = {0, 1, 0}; - if ( flags & 64 ) axis = {0, 0, 1}; - if ( flags & 128 ) axis = {1, 0, 0}; - - metadataDoor["axis"] = uf::vector::encode( axis ); - } - - metadataDoor["speed"] = metadata["speed"].as(100.0f) * scale; - metadataDoor["wait"] = metadata["wait"].as(4.0f); - metadataDoor["lip"] = metadata["lip"].as(8.0f) * scale; - metadataDoor["spawnflags"] = metadata["spawnflags"].as(0); - } - - // parse parent - auto targetname = metadata["targetname"].as(""); - if ( targetname != "" ) { - targets[targetname] = nodeID; - } - - // to-do: add additional parsing - } - - // re-parent entities - uf::stl::vector newChildren; - for ( auto nodeID : graph.root.children ) { - auto& node = graph.nodes[nodeID]; - auto& metadata = node.metadata["valve"]; - - auto parentname = metadata["parentname"].as(""); - if ( parentname != "" && targets.count(parentname) > 0 ) { - auto parentID = targets[parentname]; - auto& parentNode = graph.nodes[parentID]; - parentNode.children.emplace_back(nodeID); - - node.transform = uf::transform::relative( parentNode.transform, node.transform ); - } else { - newChildren.emplace_back(nodeID); - } - } - graph.root.children = newChildren; - } } void ext::valve::loadBsp( pod::Graph& graph, const uf::stl::string& filename, const uf::Serializer& metadata ) { @@ -780,12 +618,8 @@ void ext::valve::loadBsp( pod::Graph& graph, const uf::stl::string& filename, co return; } - if ( !graph.storage ) graph.storage = new pod::Graph::Storage(); + uf::graph::preprocess( graph, metadata, filename ); auto& storage = uf::graph::getStorage( graph ); - graph.name = filename; - graph.metadata = metadata; - graph.root.name = "%ROOT%"; - graph.root.index = -1; impl::BspContext context; impl::extractLump(buffer, header->lumps[impl::BspLump::LUMP_VERTICES], context.vertices); @@ -814,7 +648,7 @@ void ext::valve::loadBsp( pod::Graph& graph, const uf::stl::string& filename, co context.texdataToMaterial.assign( context.texdatas.size(), -1 ); // mount pakfile - size_t pakfileMount = uf::vfs::mount( ext::zlib::createZipMount(::fmt::format("pakfile://{}", filename), context.pakfile, 1000 ) ); + auto pakfileMount = uf::vfs::mount( ext::zlib::createZipMount(::fmt::format("pakfile://{}", filename), context.pakfile, 1000 ), true ); // deduce mapname uf::stl::string mapName = filename; { @@ -1116,8 +950,6 @@ void ext::valve::loadBsp( pod::Graph& graph, const uf::stl::string& filename, co auto mlets = uf::stl::values( meshlets ); auto partitioned = uf::meshgrid::partition( grid, mlets, EPS, true, true ); mesh.compile( partitioned, primitives ); - UF_MSG_DEBUG("meshlets={}, partitioned={}, primitives={}", mlets.size(), partitioned.size(), primitives.size()); - UF_ASSERT( !partitioned.empty() ); } else { mesh.compile( meshlets, primitives ); } @@ -1209,7 +1041,168 @@ void ext::valve::loadBsp( pod::Graph& graph, const uf::stl::string& filename, co } } - impl::processNodes( graph, context ); + // process nodes + { + size_t lights = 0; + + int32_t spawnID = -1; + uf::stl::vector spawns; + uf::stl::unordered_map targets; + + for ( auto nodeID : graph.root.children ) { + auto& node = graph.nodes[nodeID]; + auto& metadata = node.metadata["valve"]; + + if ( !ext::json::isObject( metadata ) ) continue; + + auto classname = metadata["classname"].as(""); + //UF_MSG_INFO("Entity found: {}", classname); + node.name = classname; + node.mesh = -1; + + // parse origin + auto origin = metadata["origin"].as(""); + if ( origin != "" ) { + auto position = impl::str2vec( origin ); + node.transform.position = impl::convertPos( position ); + } + + // parse angles + auto angles = metadata["angles"].as(""); + if ( angles != "" ) { + auto pyr = impl::str2vec( angles ) * -DEG_2_RAD; + node.transform.orientation = uf::quaternion::euler( pyr ); + } + + // parse model + auto model = metadata["model"].as(); + if ( classname == "worldspawn" ) { + node.mesh = context.modelToMesh[0]; // implicitly bind to model 0 + } else if ( model.starts_with("*") ) { + int modelID = std::stoi( model.substr(1) ); + if ( 0 <= modelID && modelID < context.modelToMesh.size() ) { + node.mesh = context.modelToMesh[modelID]; + } + } else if ( model.length() > 4 && model.ends_with(".mdl") ) { + auto it = std::find(graph.meshes.begin(), graph.meshes.end(), model); + if ( it == graph.meshes.end() ) { + auto meshID = graph.meshes.size(); + if ( ext::valve::loadMdl(graph, model) ) { + node.mesh = meshID; + } else { + uf::stl::string model = "models/error.mdl"; + auto it = std::find(graph.meshes.begin(), graph.meshes.end(), model); + + if ( it != graph.meshes.end() ) { + node.mesh = (int32_t)std::distance(graph.meshes.begin(), it); + } else if ( ext::valve::loadMdl( graph, model ) ) { + node.mesh = (int32_t)(graph.meshes.size() - 1); + } else { + node.mesh = -1; + } + } + } else { + node.mesh = (int32_t)std::distance(graph.meshes.begin(), it); + } + } + + // parse lighting info + if ( classname.starts_with("light") ) { + auto lightKeyName = ::fmt::format( "{}_{}", classname, nodeID ); + auto& light = graph.lights[lightKeyName]; + light.color = { 1.0f, 1.0f, 1.0f }; + light.intensity = 200.0f; + light.range = 0.0f; + + // read color and intensity + auto _light = metadata["_light"].as(""); + if ( _light != "" ) { + // to-do: do not use stringstream + std::istringstream stream(_light); + light.color = { 255.0f, 255.0f, 255.0f }; + + stream >> light.color.x >> light.color.y >> light.color.z; + light.color /= 255.0f; + + if (!(stream >> light.intensity)) light.intensity = 200.0f; + } + // to-do: read range + light.intensity *= 0.2f; // scale down + } + + // parse door + if ( classname.starts_with("func_door") ) { + auto& metadataDoor = metadata["door"]; + + float scale = impl::sourceToMeters; + if ( classname == "func_door" ) { + auto movedirStr = metadata["movedir"].as(""); + if ( movedirStr == "" && metadata["angle"].as() ) { + float ang = metadata["angle"].as(); + if ( ang == -1 ) movedirStr = "-90 0 0"; + else if ( ang == -2 ) movedirStr = "90 0 0"; + else movedirStr = ::fmt::format("0 {} 0", ang); + } + + if ( movedirStr != "" ) { + auto pyr = impl::str2vec( movedirStr ); + + float pitch = pyr.x * DEG_2_RAD; + float yaw = pyr.y * DEG_2_RAD; + + pod::Vector3f slideDir; + slideDir.x = cos(yaw) * cos(pitch); + slideDir.z = -(sin(yaw) * cos(pitch)); + slideDir.y = -sin(pitch); + + metadataDoor["direction"] = uf::vector::encode( uf::vector::normalize(slideDir) ); + } + } else if ( classname == "func_door_rotating" ) { + scale = 1.0f; + metadataDoor["distance"] = metadata["distance"].as(90.0f); + + int flags = metadataDoor["spawnflags"].as(); + pod::Vector3f axis = {0, 1, 0}; + if ( flags & 64 ) axis = {0, 0, 1}; + if ( flags & 128 ) axis = {1, 0, 0}; + + metadataDoor["axis"] = uf::vector::encode( axis ); + } + + metadataDoor["speed"] = metadata["speed"].as(100.0f) * scale; + metadataDoor["wait"] = metadata["wait"].as(4.0f); + metadataDoor["lip"] = metadata["lip"].as(8.0f) * scale; + metadataDoor["spawnflags"] = metadata["spawnflags"].as(0); + } + + // parse parent + auto targetname = metadata["targetname"].as(""); + if ( targetname != "" ) { + targets[targetname] = nodeID; + } + + // to-do: add additional parsing + } + + // re-parent entities + uf::stl::vector newChildren; + for ( auto nodeID : graph.root.children ) { + auto& node = graph.nodes[nodeID]; + auto& metadata = node.metadata["valve"]; + + auto parentname = metadata["parentname"].as(""); + if ( parentname != "" && targets.count(parentname) > 0 ) { + auto parentID = targets[parentname]; + auto& parentNode = graph.nodes[parentID]; + parentNode.children.emplace_back(nodeID); + + node.transform = uf::transform::relative( parentNode.transform, node.transform ); + } else { + newChildren.emplace_back(nodeID); + } + } + graph.root.children = newChildren; + } // load materials uf::stl::vector missing_pixels = { 255, 0, 255, 255, 0, 0, 0, 255, 0, 0, 0, 255, 255, 0, 255, 255 }; @@ -1317,13 +1310,11 @@ void ext::valve::loadBsp( pod::Graph& graph, const uf::stl::string& filename, co image.loadFromBuffer( missing_pixels, { 2, 2 }, 8, 4 ); } - // disable exporting if loaded from a VPK - if ( filename.starts_with("valve://") ) graph.metadata["exporter"]["enabled"] = false; - graph.metadata["exporter"]["unwrap"] = false; // not necessary to unwrap + // disable postprocessing flags + if ( filename.starts_with("valve://") ) graph.metadata["exporter"]["enabled"] = false; // disable exporting if loaded from a VPK + graph.metadata["exporter"]["unwrap"] = false; // do not unwrap UVs for baking (we already have those) + graph.metadata["baking"]["enabled"] = false; // disable lightmap baking (we already have those) uf::graph::postprocess( graph ); - - // unmount pakfile - uf::vfs::unmount( pakfileMount ); } #endif \ No newline at end of file diff --git a/engine/src/ext/valve/vpk.cpp b/engine/src/ext/valve/vpk.cpp index d8ce39db..1963ed4c 100644 --- a/engine/src/ext/valve/vpk.cpp +++ b/engine/src/ext/valve/vpk.cpp @@ -267,8 +267,25 @@ bool ext::valve::readVpk( const pod::VpkArchive& vpk, const uf::stl::string& pat return true; } -size_t ext::valve::mountVpk( const uf::stl::string& uri ) { - return uf::vfs::mount( ext::valve::createVpkMount( ::fmt::format( "valve://{}", uri ), 10 ) ); +uf::vfs::Mount ext::valve::mountVpk( const uf::stl::string& uri, bool temp ) { + return uf::vfs::mount( ext::valve::createVpkMount( ::fmt::format( "valve://{}", uri ), 10 ), temp ); +} +uf::vfs::Mount ext::valve::mountGame( const uf::stl::string& uri, bool temp ) { + uf::stl::string path = ""; + auto libraries = impl::getSteamLibraries(); + for ( const auto& lib : libraries ) { + uf::stl::string fullPath = lib + "/" + uri; + if ( !uf::io::exists(fullPath) ) { + continue; + } + path = fullPath; + break; + } + if ( path.empty() ) { + UF_MSG_ERROR("Failed to mount game: {}", uri); + return { 0 }; + } + return uf::vfs::mount( uf::vfs::createDiskMount( ::fmt::format( "game://{}", path ), 11 ), temp ); // for some reason a lower priority makes the footstep sound CS:S's } bool ext::valve::readVpkRange( const pod::VpkArchive& vpk, const uf::stl::string& path, size_t start, size_t len, uf::stl::vector& buffer ) { auto it = vpk.files.find( path ); diff --git a/engine/src/ext/vulkan/rendermodes/vr.cpp b/engine/src/ext/vulkan/rendermodes/vr.cpp index df61b9e1..7cc30311 100644 --- a/engine/src/ext/vulkan/rendermodes/vr.cpp +++ b/engine/src/ext/vulkan/rendermodes/vr.cpp @@ -164,9 +164,9 @@ void ext::vulkan::VrRenderMode::createCommandBuffers(const uf::stl::vector& buffer, const uf::stl::string& filename ) { +bool ext::zlib::decompressFromFile( uf::stl::vector& buffer, const uf::stl::string& filename, int flag ) { size_t fileSize = uf::vfs::size(filename); if (fileSize == 0) { UF_MSG_ERROR("Zlib: file not found or empty: {}", filename); @@ -24,7 +24,7 @@ bool ext::zlib::decompressFromFile( uf::stl::vector& buffer, const uf:: } z_stream strm{}; - if (inflateInit2(&strm, 15 + 32) != Z_OK) return false; + if (inflateInit2(&strm, flag) != Z_OK) return false; uint8_t outBuffer[ext::zlib::bufferSize]; @@ -50,12 +50,12 @@ bool ext::zlib::decompressFromFile( uf::stl::vector& buffer, const uf:: return success; } -bool ext::zlib::decompressFromFile( uf::stl::vector& buffer, const uf::stl::string& filename, size_t start, size_t len ) { +bool ext::zlib::decompressFromFile( uf::stl::vector& buffer, const uf::stl::string& filename, size_t start, size_t len, int flag ) { size_t fileSize = uf::vfs::size(filename); if (fileSize == 0) return false; z_stream strm{}; - if (inflateInit2(&strm, 15 + 32) != Z_OK) return false; + if (inflateInit2(&strm, flag) != Z_OK) return false; size_t uncompressedOffset = 0; uint8_t outBuffer[ext::zlib::bufferSize]; @@ -96,7 +96,7 @@ bool ext::zlib::decompressFromFile( uf::stl::vector& buffer, const uf:: return success; } -bool ext::zlib::decompressFromFile( uf::stl::vector& buffer, const uf::stl::string& filename, const uf::stl::vector& ranges ) { +bool ext::zlib::decompressFromFile( uf::stl::vector& buffer, const uf::stl::string& filename, const uf::stl::vector& ranges, int flag ) { if ( ranges.empty() ) return false; uf::stl::vector sortedRanges = ranges; @@ -106,7 +106,7 @@ bool ext::zlib::decompressFromFile( uf::stl::vector& buffer, const uf:: if (fileSize == 0) return false; z_stream strm{}; - if (inflateInit2(&strm, 15 + 32) != Z_OK) return false; + if (inflateInit2(&strm, flag) != Z_OK) return false; size_t uncompressedOffset = 0; size_t currentRangeIdx = 0; @@ -149,11 +149,11 @@ bool ext::zlib::decompressFromFile( uf::stl::vector& buffer, const uf:: return success; } -bool ext::zlib::decompressFromMemory( uf::stl::vector& dst, const void* src, size_t size, size_t usize ) { +bool ext::zlib::decompressFromMemory( uf::stl::vector& dst, const void* src, size_t size, size_t usize, int flag ) { if (size == 0) return false; z_stream strm{}; - if (inflateInit2(&strm, 15 + 32) != Z_OK) return false; + if (inflateInit2(&strm, flag) != Z_OK) return false; strm.avail_in = (uInt)size; strm.next_in = (Bytef*)src; @@ -165,10 +165,13 @@ bool ext::zlib::decompressFromMemory( uf::stl::vector& dst, const void* int ret = inflate(&strm, Z_FINISH); inflateEnd(&strm); - return (ret == Z_STREAM_END || ret == Z_OK); + if ( ret != Z_STREAM_END && ret != Z_OK ) { + UF_MSG_ERROR("Decompress encountered error: {}", ret); + } + return true; } -bool ext::zlib::decompressScatter( const uf::stl::string& filename, uf::stl::vector& requests ) { +bool ext::zlib::decompressScatter( const uf::stl::string& filename, uf::stl::vector& requests, int flag ) { if ( requests.empty() ) return true; std::sort(requests.begin(), requests.end(), [](const pod::ScatterRequest& a, const pod::ScatterRequest& b) { @@ -176,7 +179,7 @@ bool ext::zlib::decompressScatter( const uf::stl::string& filename, uf::stl::vec }); z_stream strm{}; - if ( inflateInit2(&strm, 15 + 32) != Z_OK ) return false; + if ( inflateInit2(&strm, flag) != Z_OK ) return false; size_t uncompressedOffset = 0; size_t currentReqIdx = 0; @@ -343,7 +346,7 @@ namespace { return true; } if ( entry.compressionMethod == 8 ) { - return ext::zlib::decompressFromMemory(buffer, fileData, entry.compressedSize, entry.uncompressedSize); + return ext::zlib::decompressFromMemory(buffer, fileData, entry.compressedSize, entry.uncompressedSize, -15); } // ID for lz4 return false; @@ -351,6 +354,10 @@ namespace { } pod::Mount ext::zlib::createZipMount( const uf::stl::string& uri, uf::stl::vector& buffer, int priority ) { + return ext::zlib::createZipMount(uri, std::move(buffer), priority); +} + +pod::Mount ext::zlib::createZipMount( const uf::stl::string& uri, uf::stl::vector&& buffer, int priority ) { uf::stl::string prefix; uf::stl::string path; uf::io::splitUri( uri, prefix, path ); @@ -365,10 +372,28 @@ pod::Mount ext::zlib::createZipMount( const uf::stl::string& uri, uf::stl::vecto mount.read = ::vfs_read; auto& state = uf::pointeredUserdata::get( mount.userdata ); - state.buffer = buffer; // should be a move? + state.buffer = buffer; ext::zlib::directory( state.buffer, state.entries ); + if ( mount.path.empty() ) { + mount.path = ::fmt::format( "{}/{}", uri, (void*) state.buffer.data() ); + } + +// for ( auto& [ k, v ] : state.entries ) UF_MSG_DEBUG("{} => {}{}", mount.path, uri, k); return mount; } +pod::Mount ext::zlib::createZipMount( const uf::stl::string& uri, const uf::stl::string& filename, int priority ) { + uf::stl::vector buffer; + if ( !uf::io::exists( filename ) ) { + UF_MSG_ERROR("Does not exist: {}", filename); + } + if ( !uf::io::readAsBuffer(buffer, filename) ) { + UF_MSG_ERROR("Failed to load ZIP mount from disk: {}", filename); + return pod::Mount{}; + } + + return ext::zlib::createZipMount( ::fmt::format("{}/{}", uri, filename), std::move(buffer), priority ); +} + #endif \ No newline at end of file diff --git a/engine/src/utils/io/vfs.cpp b/engine/src/utils/io/vfs.cpp index f23e1305..eb6aab4a 100644 --- a/engine/src/utils/io/vfs.cpp +++ b/engine/src/utils/io/vfs.cpp @@ -112,6 +112,10 @@ namespace { } } +uf::vfs::Mount::~Mount() { + if ( temp ) uf::vfs::unmount( hash ); +} + pod::Mount uf::vfs::createDiskMount( const uf::stl::string& uri, int priority) { uf::stl::string prefix; uf::stl::string path; @@ -135,7 +139,7 @@ pod::Mount uf::vfs::createDiskMount( const uf::stl::string& uri, int priority) { } uf::stl::vector uf::vfs::mounts; -size_t uf::vfs::mount( const pod::Mount& mount ) { +uf::vfs::Mount uf::vfs::mount( const pod::Mount& mount, bool temp ) { // compute hash size_t hash = {}; uf::hash( hash, mount.prefix, mount.path ); @@ -144,8 +148,9 @@ size_t uf::vfs::mount( const pod::Mount& mount ) { for ( auto& m : mounts ) { size_t hash2 = {}; uf::hash( hash2, m.prefix, m.path ); + if ( hash == hash2 ) { - return hash; + return uf::vfs::Mount{ hash, false }; // do not honor temp request to avoid breaking mounts in the future } } @@ -157,7 +162,7 @@ size_t uf::vfs::mount( const pod::Mount& mount ) { return a.priority > b.priority; }); - return hash; + return uf::vfs::Mount{ hash, temp }; } bool uf::vfs::unmount( size_t hash ) { @@ -175,6 +180,9 @@ bool uf::vfs::unmount( size_t hash ) { mounts.erase( it, mounts.end() ); return true; } +bool uf::vfs::unmount( const uf::vfs::Mount& mount ) { + return uf::vfs::unmount( mount.hash ); +} bool uf::vfs::unmount( const uf::stl::string& prefix, const uf::stl::string& base ) { uf::stl::string cleanBase = base; if ( !cleanBase.empty() && cleanBase.back() != '/' && cleanBase.back() != '\\' ) cleanBase += '/'; @@ -259,7 +267,7 @@ bool uf::vfs::read( const uf::stl::string& path, uf::stl::vector& buffe if ( prefix.empty() && mount.priority < 0 ) continue; if ( prefix.empty() || mount.prefix == prefix ) { bool res = mount.exists( mount, relative ); - if ( mount.exists( mount, relative ) ) return mount.read( mount, relative, buffer ); + if ( mount.exists( mount, relative ) && mount.read( mount, relative, buffer ) ) return true;; } } return false; @@ -272,7 +280,7 @@ size_t uf::vfs::write( const uf::stl::string& path, const void* buffer, size_t s for ( auto& mount : mounts ) { if ( prefix.empty() && mount.priority < 0 ) continue; if ( prefix.empty() || mount.prefix == prefix ) { - if ( mount.write ) return mount.write( mount, relative, buffer, size ); + if ( mount.write && mount.write( mount, relative, buffer, size ) ) return true; } } return 0; diff --git a/makefiles/dependencies.mk b/makefiles/dependencies.mk index a7934e84..d95e4af3 100644 --- a/makefiles/dependencies.mk +++ b/makefiles/dependencies.mk @@ -173,4 +173,8 @@ endif ifneq (,$(findstring valve,$(REQ_DEPS))) FLAGS += -DUF_USE_VALVE +endif + +ifneq (,$(findstring ttlg,$(REQ_DEPS))) + FLAGS += -DUF_USE_TTLG endif \ No newline at end of file diff --git a/makefiles/platforms/win64.mk b/makefiles/platforms/win64.mk index 5f75b3e0..007a7e5a 100644 --- a/makefiles/platforms/win64.mk +++ b/makefiles/platforms/win64.mk @@ -1,5 +1,5 @@ ifneq (,$(findstring -DUF_DEV_ENV,$(FLAGS))) - REQ_DEPS += meshoptimizer toml xatlas curl dc:texconv ffx:sdk openvr valve # vall_e cpptrace # ncurses draco discord ultralight-ux + REQ_DEPS += meshoptimizer toml xatlas curl dc:texconv ffx:sdk openvr valve ttlg # vall_e cpptrace # ncurses draco discord ultralight-ux FLAGS += -march=native -g # -flto # -g endif