added lz4 because i forgot to finish it i guess (and starting to abstract this batched-scattered read paradigm)

.gitignore

@@ -50,6 +50,7 @@ default/
 
 # Assets
 *.gz
+*.lz4
 *.png
 *.dtex
 *.glb

engine/inc/uf/engine/asset/asset.h

@@ -14,14 +14,6 @@ namespace uf {
 	namespace asset {
 		typedef uf::Hooks::name_t callback_t;
 
-		struct UF_API Job {
-			typedef uf::stl::vector<Job> container_t;
-
-			uf::asset::callback_t callback = "";
-			uf::stl::string type = "";
-			uf::asset::Payload payload = {};
-		};
-
 	#if UF_COMPONENT_POINTERED_USERDATA
 		typedef pod::PointeredUserdata userdata_t;
 	#else
@@ -32,21 +24,22 @@ namespace uf {
 		extern UF_API bool asyncQueue;
 		
 		extern UF_API uf::stl::unordered_map<uf::stl::string, uf::asset::userdata_t> map;
-		extern UF_API Job::container_t jobs;
-		extern UF_API Job::container_t finishedJobs;
+
 		extern UF_API uf::Serializer metadata;
 	
-	//	extern UF_API uf::Serializer map;
-
 		uf::asset::Payload UF_API resolveToPayload( const uf::stl::string&, const uf::stl::string& = "" );
 		bool UF_API isExpected( const uf::asset::Payload&, uf::asset::Type expected );
 
 		// URL or file path
 		void UF_API processQueue();
+        void UF_API processIO();
 
 		void UF_API cache( const uf::asset::callback_t&, const uf::asset::Payload& );
 		void UF_API load( const uf::asset::callback_t&, const uf::asset::Payload& );
 
+		void UF_API read( const uf::stl::string& filename, size_t offset, size_t length, uint8_t* dest, std::function<void()> callback = {} );
+		void UF_API stream( const uf::stl::string& filename, size_t offset, size_t length, size_t chunkSize, std::function<bool(const uint8_t* data, size_t size, size_t fileOffset)> callback );
+
 		uf::stl::string UF_API cache( uf::asset::Payload& );
 		uf::stl::string UF_API load( uf::asset::Payload& );
 
@@ -96,7 +89,6 @@ namespace uf {
 		#endif
 			return uf::asset::get<T>( url );
 		}
-
 	}
 }
 

engine/inc/uf/ext/lz4/lz4.h

@@ -0,0 +1,43 @@
+#pragma once
+
+#include <uf/config.h>
+#if UF_USE_LZ4
+
+#include <uf/utils/memory/vector.h>
+#include <uf/utils/memory/string.h>
+#include <uf/utils/memory/unordered_map.h>
+#include <uf/utils/io/file.h>
+#include <uf/utils/io/vfs.h>
+
+namespace ext {
+	namespace lz4 {
+		extern UF_API size_t bufferSize;
+		
+		bool UF_API decompressFromFile( uf::stl::vector<uint8_t>&, const uf::stl::string& );
+		bool UF_API decompressFromFile( uf::stl::vector<uint8_t>&, const uf::stl::string& filename, size_t start, size_t len );
+		bool UF_API decompressFromFile( uf::stl::vector<uint8_t>&, const uf::stl::string& filename, const uf::stl::vector<pod::Range>& ranges );
+		bool UF_API decompressFromMemory( uf::stl::vector<uint8_t>&, const void*, size_t, size_t );
+		bool UF_API decompressScatter( const uf::stl::string& filename, uf::stl::vector<pod::ScatterRequest>& requests );
+		
+		size_t UF_API compressToFile( const uf::stl::string&, const void*, size_t );
+		pod::Mount UF_API createZipMount( const uf::stl::string& uri, uf::stl::vector<uint8_t>& buffer, int priority = 0 );
+
+		inline uf::stl::vector<uint8_t> decompressFromFile( const uf::stl::string& filename ) {
+			uf::stl::vector<uint8_t> buffer;
+			decompressFromFile( buffer, filename );
+			return buffer;
+		}
+		inline uf::stl::vector<uint8_t> decompressFromFile( const uf::stl::string& filename, size_t start, size_t len ) {
+			uf::stl::vector<uint8_t> buffer;
+			decompressFromFile( buffer, filename, start, len );
+			return buffer;
+		}
+		inline uf::stl::vector<uint8_t> decompressFromFile( const uf::stl::string& filename, const uf::stl::vector<pod::Range>& ranges ) {
+			uf::stl::vector<uint8_t> buffer;
+			decompressFromFile( buffer, filename, ranges );
+			return buffer;
+		}
+	}
+}
+
+#endif

engine/src/engine/asset/asset.cpp

@@ -17,6 +17,7 @@
 #include <uf/ext/gltf/gltf.h>
 #include <uf/engine/graph/graph.h>
 #include <uf/engine/scene/scene.h>
+#include <uf/ext/zlib/zlib.h>
 
 #include <mutex>
 
@@ -38,35 +39,67 @@ namespace {
 		return true;
 	}
 
-	std::mutex mutex;
-	uint64_t uid = 0;
-	struct Job {
-		typedef uf::stl::vector<Job> container_t;
+	namespace jobs {
+		struct Job {
+			typedef uf::stl::vector<Job> container_t;
 
-		uf::asset::callback_t callback;
-		uf::stl::string type;
-		uf::asset::Payload payload;
+			uf::asset::callback_t callback = "";
+			uf::stl::string type = "";
+			uf::asset::Payload payload = {};
+		};
+
+		std::mutex mutex;
+		Job::container_t queue;
+		Job::container_t finished;
+	};
+
+	namespace io_read {
+		struct Job {
+			typedef uf::stl::unordered_map<uf::stl::string, uf::stl::vector<Job>> container_t;
+
+			size_t offset;
+			size_t length;
+			uint8_t* dest;
+			std::function<void()> callback;
+		};
+
+		std::mutex mutex;
+		Job::container_t queue;
+		Job::container_t finished;
+	};
+
+	namespace io_stream {
+		struct Job {
+			typedef uf::stl::unordered_map<uf::stl::string, uf::stl::vector<Job>> container_t;
+
+			size_t offset;
+			size_t length;
+			size_t chunkSize;
+			std::function<bool(const uint8_t* data, size_t size, size_t fileOffset)> callback;
+		};
+
+		std::mutex mutex;
+		Job::container_t queue;
+		Job::container_t finished;
 	};
 }
 
 // uf::asset uf::asset::masterAssetLoader;
 bool uf::asset::assertionLoad = true;
 bool uf::asset::asyncQueue = true;
-uf::asset::Job::container_t uf::asset::jobs;
-uf::asset::Job::container_t uf::asset::finishedJobs;
 uf::stl::unordered_map<uf::stl::string, uf::asset::userdata_t> uf::asset::map;
 uf::Serializer uf::asset::metadata;
 
 void uf::asset::processQueue() {
-	if ( uf::asset::jobs.empty() && uf::asset::finishedJobs.empty() ) return;
+	if ( ::jobs::queue.empty() && ::jobs::finished.empty() ) return;
 
-	STATIC_THREAD_LOCAL(uf::asset::Job::container_t, jobs);
-	uf::asset::Job::container_t finishedJobs;
+	STATIC_THREAD_LOCAL(::jobs::Job::container_t, jobs);
+	::jobs::Job::container_t finishedJobs;
 
-	mutex.lock();
-	std::swap( jobs, uf::asset::jobs );
-	std::swap( finishedJobs, uf::asset::finishedJobs );
-	mutex.unlock();
+	::jobs::mutex.lock();
+	std::swap( jobs, ::jobs::queue );
+	std::swap( finishedJobs, ::jobs::finished );
+	::jobs::mutex.unlock();
 
 	bool async = uf::asset::asyncQueue; // a bit buggy
 	auto tasks = uf::thread::schedule(async ? uf::thread::asyncThreadName : uf::thread::mainThreadName, !true);
@@ -88,25 +121,102 @@ void uf::asset::processQueue() {
 		uf::stl::string filename = type == "cache" ? uf::asset::cache(payload) : uf::asset::load(payload);
 		
 		if ( callback != "" && filename != "" ) {
-			mutex.lock();
-			uf::asset::finishedJobs.emplace_back( job );
-			mutex.unlock();
+			::jobs::mutex.lock();
+			::jobs::finished.emplace_back( job );
+			::jobs::mutex.unlock();
 		}
 	});
 
 	uf::thread::execute( tasks );
 }
+
+void uf::asset::processIO() {
+	STATIC_THREAD_LOCAL(::io_read::Job::container_t, pendingReads);
+	STATIC_THREAD_LOCAL(::io_stream::Job::container_t, pendingStreams);
+
+	::io_read::mutex.lock();
+	std::swap(pendingReads, ::io_read::queue);
+	::io_read::mutex.unlock();
+	::io_stream::mutex.lock();
+	std::swap(pendingStreams, ::io_stream::queue);
+	::io_stream::mutex.unlock();
+
+	if ( pendingReads.empty() && pendingStreams.empty() ) return;
+
+	bool async = uf::asset::asyncQueue;
+	auto tasks = uf::thread::schedule(async ? uf::thread::asyncThreadName : uf::thread::mainThreadName, false);
+
+	for ( auto& [filename, requests] : pendingReads ) {
+		tasks.queue([filename = filename, requests = std::move(requests)]() {
+			uf::stl::vector<pod::ScatterRequest> scatterReqs;
+			scatterReqs.reserve(requests.size());
+			for ( auto& req : requests ) scatterReqs.emplace_back(pod::ScatterRequest{ req.offset, req.length, req.dest });
+
+			uf::io::readScatter( filename, scatterReqs );
+
+			for ( auto& req : requests ) if ( req.callback ) req.callback();
+		});
+	}
+
+	for (auto& [filename, requests] : pendingStreams) {
+		tasks.queue([filename = filename, requests = std::move(requests)]() mutable {
+			size_t maxEndOffset = 0;
+			for ( auto& req : requests ) {
+				maxEndOffset = std::max(maxEndOffset, req.offset + req.length);
+			}
+
+			size_t readChunkSize = ext::zlib::bufferSize;
+			for ( auto& req : requests ) {
+				if ( req.chunkSize > 0 ) readChunkSize = std::min(readChunkSize, req.chunkSize);
+			}
+
+			size_t currentOffset = 0;
+			uf::vfs::stream( filename, readChunkSize, [&](const uint8_t* data, size_t size) -> bool {
+				size_t chunkStart = currentOffset;
+				size_t chunkEnd = currentOffset + size;
+				currentOffset += size;
+
+				bool anyActive = false;
+				for ( auto& req : requests ) {
+					if ( !req.callback ) continue;
+					anyActive = true;
+
+					if ( chunkEnd > req.offset && chunkStart < req.offset + req.length ) {
+						size_t copyStart = (chunkStart < req.offset) ? (req.offset - chunkStart) : 0;
+						size_t copyLen = std::min(size - copyStart, (req.offset + req.length) - (chunkStart + copyStart));
+
+						size_t absoluteFileOffset = chunkStart + copyStart;
+						if ( !req.callback(data + copyStart, copyLen, absoluteFileOffset) ) req.callback = nullptr;
+					}
+					if ( chunkEnd >= req.offset + req.length ) req.callback = nullptr;
+				}
+				if ( chunkEnd >= maxEndOffset || !anyActive ) return false;
+
+				return true;
+			});
+		});
+	}
+
+	uf::thread::execute(tasks);
+}
+
 void uf::asset::cache( const uf::asset::callback_t& callback, const uf::asset::Payload& payload ) {
-	mutex.lock();
-	auto& jobs = uf::asset::jobs;
-	jobs.emplace_back(Job{ callback, "cache", payload });
-	mutex.unlock();
+	std::lock_guard<std::mutex> lock(::jobs::mutex);
+	::jobs::queue.emplace_back(::jobs::Job{ callback, "cache", payload });
 }
 void uf::asset::load( const uf::asset::callback_t& callback, const uf::asset::Payload& payload ) {
-	mutex.lock();
-	auto& jobs = uf::asset::jobs;
-	jobs.emplace_back(Job{ callback, "load", payload });
-	mutex.unlock();
+	std::lock_guard<std::mutex> lock(::jobs::mutex);
+	::jobs::queue.emplace_back(::jobs::Job{ callback, "load", payload });
+}
+
+void uf::asset::read( const uf::stl::string& filename, size_t offset, size_t length, uint8_t* dest, std::function<void()> callback ) {
+	std::lock_guard<std::mutex> lock(::io_read::mutex);
+	::io_read::queue[filename].emplace_back(::io_read::Job{ offset, length, dest, callback });
+}
+
+void uf::asset::stream( const uf::stl::string& filename, size_t offset, size_t length, size_t chunkSize, std::function<bool(const uint8_t* data, size_t size, size_t fileOffset)> callback ) {
+	std::lock_guard<std::mutex> lock(::io_stream::mutex);
+	::io_stream::queue[filename].emplace_back(::io_stream::Job{ offset, length, chunkSize, callback });
 }
 
 uf::asset::Payload uf::asset::resolveToPayload( const uf::stl::string& uri, const uf::stl::string& mime ) {

engine/src/engine/ext/scene/behavior.cpp

@@ -282,7 +282,7 @@ void ext::ExtSceneBehavior::initialize( uf::Object& self ) {
 }
 void ext::ExtSceneBehavior::tick( uf::Object& self ) {
 	uf::asset::processQueue();
-//	auto& assetLoader = this->getComponent<uf::asset>();
+	uf::asset::processIO();
 	
 	auto& metadata = this->getComponent<ext::ExtSceneBehavior::Metadata>();
 	auto& metadataJson = this->getComponent<uf::Serializer>();

engine/src/ext/lz4/lz4.cpp

@@ -0,0 +1,232 @@
+#include <uf/ext/lz4/lz4.h>
+#include <uf/utils/io/file.h>
+#include <uf/utils/io/vfs.h>
+#include <lz4frame.h>
+#include <algorithm>
+#include <cstring>
+
+size_t ext::lz4::bufferSize = 16384;
+
+bool ext::lz4::decompressFromFile( uf::stl::vector<uint8_t>& buffer, const uf::stl::string& filename ) {
+	LZ4F_dctx* dctx;
+	if ( LZ4F_isError( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION)) ) return false;
+
+	uint8_t outBuffer[ext::lz4::bufferSize];
+	bool success = uf::vfs::stream(filename, ext::lz4::bufferSize, [&](const uint8_t* data, size_t size) -> bool {
+		size_t srcSize = size;
+		const uint8_t* srcPtr = data;
+
+		while (srcSize > 0) {
+			size_t dstSize = sizeof(outBuffer);
+			size_t consumed = srcSize;
+			size_t ret = LZ4F_decompress(dctx, outBuffer, &dstSize, srcPtr, &consumed, nullptr);
+
+			if (LZ4F_isError(ret)) return false;
+
+			buffer.insert(buffer.end(), outBuffer, outBuffer + dstSize);
+			srcPtr += consumed;
+			srcSize -= consumed;
+		}
+		return true;
+	});
+
+	LZ4F_freeDecompressionContext(dctx);
+	return success;
+}
+
+bool ext::lz4::decompressFromFile( uf::stl::vector<uint8_t>& buffer, const uf::stl::string& filename, size_t start, size_t len ) {
+	size_t fileSize = uf::vfs::size(filename);
+	if ( fileSize == 0 ) return false;
+
+	LZ4F_dctx* dctx;
+	if ( LZ4F_isError( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION)) ) return false;
+
+	size_t uncompressedOffset = 0;
+	uint8_t outBuffer[ext::lz4::bufferSize];
+
+	bool success = uf::vfs::stream(filename, ext::lz4::bufferSize, [&](const uint8_t* data, size_t size) -> bool {
+		size_t srcSize = size;
+		const uint8_t* srcPtr = data;
+
+		while ( srcSize > 0 ) {
+			size_t dstSize = sizeof(outBuffer);
+			size_t consumed = srcSize;
+			size_t ret = LZ4F_decompress(dctx, outBuffer, &dstSize, srcPtr, &consumed, nullptr);
+
+			if ( LZ4F_isError(ret) ) return false;
+
+			if ( dstSize > 0 ) {
+				size_t chunkStart = uncompressedOffset;
+				size_t chunkEnd = uncompressedOffset + dstSize;
+
+				if ( chunkEnd > start && chunkStart < start + len ) {
+					size_t copyStart = (chunkStart < start) ? (start - chunkStart) : 0;
+					size_t copyLen = std::min(dstSize - copyStart, (start + len) - (chunkStart + copyStart));
+					buffer.insert(buffer.end(), outBuffer + copyStart, outBuffer + copyStart + copyLen);
+				}
+
+				uncompressedOffset += dstSize;
+
+				if ( uncompressedOffset >= start + len ) return false;
+			}
+
+			srcPtr += consumed;
+			srcSize -= consumed;
+		}
+		return true;
+	});
+
+	LZ4F_freeDecompressionContext(dctx);
+
+	if ( !success && uncompressedOffset >= start + len ) return true;
+
+	return success;
+}
+
+bool ext::lz4::decompressFromFile( uf::stl::vector<uint8_t>& buffer, const uf::stl::string& filename, const uf::stl::vector<pod::Range>& ranges ) {
+	if ( ranges.empty() ) return false;
+
+	uf::stl::vector<pod::Range> sortedRanges = ranges;
+	std::sort( sortedRanges.begin(), sortedRanges.end(), [](const pod::Range& a, const pod::Range& b) { return a.start < b.start; } );
+
+	size_t fileSize = uf::vfs::size(filename);
+	if ( fileSize == 0 ) return false;
+
+	LZ4F_dctx* dctx;
+	if ( LZ4F_isError( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION)) ) return false;
+
+	size_t uncompressedOffset = 0;
+	size_t currentRangeIdx = 0;
+	uint8_t outBuffer[ext::lz4::bufferSize];
+
+	bool success = uf::vfs::stream(filename, ext::lz4::bufferSize, [&](const uint8_t* data, size_t size) -> bool {
+		size_t srcSize = size;
+		const uint8_t* srcPtr = data;
+
+		while ( srcSize > 0 ) {
+			size_t dstSize = sizeof(outBuffer);
+			size_t consumed = srcSize;
+			size_t ret = LZ4F_decompress(dctx, outBuffer, &dstSize, srcPtr, &consumed, nullptr);
+
+			if ( LZ4F_isError(ret) ) return false;
+
+			if ( dstSize > 0 ) {
+				size_t chunkStart = uncompressedOffset;
+				size_t chunkEnd = uncompressedOffset + dstSize;
+
+				for ( size_t i = currentRangeIdx; i < sortedRanges.size(); ++i ) {
+					const auto& r = sortedRanges[i];
+					if ( chunkEnd > r.start && chunkStart < r.start + r.len ) {
+						size_t copyStart = (chunkStart < r.start) ? (r.start - chunkStart) : 0;
+						size_t copyLen = std::min(dstSize - copyStart, (r.start + r.len) - (chunkStart + copyStart));
+						buffer.insert(buffer.end(), outBuffer + copyStart, outBuffer + copyStart + copyLen);
+					}
+					if ( chunkEnd >= r.start + r.len ) {
+						currentRangeIdx = i + 1;
+					}
+				}
+
+				uncompressedOffset += dstSize;
+
+				if ( currentRangeIdx >= sortedRanges.size() ) return false;
+			}
+
+			srcPtr += consumed;
+			srcSize -= consumed;
+		}
+		return true;
+	});
+
+	LZ4F_freeDecompressionContext(dctx);
+
+	if ( !success && currentRangeIdx >= sortedRanges.size() ) return true;
+
+	return success;
+}
+
+bool ext::lz4::decompressScatter( const uf::stl::string& filename, uf::stl::vector<pod::ScatterRequest>& requests ) {
+	if (requests.empty()) return true;
+
+	std::sort(requests.begin(), requests.end(), [](const pod::ScatterRequest& a, const pod::ScatterRequest& b) {
+		return a.start < b.start;
+	});
+
+	LZ4F_dctx* dctx;
+	if ( LZ4F_isError( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION)) ) return false;
+
+	size_t uncompressedOffset = 0;
+	size_t currentReqIdx = 0;
+	uint8_t outBuffer[ext::lz4::bufferSize];
+
+	bool success = uf::vfs::stream(filename, ext::lz4::bufferSize, [&](const uint8_t* data, size_t size) -> bool {
+		size_t srcSize = size;
+		const uint8_t* srcPtr = data;
+
+		while ( srcSize > 0 ) {
+			size_t dstSize = sizeof(outBuffer);
+			size_t consumed = srcSize;
+			size_t ret = LZ4F_decompress(dctx, outBuffer, &dstSize, srcPtr, &consumed, nullptr);
+
+			if ( LZ4F_isError(ret) ) return false;
+
+			if ( dstSize > 0 ) {
+				size_t chunkStart = uncompressedOffset;
+				size_t chunkEnd = uncompressedOffset + dstSize;
+
+				for ( size_t i = currentReqIdx; i < requests.size(); ++i ) {
+					auto& req = requests[i];
+					if ( chunkEnd > req.start && chunkStart < req.start + req.len ) {
+						size_t copyStart = (chunkStart < req.start) ? (req.start - chunkStart) : 0;
+						size_t copyLen = std::min(dstSize - copyStart, (req.start + req.len)  - (chunkStart + copyStart));
+
+						size_t destOffset = (chunkStart + copyStart) - req.start;
+						std::memcpy(req.dest + destOffset, outBuffer + copyStart, copyLen);
+					}
+					if ( chunkEnd >= req.start + req.len && i == currentReqIdx ) {
+						currentReqIdx++;
+					}
+				}
+				uncompressedOffset += dstSize;
+				if ( currentReqIdx >= requests.size() ) return false;
+			}
+
+			srcPtr += consumed;
+			srcSize -= consumed;
+		}
+		return true;
+	});
+
+	LZ4F_freeDecompressionContext(dctx);
+	return success;
+}
+
+bool ext::lz4::decompressFromMemory( uf::stl::vector<uint8_t>& dst, const void* src, size_t compressedSize, size_t uncompressedSize ) {
+	LZ4F_dctx* dctx;
+	if ( LZ4F_isError( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION)) ) return false;
+
+	dst.resize(uncompressedSize);
+
+	size_t dstSize = uncompressedSize;
+	size_t srcSize = compressedSize;
+	size_t ret = LZ4F_decompress(dctx, dst.data(), &dstSize, src, &srcSize, nullptr);
+
+	LZ4F_freeDecompressionContext(dctx);
+
+	if ( LZ4F_isError(ret) || dstSize != uncompressedSize ) {
+		dst.clear();
+		return false;
+	}
+	return true;
+}
+
+size_t ext::lz4::compressToFile( const uf::stl::string& filename, const void* data, size_t size ) {
+	LZ4F_preferences_t prefs = {};
+	size_t bound = LZ4F_compressFrameBound(size, &prefs);
+
+	uf::stl::vector<uint8_t> compressedData(bound);
+	size_t compressedSize = LZ4F_compressFrame(compressedData.data(), bound, data, size, &prefs);
+
+	if ( LZ4F_isError(compressedSize) ) return 0;
+
+	return uf::vfs::write( filename, compressedData.data(), compressedSize );
+}

engine/src/ext/zlib/zlib.cpp

@@ -345,6 +345,7 @@ namespace {
 		if ( entry.compressionMethod == 8 ) {
 			return ext::zlib::decompressFromMemory(buffer, fileData, entry.compressedSize, entry.uncompressedSize);
 		}
+		// ID for lz4
 		return false;
 	};
 }

engine/src/utils/io/file.cpp

@@ -10,8 +10,7 @@
 #include <uf/utils/io/file.h>
 #include <uf/utils/io/iostream.h>
 #include <uf/ext/zlib/zlib.h>
-// #include <uf/ext/lz4/lz4.h>
-// #include <uf/ext/xz/xz.h>
+#include <uf/ext/lz4/lz4.h>
 
 #ifdef WINDOWS
 	#include <direct.h>
@@ -86,7 +85,7 @@ bool uf::io::readAsString( uf::stl::string& buffer, const uf::stl::string& _file
 	uf::stl::string filename = uf::io::normalize( _filename );
 	uf::stl::string extension = uf::io::extension( filename );
 
-	if ( extension == "gz" ) {
+	if ( extension == "gz" || extension == "lz4" ) {
 		auto decompressed = uf::io::decompress( filename );
 		buffer.resize(decompressed.size());
 		buffer.assign(decompressed.begin(), decompressed.end());
@@ -178,12 +177,11 @@ bool uf::io::readScatter( const uf::stl::string& filename, uf::stl::vector<pod::
 #endif
 
 	uf::stl::string extension = uf::io::extension(filename);
-	bool isZlib = (extension == "gz");
 	size_t fileSize = uf::io::size(filename);
 
 	if ( 0 < fileSize && fileSize <= THRESHOLD ) {
 		uf::stl::vector<uint8_t> fullBuffer;
-		if ( isZlib ) ext::zlib::decompressFromFile(fullBuffer, filename);
+		if ( extension == "gz" || extension == "lz4" ) uf::io::decompress(fullBuffer, filename);
 		else uf::vfs::read(filename, fullBuffer);
 
 		for ( auto& req : requests ) {
@@ -194,7 +192,8 @@ bool uf::io::readScatter( const uf::stl::string& filename, uf::stl::vector<pod::
 		return true;
 	}
 
-	if ( isZlib ) return ext::zlib::decompressScatter(filename, requests);
+	if ( extension == "gz" ) return ext::zlib::decompressScatter(filename, requests);
+	if ( extension == "lz4" ) return ext::lz4::decompressScatter(filename, requests);
 
 	for ( auto& req : requests ) {
 		uf::stl::vector<uint8_t> temp;
@@ -215,28 +214,28 @@ size_t uf::io::write( const uf::stl::string& filename, const void* buffer, size_
 bool uf::io::decompress( uf::stl::vector<uint8_t>& buffer, const uf::stl::string& filename ) {
 	uf::stl::string extension = uf::io::extension( filename );
 	if ( extension == "gz" ) return ext::zlib::decompressFromFile( buffer, filename );
-//	if ( extension == "lz4" ) return ext::lz4::decompressFromFile( buffer, filename );
+	if ( extension == "lz4" ) return ext::lz4::decompressFromFile( buffer, filename );
 	UF_MSG_ERROR("unsupported compression format requested: {}", extension);
 	return false;
 }
 bool uf::io::decompress( uf::stl::vector<uint8_t>& buffer, const uf::stl::string& filename, size_t start, size_t len ) {
 	uf::stl::string extension = uf::io::extension( filename );
 	if ( extension == "gz" ) return ext::zlib::decompressFromFile( buffer, filename, start, len );
-//	if ( extension == "lz4" ) return ext::lz4::decompressFromFile( buffer, filename, start, len );
+	if ( extension == "lz4" ) return ext::lz4::decompressFromFile( buffer, filename, start, len );
 	UF_MSG_ERROR("unsupported compression format requested: {}", extension);
 	return false;
 }
 bool uf::io::decompress( uf::stl::vector<uint8_t>& buffer, const uf::stl::string& filename, const uf::stl::vector<pod::Range>& ranges ) {
 	uf::stl::string extension = uf::io::extension( filename );
 	if ( extension == "gz" ) return ext::zlib::decompressFromFile( buffer, filename, ranges );
-//	if ( extension == "lz4" ) return ext::lz4::decompressFromFile( buffer, filename, ranges );
+	if ( extension == "lz4" ) return ext::lz4::decompressFromFile( buffer, filename, ranges );
 	UF_MSG_ERROR("unsupported compression format requested: {}", extension);
 	return false;
 }
 size_t uf::io::compress( const uf::stl::string& filename, const void* buffer, size_t size ) {
 	uf::stl::string extension = uf::io::extension( filename );
 	if ( extension == "gz" ) return ext::zlib::compressToFile( filename, buffer, size );
-//	if ( extension == "lz4" ) return ext::lz4::compressToFile( filename, buffer, size );
+	if ( extension == "lz4" ) return ext::lz4::compressToFile( filename, buffer, size );
 	UF_MSG_ERROR("unsupported compression format requested: {}", extension);
 	return 0;
 }

makefiles/platforms/dreamcast.mk

@@ -1,5 +1,5 @@
 FLAGS 				+= -DUF_ENV_DREAMCAST
-REQ_DEPS 			+= opengl gldc json:nlohmann zlib lua simd ctti fmt freetype openal aldc ogg wav png
+REQ_DEPS 			+= opengl gldc json:nlohmann zlib lz4 lua simd ctti fmt freetype openal aldc ogg wav png
 
 INCS 				:= -I./dep/dreamcast/include $(INCS)
 

makefiles/platforms/linux.mk

@@ -3,7 +3,7 @@ ifneq (,$(findstring -DUF_DEV_ENV,$(FLAGS)))
 	FLAGS 				+= -march=native -g # -flto # -g
 endif
 
-REQ_DEPS 			+= $(RENDERER) json:nlohmann zlib luajit simd ctti gltf imgui fmt freetype openal ogg wav
+REQ_DEPS 			+= $(RENDERER) json:nlohmann zlib lz4 luajit simd ctti gltf imgui fmt freetype openal ogg wav
 FLAGS 				+= -DUF_ENV_LINUX -fPIC
 DEPS 				+= -pthread -ldl -lX11 -lXrandr
 INCS 				:= -I./dep/master/include $(INCS)

makefiles/platforms/win64.mk

@@ -3,7 +3,7 @@ ifneq (,$(findstring -DUF_DEV_ENV,$(FLAGS)))
 	FLAGS 				+= -march=native -g # -flto # -g
 endif
 
-REQ_DEPS 			+= $(RENDERER) json:nlohmann zlib luajit simd ctti gltf imgui fmt freetype openal ogg wav
+REQ_DEPS 			+= $(RENDERER) json:nlohmann zlib lz4 luajit simd ctti gltf imgui fmt freetype openal ogg wav
 FLAGS 				+= -DUF_ENV_WINDOWS -DUF_ENV_WIN64 -DWIN32_LEAN_AND_MEAN
 DEPS 				+= -lgdi32 -ldwmapi
 LINKS 				+= #-Wl,-subsystem,windows