ecker/engine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

added wav (from file) and PCM (from memory buffer) playback, modified hook to VALL-E TTS to now either directly play the audio or invoke a callback (currently only hooked via sound emitter)

Browse Source
This commit is contained in:
ecker 2025-08-03 13:45:31 -05:00
parent 73ca9bb168
commit 81da764d6b
28 changed files with 9871 additions and 6808 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
Makefile
View File

@ -62,14 +62,14 @@ LIBS += -L$(ENGINE_LIB_DIR) -L$(LIB_DIR)/$(PREFIX_PATH) -L$(LIB_DIR)/$(ARCH
LINKS += $(UF_LIBS) $(EXT_LIBS) $(DEPS)
DEPS +=
FLAGS += # -DUF_DEBUG
FLAGS += -DUF_DEBUG
ifneq (,$(findstring -DUF_DEBUG,$(FLAGS)))
REQ_DEPS += meshoptimizer toml xatlas curl ffx:fsr cpptrace vall_e # ncurses openvr draco discord bullet ultralight-ux
FLAGS += -g
endif
ifneq (,$(findstring win64,$(ARCH)))
REQ_DEPS += $(RENDERER) json:nlohmann png zlib luajit reactphysics simd ctti gltf imgui fmt freetype openal ogg # meshoptimizer toml xatlas curl ffx:fsr cpptrace # ncurses openvr draco discord bullet ultralight-ux
REQ_DEPS += $(RENDERER) json:nlohmann png zlib luajit reactphysics simd ctti gltf imgui fmt freetype openal ogg wav # meshoptimizer toml xatlas curl ffx:fsr cpptrace # ncurses openvr draco discord bullet ultralight-ux
FLAGS += -DUF_ENV_WINDOWS -DUF_ENV_WIN64 -DWIN32_LEAN_AND_MEAN
DEPS += -lgdi32 -ldwmapi
LINKS += #-Wl,-subsystem,windows
@ -172,6 +172,9 @@ ifneq (,$(findstring ogg,$(REQ_DEPS)))
DEPS += -lvorbis -lvorbisfile -logg
endif
endif
ifneq (,$(findstring wav,$(REQ_DEPS)))
FLAGS += -DUF_USE_WAV
endif
ifneq (,$(findstring freetype,$(REQ_DEPS)))
FLAGS += -DUF_USE_FREETYPE
DEPS += -lfreetype -lbz2

2
bin/data/config.json
View File

@ -292,7 +292,7 @@
"enabled": true
},
"imgui": {
"enabled": true
"enabled": false
},
"fsr": {
"enabled": true,

3
bin/data/entities/craeture.json
View File

@ -7,7 +7,8 @@
"./scripts/craeture.lua"
],
"behaviors": [
"CraetureBehavior"
"CraetureBehavior",
"SoundEmitterBehavior"
],
"transform": {
//"position": [ 0, 1.5, 21 ],

3
bin/data/entities/craeture2.json
View File

@ -7,7 +7,8 @@
"./scripts/craeture.lua"
],
"behaviors": [
"CraetureBehavior"
"CraetureBehavior",
"SoundEmitterBehavior"
],
"transform": {
//"position": [ 0, 1.5, 21 ],

7
bin/data/entities/scripts/craeture.lua
View File

@ -63,6 +63,7 @@ end
local collider = ent:getComponent("PhysicsState")
local target_transform = nil
local soundEmitter = ent
-- on tick
ent:bind( "tick", function(self)
-- rotate to target
@ -127,6 +128,12 @@ ent:addHook( "entity:Use.%UID%", function( payload )
local text = texts[math.random( #texts )] or "!! Test Message !!"
ent:callHook("llm:VALL-E.synthesize", {
text = text,
prom = "./data/tmp/prom.wav",
callback = soundEmitter:formatHookName("sound:Emit.%UID%")
} )
local forward = {
name = "dialogue",
metadata = {

2
bin/data/scenes/sourceengine/mds_mcdonalds.json
View File

@ -7,7 +7,7 @@
],
"metadata": {
"graph": {
"bgm": "./audio/soundscape/sh2_ambience.ogg",
"bgm": "./audio/soundscape/sh2_ambience.wav",
"tags": {
// exact matches
"func_door_rotating_5473": { "action": "load", "payload": { "import": "/door.json", "metadata": { "angle":-1.570795, "normal": [1,0,0] } } },

4
bin/data/scenes/sourceengine/sourceengine.json
View File

@ -2,7 +2,7 @@
// "import": "./rp_downtown_v2.json"
// "import": "./ss2_medsci1.json"
// "import": "./sh2_mcdonalds.json"
// "import": "./animal_crossing.json"
"import": "./mds_mcdonalds.json"
"import": "./animal_crossing.json"
// "import": "./mds_mcdonalds.json"
// "import": "./gm_construct.json"
}

15435
dep/include/vall_e.cpp/dr_wav.h → dep/include/dr_wav.h
View File

File diff suppressed because it is too large Load Diff

15
engine/inc/uf/ext/audio/pcm.h Normal file
View File

@ -0,0 +1,15 @@
#pragma once
#include <uf/config.h>
#include <uf/utils/audio/audio.h>
namespace ext {
namespace pcm {
void UF_API open( uf::Audio::Metadata&, const pod::PCM& );
void UF_API load( uf::Audio::Metadata& );
void UF_API stream( uf::Audio::Metadata& );
void UF_API update( uf::Audio::Metadata& );
void UF_API close( uf::Audio::Metadata& );
}
}

0
engine/inc/uf/ext/vorbis/vorbis.h → engine/inc/uf/ext/audio/vorbis.h
View File

17
engine/inc/uf/ext/audio/wav.h Normal file
View File

@ -0,0 +1,17 @@
#pragma once
#include <uf/config.h>
#if UF_USE_WAV
#include <uf/utils/audio/audio.h>
namespace ext {
namespace wav {
void UF_API open( uf::Audio::Metadata& );
void UF_API load( uf::Audio::Metadata& );
void UF_API stream( uf::Audio::Metadata& );
void UF_API update( uf::Audio::Metadata& );
void UF_API close( uf::Audio::Metadata& );
}
}
#endif

5
engine/inc/uf/ext/oal/oal.h
View File

@ -54,10 +54,15 @@ namespace ext {
uf::stl::string UF_API getError( ALenum = 0 );
uf::audio::Metadata* UF_API create( const uf::stl::string&, bool, uint8_t );
uf::audio::Metadata* UF_API create( const pod::PCM&, bool, uint8_t );
uf::audio::Metadata* UF_API open( const uf::stl::string& );
uf::audio::Metadata* UF_API open( const uf::stl::string&, bool );
uf::audio::Metadata* UF_API open( const pod::PCM& );
uf::audio::Metadata* UF_API open( const pod::PCM&, bool );
uf::audio::Metadata* UF_API load( const uf::stl::string& );
uf::audio::Metadata* UF_API load( const pod::PCM& );
uf::audio::Metadata* UF_API stream( const uf::stl::string& );
uf::audio::Metadata* UF_API stream( const pod::PCM& );
void UF_API update( uf::audio::Metadata& );
void UF_API close( uf::audio::Metadata* );

4
engine/inc/uf/ext/vall_e/vall_e.h
View File

@ -5,10 +5,12 @@
#if UF_USE_VALL_E
#include <vall_e.cpp/vall_e.h>
#include <uf/utils/audio/audio.h>
namespace ext {
namespace vall_e {
void UF_API initialize( const std::string& model_path = "", const std::string& encodec_path = "" );
std::string UF_API generate( const std::string& text, const std::string& prom, const std::string& lang = "en" );
pod::PCM UF_API generate( const std::string& text, const std::string& prom, const std::string& lang = "en" );
void UF_API terminate();
}
}

4
engine/inc/uf/utils/audio/audio.h
View File

@ -50,8 +50,12 @@ namespace uf {
void open( const uf::stl::string& );
void open( const uf::stl::string&, bool );
void open( const pod::PCM& );
void open( const pod::PCM&, bool );
void load( const uf::stl::string& );
void load( const pod::PCM& );
void stream( const uf::stl::string& );
void stream( const pod::PCM& );
void update();
void destroy();

13
engine/inc/uf/utils/audio/metadata.h
View File

@ -1,11 +1,22 @@
#pragma once
#include <uf/utils/memory/string.h>
#include <fstream>
#include <uf/utils/memory/string.h>
#include <uf/utils/memory/vector.h>
#include <uf/ext/oal/source.h>
#include <uf/ext/oal/buffer.h>
#include <uf/utils/time/time.h>
namespace pod {
struct UF_API PCM {
uf::stl::vector<float> waveform;
uint16_t sampleRate = 24000;
uint16_t channels = 1;
};
}
namespace uf {
namespace audio {
struct UF_API Metadata {

1
engine/src/engine/asset/asset.cpp
View File

@ -107,6 +107,7 @@ uf::asset::Payload uf::asset::resolveToPayload( const uf::stl::string& uri, cons
{ "png", uf::asset::Type::IMAGE },
{ "ogg", uf::asset::Type::AUDIO },
{ "wav", uf::asset::Type::AUDIO },
{ "json", uf::asset::Type::JSON },
{ "bson", uf::asset::Type::JSON },

149
engine/src/ext/audio/pcm.cpp Normal file
View File

@ -0,0 +1,149 @@
#include <uf/config.h>
#if UF_USE_WAV
#if UF_USE_OPENAL
#include <uf/ext/oal/oal.h>
#endif
#include <uf/ext/audio/pcm.h>
#include <uf/utils/memory/pool.h>
#include <iostream>
#include <cstdio>
void ext::pcm::open( uf::Audio::Metadata& metadata, const pod::PCM& pcm ) {
metadata.info.channels = pcm.channels;
metadata.info.bitDepth = 16;
metadata.info.frequency = pcm.sampleRate;
metadata.info.duration = double(pcm.waveform.size()) / pcm.channels / pcm.sampleRate;
metadata.info.size = pcm.waveform.size() * sizeof(int16_t);
// Determine OpenAL format
if (pcm.channels == 1)
metadata.info.format = AL_FORMAT_MONO16;
else if (pcm.channels == 2)
metadata.info.format = AL_FORMAT_STEREO16;
else {
UF_MSG_ERROR("PCM: Only mono or stereo supported ({} channels)", pcm.channels);
return;
}
metadata.stream.handle = malloc( metadata.info.size );
metadata.stream.consumed = 0;
// Convert float waveform to int16_t PCM
int16_t* pcm16 = (int16_t*) metadata.stream.handle;
for (size_t i = 0; i < pcm.waveform.size(); ++i) {
float sample = std::clamp(pcm.waveform[i], -1.0f, 1.0f);
pcm16[i] = static_cast<int16_t>(sample * 32767.0f);
}
// choose load or stream
return metadata.settings.streamed ? ext::pcm::stream(metadata) : ext::pcm::load(metadata);
}
void ext::pcm::load( uf::Audio::Metadata& metadata ) {
if ( metadata.settings.streamed ) return ext::pcm::stream( metadata );
// Upload to OpenAL buffer
metadata.al.buffer.buffer(metadata.info.format, metadata.stream.handle, (ALsizei) metadata.info.size, metadata.info.frequency);
metadata.al.source.set(AL_BUFFER, (ALint) metadata.al.buffer.getIndex());
}
void ext::pcm::stream(uf::Audio::Metadata& metadata) {
if ( !metadata.settings.streamed ) return ext::pcm::load( metadata );
int16_t* pcm16 = (int16_t*) metadata.stream.handle;
size_t frameSize = metadata.info.channels * sizeof(int16_t);
size_t totalFrames = metadata.info.size / frameSize;
size_t bufferFrames = uf::audio::bufferSize / frameSize;
uint8_t queuedBuffers = 0;
size_t offset = 0;
for (; queuedBuffers < metadata.settings.buffers; ++queuedBuffers) {
size_t framesToRead = std::min(bufferFrames, totalFrames - offset);
size_t bytesToRead = framesToRead * frameSize;
if (framesToRead == 0) {
if (metadata.settings.loop) {
offset = 0;
framesToRead = std::min(bufferFrames, totalFrames);
bytesToRead = framesToRead * frameSize;
} else {
break;
}
}
AL_CHECK_RESULT(alBufferData(metadata.al.buffer.getIndex(queuedBuffers), metadata.info.format,
pcm16 + offset * metadata.info.channels, (ALsizei)bytesToRead, metadata.info.frequency));
offset += framesToRead;
}
metadata.stream.consumed = offset * frameSize;
AL_CHECK_RESULT(alSourceQueueBuffers(metadata.al.source.getIndex(), queuedBuffers, &metadata.al.buffer.getIndex()));
// Switch to soft looping if needed
if (queuedBuffers >= metadata.settings.buffers) {
metadata.settings.loopMode = 1;
metadata.al.source.set(AL_LOOPING, AL_FALSE);
}
}
void ext::pcm::update(uf::Audio::Metadata& metadata) {
if (!metadata.settings.streamed) return;
if (metadata.settings.loopMode == 1) metadata.al.source.set(AL_LOOPING, AL_FALSE);
ALint state;
metadata.al.source.get(AL_SOURCE_STATE, state);
if (state != AL_PLAYING) {
if (!metadata.settings.loop && metadata.stream.consumed >= metadata.info.size) {
// stream finished
return;
}
metadata.al.source.play();
}
ALint processed = 0;
metadata.al.source.get(AL_BUFFERS_PROCESSED, processed);
if (processed <= 0) return;
int16_t* pcm16 = (int16_t*) metadata.stream.handle;
size_t frameSize = metadata.info.channels * sizeof(int16_t);
size_t totalFrames = metadata.info.size / frameSize;
size_t bufferFrames = uf::audio::bufferSize / frameSize;
ALuint index;
while (processed--) {
AL_CHECK_RESULT(alSourceUnqueueBuffers(metadata.al.source.getIndex(), 1, &index));
size_t offset = metadata.stream.consumed / frameSize;
size_t framesToRead = std::min(bufferFrames, totalFrames - offset);
size_t bytesToRead = framesToRead * frameSize;
if (framesToRead == 0) {
if (!metadata.settings.loop) break;
offset = 0;
framesToRead = std::min(bufferFrames, totalFrames);
bytesToRead = framesToRead * frameSize;
}
if (framesToRead > 0) {
AL_CHECK_RESULT(alBufferData(index, metadata.info.format,
pcm16 + offset * metadata.info.channels, (ALsizei)bytesToRead, metadata.info.frequency));
AL_CHECK_RESULT(alSourceQueueBuffers(metadata.al.source.getIndex(), 1, &index));
metadata.stream.consumed = (offset + framesToRead) * frameSize;
}
if (metadata.settings.loop && bytesToRead < uf::audio::bufferSize) {
UF_MSG_ERROR("PCM: missing data: {}", metadata.filename);
}
}
if (metadata.settings.loopMode == 1) metadata.al.source.set(AL_LOOPING, AL_TRUE);
}
void ext::pcm::close(uf::Audio::Metadata& metadata) {
if ( metadata.stream.handle ) {
free( metadata.stream.handle );
metadata.stream.handle = NULL;
}
}
#endif

302
engine/src/ext/audio/vorbis.cpp Normal file
View File

@ -0,0 +1,302 @@
#include <uf/config.h>
#if UF_USE_VORBIS
#if UF_USE_OPENAL
#include <uf/ext/oal/oal.h>
#endif
#include <uf/ext/audio/vorbis.h>
#include <uf/utils/memory/pool.h>
#include <iostream>
#include <fstream>
#include <vector>
#include <cstdio>
#include <cstring>
namespace {
constexpr int endian = 0; // 0 = little endian
namespace funs {
size_t read(void* destination, size_t size, size_t nmemb, void* userdata) {
uf::Audio::Metadata& metadata = *((uf::Audio::Metadata*)userdata);
std::ifstream& file = *metadata.stream.file;
size_t length = size * nmemb;
if (metadata.stream.consumed + length > metadata.info.size)
length = metadata.info.size - metadata.stream.consumed;
if (!file.is_open()) {
file.open(metadata.filename, std::ios::binary);
if (!file.is_open()) {
UF_MSG_ERROR("Could not open file: {}", metadata.filename);
return 0;
}
}
file.clear();
file.seekg(metadata.stream.consumed);
uf::stl::vector<char> data(length);
if (!file.read(data.data(), length)) {
if (file.eof()) file.clear();
else {
UF_MSG_ERROR("File stream error: {}", metadata.filename);
file.clear();
return 0;
}
}
memcpy(destination, data.data(), length);
metadata.stream.consumed += length;
return length;
}
int seek(void* userdata, ogg_int64_t to, int type) {
uf::Audio::Metadata& metadata = *((uf::Audio::Metadata*)userdata);
switch (type) {
case SEEK_CUR: metadata.stream.consumed += to; break;
case SEEK_END: metadata.stream.consumed = metadata.info.size - to; break;
case SEEK_SET: metadata.stream.consumed = to; break;
default: return -1;
}
if (metadata.stream.consumed < 0) metadata.stream.consumed = 0;
if (metadata.stream.consumed > metadata.info.size) metadata.stream.consumed = metadata.info.size;
return 0;
}
int close(void* userdata) {
uf::Audio::Metadata& metadata = *((uf::Audio::Metadata*)userdata);
if (metadata.stream.file) {
std::ifstream& file = *metadata.stream.file;
if (file.is_open()) file.close();
delete metadata.stream.file;
metadata.stream.file = NULL;
}
return 0;
}
long tell(void* userdata) {
uf::Audio::Metadata& metadata = *((uf::Audio::Metadata*)userdata);
return metadata.stream.consumed;
}
}
inline bool format(uf::Audio::Metadata& metadata, int channels, int bitDepth) {
if (channels == 1 && bitDepth == 8) metadata.info.format = AL_FORMAT_MONO8;
else if (channels == 1 && bitDepth == 16) metadata.info.format = AL_FORMAT_MONO16;
else if (channels == 2 && bitDepth == 8) metadata.info.format = AL_FORMAT_STEREO8;
else if (channels == 2 && bitDepth == 16) metadata.info.format = AL_FORMAT_STEREO16;
else {
UF_MSG_ERROR("Vorbis: unrecognized OGG format: {} channels, {} bps", channels, bitDepth);
return false;
}
return true;
}
}
void ext::vorbis::open(uf::Audio::Metadata& metadata) {
if (metadata.settings.streamed)
stream(metadata);
else
load(metadata);
}
void ext::vorbis::load(uf::Audio::Metadata& metadata) {
if (metadata.settings.streamed) return stream(metadata);
FILE* file = fopen(metadata.filename.c_str(), "rb");
if (!file) {
UF_MSG_ERROR("Vorbis: failed to open {}. File error.", metadata.filename);
return;
}
fseek(file, 0, SEEK_END);
metadata.info.size = ftell(file);
fseek(file, 0, SEEK_SET);
metadata.stream.consumed = 0;
OggVorbis_File vorbisFile;
if (ov_open_callbacks(file, &vorbisFile, NULL, 0, OV_CALLBACKS_DEFAULT) < 0) {
UF_MSG_ERROR("Vorbis: failed to open {}. Not Ogg.", metadata.filename);
fclose(file);
return;
}
vorbis_info* info = ov_info(&vorbisFile, -1);
metadata.info.channels = info->channels;
metadata.info.bitDepth = 16;
metadata.info.frequency = info->rate;
metadata.info.duration = ov_time_total(&vorbisFile, -1);
if (!format(metadata, info->channels, 16)) {
ov_clear(&vorbisFile);
return;
}
uf::stl::vector<char> bytes;
char buffer[uf::audio::bufferSize];
int bitStream = 0;
int read = 0;
do {
read = ov_read(&vorbisFile, buffer, uf::audio::bufferSize, endian, 2, 1, &bitStream);
if (read > 0)
bytes.insert(bytes.end(), buffer, buffer + read);
} while (read > 0);
metadata.al.buffer.buffer(metadata.info.format, bytes.data(), (ALsizei)bytes.size(), metadata.info.frequency);
metadata.al.source.set(AL_BUFFER, (ALint)metadata.al.buffer.getIndex());
ov_clear(&vorbisFile);
}
void ext::vorbis::stream(uf::Audio::Metadata& metadata) {
if (!metadata.settings.streamed) return load(metadata);
if (!metadata.stream.file) metadata.stream.file = new std::ifstream;
if (!metadata.stream.handle) metadata.stream.handle = new OggVorbis_File;
std::ifstream& file = *metadata.stream.file;
OggVorbis_File& vorbisFile = *(OggVorbis_File*)metadata.stream.handle;
file.open(metadata.filename, std::ios::binary);
if (!file.is_open()) {
UF_MSG_ERROR("Vorbis: failed to open file stream: {}", metadata.filename);
return;
}
file.seekg(0, std::ios::end);
metadata.info.size = file.tellg();
file.seekg(0, std::ios::beg);
metadata.stream.consumed = 0;
ov_callbacks callbacks;
callbacks.read_func = funs::read;
callbacks.seek_func = funs::seek;
callbacks.close_func = funs::close;
callbacks.tell_func = funs::tell;
if (ov_open_callbacks((void*)&metadata, &vorbisFile, NULL, -1, callbacks) < 0) {
UF_MSG_ERROR("Vorbis: failed call to ov_open_callbacks: {}", metadata.filename);
return;
}
vorbis_info* info = ov_info(&vorbisFile, -1);
metadata.info.channels = info->channels;
metadata.info.bitDepth = 16;
metadata.info.frequency = info->rate;
metadata.info.duration = ov_time_total(&vorbisFile, -1);
if (!format(metadata, info->channels, 16)) {
ov_clear(&vorbisFile);
return;
}
// Fill and queue initial buffers
char buffer[uf::audio::bufferSize];
uint8_t queuedBuffers = 0;
int bitStream = 0;
for (; queuedBuffers < metadata.settings.buffers; ++queuedBuffers) {
int totalRead = 0;
while (totalRead < uf::audio::bufferSize) {
int result = ov_read(&vorbisFile, buffer + totalRead, uf::audio::bufferSize - totalRead, endian, 2, 1, &bitStream);
if (result <= 0) {
if (result == 0 && metadata.settings.loop) {
if (ov_raw_seek(&vorbisFile, 0) != 0) {
UF_MSG_ERROR("Vorbis: failed to loop (seek to start): {}", metadata.filename);
break;
}
continue;
}
if (result == OV_HOLE) UF_MSG_ERROR("Vorbis: OV_HOLE in buffer read: {}", metadata.filename);
if (result == OV_EBADLINK) UF_MSG_ERROR("Vorbis: OV_EBADLINK in buffer read: {}", metadata.filename);
if (result == OV_EINVAL) UF_MSG_ERROR("Vorbis: OV_EINVAL in buffer read: {}", metadata.filename);
break;
}
totalRead += result;
}
if (totalRead == 0) {
UF_MSG_WARNING("Vorbis: consumed file stream before buffers are filled: {} {}", (int)queuedBuffers, metadata.filename);
break;
}
AL_CHECK_RESULT(alBufferData(metadata.al.buffer.getIndex(queuedBuffers), metadata.info.format, buffer, totalRead, metadata.info.frequency));
}
AL_CHECK_RESULT(alSourceQueueBuffers(metadata.al.source.getIndex(), queuedBuffers, &metadata.al.buffer.getIndex()));
if (queuedBuffers >= metadata.settings.buffers) {
metadata.settings.loopMode = 1;
metadata.al.source.set(AL_LOOPING, AL_FALSE);
}
}
void ext::vorbis::update(uf::Audio::Metadata& metadata) {
if (!metadata.settings.streamed) return;
if (metadata.settings.loopMode == 1)
metadata.al.source.set(AL_LOOPING, AL_FALSE);
ALint state;
metadata.al.source.get(AL_SOURCE_STATE, state);
if (state != AL_PLAYING) {
if (!metadata.settings.loop && metadata.stream.consumed >= metadata.info.size) {
// Stream finished
return;
}
metadata.al.source.play();
}
ALint processed = 0;
metadata.al.source.get(AL_BUFFERS_PROCESSED, processed);
if (processed <= 0) return;
OggVorbis_File& vorbisFile = *(OggVorbis_File*)metadata.stream.handle;
int bitStream = metadata.stream.bitStream;
ALuint index;
char buffer[uf::audio::bufferSize];
while (processed--) {
memset(buffer, 0, uf::audio::bufferSize);
AL_CHECK_RESULT(alSourceUnqueueBuffers(metadata.al.source.getIndex(), 1, &index));
int totalRead = 0;
while (totalRead < uf::audio::bufferSize) {
int result = ov_read(&vorbisFile, buffer + totalRead, uf::audio::bufferSize - totalRead, endian, 2, 1, &bitStream);
if (result <= 0) {
if (result == 0 && metadata.settings.loop) {
if (ov_raw_seek(&vorbisFile, 0) != 0) {
UF_MSG_ERROR("Vorbis: failed to loop (seek to start): {}", metadata.filename);
break;
}
continue;
}
if (result == OV_HOLE) UF_MSG_ERROR("Vorbis: OV_HOLE in buffer read: {}", metadata.filename);
if (result == OV_EBADLINK) UF_MSG_ERROR("Vorbis: OV_EBADLINK in buffer read: {}", metadata.filename);
if (result == OV_EINVAL) UF_MSG_ERROR("Vorbis: OV_EINVAL in buffer read: {}", metadata.filename);
break;
}
totalRead += result;
}
if (totalRead > 0) {
AL_CHECK_RESULT(alBufferData(index, metadata.info.format, buffer, totalRead, metadata.info.frequency));
AL_CHECK_RESULT(alSourceQueueBuffers(metadata.al.source.getIndex(), 1, &index));
}
if (metadata.settings.loop && totalRead < uf::audio::bufferSize) {
UF_MSG_ERROR("Vorbis: missing data: {}", metadata.filename);
}
}
if (metadata.settings.loopMode == 1)
metadata.al.source.set(AL_LOOPING, AL_TRUE);
}
void ext::vorbis::close(uf::Audio::Metadata& metadata) {
if (metadata.stream.handle) {
OggVorbis_File* file = (OggVorbis_File*) metadata.stream.handle;
ov_clear(file);
delete file;
metadata.stream.handle = NULL;
}
if (metadata.stream.file) {
if (metadata.stream.file->is_open()) metadata.stream.file->close();
delete metadata.stream.file;
metadata.stream.file = NULL;
}
}
#endif

226
engine/src/ext/audio/wav.cpp Normal file
View File

@ -0,0 +1,226 @@
#include <uf/config.h>
#if UF_USE_WAV
#if UF_USE_OPENAL
#include <uf/ext/oal/oal.h>
#endif
#include <uf/ext/audio/wav.h>
#include <uf/utils/memory/pool.h>
#include <iostream>
#include <cstdio>
#define DR_WAV_IMPLEMENTATION
#include "dr_wav.h"
namespace {
namespace funs {
size_t wav_read(void* destination, size_t size, size_t nmemb, void* userdata) {
uf::Audio::Metadata& metadata = *((uf::Audio::Metadata*)userdata);
drwav* wav = (drwav*) metadata.stream.handle;
size_t bytesRequested = size * nmemb;
size_t frameSize = wav->channels * (wav->bitsPerSample / 8);
size_t framesToRead = bytesRequested / frameSize;
drwav_uint64 framesRead = drwav_read_pcm_frames(wav, framesToRead, destination);
size_t bytesRead = framesRead * frameSize;
metadata.stream.consumed += bytesRead;
return bytesRead;
}
int wav_seek(void* userdata, int64_t to, int type) {
uf::Audio::Metadata& metadata = *((uf::Audio::Metadata*)userdata);
drwav* wav = (drwav*) metadata.stream.handle;
drwav_uint64 targetFrame = 0;
switch (type) {
case SEEK_CUR: targetFrame = metadata.stream.consumed / (wav->channels * (wav->bitsPerSample / 8)) + to; break;
case SEEK_END: targetFrame = wav->totalPCMFrameCount - to; break;
case SEEK_SET: targetFrame = to; break;
default: return -1;
}
if (!drwav_seek_to_pcm_frame(wav, targetFrame)) return -1;
metadata.stream.consumed = (size_t)(targetFrame * wav->channels * (wav->bitsPerSample / 8));
return 0;
}
int wav_close(void* userdata) {
uf::Audio::Metadata& metadata = *((uf::Audio::Metadata*)userdata);
drwav* wav = (drwav*) metadata.stream.handle;
if (wav) {
drwav_uninit(wav);
delete wav;
metadata.stream.handle = nullptr;
}
return 0;
}
long wav_tell(void* userdata) {
uf::Audio::Metadata& metadata = *((uf::Audio::Metadata*)userdata);
return metadata.stream.consumed;
}
}
}
void ext::wav::open(uf::Audio::Metadata& metadata) {
// open file
drwav* wav = new drwav;
if (!drwav_init_file(wav, metadata.filename.c_str(), nullptr)) {
UF_MSG_ERROR("Could not open WAV file: {}", metadata.filename);
delete wav;
return;
}
// fill out metadata
metadata.stream.handle = wav;
metadata.info.size = wav->totalPCMFrameCount * wav->channels * (wav->bitsPerSample / 8);
metadata.stream.consumed = 0;
metadata.info.channels = wav->channels;
metadata.info.bitDepth = wav->bitsPerSample;
metadata.info.frequency = wav->sampleRate;
metadata.info.duration = (double) wav->totalPCMFrameCount / wav->sampleRate;
// determine OpenAL format
if (wav->channels == 1 && wav->bitsPerSample == 8)
metadata.info.format = AL_FORMAT_MONO8;
else if (wav->channels == 1 && wav->bitsPerSample == 16)
metadata.info.format = AL_FORMAT_MONO16;
else if (wav->channels == 2 && wav->bitsPerSample == 8)
metadata.info.format = AL_FORMAT_STEREO8;
else if (wav->channels == 2 && wav->bitsPerSample == 16)
metadata.info.format = AL_FORMAT_STEREO16;
else {
UF_MSG_ERROR("WAV: unrecognized format: {} channels, {} bps", wav->channels, wav->bitsPerSample);
funs::wav_close(&metadata);
return;
}
// choose load or stream
return metadata.settings.streamed ? ext::wav::stream(metadata) : ext::wav::load(metadata);
}
void ext::wav::load(uf::Audio::Metadata& metadata) {
// if streaming is requested, use streaming function
if (metadata.settings.streamed) return ext::wav::stream(metadata);
drwav* wav = (drwav*) metadata.stream.handle;
// read all PCM data
size_t totalBytes = (size_t) metadata.info.size;
std::vector<uint8_t> bytes(totalBytes);
// Use funs::wav_read instead of drwav_read_pcm_frames
size_t bytesRead = funs::wav_read(bytes.data(), 1, totalBytes, &metadata);
if (bytesRead < totalBytes) {
bytes.resize(bytesRead); // in case file is truncated
}
// upload to OpenAL buffer
metadata.al.buffer.buffer(metadata.info.format, bytes.data(), (ALsizei) bytes.size(), metadata.info.frequency);
metadata.al.source.set(AL_BUFFER, (ALint) metadata.al.buffer.getIndex());
funs::wav_close(&metadata);
}
void ext::wav::stream(uf::Audio::Metadata& metadata) {
if (!metadata.settings.streamed) return ext::wav::load(metadata);
// Ensure we're at the start
funs::wav_seek(&metadata, 0, SEEK_SET);
drwav* wav = (drwav*) metadata.stream.handle;
char buffer[uf::audio::bufferSize];
uint8_t queuedBuffers = 0;
for (; queuedBuffers < metadata.settings.buffers; ++queuedBuffers) {
size_t bytesRead = funs::wav_read(buffer, 1, uf::audio::bufferSize, &metadata);
if (bytesRead == 0) {
if (metadata.settings.loop) {
metadata.stream.consumed = 0;
if (funs::wav_seek(&metadata, 0, SEEK_SET) != 0) {
UF_MSG_ERROR("WAV: failed to loop (seek to start): {}", metadata.filename);
break;
}
bytesRead = funs::wav_read(buffer, 1, uf::audio::bufferSize, &metadata);
}
}
if (bytesRead == 0) {
UF_MSG_WARNING("WAV: consumed file stream before buffers are filled: {} {}", (int)queuedBuffers, metadata.filename);
break;
}
AL_CHECK_RESULT(alBufferData(metadata.al.buffer.getIndex(queuedBuffers), metadata.info.format, buffer, bytesRead, metadata.info.frequency));
}
AL_CHECK_RESULT(alSourceQueueBuffers(metadata.al.source.getIndex(), queuedBuffers, &metadata.al.buffer.getIndex()));
if (queuedBuffers >= metadata.settings.buffers) {
metadata.settings.loopMode = 1;
metadata.al.source.set(AL_LOOPING, AL_FALSE);
}
}
void ext::wav::update(uf::Audio::Metadata& metadata) {
if (!metadata.settings.streamed) return;
// disable hard looping temporarily
if (metadata.settings.loopMode == 1) metadata.al.source.set(AL_LOOPING, AL_FALSE);
ALint state;
metadata.al.source.get(AL_SOURCE_STATE, state);
if (state != AL_PLAYING) {
if (!metadata.settings.loop && metadata.stream.consumed >= metadata.info.size) {
// stream finished
return;
}
// stream stalled, restart it
metadata.al.source.play();
}
ALint processed = 0;
metadata.al.source.get(AL_BUFFERS_PROCESSED, processed);
if (processed <= 0) return;
drwav* wav = (drwav*) metadata.stream.handle;
ALuint index;
char buffer[uf::audio::bufferSize];
while (processed--) {
memset(buffer, 0, uf::audio::bufferSize);
AL_CHECK_RESULT(alSourceUnqueueBuffers(metadata.al.source.getIndex(), 1, &index));
// Use funs::wav_read instead of drwav_read_pcm_frames
size_t bytesRead = funs::wav_read(buffer, 1, uf::audio::bufferSize, &metadata);
if (bytesRead == 0) {
// no more data left to read, reset file stream if we're looping
if (!metadata.settings.loop) break;
if (funs::wav_seek(&metadata, 0, SEEK_SET) != 0) {
UF_MSG_ERROR("WAV: failed to loop (seek to start): {}", metadata.filename);
break;
}
bytesRead = funs::wav_read(buffer, 1, uf::audio::bufferSize, &metadata);
if (bytesRead == 0) {
UF_MSG_ERROR("WAV: failed to read after looping: {}", metadata.filename);
break;
}
}
if (bytesRead > 0) {
AL_CHECK_RESULT(alBufferData(index, metadata.info.format, buffer, bytesRead, metadata.info.frequency));
AL_CHECK_RESULT(alSourceQueueBuffers(metadata.al.source.getIndex(), 1, &index));
}
if (metadata.settings.loop && bytesRead < uf::audio::bufferSize) {
// should never actually reach here
UF_MSG_ERROR("WAV: missing data: {}", metadata.filename);
}
}
// enable hard looping for if we aren't able to call an update in a timely manner
if (metadata.settings.loopMode == 1) metadata.al.source.set(AL_LOOPING, AL_TRUE);
}
void ext::wav::close(uf::Audio::Metadata& metadata) {
if (metadata.stream.handle) {
funs::wav_close(&metadata);
}
metadata.stream.handle = nullptr;
}
#endif

5
engine/src/ext/imgui/imgui.cpp
View File

@ -118,11 +118,14 @@ namespace {
this->scroll.bottom = true;
reclaimFocus = true;
// to-do: add a way to either asynchronously invoke commands or not
// to-do: add a way to either asynchronously invoke commands or not
uf::console::execute( command );
/*
uf::thread::queue( uf::thread::asyncThreadName, [=](){
uf::console::execute( command );
});
*/
/*
// this blocks
uf::thread::queue( uf::thread::fetchWorker(), [=](){

53
engine/src/ext/oal/oal.cpp
View File

@ -1,10 +1,13 @@
#include <uf/config.h>
#if defined(UF_USE_OPENAL)
#if UF_USE_OPENAL
#include <uf/ext/oal/oal.h>
#include <uf/utils/memory/pool.h>
#include <uf/utils/string/io.h>
#include <uf/ext/vorbis/vorbis.h>
#include <uf/ext/audio/vorbis.h>
#include <uf/ext/audio/wav.h>
#include <uf/ext/audio/pcm.h>
#include <uf/utils/audio/audio.h>
#include <iostream>
@ -116,17 +119,51 @@ uf::audio::Metadata* ext::al::create( const uf::stl::string& filename, bool stre
metadata.al.source.set( AL_LOOPING, metadata.settings.loop ? AL_TRUE : AL_FALSE );
return pointer;
}
uf::audio::Metadata* ext::al::create( const pod::PCM& buffer, bool streamed, uint8_t buffers ) {
#if UF_MEMORYPOOL_INVALID_MALLOC
uf::audio::Metadata* pointer = &uf::memoryPool::global.alloc<uf::audio::Metadata>();
#else
uf::MemoryPool* memoryPool = uf::memoryPool::global.size() > 0 ? &uf::memoryPool::global : NULL;
uf::audio::Metadata* pointer = (memoryPool) ? &memoryPool->alloc<uf::audio::Metadata>() : new uf::audio::Metadata;
#endif
uf::audio::Metadata& metadata = *pointer;
metadata.filename = "tmp.pcm"; // probably just stringify the pointer
metadata.settings.streamed = streamed;
metadata.settings.buffers = buffers;
metadata.extension = uf::io::extension( metadata.filename );
metadata.al.buffer.initialize( metadata.settings.buffers );
metadata.al.source.initialize();
metadata.al.source.set( AL_PITCH, 1.0f );
metadata.al.source.set( AL_GAIN, 1.0f );
metadata.al.source.set( AL_LOOPING, metadata.settings.loop ? AL_TRUE : AL_FALSE );
return pointer;
}
uf::audio::Metadata* ext::al::open( const uf::stl::string& filename ) {
return ext::al::open( filename, uf::audio::streamsByDefault );
}
uf::audio::Metadata* ext::al::open( const pod::PCM& buffer ) {
return ext::al::open( buffer, uf::audio::streamsByDefault );
}
uf::audio::Metadata* ext::al::open( const uf::stl::string& filename, bool streams ) {
return streams ? ext::al::stream( filename ) : ext::al::load( filename );
}
uf::audio::Metadata* ext::al::open( const pod::PCM& buffer, bool streams ) {
return streams ? ext::al::stream( buffer ) : ext::al::load( buffer );
}
uf::audio::Metadata* ext::al::load( const uf::stl::string& filename ) {
uf::audio::Metadata* pointer = ext::al::create( filename, false, 1 );
uf::audio::Metadata& metadata = *pointer;
if ( metadata.extension == "ogg" ) ext::vorbis::open( metadata );
else if ( metadata.extension == "wav" ) ext::wav::open( metadata );
return pointer;
}
uf::audio::Metadata* ext::al::load( const pod::PCM& buffer ) {
uf::audio::Metadata* pointer = ext::al::create( buffer, false, 1 );
uf::audio::Metadata& metadata = *pointer;
ext::pcm::open( metadata, buffer );
return pointer;
}
@ -135,10 +172,20 @@ uf::audio::Metadata* ext::al::stream( const uf::stl::string& filename ) {
uf::audio::Metadata& metadata = *pointer;
if ( metadata.extension == "ogg" ) ext::vorbis::open( metadata );
else if ( metadata.extension == "wav" ) ext::wav::open( metadata );
return pointer;
}
uf::audio::Metadata* ext::al::stream( const pod::PCM& buffer ) {
uf::audio::Metadata* pointer = ext::al::create( buffer, true, uf::audio::buffers );
uf::audio::Metadata& metadata = *pointer;
ext::pcm::open( metadata, buffer );
return pointer;
}
void ext::al::update( uf::audio::Metadata& metadata ) {
if ( metadata.extension == "ogg" ) return ext::vorbis::update( metadata );
if ( metadata.extension == "wav" ) return ext::wav::update( metadata );
if ( metadata.extension == "pcm" ) return ext::pcm::update( metadata );
}
void ext::al::close( uf::audio::Metadata* metadata ) {
if ( !metadata ) return;
@ -157,6 +204,8 @@ void ext::al::close( uf::audio::Metadata& metadata ) {
metadata.al.source.destroy();
metadata.al.buffer.destroy();
if ( metadata.extension == "ogg" ) return ext::vorbis::close( metadata );
if ( metadata.extension == "wav" ) return ext::wav::close( metadata );
if ( metadata.extension == "pcm" ) return ext::pcm::close( metadata );
}
void ext::al::listener( const pod::Transform<>& transform ) {

15
engine/src/ext/vall_e/vall_e.cpp
View File

@ -21,15 +21,14 @@ void ext::vall_e::initialize( const std::string& model_path, const std::string&
return;
}
}
std::string ext::vall_e::generate( const std::string& text, const std::string& prom, const std::string& lang ) {
if ( !::ctx ) return "";
pod::PCM ext::vall_e::generate( const std::string& text, const std::string& prom, const std::string& lang ) {
pod::PCM pcm;
std::string path = "./data/tmp/" + std::to_string(uf::time::time()) + ".wav";
if ( !::ctx ) return pcm;
vall_e_args_t args;
args.text = text;
args.prompt_path = prom;
args.output_path = path;
args.language = lang == "" ? "en" : lang;
args.task = "tts";
args.modality = MODALITY_NAR_LEN;
@ -39,10 +38,12 @@ std::string ext::vall_e::generate( const std::string& text, const std::string& p
auto inputs = vall_e_prepare_inputs( ::ctx, args.text, args.prompt_path, args.language );
auto output_audio_codes = vall_e_generate( ::ctx, inputs, args.max_steps, args.max_duration, args.modality );
auto waveform = decode_audio( ::ctx->encodec.ctx, output_audio_codes );
write_audio_to_disk( waveform, args.output_path );
//UF_MSG_DEBUG("Generated to {}", path);
pcm.waveform.insert( pcm.waveform.end(), waveform.begin(), waveform.end() ); // because technically im using different vector classes
pcm.sampleRate = 24000; // should deduce from the backend in the event I ever get around to porting the other models
pcm.channels = 1;
return path;
return pcm;
}
void ext::vall_e::terminate() {
if ( !::ctx ) return;

348
engine/src/ext/vorbis/vorbis.cpp
View File

@ -1,348 +0,0 @@
#include <uf/config.h>
#if UF_USE_VORBIS
#if UF_USE_OPENAL
#include <uf/ext/oal/oal.h>
#endif
#include <uf/ext/vorbis/vorbis.h>
#include <uf/utils/memory/pool.h>
#include <iostream>
#include <cstdio>
namespace {
int endian = 0;
namespace funs {
size_t read( void* destination, size_t size, size_t nmemb, void* userdata ) {
//UF_MSG_DEBUG( size * nmemb );
uf::Audio::Metadata& metadata = *((uf::Audio::Metadata*) userdata);
std::ifstream& file = *metadata.stream.file;
ALsizei length = size * nmemb; // chunk size * chunk count
if ( metadata.stream.consumed + length > metadata.info.size ) {
length = metadata.info.size - metadata.stream.consumed;
}
// reopen file handle if necessary
if ( !file.is_open() ) {
file.open(metadata.filename, std::ios::binary);
if ( !file.is_open() ) {
UF_MSG_ERROR("Could not open file: {}", metadata.filename);
return 0;
}
}
// (re)seek to current position
file.clear();
file.seekg(metadata.stream.consumed);
// setup read buffer
char data[length];
if ( !file.read(&data[0], length) ) {
if ( file.eof() ) file.clear();
else if ( file.fail() ) {
UF_MSG_ERROR("File stream has fail bit set: {}", metadata.filename );
file.clear();
return 0;
}
else if ( file.bad() ) {
UF_MSG_ERROR("File stream has bad bit set: {}", metadata.filename );
file.clear();
return 0;
}
} else file.clear();
// copy from temp buffer
metadata.stream.consumed += length;
memcpy( destination, &data[0], length );
return length;
}
int seek( void* userdata, ogg_int64_t to, int type ) {
//UF_MSG_DEBUG(type << " " << to);
uf::Audio::Metadata& metadata = *((uf::Audio::Metadata*) userdata);
switch ( type ) {
case SEEK_CUR: metadata.stream.consumed += to; break; // increment
case SEEK_END: metadata.stream.consumed = metadata.info.size - to; break; // from the end
case SEEK_SET: metadata.stream.consumed = to; break; // from the start
default: return -1;
}
// clamp
if ( metadata.stream.consumed < 0 ) {
metadata.stream.consumed = 0;
return -1;
}
if ( metadata.stream.consumed > metadata.info.size ) {
metadata.stream.consumed = metadata.info.size;
return -1;
}
return 0;
}
int close( void* userdata ) {
//UF_MSG_DEBUG( userdata );
uf::Audio::Metadata& metadata = *((uf::Audio::Metadata*) userdata);
if ( !metadata.stream.file ) return 0;
std::ifstream& file = *metadata.stream.file;
if ( file.is_open() ) file.close();
delete metadata.stream.file;
metadata.stream.file = NULL;
return 0;
}
long tell( void* userdata ) {
//UF_MSG_DEBUG( userdata );
uf::Audio::Metadata& metadata = *((uf::Audio::Metadata*) userdata);
return metadata.stream.consumed;
}
}
}
void ext::vorbis::open( uf::Audio::Metadata& metadata ) {
// //UF_MSG_INFO( "Vorbis {} opened: {}", (metadata.settings.streamed ? "stream" : "load"), metadata.filename );
return metadata.settings.streamed ? ext::vorbis::stream( metadata ) : ext::vorbis::load( metadata );
}
void ext::vorbis::load( uf::Audio::Metadata& metadata ) {
// use correct function
if ( metadata.settings.streamed ) return ext::vorbis::stream( metadata );
// create file handles
FILE* file = fopen( metadata.filename.c_str(), "rb" );
OggVorbis_File vorbisFile;
if ( !file ) {
UF_MSG_ERROR("Vorbis: failed to open {}. File error.", metadata.filename);
return;
}
// get total file size
fseek(file, 0L, SEEK_END);
metadata.info.size = ftell(file);
metadata.stream.consumed = 0;
fseek(file, 0L, SEEK_SET);
// create oggvorbis handle
if( ov_open_callbacks(file, &vorbisFile, NULL, 0, OV_CALLBACKS_DEFAULT) < 0 ) {
UF_MSG_ERROR("Vorbis: failed to open {}. Not Ogg.", metadata.filename);
return;
}
// grab metadata
vorbis_info* info = ov_info(&vorbisFile, -1);
metadata.info.channels = info->channels;
metadata.info.bitDepth = 16;
metadata.info.frequency = info->rate;
metadata.info.duration = ov_time_total(&vorbisFile, -1);
if ( metadata.info.channels == 1 && metadata.info.bitDepth == 8 ) metadata.info.format = AL_FORMAT_MONO8;
else if ( metadata.info.channels == 1 && metadata.info.bitDepth == 16 ) metadata.info.format = AL_FORMAT_MONO16;
else if ( metadata.info.channels == 2 && metadata.info.bitDepth == 8 ) metadata.info.format = AL_FORMAT_STEREO8;
else if ( metadata.info.channels == 2 && metadata.info.bitDepth == 16 ) metadata.info.format = AL_FORMAT_STEREO16;
else {
UF_MSG_ERROR("Vorbis: unrecognized OGG format: {} channels, {} bps", metadata.info.channels, metadata.info.bitDepth);
return;
}
int32_t read;
char buffer[uf::audio::bufferSize];
uf::stl::vector<char> bytes;
do {
read = ov_read( &vorbisFile, buffer, uf::audio::bufferSize, endian, 2, 1, &metadata.stream.bitStream );
bytes.insert( bytes.end(), buffer, buffer + read );
} while ( read > 0 );
metadata.al.buffer.buffer( metadata.info.format, (char*) &bytes[0], bytes.size(), metadata.info.frequency );
metadata.al.source.set( AL_BUFFER, (ALint) metadata.al.buffer.getIndex() );
ov_clear(&vorbisFile);
}
void ext::vorbis::stream( uf::Audio::Metadata& metadata ) {
// use correct function
if ( !metadata.settings.streamed ) return ext::vorbis::load( metadata );
if ( !metadata.stream.file ) metadata.stream.file = new std::ifstream;
if ( !metadata.stream.handle ) metadata.stream.handle = (void*) new OggVorbis_File;
// create file handles
std::ifstream& file = *metadata.stream.file;
OggVorbis_File& vorbisFile = *((OggVorbis_File*) metadata.stream.handle);
file.open( metadata.filename, std::ios::binary );
if ( !file.is_open() ) {
UF_MSG_ERROR("Vorbis: failed to open file stream: {}", metadata.filename);
return;
}
if ( file.eof() ) {
UF_MSG_ERROR("Vorbis: file stream EOF bit set: {}", metadata.filename);
return;
}
if ( file.fail() ) {
UF_MSG_ERROR("Vorbis: file stream fail bit set: {}", metadata.filename);
return;
}
if ( !file ) {
UF_MSG_ERROR("Vorbis: file is false: {}", metadata.filename);
return;
}
// get total file size
file.seekg(0, std::ios::end);
metadata.info.size = file.tellg();
metadata.stream.consumed = 0;
file.seekg(0, std::ios::beg);
// set our custom callbacks
ov_callbacks callbacks;
callbacks.read_func = ::funs::read;
callbacks.seek_func = ::funs::seek;
callbacks.close_func = ::funs::close;
callbacks.tell_func = ::funs::tell;
// create oggvorbis handle
if ( ov_open_callbacks((void*) &metadata, &vorbisFile, NULL, -1, callbacks) < 0 ) {
UF_MSG_ERROR("Vorbis: failed call to ov_open_callbacks: {}", metadata.filename);
return;
}
// grab metadata
vorbis_info* info = ov_info(&vorbisFile, -1);
metadata.info.channels = info->channels;
metadata.info.bitDepth = 16;
metadata.info.frequency = info->rate;
metadata.info.duration = ov_time_total(&vorbisFile, -1);
if ( metadata.info.channels == 1 && metadata.info.bitDepth == 8 ) metadata.info.format = AL_FORMAT_MONO8;
else if ( metadata.info.channels == 1 && metadata.info.bitDepth == 16 ) metadata.info.format = AL_FORMAT_MONO16;
else if ( metadata.info.channels == 2 && metadata.info.bitDepth == 8 ) metadata.info.format = AL_FORMAT_STEREO8;
else if ( metadata.info.channels == 2 && metadata.info.bitDepth == 16 ) metadata.info.format = AL_FORMAT_STEREO16;
else {
UF_MSG_ERROR("Vorbis: unrecognized OGG format: {} channels, {} bps", (int) metadata.info.channels, (int) metadata.info.bitDepth);
return;
}
// UF_MSG_DEBUG( "filename\tchannels\tbitDepth\tfrequency\tduration\tchannels" );
// UF_MSG_DEBUG( "{}\t{}\t{}\t{}\t{}\t{}", metadata.filename, (int) metadata.info.channels, (int) metadata.info.bitDepth, (int) metadata.info.frequency, metadata.info.duration, (int) metadata.info.channels );
// fill and queue initial buffers
char buffer[uf::audio::bufferSize];
uint8_t queuedBuffers = 0;
for ( ; queuedBuffers < metadata.settings.buffers; ++queuedBuffers ) {
int32_t read = 0;
while ( read < uf::audio::bufferSize ) {
int32_t result = ov_read( &vorbisFile, &buffer[read], uf::audio::bufferSize - read, endian, 2, 1, &metadata.stream.bitStream );
if ( result == OV_HOLE ) {
UF_MSG_ERROR("Vorbis: OV_HOLE found in buffer read: {} {}", (int) queuedBuffers, metadata.filename);
break;
} else if ( result == OV_EBADLINK ) {
UF_MSG_ERROR("Vorbis: OV_EBADLINK found in buffer read: {} {}", (int) queuedBuffers, metadata.filename);
break;
} else if ( result == OV_EINVAL ) {
UF_MSG_ERROR("Vorbis: OV_EINVAL found in buffer read: {} {}", (int) queuedBuffers, metadata.filename);
break;
} else if ( result == 0 ) {
if ( !metadata.settings.loop ) break;
std::int32_t seek = ov_raw_seek( &vorbisFile, 0 );
// UF_MSG_ERROR("Vorbis: EOF found in buffer read: {} {}", (int) queuedBuffers, metadata.filename); break;
switch ( seek ) {
case OV_ENOSEEK: UF_MSG_ERROR("Vorbis: OV_ENOSEEK found in buffer loop: {}", metadata.filename); break;
case OV_EINVAL: UF_MSG_ERROR("Vorbis: OV_EINVAL found in buffer loop: {}", metadata.filename); break;
case OV_EREAD: UF_MSG_ERROR("Vorbis: OV_EREAD found in buffer loop: {}", metadata.filename); break;
case OV_EFAULT: UF_MSG_ERROR("Vorbis: OV_EFAULT found in buffer loop: {}", metadata.filename); break;
case OV_EOF: UF_MSG_ERROR("Vorbis: OV_EOF found in buffer loop: {}", metadata.filename); break;
case OV_EBADLINK: UF_MSG_ERROR("Vorbis: OV_EBADLINK found in buffer loop: {}", metadata.filename); break;
}
}
read += result;
}
if ( read == 0 ) {
UF_MSG_WARNING("Vorbis: consumed file stream before buffers are filled: {} {}", (int) queuedBuffers, metadata.filename);
// if ( metadata.settings.loopMode == 0 ) metadata.settings.loopMode = 1;
// if ( metadata.settings.loop ) metadata.al.source.set( AL_LOOPING, AL_TRUE );
break;
}
AL_CHECK_RESULT(alBufferData(metadata.al.buffer.getIndex(queuedBuffers), metadata.info.format, buffer, read, metadata.info.frequency ));
}
AL_CHECK_RESULT(alSourceQueueBuffers(metadata.al.source.getIndex(), queuedBuffers, &metadata.al.buffer.getIndex()));
// switch to soft looping
if ( queuedBuffers >= metadata.settings.buffers ) {
//UF_MSG_WARNING("Vorbis: file not completely consumed from initial buffer filled, yet looping is enabled. Soft looping...: {}", metadata.filename );
metadata.settings.loopMode = 1;
metadata.al.source.set( AL_LOOPING, AL_FALSE );
}
}
void ext::vorbis::update( uf::Audio::Metadata& metadata ) {
if ( !metadata.settings.streamed ) return;
// disable hard looping temporarily
if ( metadata.settings.loopMode == 1 ) metadata.al.source.set( AL_LOOPING, AL_FALSE );
ALint state;
// metadata.al.source.get( AL_SOURCE_STATE, &state );
metadata.al.source.get( AL_SOURCE_STATE, state );
if ( state != AL_PLAYING ) {
if ( !metadata.settings.loop && metadata.stream.consumed >= metadata.info.size ) {
//UF_MSG_INFO("Vorbis stream finished: {}", metadata.filename);
//metadata.al.source.stop();
return;
}
// stream stalled, restart it
//UF_MSG_INFO("Vorbis stream stalled: {}", metadata.filename);
metadata.al.source.play();
}
ALint processed = 0;
// metadata.al.source.get(AL_BUFFERS_PROCESSED, &processed);
metadata.al.source.get(AL_BUFFERS_PROCESSED, processed);
// no work need to be done
if ( processed <= 0 ) return;
OggVorbis_File& vorbisFile = *((OggVorbis_File*) metadata.stream.handle);
ALuint index;
char buffer[uf::audio::bufferSize];
while ( processed-- ) {
// clear read buffer
memset( buffer, 0, uf::audio::bufferSize );
// grab an available, unqueued buffer
AL_CHECK_RESULT(alSourceUnqueueBuffers(metadata.al.source.getIndex(), 1, &index));
// decode in chunks
int32_t read = 0;
while ( read < uf::audio::bufferSize ) {
int32_t result = ov_read( &vorbisFile, &buffer[read], uf::audio::bufferSize - read, endian, 2, 1, &metadata.stream.bitStream );
if ( result == OV_HOLE ) {
UF_MSG_ERROR("Vorbis: OV_HOLE found in buffer read: {}", metadata.filename);
break;
} else if ( result == OV_EBADLINK ) {
UF_MSG_ERROR("Vorbis: OV_EBADLINK found in buffer read: {}", metadata.filename);
break;
} else if ( result == OV_EINVAL ) {
UF_MSG_ERROR("Vorbis: OV_EINVAL found in buffer read: {}", metadata.filename);
break;
} else if ( result == 0 ) {
// no more data left to read, reset file stream if we're looping
if ( !metadata.settings.loop ) break;
std::int32_t seek = ov_raw_seek( &vorbisFile, 0 );
switch ( seek ) {
case OV_ENOSEEK: UF_MSG_ERROR("Vorbis: OV_ENOSEEK found in buffer loop: {}", metadata.filename); break;
case OV_EINVAL: UF_MSG_ERROR("Vorbis: OV_EINVAL found in buffer loop: {}", metadata.filename); break;
case OV_EREAD: UF_MSG_ERROR("Vorbis: OV_EREAD found in buffer loop: {}", metadata.filename); break;
case OV_EFAULT: UF_MSG_ERROR("Vorbis: OV_EFAULT found in buffer loop: {}", metadata.filename); break;
case OV_EOF: UF_MSG_ERROR("Vorbis: OV_EOF found in buffer loop: {}", metadata.filename); break;
case OV_EBADLINK: UF_MSG_ERROR("Vorbis: OV_EBADLINK found in buffer loop: {}", metadata.filename); break;
}
if( seek != 0 ) {
UF_MSG_ERROR("Vorbis: Unknown error in ov_raw_seek: {}", metadata.filename);
return;
}
}
read += result;
}
// buffer more data
if ( read > 0 ) {
AL_CHECK_RESULT(alBufferData(index, metadata.info.format, &buffer[0], read, metadata.info.frequency));
AL_CHECK_RESULT(alSourceQueueBuffers(metadata.al.source.getIndex(), 1, &index));
}
if ( metadata.settings.loop && read < uf::audio::bufferSize ) {
// should never actually reach here
UF_MSG_ERROR("Vorbis: missing data: {}", metadata.filename);
}
}
// enable hard looping for if we aren't able to call an update in a timely manner
if ( metadata.settings.loopMode == 1 ) metadata.al.source.set( AL_LOOPING, AL_TRUE );
}
void ext::vorbis::close( uf::Audio::Metadata& metadata ) {
//UF_MSG_INFO("Vorbis {} closed: {}", ( metadata.settings.streamed ? "stream" : "load" ), metadata.filename);
if ( metadata.stream.handle ) {
ov_clear((OggVorbis_File*) metadata.stream.handle);
delete (OggVorbis_File*) metadata.stream.handle;
}
if ( metadata.stream.file ) {
if ( metadata.stream.file->is_open() ) metadata.stream.file->close();
delete metadata.stream.file;
}
metadata.stream.file = NULL;
metadata.stream.handle = NULL;
}
#endif

24
engine/src/utils/audio/audio.cpp
View File

@ -2,7 +2,9 @@
#include <uf/utils/string/ext.h>
#if UF_USE_OPENAL
#include <uf/ext/vorbis/vorbis.h>
#include <uf/ext/audio/vorbis.h>
#include <uf/ext/audio/wav.h>
#include <uf/ext/audio/pcm.h>
#include <uf/ext/oal/oal.h>
#endif
@ -46,6 +48,12 @@ void uf::Audio::open( const uf::stl::string& filename ) {
void uf::Audio::open( const uf::stl::string& filename, bool streamed ) {
streamed ? stream( filename ) : load( filename );
}
void uf::Audio::open( const pod::PCM& buffer ) {
this->open( buffer, uf::audio::streamsByDefault );
}
void uf::Audio::open( const pod::PCM& buffer, bool streamed ) {
streamed ? stream( buffer ) : load( buffer );
}
void uf::Audio::load( const uf::stl::string& filename ) {
if ( uf::audio::muted ) return;
#if UF_USE_OPENAL
@ -53,6 +61,13 @@ void uf::Audio::load( const uf::stl::string& filename ) {
this->m_metadata = ext::al::load( filename );
#endif
}
void uf::Audio::load( const pod::PCM& buffer ) {
if ( uf::audio::muted ) return;
#if UF_USE_OPENAL
if ( this->m_metadata ) ext::al::close( *this->m_metadata );
this->m_metadata = ext::al::load( buffer );
#endif
}
void uf::Audio::stream( const uf::stl::string& filename ) {
if ( uf::audio::muted ) return;
#if UF_USE_OPENAL
@ -60,6 +75,13 @@ void uf::Audio::stream( const uf::stl::string& filename ) {
this->m_metadata = ext::al::stream( filename );
#endif
}
void uf::Audio::stream( const pod::PCM& buffer ) {
if ( uf::audio::muted ) return;
#if UF_USE_OPENAL
if ( this->m_metadata ) ext::al::close( *this->m_metadata );
this->m_metadata = ext::al::stream( buffer );
#endif
}
void uf::Audio::update() {
#if UF_USE_OPENAL
if ( !this->m_metadata ) return;

1
ext/behaviors/gui/manager/behavior.cpp
View File

@ -46,7 +46,6 @@ void ext::GuiManagerBehavior::tick( uf::Object& self ) {
// and this oversight seems to only happen when registerRenderMode = false
auto& metadata = this->getComponent<ext::GuiManagerBehavior::Metadata>();
if ( !metadata.boundGui && !uf::renderer::hasRenderMode( "Gui", true ) ) {
UF_MSG_DEBUG("ADDING RENDER MODE");
auto& renderMode = this->getComponent<uf::renderer::RenderTargetRenderMode>();
uf::stl::string name = "Gui";
renderMode.blitter.descriptor.renderMode = "Swapchain";

9
ext/behaviors/scene/behavior.cpp
View File

@ -69,6 +69,15 @@ void ext::ExtSceneBehavior::initialize( uf::Object& self ) {
metadataJson.import( payload.metadata );
}
gui.initialize();
// an example to synthesize and playback speech
/*
ext::json::Value payload;
payload["text"] = "Opening menu.";
payload["prom"] = "./data/tmp/prom.wav";
payload["callback"] = this->formatHookName("sound:Emit.%UID%");
uf::hooks.call("llm:VALL-E.synthesize", payload);
*/
};
});
this->addHook( "world:Entity.LoadAsset", [&](pod::payloads::assetLoad& payload){

5
ext/behaviors/sound-emitter/behavior.cpp
View File

@ -31,6 +31,11 @@ void ext::SoundEmitterBehavior::initialize( uf::Object& self ) {
metadata["sounds"].erase(i);
}
});
this->addHook( "sound:Emit.%UID%", [&]( pod::PCM& waveform ){
uf::Audio& audio = emitter.add();
audio.load( waveform );
audio.play();
});
this->addHook( "sound:Emit.%UID%", [&](ext::json::Value& json){
if ( ext::json::isNull(json["volume"]) ) json["volume"] = metadata["audio"]["volume"];
if ( ext::json::isNull(json["pitch"]) ) json["pitch"] = metadata["audio"]["pitch"];

20
ext/main.cpp
View File

@ -733,12 +733,22 @@ void EXT_API ext::initialize() {
uf::hooks.addHook( "llm:VALL-E.synthesize", [&](ext::json::Value& json){
auto text = json["text"].as<uf::stl::string>();
auto prom = json["prom"].as<uf::stl::string>();
auto path = ext::vall_e::generate( text, prom );
UF_MSG_DEBUG("Called {} {}: {}", text, prom, path);
auto play = json["play"].as<bool>();
auto callback = json["callback"].as<uf::stl::string>("");
return path;
uf::thread::queue( uf::thread::asyncThreadName, [=](){
auto waveform = ext::vall_e::generate( text, prom );
if ( callback != "" ) {
UF_MSG_DEBUG("Calling hook: {}", callback);
uf::hooks.call( callback, waveform );
}
if ( play ) {
uf::Audio audio;
audio.load( waveform );
audio.setVolume( 4.0f );
audio.play();
}
});
});
}
#endif