462 lines
12 KiB
C
462 lines
12 KiB
C
#include "../../media-io/audio-resampler.h"
|
|
#include "../../util/circlebuf.h"
|
|
#include "../../util/platform.h"
|
|
#include "../../util/darray.h"
|
|
#include "../../util/util_uint64.h"
|
|
#include "../../obs-internal.h"
|
|
|
|
#include "wasapi-output.h"
|
|
|
|
#define ACTUALLY_DEFINE_GUID(name, l, w1, w2, b1, b2, b3, b4, b5, b6, b7, b8) \
|
|
EXTERN_C const GUID DECLSPEC_SELECTANY name = { \
|
|
l, w1, w2, {b1, b2, b3, b4, b5, b6, b7, b8}}
|
|
|
|
#define do_log(level, format, ...) \
|
|
blog(level, "[audio monitoring: '%s'] " format, \
|
|
obs_source_get_name(monitor->source), ##__VA_ARGS__)
|
|
|
|
#define warn(format, ...) do_log(LOG_WARNING, format, ##__VA_ARGS__)
|
|
#define info(format, ...) do_log(LOG_INFO, format, ##__VA_ARGS__)
|
|
#define debug(format, ...) do_log(LOG_DEBUG, format, ##__VA_ARGS__)
|
|
|
|
ACTUALLY_DEFINE_GUID(CLSID_MMDeviceEnumerator, 0xBCDE0395, 0xE52F, 0x467C, 0x8E,
|
|
0x3D, 0xC4, 0x57, 0x92, 0x91, 0x69, 0x2E);
|
|
ACTUALLY_DEFINE_GUID(IID_IMMDeviceEnumerator, 0xA95664D2, 0x9614, 0x4F35, 0xA7,
|
|
0x46, 0xDE, 0x8D, 0xB6, 0x36, 0x17, 0xE6);
|
|
ACTUALLY_DEFINE_GUID(IID_IAudioClient, 0x1CB9AD4C, 0xDBFA, 0x4C32, 0xB1, 0x78,
|
|
0xC2, 0xF5, 0x68, 0xA7, 0x03, 0xB2);
|
|
ACTUALLY_DEFINE_GUID(IID_IAudioRenderClient, 0xF294ACFC, 0x3146, 0x4483, 0xA7,
|
|
0xBF, 0xAD, 0xDC, 0xA7, 0xC2, 0x60, 0xE2);
|
|
|
|
struct audio_monitor {
|
|
obs_source_t *source;
|
|
IMMDevice *device;
|
|
IAudioClient *client;
|
|
IAudioRenderClient *render;
|
|
|
|
uint64_t last_recv_time;
|
|
uint64_t prev_video_ts;
|
|
uint64_t time_since_prev;
|
|
audio_resampler_t *resampler;
|
|
uint32_t sample_rate;
|
|
uint32_t channels;
|
|
bool source_has_video;
|
|
bool ignore;
|
|
|
|
int64_t lowest_audio_offset;
|
|
struct circlebuf delay_buffer;
|
|
uint32_t delay_size;
|
|
|
|
DARRAY(float) buf;
|
|
pthread_mutex_t playback_mutex;
|
|
};
|
|
|
|
/* #define DEBUG_AUDIO */
|
|
|
|
static bool process_audio_delay(struct audio_monitor *monitor, float **data,
|
|
uint32_t *frames, uint64_t ts, uint32_t pad)
|
|
{
|
|
obs_source_t *s = monitor->source;
|
|
uint64_t last_frame_ts = s->last_frame_ts;
|
|
uint64_t cur_time = os_gettime_ns();
|
|
uint64_t front_ts;
|
|
uint64_t cur_ts;
|
|
int64_t diff;
|
|
uint32_t blocksize = monitor->channels * sizeof(float);
|
|
|
|
/* cut off audio if long-since leftover audio in delay buffer */
|
|
if (cur_time - monitor->last_recv_time > 1000000000)
|
|
circlebuf_free(&monitor->delay_buffer);
|
|
monitor->last_recv_time = cur_time;
|
|
|
|
ts += monitor->source->sync_offset;
|
|
|
|
circlebuf_push_back(&monitor->delay_buffer, &ts, sizeof(ts));
|
|
circlebuf_push_back(&monitor->delay_buffer, frames, sizeof(*frames));
|
|
circlebuf_push_back(&monitor->delay_buffer, *data, *frames * blocksize);
|
|
|
|
if (!monitor->prev_video_ts) {
|
|
monitor->prev_video_ts = last_frame_ts;
|
|
|
|
} else if (monitor->prev_video_ts == last_frame_ts) {
|
|
monitor->time_since_prev += util_mul_div64(
|
|
*frames, 1000000000ULL, monitor->sample_rate);
|
|
} else {
|
|
monitor->time_since_prev = 0;
|
|
}
|
|
|
|
while (monitor->delay_buffer.size != 0) {
|
|
size_t size;
|
|
bool bad_diff;
|
|
|
|
circlebuf_peek_front(&monitor->delay_buffer, &cur_ts,
|
|
sizeof(ts));
|
|
front_ts = cur_ts - util_mul_div64(pad, 1000000000ULL,
|
|
monitor->sample_rate);
|
|
diff = (int64_t)front_ts - (int64_t)last_frame_ts;
|
|
bad_diff = !last_frame_ts || llabs(diff) > 5000000000 ||
|
|
monitor->time_since_prev > 100000000ULL;
|
|
|
|
/* delay audio if rushing */
|
|
if (!bad_diff && diff > 75000000) {
|
|
#ifdef DEBUG_AUDIO
|
|
blog(LOG_INFO,
|
|
"audio rushing, cutting audio, "
|
|
"diff: %lld, delay buffer size: %lu, "
|
|
"v: %llu: a: %llu",
|
|
diff, (int)monitor->delay_buffer.size,
|
|
last_frame_ts, front_ts);
|
|
#endif
|
|
return false;
|
|
}
|
|
|
|
circlebuf_pop_front(&monitor->delay_buffer, NULL, sizeof(ts));
|
|
circlebuf_pop_front(&monitor->delay_buffer, frames,
|
|
sizeof(*frames));
|
|
|
|
size = *frames * blocksize;
|
|
da_resize(monitor->buf, size);
|
|
circlebuf_pop_front(&monitor->delay_buffer, monitor->buf.array,
|
|
size);
|
|
|
|
/* cut audio if dragging */
|
|
if (!bad_diff && diff < -75000000 &&
|
|
monitor->delay_buffer.size > 0) {
|
|
#ifdef DEBUG_AUDIO
|
|
blog(LOG_INFO,
|
|
"audio dragging, cutting audio, "
|
|
"diff: %lld, delay buffer size: %lu, "
|
|
"v: %llu: a: %llu",
|
|
diff, (int)monitor->delay_buffer.size,
|
|
last_frame_ts, front_ts);
|
|
#endif
|
|
continue;
|
|
}
|
|
|
|
*data = monitor->buf.array;
|
|
return true;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
static void on_audio_playback(void *param, obs_source_t *source,
|
|
const struct audio_data *audio_data, bool muted)
|
|
{
|
|
struct audio_monitor *monitor = param;
|
|
IAudioRenderClient *render = monitor->render;
|
|
uint8_t *resample_data[MAX_AV_PLANES];
|
|
float vol = source->user_volume;
|
|
uint32_t resample_frames;
|
|
uint64_t ts_offset;
|
|
bool success;
|
|
BYTE *output;
|
|
|
|
if (pthread_mutex_trylock(&monitor->playback_mutex) != 0) {
|
|
return;
|
|
}
|
|
if (os_atomic_load_long(&source->activate_refs) == 0) {
|
|
goto unlock;
|
|
}
|
|
|
|
success = audio_resampler_resample(
|
|
monitor->resampler, resample_data, &resample_frames, &ts_offset,
|
|
(const uint8_t *const *)audio_data->data,
|
|
(uint32_t)audio_data->frames);
|
|
if (!success) {
|
|
goto unlock;
|
|
}
|
|
|
|
UINT32 pad = 0;
|
|
monitor->client->lpVtbl->GetCurrentPadding(monitor->client, &pad);
|
|
|
|
bool decouple_audio = source->async_unbuffered &&
|
|
source->async_decoupled;
|
|
|
|
if (monitor->source_has_video && !decouple_audio) {
|
|
uint64_t ts = audio_data->timestamp - ts_offset;
|
|
|
|
if (!process_audio_delay(monitor, (float **)(&resample_data[0]),
|
|
&resample_frames, ts, pad)) {
|
|
goto unlock;
|
|
}
|
|
}
|
|
|
|
HRESULT hr =
|
|
render->lpVtbl->GetBuffer(render, resample_frames, &output);
|
|
if (FAILED(hr)) {
|
|
goto unlock;
|
|
}
|
|
|
|
if (!muted) {
|
|
/* apply volume */
|
|
if (!close_float(vol, 1.0f, EPSILON)) {
|
|
register float *cur = (float *)resample_data[0];
|
|
register float *end =
|
|
cur + resample_frames * monitor->channels;
|
|
|
|
while (cur < end)
|
|
*(cur++) *= vol;
|
|
}
|
|
memcpy(output, resample_data[0],
|
|
resample_frames * monitor->channels * sizeof(float));
|
|
}
|
|
|
|
render->lpVtbl->ReleaseBuffer(render, resample_frames,
|
|
muted ? AUDCLNT_BUFFERFLAGS_SILENT : 0);
|
|
|
|
unlock:
|
|
pthread_mutex_unlock(&monitor->playback_mutex);
|
|
}
|
|
|
|
static inline void audio_monitor_free(struct audio_monitor *monitor)
|
|
{
|
|
if (monitor->ignore)
|
|
return;
|
|
|
|
if (monitor->source) {
|
|
obs_source_remove_audio_capture_callback(
|
|
monitor->source, on_audio_playback, monitor);
|
|
}
|
|
|
|
if (monitor->client)
|
|
monitor->client->lpVtbl->Stop(monitor->client);
|
|
|
|
safe_release(monitor->device);
|
|
safe_release(monitor->client);
|
|
safe_release(monitor->render);
|
|
audio_resampler_destroy(monitor->resampler);
|
|
circlebuf_free(&monitor->delay_buffer);
|
|
da_free(monitor->buf);
|
|
}
|
|
|
|
static enum speaker_layout convert_speaker_layout(DWORD layout, WORD channels)
|
|
{
|
|
switch (layout) {
|
|
case KSAUDIO_SPEAKER_2POINT1:
|
|
return SPEAKERS_2POINT1;
|
|
case KSAUDIO_SPEAKER_SURROUND:
|
|
return SPEAKERS_4POINT0;
|
|
case KSAUDIO_SPEAKER_4POINT1:
|
|
return SPEAKERS_4POINT1;
|
|
case KSAUDIO_SPEAKER_5POINT1:
|
|
return SPEAKERS_5POINT1;
|
|
case KSAUDIO_SPEAKER_7POINT1:
|
|
return SPEAKERS_7POINT1;
|
|
}
|
|
|
|
return (enum speaker_layout)channels;
|
|
}
|
|
|
|
extern bool devices_match(const char *id1, const char *id2);
|
|
|
|
static bool audio_monitor_init(struct audio_monitor *monitor,
|
|
obs_source_t *source)
|
|
{
|
|
IMMDeviceEnumerator *immde = NULL;
|
|
WAVEFORMATEX *wfex = NULL;
|
|
bool success = false;
|
|
UINT32 frames;
|
|
HRESULT hr;
|
|
|
|
pthread_mutex_init_value(&monitor->playback_mutex);
|
|
|
|
monitor->source = source;
|
|
|
|
const char *id = obs->audio.monitoring_device_id;
|
|
if (!id) {
|
|
warn("%s: No device ID set", __FUNCTION__);
|
|
return false;
|
|
}
|
|
|
|
if (source->info.output_flags & OBS_SOURCE_DO_NOT_SELF_MONITOR) {
|
|
obs_data_t *s = obs_source_get_settings(source);
|
|
const char *s_dev_id = obs_data_get_string(s, "device_id");
|
|
bool match = devices_match(s_dev_id, id);
|
|
obs_data_release(s);
|
|
|
|
if (match) {
|
|
monitor->ignore = true;
|
|
return true;
|
|
}
|
|
}
|
|
|
|
/* ------------------------------------------ *
|
|
* Init device */
|
|
|
|
hr = CoCreateInstance(&CLSID_MMDeviceEnumerator, NULL, CLSCTX_ALL,
|
|
&IID_IMMDeviceEnumerator, (void **)&immde);
|
|
if (FAILED(hr)) {
|
|
warn("%s: Failed to create IMMDeviceEnumerator: %08lX",
|
|
__FUNCTION__, hr);
|
|
return false;
|
|
}
|
|
|
|
if (strcmp(id, "default") == 0) {
|
|
hr = immde->lpVtbl->GetDefaultAudioEndpoint(
|
|
immde, eRender, eConsole, &monitor->device);
|
|
} else {
|
|
wchar_t w_id[512];
|
|
os_utf8_to_wcs(id, 0, w_id, 512);
|
|
|
|
hr = immde->lpVtbl->GetDevice(immde, w_id, &monitor->device);
|
|
}
|
|
|
|
if (FAILED(hr)) {
|
|
warn("%s: Failed to get device: %08lX", __FUNCTION__, hr);
|
|
goto fail;
|
|
}
|
|
|
|
/* ------------------------------------------ *
|
|
* Init client */
|
|
|
|
hr = monitor->device->lpVtbl->Activate(monitor->device,
|
|
&IID_IAudioClient, CLSCTX_ALL,
|
|
NULL, (void **)&monitor->client);
|
|
if (FAILED(hr)) {
|
|
warn("%s: Failed to activate device: %08lX", __FUNCTION__, hr);
|
|
goto fail;
|
|
}
|
|
|
|
hr = monitor->client->lpVtbl->GetMixFormat(monitor->client, &wfex);
|
|
if (FAILED(hr)) {
|
|
warn("%s: Failed to get mix format: %08lX", __FUNCTION__, hr);
|
|
goto fail;
|
|
}
|
|
|
|
hr = monitor->client->lpVtbl->Initialize(monitor->client,
|
|
AUDCLNT_SHAREMODE_SHARED, 0,
|
|
10000000, 0, wfex, NULL);
|
|
if (FAILED(hr)) {
|
|
warn("%s: Failed to initialize: %08lX", __FUNCTION__, hr);
|
|
goto fail;
|
|
}
|
|
|
|
/* ------------------------------------------ *
|
|
* Init resampler */
|
|
|
|
const struct audio_output_info *info =
|
|
audio_output_get_info(obs->audio.audio);
|
|
WAVEFORMATEXTENSIBLE *ext = (WAVEFORMATEXTENSIBLE *)wfex;
|
|
struct resample_info from;
|
|
struct resample_info to;
|
|
|
|
from.samples_per_sec = info->samples_per_sec;
|
|
from.speakers = info->speakers;
|
|
from.format = AUDIO_FORMAT_FLOAT_PLANAR;
|
|
|
|
to.samples_per_sec = (uint32_t)wfex->nSamplesPerSec;
|
|
to.speakers =
|
|
convert_speaker_layout(ext->dwChannelMask, wfex->nChannels);
|
|
to.format = AUDIO_FORMAT_FLOAT;
|
|
|
|
monitor->sample_rate = (uint32_t)wfex->nSamplesPerSec;
|
|
monitor->channels = wfex->nChannels;
|
|
monitor->resampler = audio_resampler_create(&to, &from);
|
|
if (!monitor->resampler) {
|
|
goto fail;
|
|
}
|
|
|
|
/* ------------------------------------------ *
|
|
* Init client */
|
|
|
|
hr = monitor->client->lpVtbl->GetBufferSize(monitor->client, &frames);
|
|
if (FAILED(hr)) {
|
|
warn("%s: Failed to get buffer size: %08lX", __FUNCTION__, hr);
|
|
goto fail;
|
|
}
|
|
|
|
hr = monitor->client->lpVtbl->GetService(monitor->client,
|
|
&IID_IAudioRenderClient,
|
|
(void **)&monitor->render);
|
|
if (FAILED(hr)) {
|
|
warn("%s: Failed to get IAudioRenderClient: %08lX",
|
|
__FUNCTION__, hr);
|
|
goto fail;
|
|
}
|
|
|
|
if (pthread_mutex_init(&monitor->playback_mutex, NULL) != 0) {
|
|
warn("%s: Failed to initialize mutex", __FUNCTION__);
|
|
goto fail;
|
|
}
|
|
|
|
hr = monitor->client->lpVtbl->Start(monitor->client);
|
|
if (FAILED(hr)) {
|
|
warn("%s: Failed to start audio: %08lX", __FUNCTION__, hr);
|
|
goto fail;
|
|
}
|
|
|
|
success = true;
|
|
|
|
fail:
|
|
safe_release(immde);
|
|
if (wfex)
|
|
CoTaskMemFree(wfex);
|
|
return success;
|
|
}
|
|
|
|
static void audio_monitor_init_final(struct audio_monitor *monitor)
|
|
{
|
|
if (monitor->ignore)
|
|
return;
|
|
|
|
monitor->source_has_video =
|
|
(monitor->source->info.output_flags & OBS_SOURCE_VIDEO) != 0;
|
|
obs_source_add_audio_capture_callback(monitor->source,
|
|
on_audio_playback, monitor);
|
|
}
|
|
|
|
struct audio_monitor *audio_monitor_create(obs_source_t *source)
|
|
{
|
|
struct audio_monitor monitor = {0};
|
|
struct audio_monitor *out;
|
|
|
|
if (!audio_monitor_init(&monitor, source)) {
|
|
goto fail;
|
|
}
|
|
|
|
out = bmemdup(&monitor, sizeof(monitor));
|
|
|
|
pthread_mutex_lock(&obs->audio.monitoring_mutex);
|
|
da_push_back(obs->audio.monitors, &out);
|
|
pthread_mutex_unlock(&obs->audio.monitoring_mutex);
|
|
|
|
audio_monitor_init_final(out);
|
|
return out;
|
|
|
|
fail:
|
|
audio_monitor_free(&monitor);
|
|
return NULL;
|
|
}
|
|
|
|
void audio_monitor_reset(struct audio_monitor *monitor)
|
|
{
|
|
struct audio_monitor new_monitor = {0};
|
|
bool success;
|
|
|
|
pthread_mutex_lock(&monitor->playback_mutex);
|
|
success = audio_monitor_init(&new_monitor, monitor->source);
|
|
pthread_mutex_unlock(&monitor->playback_mutex);
|
|
|
|
if (success) {
|
|
obs_source_t *source = monitor->source;
|
|
audio_monitor_free(monitor);
|
|
*monitor = new_monitor;
|
|
audio_monitor_init_final(monitor);
|
|
} else {
|
|
audio_monitor_free(&new_monitor);
|
|
}
|
|
}
|
|
|
|
void audio_monitor_destroy(struct audio_monitor *monitor)
|
|
{
|
|
if (monitor) {
|
|
audio_monitor_free(monitor);
|
|
|
|
pthread_mutex_lock(&obs->audio.monitoring_mutex);
|
|
da_erase_item(obs->audio.monitors, &monitor);
|
|
pthread_mutex_unlock(&obs->audio.monitoring_mutex);
|
|
|
|
bfree(monitor);
|
|
}
|
|
}
|