Skip to content

Commit

Permalink
[libav] More audio encoding work
Browse files Browse the repository at this point in the history
  • Loading branch information
@jcelerier
jcelerier committed Jan 22, 2024
1 parent cc7e6ed commit b667fb7
Show file tree
Hide file tree
Showing 5 changed files with 300 additions and 134 deletions.
169 changes: 169 additions & 0 deletions src/plugins/score-plugin-gfx/Gfx/Libav/AudioFrameEncoder.hpp
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
#pragma once

#include <Media/Libav.hpp>
#if SCORE_HAS_LIBAV
extern "C" {
#include <libavutil/frame.h>
}
#define SAMPLE_RATE_TEST 44100
#define BUFFER_SIZE_TEST 512

#include <ossia/dataflow/sample_to_float.hpp>

namespace Gfx
{

struct AudioFrameEncoder
{
AudioFrameEncoder(int target_buffer_size)
: target_buffer_size{target_buffer_size}
{
}

// We assume that vec has correct channel count here
// Also that vec.size() > 0
virtual void add_frame(AVFrame& frame, const tcb::span<ossia::float_vector> vec) = 0;

int target_buffer_size{};
};

struct S16IAudioFrameEncoder : AudioFrameEncoder
{
using AudioFrameEncoder::AudioFrameEncoder;

boost::container::vector<int16_t> data;

void add_frame(AVFrame& frame, const tcb::span<ossia::float_vector> vec) override
{
const int channels = vec.size();
const int frames = vec[0].size();
data.clear();
data.resize(frames * channels, boost::container::default_init);
auto ptr = data.data();
for(int i = 0; i < frames; i++)
for(int c = 0; c < channels; c++)
*ptr++ = ossia::float_to_sample<int16_t, 16>(vec[c][i]);

frame.data[0] = (uint8_t*)data.data();
frame.data[1] = nullptr;
}
};

struct S24IAudioFrameEncoder : AudioFrameEncoder
{
using AudioFrameEncoder::AudioFrameEncoder;

boost::container::vector<int32_t> data;
void add_frame(AVFrame& frame, const tcb::span<ossia::float_vector> vec) override
{
const int channels = vec.size();
const int frames = vec[0].size();
data.clear();
data.resize(frames * channels, boost::container::default_init);
auto ptr = data.data();
for(int i = 0; i < frames; i++)
for(int c = 0; c < channels; c++)
*ptr++ = ossia::float_to_sample<int32_t, 24>(vec[c][i]);

frame.data[0] = (uint8_t*)data.data();
frame.data[1] = nullptr;
}
};

struct S32IAudioFrameEncoder : AudioFrameEncoder
{
using AudioFrameEncoder::AudioFrameEncoder;

boost::container::vector<int32_t> data;
void add_frame(AVFrame& frame, const tcb::span<ossia::float_vector> vec) override
{
const int channels = vec.size();
const int frames = vec[0].size();
data.clear();
data.resize(frames * channels, boost::container::default_init);
auto ptr = data.data();
for(int i = 0; i < frames; i++)
for(int c = 0; c < channels; c++)
*ptr++ = ossia::float_to_sample<int32_t, 32>(vec[c][i]);

frame.data[0] = (uint8_t*)data.data();
frame.data[1] = nullptr;
}
};

struct FltIAudioFrameEncoder : AudioFrameEncoder
{
using AudioFrameEncoder::AudioFrameEncoder;

boost::container::vector<float> data;

void add_frame(AVFrame& frame, const tcb::span<ossia::float_vector> vec) override
{
const int channels = vec.size();
const int frames = vec[0].size();
data.clear();
data.resize(frames * channels, boost::container::default_init);
auto ptr = data.data();
for(int i = 0; i < frames; i++)
for(int c = 0; c < channels; c++)
*ptr++ = vec[c][i];

frame.data[0] = (uint8_t*)data.data();
frame.data[1] = nullptr;
}
};

struct DblIAudioFrameEncoder : AudioFrameEncoder
{
using AudioFrameEncoder::AudioFrameEncoder;

boost::container::vector<double> data;

void add_frame(AVFrame& frame, const tcb::span<ossia::float_vector> vec) override
{
const int channels = vec.size();
const int frames = vec[0].size();
data.clear();
data.resize(frames * channels, boost::container::default_init);
auto ptr = data.data();
for(int i = 0; i < frames; i++)
for(int c = 0; c < channels; c++)
*ptr++ = vec[c][i];

frame.data[0] = (uint8_t*)data.data();
frame.data[1] = nullptr;
}
};

struct FltPAudioFrameEncoder : AudioFrameEncoder
{
using AudioFrameEncoder::AudioFrameEncoder;

void add_frame(AVFrame& frame, const tcb::span<ossia::float_vector> vec) override
{
const int channels = vec.size();
if(channels <= AV_NUM_DATA_POINTERS)
{
for(int i = 0; i < channels; ++i)
{
frame.data[i] = reinterpret_cast<uint8_t*>(vec[i].data());
}
}
else
{
// FIXME where does this get freed???
frame.extended_data
= static_cast<uint8_t**>(av_malloc(channels * sizeof(*frame.extended_data)));
int i = 0;
for(; i < AV_NUM_DATA_POINTERS; ++i)
{
frame.data[i] = reinterpret_cast<uint8_t*>(vec[i].data());
frame.extended_data[i] = reinterpret_cast<uint8_t*>(vec[i].data());
}
for(; i < channels; ++i)
frame.extended_data[i] = reinterpret_cast<uint8_t*>(vec[i].data());
}
}
};
}
#endif
56 changes: 26 additions & 30 deletions src/plugins/score-plugin-gfx/Gfx/Libav/LibavEncoder.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,49 +112,45 @@ int LibavEncoder::add_frame(tcb::span<ossia::float_vector> vec)
{
if(!m_formatContext)
return 1;

const int channels = vec.size();
if(channels == 0) // Write silence?
return 1;

auto& stream = streams[audio_stream_index];
AVFrame* next_frame = stream.get_audio_frame();

next_frame->sample_rate = SAMPLE_RATE_TEST;
next_frame->format = SAMPLE_FORMAT_TEST;
next_frame->format = stream.enc->sample_fmt;
next_frame->nb_samples = vec[0].size();
next_frame->ch_layout.nb_channels = vec.size();
next_frame->ch_layout.nb_channels = channels;
next_frame->ch_layout.order = AV_CHANNEL_ORDER_UNSPEC;

std::vector<int16_t> data;
data.reserve(1024);
for(int i = 0; i < BUFFER_SIZE_TEST; i++)
for(int c = 0; c < CHANNELS_TEST; c++)
data.push_back(vec[c][i] * 32768.f);

next_frame->data[0] = (uint8_t*)data.data();
next_frame->data[1] = nullptr;
#if 0
// Write the data
if(stream.resamplers.empty())
{
// Encode directly
stream.encoder->add_frame(*next_frame, vec);
}
else
{
auto& frame = next_frame;
int channels = vec.size();
if(channels <= AV_NUM_DATA_POINTERS)
if(vec.size() != stream.resamplers.size())
{
for(int i = 0; i < channels; ++i)
{
frame->data[i] = reinterpret_cast<uint8_t*>(vec[i].data());
}
qDebug() << "Error: invalid channel count for expected resampling.";
return 1;
}
else

std::vector<std::vector<double>> resample_in(channels);
std::vector<ossia::float_vector> resample_outf(channels);
for(int c = 0; c < channels; c++)
{
frame->extended_data
= static_cast<uint8_t**>(av_malloc(channels * sizeof(*frame->extended_data)));
int i = 0;
for(; i < AV_NUM_DATA_POINTERS; ++i)
{
frame->data[i] = reinterpret_cast<uint8_t*>(vec[i].data());
frame->extended_data[i] = reinterpret_cast<uint8_t*>(vec[i].data());
}
for(; i < channels; ++i)
frame->extended_data[i] = reinterpret_cast<uint8_t*>(vec[i].data());
double* ret{};
int res = stream.resamplers[c]->process(resample_in[c].data(), vec[c].size(), ret);
resample_outf[c].assign(ret, ret + res);
}

stream.encoder->add_frame(*next_frame, resample_outf);
}
#endif
return stream.write_audio_frame(m_formatContext, next_frame);
}

Expand Down
31 changes: 28 additions & 3 deletions src/plugins/score-plugin-gfx/Gfx/Libav/LibavOutputDevice.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
#include <State/MessageListSerialization.hpp>
#include <State/Widgets/AddressFragmentLineEdit.hpp>

#include <Audio/Settings/Model.hpp>
#include <Gfx/GfxApplicationPlugin.hpp>
#include <Gfx/GfxParameter.hpp>
#include <Gfx/Libav/LibavEncoder.hpp>
Expand Down Expand Up @@ -148,11 +149,12 @@ class libav_output_device : public ossia::net::device_base
auto& p = *static_cast<gfx_protocol_base*>(m_protocol.get());
auto node = new LibavEncoderNode{set, enc, 0};
root.add_child(std::make_unique<gfx_node_base>(*this, p, node, "Video"));

auto& audio_stgs = score::AppContext().settings<Audio::Settings::Model>();
auto audio = root.add_child(
std::make_unique<ossia::net::generic_node>("Audio", *this, root));
SCORE_ASSERT(audio);
audio->set_parameter(std::make_unique<record_audio_parameter>(
enc, CHANNELS_TEST, BUFFER_SIZE_TEST, *audio));
enc, set.audio_channels, audio_stgs.getBufferSize(), *audio));
}

const ossia::net::generic_node& get_root_node() const override { return root; }
Expand Down Expand Up @@ -201,6 +203,8 @@ class LibavOutputSettingsWidget final : public Gfx::SharedOutputSettingsWidget
LibavOutputDevice::~LibavOutputDevice() { }

static const std::map<QString, LibavOutputSettings> libav_preset_list{
// Play with:
// ffplay -an -fflags nobuffer -flags low_delay -probesize 32 -analyzeduration 1 -strict experimental -framedrop -vf setpts=0 'udp://127.0.0.1:1234'
{"UDP MJPEG streaming",
LibavOutputSettings{
{
Expand All @@ -213,7 +217,10 @@ static const std::map<QString, LibavOutputSettings> libav_preset_list{
.video_render_pixfmt = "rgba",
.video_converted_pixfmt = "yuv420p",
.muxer = "mjpeg",
.options = {{"fflags", "+nobuffer+genpts"}, {"flags", "+low_delay"}}}},
.options
= {{"fflags", "+nobuffer+genpts"},
{"flags", "+low_delay"},
{"flush_packets", "1"}}}},

{"MKV H.264 recording",
LibavOutputSettings{
Expand All @@ -228,6 +235,24 @@ static const std::map<QString, LibavOutputSettings> libav_preset_list{
.video_converted_pixfmt = "yuv420p",
.muxer = "matroska"}},

// clang-format off
// Read with:
// ffplay -an -fflags nobuffer -flags low_delay -probesize 32 -analyzeduration 1 -strict experimental -framedrop -vf setpts=0 'srt://127.0.0.1:40052?mode=caller
{"SRT streaming",
LibavOutputSettings{
{
.path = "srt://:40052?mode=listener&latency=2000&transtype=live&recv_buffer_size=0",
.width = 1280,
.height = 720,
.rate = 30,
},
.video_encoder_short = "libx264",
.video_render_pixfmt = "rgba",
.video_converted_pixfmt = "yuv420p",
.muxer = "mpegts",
.options = { {"preset", "ultrafast"}, {"tune", "zerolatency"}, {"flush_packets", "1"}}}},
// clang-format on

{"WAV recording",
LibavOutputSettings{
{
Expand Down
Loading

0 comments on commit b667fb7

Please sign in to comment.