diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index ef43db4b..4bb076ee 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -22,10 +22,10 @@ jobs: submodules: recursive - name: Install Dependencies - run: sudo apt update && sudo apt -y install libsodium-dev cmake + run: sudo apt update && sudo apt -y install libsodium-dev cmake libvpx-dev libopus-dev - name: Configure CMake - run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} + run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DTOMATO_TOX_AV=ON - name: Build run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} -j 4 -t tomato @@ -101,7 +101,7 @@ jobs: - name: Configure CMake env: ANDROID_NDK_HOME: ${{steps.setup_ndk.outputs.ndk-path}} - run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_TOOLCHAIN_FILE=/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=${{matrix.platform.vcpkg_toolkit}} -DANDROID=1 -DANDROID_PLATFORM=23 -DANDROID_ABI=${{matrix.platform.ndk_abi}} -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=${{steps.setup_ndk.outputs.ndk-path}}/build/cmake/android.toolchain.cmake -DSDLIMAGE_JPG_SHARED=OFF -DSDLIMAGE_PNG_SHARED=OFF -DTOMATO_MAIN_SO=ON + run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_TOOLCHAIN_FILE=/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=${{matrix.platform.vcpkg_toolkit}} -DANDROID=1 -DANDROID_PLATFORM=23 -DANDROID_ABI=${{matrix.platform.ndk_abi}} -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=${{steps.setup_ndk.outputs.ndk-path}}/build/cmake/android.toolchain.cmake -DSDLIMAGE_JPG_SHARED=OFF -DSDLIMAGE_PNG_SHARED=OFF -DTOMATO_MAIN_SO=ON -DTOMATO_TOX_AV=ON - name: Build (tomato) run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} -j 4 -t tomato @@ -164,7 +164,7 @@ jobs: #- uses: ilammy/setup-nasm@v1 - name: Configure CMake - run: cmake -G Ninja -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=x64-windows-static -DSDLIMAGE_VENDORED=ON -DSDLIMAGE_DEPS_SHARED=ON -DSDLIMAGE_JXL=OFF -DSDLIMAGE_AVIF=OFF -DPKG_CONFIG_EXECUTABLE=C:/vcpkg/installed/x64-windows/tools/pkgconf/pkgconf.exe + run: cmake -G Ninja -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=x64-windows-static -DSDLIMAGE_VENDORED=ON -DSDLIMAGE_DEPS_SHARED=ON -DSDLIMAGE_JXL=OFF -DSDLIMAGE_AVIF=OFF -DPKG_CONFIG_EXECUTABLE=C:/vcpkg/installed/x64-windows/tools/pkgconf/pkgconf.exe -DTOMATO_TOX_AV=ON - name: Build run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} -t tomato @@ -229,7 +229,7 @@ jobs: #- uses: ilammy/setup-nasm@v1 - name: Configure CMake - run: cmake -G Ninja -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=x64-windows-static -DTOMATO_ASAN=ON -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded -DSDLIMAGE_VENDORED=ON -DSDLIMAGE_DEPS_SHARED=ON -DSDLIMAGE_JXL=OFF -DSDLIMAGE_AVIF=OFF -DPKG_CONFIG_EXECUTABLE=C:/vcpkg/installed/x64-windows/tools/pkgconf/pkgconf.exe + run: cmake -G Ninja -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=x64-windows-static -DTOMATO_ASAN=ON -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded -DSDLIMAGE_VENDORED=ON -DSDLIMAGE_DEPS_SHARED=ON -DSDLIMAGE_JXL=OFF -DSDLIMAGE_AVIF=OFF -DPKG_CONFIG_EXECUTABLE=C:/vcpkg/installed/x64-windows/tools/pkgconf/pkgconf.exe -DTOMATO_TOX_AV=ON - name: Build run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} -j 4 -t tomato diff --git a/flake.nix b/flake.nix index bbb2ea11..433f33ca 100644 --- a/flake.nix +++ b/flake.nix @@ -80,6 +80,7 @@ ] ++ self.packages.${system}.default.dlopenBuildInputs; cmakeFlags = [ + "-DTOMATO_TOX_AV=ON" "-DTOMATO_ASAN=OFF" "-DCMAKE_BUILD_TYPE=RelWithDebInfo" #"-DCMAKE_BUILD_TYPE=Debug" diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1bfe5e64..036ab969 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -107,6 +107,10 @@ target_sources(tomato PUBLIC ./frame_streams/audio_stream2.hpp ./frame_streams/stream_manager.hpp ./frame_streams/stream_manager.cpp + ./frame_streams/locked_frame_stream.hpp + ./frame_streams/multi_source.hpp + + ./frame_streams/voip_model.hpp ./frame_streams/sdl/sdl_audio2_frame_stream2.hpp ./frame_streams/sdl/sdl_audio2_frame_stream2.cpp @@ -123,6 +127,9 @@ if (TOMATO_TOX_AV) target_sources(tomato PUBLIC ./tox_av.hpp ./tox_av.cpp + + ./tox_av_voip_model.hpp + ./tox_av_voip_model.cpp ) target_compile_definitions(tomato PUBLIC TOMATO_TOX_AV) @@ -162,3 +169,18 @@ target_link_libraries(tomato PUBLIC set_target_properties(tomato PROPERTIES POSITION_INDEPENDENT_CODE ON) +######################################## + +add_executable(test_frame_stream2_pop_reframer EXCLUDE_FROM_ALL + ./frame_streams/frame_stream2.hpp + ./frame_streams/audio_stream2.hpp + ./frame_streams/locked_frame_stream.hpp + ./frame_streams/multi_source.hpp + + ./frame_streams/test_pop_reframer.cpp +) + +target_link_libraries(test_frame_stream2_pop_reframer + solanaceae_util +) + diff --git a/src/chat_gui4.cpp b/src/chat_gui4.cpp index b348ae42..1cb34eec 100644 --- a/src/chat_gui4.cpp +++ b/src/chat_gui4.cpp @@ -9,9 +9,13 @@ #include #include +#include "./frame_streams/voip_model.hpp" + // HACK: remove them #include +#include + #include #include @@ -30,6 +34,7 @@ #include #include #include +#include #include namespace Components { @@ -257,6 +262,97 @@ float ChatGui4::render(float time_delta) { if (ImGui::BeginChild(chat_label.c_str(), {0, 0}, ImGuiChildFlags_Border, ImGuiWindowFlags_MenuBar)) { if (ImGui::BeginMenuBar()) { + // check if contact has voip model + // use activesessioncomp instead? + if (_cr.all_of(*_selected_contact)) { + if (ImGui::BeginMenu("VoIP")) { + auto* voip_model = _cr.get(*_selected_contact); + + std::vector contact_sessions; + std::vector acceptable_sessions; + for (const auto& [ov, o_vm, sc] : _os.registry().view().each()) { + if (o_vm != voip_model) { + continue; + } + if (sc.c != *_selected_contact) { + continue; + } + + auto o = _os.objectHandle(ov); + contact_sessions.push_back(o); + + if (!o.all_of()) { + continue; // not incoming + } + + // state is ringing/not yet accepted + const auto* session_state = o.try_get(); + if (session_state == nullptr) { + continue; + } + + if (session_state->state != Components::VoIP::SessionState::State::RINGING) { + continue; + } + acceptable_sessions.push_back(o); + } + + static Components::VoIP::DefaultConfig g_default_connections{}; + + if (ImGui::BeginMenu("default connections")) { + ImGui::MenuItem("incoming audio", nullptr, &g_default_connections.incoming_audio); + ImGui::MenuItem("incoming video", nullptr, &g_default_connections.incoming_video); + ImGui::Separator(); + ImGui::MenuItem("outgoing audio", nullptr, &g_default_connections.outgoing_audio); + ImGui::MenuItem("outgoing video", nullptr, &g_default_connections.outgoing_video); + ImGui::EndMenu(); + } + + if (acceptable_sessions.size() < 2) { + if (ImGui::MenuItem("accept call", nullptr, false, !acceptable_sessions.empty())) { + voip_model->accept(acceptable_sessions.front(), g_default_connections); + } + } else { + if (ImGui::BeginMenu("accept call", !acceptable_sessions.empty())) { + for (const auto o : acceptable_sessions) { + std::string label = "accept #"; + label += std::to_string(entt::to_integral(entt::to_entity(o.entity()))); + + if (ImGui::MenuItem(label.c_str())) { + voip_model->accept(o, g_default_connections); + } + } + ImGui::EndMenu(); + } + } + + // TODO: disable if already in call? + if (ImGui::Button(" call ")) { + voip_model->enter(*_selected_contact, g_default_connections); + } + + if (contact_sessions.size() < 2) { + if (ImGui::MenuItem("leave/reject call", nullptr, false, !contact_sessions.empty())) { + voip_model->leave(contact_sessions.front()); + } + } else { + if (ImGui::BeginMenu("leave/reject call")) { + // list + for (const auto o : contact_sessions) { + std::string label = "end #"; + label += std::to_string(entt::to_integral(entt::to_entity(o.entity()))); + + if (ImGui::MenuItem(label.c_str())) { + voip_model->leave(o); + } + } + ImGui::EndMenu(); + } + } + + ImGui::EndMenu(); + } + } if (ImGui::BeginMenu("debug")) { ImGui::Checkbox("show extra info", &_show_chat_extra_info); ImGui::Checkbox("show avatar transfers", &_show_chat_avatar_tf); diff --git a/src/frame_streams/audio_stream_pop_reframer.hpp b/src/frame_streams/audio_stream_pop_reframer.hpp new file mode 100644 index 00000000..fdb44bdb --- /dev/null +++ b/src/frame_streams/audio_stream_pop_reframer.hpp @@ -0,0 +1,103 @@ +#pragma once + +#include "./audio_stream2.hpp" + +// reframes audio frames to a specified size in ms +// TODO: use absolute sample count instead?? +template +struct AudioStreamPopReFramer : public FrameStream2I { + uint32_t _frame_length_ms {20}; + + // gotta be careful of the multithreaded nature + // and false(true) sharing + uint64_t _pad0{}; + RealAudioStream _stream; + uint64_t _pad1{}; + + // dequeue? + std::vector _buffer; + + uint32_t _sample_rate {48'000}; + size_t _channels {0}; + + AudioStreamPopReFramer(uint32_t frame_length_ms = 20) + : _frame_length_ms(frame_length_ms) { + } + + AudioStreamPopReFramer(uint32_t frame_length_ms, FrameStream2I&& stream) + : _frame_length_ms(frame_length_ms), _stream(std::move(stream)) { + } + + ~AudioStreamPopReFramer(void) {} + + size_t getDesiredSize(void) const { + return _frame_length_ms * _sample_rate * _channels / 1000; + } + + int32_t size(void) override { return -1; } + + std::optional pop(void) override { + do { + auto new_in = _stream.pop(); + if (new_in.has_value()) { + auto& new_value = new_in.value(); + + // changed + if (_sample_rate != new_value.sample_rate || _channels != new_value.channels) { + //if (_channels != 0) { + // std::cerr << "ReFramer warning: reconfiguring, dropping buffer\n"; + //} + _sample_rate = new_value.sample_rate; + _channels = new_value.channels; + + // buffer does not exist or config changed and we discard + _buffer = {}; + } + + //std::cout << "new incoming frame is " << new_value.getSpan().size/new_value.channels*1000/new_value.sample_rate << "ms\n"; + + auto new_span = new_value.getSpan(); + + if (_buffer.empty()) { + _buffer = {new_span.cbegin(), new_span.cend()}; + } else { + _buffer.insert(_buffer.cend(), new_span.cbegin(), new_span.cend()); + } + } else if (_buffer.empty()) { + // first pop might result in invalid state + return std::nullopt; + } else { + // inner stream pop did not give a new value + break; // out of loop + } + } while (_buffer.size() < getDesiredSize()); + + const auto desired_size = getDesiredSize(); + + // > threshold? + if (_buffer.size() < desired_size) { + return std::nullopt; + } + + // copy data + std::vector return_buffer(_buffer.cbegin(), _buffer.cbegin()+desired_size); + + // now crop buffer (meh) + // move data from back to front + _buffer.erase(_buffer.cbegin(), _buffer.cbegin() + desired_size); + + return AudioFrame2{ + _sample_rate, + _channels, + std::move(return_buffer), + }; + } + + bool push(const AudioFrame2& value) override { + // might be worth it to instead do the work on push + //assert(false && "push reframing not implemented"); + // passthrough + return _stream.push(value); + } +}; + diff --git a/src/frame_streams/locked_frame_stream.hpp b/src/frame_streams/locked_frame_stream.hpp new file mode 100644 index 00000000..5a095694 --- /dev/null +++ b/src/frame_streams/locked_frame_stream.hpp @@ -0,0 +1,46 @@ +#pragma once + +#include "./frame_stream2.hpp" + +#include +#include + +// threadsafe queue frame stream +// protected by a simple mutex lock +// prefer lockless queue implementations, when available +template +struct LockedFrameStream2 : public FrameStream2I { + std::mutex _lock; + + std::deque _frames; + + ~LockedFrameStream2(void) {} + + int32_t size(void) { return -1; } + + std::optional pop(void) { + std::lock_guard lg{_lock}; + + if (_frames.empty()) { + return std::nullopt; + } + + FrameType new_frame = std::move(_frames.front()); + _frames.pop_front(); + + return std::move(new_frame); + } + + bool push(const FrameType& value) { + std::lock_guard lg{_lock}; + + if (_frames.size() > 1024) { + return false; // hard limit + } + + _frames.push_back(value); + + return true; + } +}; + diff --git a/src/frame_streams/multi_source.hpp b/src/frame_streams/multi_source.hpp new file mode 100644 index 00000000..cce966cf --- /dev/null +++ b/src/frame_streams/multi_source.hpp @@ -0,0 +1,62 @@ +#pragma once + +#include "./locked_frame_stream.hpp" + +#include + +// implements a stream that pushes to all sub streams +template> +struct FrameStream2MultiSource : public FrameStream2SourceI, public FrameStream2I { + using sub_stream_type_t = SubStreamType; + + // pointer stability + std::vector> _sub_streams; + std::mutex _sub_stream_lock; // accessing the _sub_streams array needs to be exclusive + // a simple lock here is ok, since this tends to be a rare operation, + // except for the push, which is always on the same thread + + virtual ~FrameStream2MultiSource(void) {} + + // TODO: forward args instead + std::shared_ptr> subscribe(void) override { + std::lock_guard lg{_sub_stream_lock}; + return _sub_streams.emplace_back(std::make_unique()); + } + + bool unsubscribe(const std::shared_ptr>& sub) override { + std::lock_guard lg{_sub_stream_lock}; + for (auto it = _sub_streams.begin(); it != _sub_streams.end(); it++) { + if (*it == sub) { + _sub_streams.erase(it); + return true; + } + } + return false; // ? + } + + // stream interface + + int32_t size(void) override { + // TODO: return something sensible? + return -1; + } + + std::optional pop(void) override { + // nope + assert(false && "this logic is very frame type specific, provide an impl"); + return std::nullopt; + } + + // returns true if there are readers + bool push(const FrameType& value) override { + std::lock_guard lg{_sub_stream_lock}; + bool have_readers{false}; + for (auto& it : _sub_streams) { + [[maybe_unused]] auto _ = it->push(value); + have_readers = true; // even if queue full, we still continue believing in them + // maybe consider push return value? + } + return have_readers; + } +}; + diff --git a/src/frame_streams/sdl/sdl_audio2_frame_stream2.cpp b/src/frame_streams/sdl/sdl_audio2_frame_stream2.cpp index 792f76ba..b70e4207 100644 --- a/src/frame_streams/sdl/sdl_audio2_frame_stream2.cpp +++ b/src/frame_streams/sdl/sdl_audio2_frame_stream2.cpp @@ -4,6 +4,8 @@ #include #include +#include "../audio_stream_pop_reframer.hpp" + // "thin" wrapper around sdl audio streams // we dont needs to get fance, as they already provide everything we need struct SDLAudio2StreamReader : public AudioFrame2Stream2I { @@ -58,7 +60,7 @@ struct SDLAudio2StreamReader : public AudioFrame2Stream2I { return std::nullopt; } - return AudioFrame2 { + return AudioFrame2{ _sample_rate, _channels, Span(_buffer.data(), read_bytes/sizeof(int16_t)), }; @@ -127,11 +129,17 @@ std::shared_ptr> SDLAudio2InputDevice::subscribe(void // error check SDL_BindAudioStream(_virtual_device_id, sdl_stream); - auto new_stream = std::make_shared(); - // TODO: move to ctr - new_stream->_stream = {sdl_stream, &SDL_DestroyAudioStream}; - new_stream->_sample_rate = spec.freq; - new_stream->_channels = spec.channels; + //auto new_stream = std::make_shared(); + //// TODO: move to ctr + //new_stream->_stream = {sdl_stream, &SDL_DestroyAudioStream}; + //new_stream->_sample_rate = spec.freq; + //new_stream->_channels = spec.channels; + + auto new_stream = std::make_shared>(); + new_stream->_stream._stream = {sdl_stream, &SDL_DestroyAudioStream}; + new_stream->_stream._sample_rate = spec.freq; + new_stream->_stream._channels = spec.channels; + new_stream->_frame_length_ms = 5; // WHY DOES THIS FIX MY ISSUE !!! _streams.emplace_back(new_stream); diff --git a/src/frame_streams/test_pop_reframer.cpp b/src/frame_streams/test_pop_reframer.cpp new file mode 100644 index 00000000..54ae6add --- /dev/null +++ b/src/frame_streams/test_pop_reframer.cpp @@ -0,0 +1,155 @@ +#include + +#include "./audio_stream_pop_reframer.hpp" +#include "./locked_frame_stream.hpp" + +#include + +int main(void) { + { // pump perfect + AudioStreamPopReFramer> stream; + stream._frame_length_ms = 10; + + AudioFrame2 f1 { + 48'000, + 1, + {}, + }; + f1.buffer = std::vector( + // perfect size + stream._frame_length_ms * f1.sample_rate * f1.channels / 1000, + 0 + ); + + { // fill with sequential value + int16_t seq = 0; + for (auto& v : std::get>(f1.buffer)) { + v = seq++; + } + } + + stream.push(f1); + + auto ret_opt = stream.pop(); + assert(ret_opt); + + auto& ret = ret_opt.value(); + assert(ret.sample_rate == f1.sample_rate); + assert(ret.channels == f1.channels); + assert(ret.getSpan().size == f1.getSpan().size); + { + int16_t seq = 0; + for (const auto v : ret.getSpan()) { + assert(v == seq++); + } + } + } + + { // pump half + AudioStreamPopReFramer> stream; + stream._frame_length_ms = 10; + + AudioFrame2 f1 { + 48'000, + 1, + {}, + }; + f1.buffer = std::vector( + // perfect size + (stream._frame_length_ms * f1.sample_rate * f1.channels / 1000) / 2, + 0 + ); + AudioFrame2 f2 { + 48'000, + 1, + {}, + }; + f2.buffer = std::vector( + // perfect size + (stream._frame_length_ms * f1.sample_rate * f1.channels / 1000) / 2, + 0 + ); + + { // fill with sequential value + int16_t seq = 0; + for (auto& v : std::get>(f1.buffer)) { + v = seq++; + } + for (auto& v : std::get>(f2.buffer)) { + v = seq++; + } + } + + stream.push(f1); + stream.push(f2); + + // supposed to combine both + auto ret_opt = stream.pop(); + assert(ret_opt); + + auto& ret = ret_opt.value(); + assert(ret.sample_rate == f1.sample_rate); + assert(ret.channels == f1.channels); + assert(ret.getSpan().size == stream._frame_length_ms * f1.sample_rate * f1.channels / 1000); + { + int16_t seq = 0; + for (const auto v : ret.getSpan()) { + assert(v == seq++); + } + } + } + + { // pump double + AudioStreamPopReFramer> stream; + stream._frame_length_ms = 20; + + AudioFrame2 f1 { + 48'000, + 2, + {}, + }; + f1.buffer = std::vector( + // perfect size + (stream._frame_length_ms * f1.sample_rate * f1.channels / 1000) * 2, + 0 + ); + { // fill with sequential value + int16_t seq = 0; + for (auto& v : std::get>(f1.buffer)) { + v = seq++; + } + } + + stream.push(f1); + + // pop 2x + int16_t seq = 0; + { + auto ret_opt = stream.pop(); + assert(ret_opt); + + auto& ret = ret_opt.value(); + assert(ret.sample_rate == f1.sample_rate); + assert(ret.channels == f1.channels); + assert(ret.getSpan().size == stream._frame_length_ms * f1.sample_rate * f1.channels / 1000); + for (const auto v : ret.getSpan()) { + assert(v == seq++); + } + } + + { + auto ret_opt = stream.pop(); + assert(ret_opt); + + auto& ret = ret_opt.value(); + assert(ret.sample_rate == f1.sample_rate); + assert(ret.channels == f1.channels); + assert(ret.getSpan().size == stream._frame_length_ms * f1.sample_rate * f1.channels / 1000); + for (const auto v : ret.getSpan()) { + assert(v == seq++); + } + } + } + + return 0; +} diff --git a/src/frame_streams/voip_model.hpp b/src/frame_streams/voip_model.hpp new file mode 100644 index 00000000..573085f7 --- /dev/null +++ b/src/frame_streams/voip_model.hpp @@ -0,0 +1,77 @@ +#pragma once + +#include +#include + +struct VoIPModelI; + +namespace Components::VoIP { + + struct TagVoIPSession {}; + + // getting called or invited by + struct Incoming { + Contact3 c{entt::null}; + }; + + struct DefaultConfig { + bool incoming_audio {true}; + bool incoming_video {true}; + bool outgoing_audio {true}; + bool outgoing_video {true}; + }; + + // to talk to the model handling this session + //struct VoIPModel { + //VoIPModelI* ptr {nullptr}; + //}; + + struct SessionState { + // ???? + // incoming + // outgoing + enum class State { + RINGING, + CONNECTED, + } state; + }; + + struct SessionContact { + Contact3 c{entt::null}; + }; + + struct StreamSources { + // list of all stream sources originating from this VoIP session + std::vector streams; + }; + + struct StreamSinks { + // list of all stream sinks going to this VoIP session + std::vector streams; + }; + +} // Components::VoIP + +// TODO: events? piggyback on objects? + +// stream model instead?? -> more generic than "just" audio and video? +// or specialized like this +// streams abstract type in a nice way +struct VoIPModelI { + virtual ~VoIPModelI(void) {} + + // enters a call/voicechat/videocall ??? + // - contact + // - default stream sources/sinks ? + // - enable a/v ? -> done on connecting to sources + // returns object tieing together the VoIP session + virtual ObjectHandle enter(const Contact3 c, const Components::VoIP::DefaultConfig& defaults = {true, true, true, true}) { (void)c,(void)defaults; return {}; } + + // accept/join an invite to a session + virtual bool accept(ObjectHandle session, const Components::VoIP::DefaultConfig& defaults = {true, true, true, true}) { (void)session,(void)defaults; return false; } + + // leaves a call + // - VoIP session object + virtual bool leave(ObjectHandle session) { (void)session; return false; } +}; + diff --git a/src/main_screen.cpp b/src/main_screen.cpp index c6f70605..23ab68b6 100644 --- a/src/main_screen.cpp +++ b/src/main_screen.cpp @@ -25,13 +25,14 @@ MainScreen::MainScreen(SimpleConfigModel&& conf_, SDL_Renderer* renderer_, Theme tc(save_path, save_password), tpi(tc.getTox()), ad(tc), -#if TOMATO_TOX_AV - tav(tc.getTox()), -#endif tcm(cr, tc, tc), tmm(rmm, cr, tcm, tc, tc), ttm(rmm, cr, tcm, tc, tc, os), tffom(cr, rmm, tcm, tc, tc), +#if TOMATO_TOX_AV + tav(tc.getTox()), + tavvoip(os, tav, cr, tcm), +#endif theme(theme_), mmil(rmm), tam(/*rmm, */ os, cr, conf), @@ -80,7 +81,7 @@ MainScreen::MainScreen(SimpleConfigModel&& conf_, SDL_Renderer* renderer_, Theme g_provideInstance("ToxPrivateI", "host", &tpi); g_provideInstance("ToxEventProviderI", "host", &tc); #if TOMATO_TOX_AV - g_provideInstance("ToxAV", "host", &tav); + g_provideInstance("ToxAVI", "host", &tav); #endif g_provideInstance("ToxContactModel2", "host", &tcm); @@ -526,8 +527,12 @@ Screen* MainScreen::tick(float time_delta, bool& quit) { #if TOMATO_TOX_AV tav.toxavIterate(); + // breaks it // HACK: pow by 1.18 to increase 200 -> ~500 - const float av_interval = std::pow(tav.toxavIterationInterval(), 1.18)/1000.f; + //const float av_interval = std::pow(tav.toxavIterationInterval(), 1.18)/1000.f; + const float av_interval = tav.toxavIterationInterval()/1000.f; + + tavvoip.tick(); #endif tcm.iterate(time_delta); // compute diff --git a/src/main_screen.hpp b/src/main_screen.hpp index cd985a9a..daa3ab25 100644 --- a/src/main_screen.hpp +++ b/src/main_screen.hpp @@ -39,6 +39,7 @@ #if TOMATO_TOX_AV #include "./tox_av.hpp" +#include "./tox_av_voip_model.hpp" #endif #include @@ -67,13 +68,14 @@ struct MainScreen final : public Screen { ToxClient tc; ToxPrivateImpl tpi; AutoDirty ad; -#if TOMATO_TOX_AV - ToxAV tav; -#endif ToxContactModel2 tcm; ToxMessageManager tmm; ToxTransferManager ttm; ToxFriendFauxOfflineMessaging tffom; +#if TOMATO_TOX_AV + ToxAVI tav; + ToxAVVoIPModel tavvoip; +#endif Theme& theme; diff --git a/src/tox_av.cpp b/src/tox_av.cpp index 695744c3..4e6c044b 100644 --- a/src/tox_av.cpp +++ b/src/tox_av.cpp @@ -7,18 +7,7 @@ // https://almogfx.bandcamp.com/track/crushed-w-cassade -struct ToxAVFriendCallState final { - const uint32_t state {TOXAV_FRIEND_CALL_STATE_NONE}; - - [[nodiscard]] bool is_error(void) const { return state & TOXAV_FRIEND_CALL_STATE_ERROR; } - [[nodiscard]] bool is_finished(void) const { return state & TOXAV_FRIEND_CALL_STATE_FINISHED; } - [[nodiscard]] bool is_sending_a(void) const { return state & TOXAV_FRIEND_CALL_STATE_SENDING_A; } - [[nodiscard]] bool is_sending_v(void) const { return state & TOXAV_FRIEND_CALL_STATE_SENDING_V; } - [[nodiscard]] bool is_accepting_a(void) const { return state & TOXAV_FRIEND_CALL_STATE_ACCEPTING_A; } - [[nodiscard]] bool is_accepting_v(void) const { return state & TOXAV_FRIEND_CALL_STATE_ACCEPTING_V; } -}; - -ToxAV::ToxAV(Tox* tox) : _tox(tox) { +ToxAVI::ToxAVI(Tox* tox) : _tox(tox) { Toxav_Err_New err_new {TOXAV_ERR_NEW_OK}; _tox_av = toxav_new(_tox, &err_new); // TODO: throw @@ -28,7 +17,7 @@ ToxAV::ToxAV(Tox* tox) : _tox(tox) { _tox_av, +[](ToxAV*, uint32_t friend_number, bool audio_enabled, bool video_enabled, void *user_data) { assert(user_data != nullptr); - static_cast(user_data)->cb_call(friend_number, audio_enabled, video_enabled); + static_cast(user_data)->cb_call(friend_number, audio_enabled, video_enabled); }, this ); @@ -36,7 +25,7 @@ ToxAV::ToxAV(Tox* tox) : _tox(tox) { _tox_av, +[](ToxAV*, uint32_t friend_number, uint32_t state, void *user_data) { assert(user_data != nullptr); - static_cast(user_data)->cb_call_state(friend_number, state); + static_cast(user_data)->cb_call_state(friend_number, state); }, this ); @@ -44,7 +33,7 @@ ToxAV::ToxAV(Tox* tox) : _tox(tox) { _tox_av, +[](ToxAV*, uint32_t friend_number, uint32_t audio_bit_rate, void *user_data) { assert(user_data != nullptr); - static_cast(user_data)->cb_audio_bit_rate(friend_number, audio_bit_rate); + static_cast(user_data)->cb_audio_bit_rate(friend_number, audio_bit_rate); }, this ); @@ -52,7 +41,7 @@ ToxAV::ToxAV(Tox* tox) : _tox(tox) { _tox_av, +[](ToxAV*, uint32_t friend_number, uint32_t video_bit_rate, void *user_data) { assert(user_data != nullptr); - static_cast(user_data)->cb_video_bit_rate(friend_number, video_bit_rate); + static_cast(user_data)->cb_video_bit_rate(friend_number, video_bit_rate); }, this ); @@ -60,7 +49,7 @@ ToxAV::ToxAV(Tox* tox) : _tox(tox) { _tox_av, +[](ToxAV*, uint32_t friend_number, const int16_t pcm[], size_t sample_count, uint8_t channels, uint32_t sampling_rate, void *user_data) { assert(user_data != nullptr); - static_cast(user_data)->cb_audio_receive_frame(friend_number, pcm, sample_count, channels, sampling_rate); + static_cast(user_data)->cb_audio_receive_frame(friend_number, pcm, sample_count, channels, sampling_rate); }, this ); @@ -75,83 +64,83 @@ ToxAV::ToxAV(Tox* tox) : _tox(tox) { void *user_data ) { assert(user_data != nullptr); - static_cast(user_data)->cb_video_receive_frame(friend_number, width, height, y, u, v, ystride, ustride, vstride); + static_cast(user_data)->cb_video_receive_frame(friend_number, width, height, y, u, v, ystride, ustride, vstride); }, this ); } -ToxAV::~ToxAV(void) { +ToxAVI::~ToxAVI(void) { toxav_kill(_tox_av); } -uint32_t ToxAV::toxavIterationInterval(void) const { +uint32_t ToxAVI::toxavIterationInterval(void) const { return toxav_iteration_interval(_tox_av); } -void ToxAV::toxavIterate(void) { +void ToxAVI::toxavIterate(void) { toxav_iterate(_tox_av); } -uint32_t ToxAV::toxavAudioIterationInterval(void) const { +uint32_t ToxAVI::toxavAudioIterationInterval(void) const { return toxav_audio_iteration_interval(_tox_av); } -void ToxAV::toxavAudioIterate(void) { +void ToxAVI::toxavAudioIterate(void) { toxav_audio_iterate(_tox_av); } -uint32_t ToxAV::toxavVideoIterationInterval(void) const { +uint32_t ToxAVI::toxavVideoIterationInterval(void) const { return toxav_video_iteration_interval(_tox_av); } -void ToxAV::toxavVideoIterate(void) { +void ToxAVI::toxavVideoIterate(void) { toxav_video_iterate(_tox_av); } -Toxav_Err_Call ToxAV::toxavCall(uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate) { +Toxav_Err_Call ToxAVI::toxavCall(uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate) { Toxav_Err_Call err {TOXAV_ERR_CALL_OK}; toxav_call(_tox_av, friend_number, audio_bit_rate, video_bit_rate, &err); return err; } -Toxav_Err_Answer ToxAV::toxavAnswer(uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate) { +Toxav_Err_Answer ToxAVI::toxavAnswer(uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate) { Toxav_Err_Answer err {TOXAV_ERR_ANSWER_OK}; toxav_answer(_tox_av, friend_number, audio_bit_rate, video_bit_rate, &err); return err; } -Toxav_Err_Call_Control ToxAV::toxavCallControl(uint32_t friend_number, Toxav_Call_Control control) { +Toxav_Err_Call_Control ToxAVI::toxavCallControl(uint32_t friend_number, Toxav_Call_Control control) { Toxav_Err_Call_Control err {TOXAV_ERR_CALL_CONTROL_OK}; toxav_call_control(_tox_av, friend_number, control, &err); return err; } -Toxav_Err_Send_Frame ToxAV::toxavAudioSendFrame(uint32_t friend_number, const int16_t pcm[], size_t sample_count, uint8_t channels, uint32_t sampling_rate) { +Toxav_Err_Send_Frame ToxAVI::toxavAudioSendFrame(uint32_t friend_number, const int16_t pcm[], size_t sample_count, uint8_t channels, uint32_t sampling_rate) { Toxav_Err_Send_Frame err {TOXAV_ERR_SEND_FRAME_OK}; toxav_audio_send_frame(_tox_av, friend_number, pcm, sample_count, channels, sampling_rate, &err); return err; } -Toxav_Err_Bit_Rate_Set ToxAV::toxavAudioSetBitRate(uint32_t friend_number, uint32_t bit_rate) { +Toxav_Err_Bit_Rate_Set ToxAVI::toxavAudioSetBitRate(uint32_t friend_number, uint32_t bit_rate) { Toxav_Err_Bit_Rate_Set err {TOXAV_ERR_BIT_RATE_SET_OK}; toxav_audio_set_bit_rate(_tox_av, friend_number, bit_rate, &err); return err; } -Toxav_Err_Send_Frame ToxAV::toxavVideoSendFrame(uint32_t friend_number, uint16_t width, uint16_t height, const uint8_t y[], const uint8_t u[], const uint8_t v[]) { +Toxav_Err_Send_Frame ToxAVI::toxavVideoSendFrame(uint32_t friend_number, uint16_t width, uint16_t height, const uint8_t y[], const uint8_t u[], const uint8_t v[]) { Toxav_Err_Send_Frame err {TOXAV_ERR_SEND_FRAME_OK}; toxav_video_send_frame(_tox_av, friend_number, width, height, y, u, v, &err); return err; } -Toxav_Err_Bit_Rate_Set ToxAV::toxavVideoSetBitRate(uint32_t friend_number, uint32_t bit_rate) { +Toxav_Err_Bit_Rate_Set ToxAVI::toxavVideoSetBitRate(uint32_t friend_number, uint32_t bit_rate) { Toxav_Err_Bit_Rate_Set err {TOXAV_ERR_BIT_RATE_SET_OK}; toxav_video_set_bit_rate(_tox_av, friend_number, bit_rate, &err); return err; } -void ToxAV::cb_call(uint32_t friend_number, bool audio_enabled, bool video_enabled) { +void ToxAVI::cb_call(uint32_t friend_number, bool audio_enabled, bool video_enabled) { std::cerr << "TOXAV: receiving call f:" << friend_number << " a:" << audio_enabled << " v:" << video_enabled << "\n"; //Toxav_Err_Answer err_answer { TOXAV_ERR_ANSWER_OK }; //toxav_answer(_tox_av, friend_number, 0, 0, &err_answer); @@ -169,7 +158,7 @@ void ToxAV::cb_call(uint32_t friend_number, bool audio_enabled, bool video_enabl ); } -void ToxAV::cb_call_state(uint32_t friend_number, uint32_t state) { +void ToxAVI::cb_call_state(uint32_t friend_number, uint32_t state) { //ToxAVFriendCallState w_state{state}; //w_state.is_error(); @@ -185,7 +174,7 @@ void ToxAV::cb_call_state(uint32_t friend_number, uint32_t state) { ); } -void ToxAV::cb_audio_bit_rate(uint32_t friend_number, uint32_t audio_bit_rate) { +void ToxAVI::cb_audio_bit_rate(uint32_t friend_number, uint32_t audio_bit_rate) { std::cerr << "TOXAV: audio bitrate f:" << friend_number << " abr:" << audio_bit_rate << "\n"; dispatch( @@ -197,7 +186,7 @@ void ToxAV::cb_audio_bit_rate(uint32_t friend_number, uint32_t audio_bit_rate) { ); } -void ToxAV::cb_video_bit_rate(uint32_t friend_number, uint32_t video_bit_rate) { +void ToxAVI::cb_video_bit_rate(uint32_t friend_number, uint32_t video_bit_rate) { std::cerr << "TOXAV: video bitrate f:" << friend_number << " vbr:" << video_bit_rate << "\n"; dispatch( @@ -209,7 +198,7 @@ void ToxAV::cb_video_bit_rate(uint32_t friend_number, uint32_t video_bit_rate) { ); } -void ToxAV::cb_audio_receive_frame(uint32_t friend_number, const int16_t pcm[], size_t sample_count, uint8_t channels, uint32_t sampling_rate) { +void ToxAVI::cb_audio_receive_frame(uint32_t friend_number, const int16_t pcm[], size_t sample_count, uint8_t channels, uint32_t sampling_rate) { //std::cerr << "TOXAV: audio frame f:" << friend_number << " sc:" << sample_count << " ch:" << (int)channels << " sr:" << sampling_rate << "\n"; dispatch( @@ -223,7 +212,7 @@ void ToxAV::cb_audio_receive_frame(uint32_t friend_number, const int16_t pcm[], ); } -void ToxAV::cb_video_receive_frame( +void ToxAVI::cb_video_receive_frame( uint32_t friend_number, uint16_t width, uint16_t height, const uint8_t y[/*! max(width, abs(ystride)) * height */], diff --git a/src/tox_av.hpp b/src/tox_av.hpp index bc00ce0f..7771118d 100644 --- a/src/tox_av.hpp +++ b/src/tox_av.hpp @@ -82,14 +82,15 @@ struct ToxAVEventI { }; using ToxAVEventProviderI = EventProviderI; -struct ToxAV : public ToxAVEventProviderI{ +// TODO: seperate out implementation from interface +struct ToxAVI : public ToxAVEventProviderI { Tox* _tox = nullptr; ToxAV* _tox_av = nullptr; static constexpr const char* version {"0"}; - ToxAV(Tox* tox); - virtual ~ToxAV(void); + ToxAVI(Tox* tox); + virtual ~ToxAVI(void); // interface // if iterate is called on a different thread, it will fire events there @@ -134,3 +135,14 @@ struct ToxAV : public ToxAVEventProviderI{ ); }; +struct ToxAVFriendCallState final { + const uint32_t state {TOXAV_FRIEND_CALL_STATE_NONE}; + + [[nodiscard]] bool is_error(void) const { return state & TOXAV_FRIEND_CALL_STATE_ERROR; } + [[nodiscard]] bool is_finished(void) const { return state & TOXAV_FRIEND_CALL_STATE_FINISHED; } + [[nodiscard]] bool is_sending_a(void) const { return state & TOXAV_FRIEND_CALL_STATE_SENDING_A; } + [[nodiscard]] bool is_sending_v(void) const { return state & TOXAV_FRIEND_CALL_STATE_SENDING_V; } + [[nodiscard]] bool is_accepting_a(void) const { return state & TOXAV_FRIEND_CALL_STATE_ACCEPTING_A; } + [[nodiscard]] bool is_accepting_v(void) const { return state & TOXAV_FRIEND_CALL_STATE_ACCEPTING_V; } +}; + diff --git a/src/tox_av_voip_model.cpp b/src/tox_av_voip_model.cpp new file mode 100644 index 00000000..9da602a1 --- /dev/null +++ b/src/tox_av_voip_model.cpp @@ -0,0 +1,476 @@ +#include "./tox_av_voip_model.hpp" + +#include +#include + +#include "./frame_streams/stream_manager.hpp" +#include "./frame_streams/audio_stream2.hpp" +#include "./frame_streams/locked_frame_stream.hpp" +#include "./frame_streams/multi_source.hpp" +#include "./frame_streams/audio_stream_pop_reframer.hpp" + +#include + +namespace Components { + struct ToxAVIncomingAV { + bool incoming_audio {false}; + bool incoming_video {false}; + }; + + struct ToxAVAudioSink { + ObjectHandle o; + // ptr? + }; + // vid + struct ToxAVAudioSource { + ObjectHandle o; + // ptr? + }; + // vid +} // Components + +struct ToxAVCallAudioSink : public FrameStream2SinkI { + ToxAVI& _toxav; + + // bitrate for enabled state + uint32_t _audio_bitrate {32}; + + uint32_t _fid; + std::shared_ptr>> _writer; + + ToxAVCallAudioSink(ToxAVI& toxav, uint32_t fid) : _toxav(toxav), _fid(fid) {} + ~ToxAVCallAudioSink(void) { + if (_writer) { + _writer = nullptr; + _toxav.toxavAudioSetBitRate(_fid, 0); + } + } + + // sink + std::shared_ptr> subscribe(void) override { + if (_writer) { + // max 1 (exclusive for now) + return nullptr; + } + + auto err = _toxav.toxavAudioSetBitRate(_fid, _audio_bitrate); + if (err != TOXAV_ERR_BIT_RATE_SET_OK) { + return nullptr; + } + + // 20ms for now, 10ms would work too, further investigate stutters at 5ms (probably too slow interval rate) + _writer = std::make_shared>>(20); + + return _writer; + } + + bool unsubscribe(const std::shared_ptr>& sub) override { + if (!sub || !_writer) { + // nah + return false; + } + + if (sub == _writer) { + _writer = nullptr; + + /*auto err = */_toxav.toxavAudioSetBitRate(_fid, 0); + // print warning? on error? + + return true; + } + + // what + return false; + } +}; + +void ToxAVVoIPModel::addAudioSource(ObjectHandle session, uint32_t friend_number) { + auto& stream_source = session.get_or_emplace().streams; + + ObjectHandle incoming_audio {_os.registry(), _os.registry().create()}; + + auto new_asrc = std::make_unique>(); + incoming_audio.emplace*>(new_asrc.get()); + incoming_audio.emplace>(std::move(new_asrc)); + incoming_audio.emplace(Components::StreamSource::create("ToxAV Friend Call Incoming Audio")); + + std::cout << "new incoming audio\n"; + if ( + const auto* defaults = session.try_get(); + defaults != nullptr && defaults->incoming_audio + ) { + incoming_audio.emplace(); // depends on what was specified in enter() + std::cout << "with default\n"; + } + + stream_source.push_back(incoming_audio); + session.emplace(incoming_audio); + // TODO: tie session to stream + + _audio_sources[friend_number] = incoming_audio; + + _os.throwEventConstruct(incoming_audio); +} + +void ToxAVVoIPModel::addAudioSink(ObjectHandle session, uint32_t friend_number) { + auto& stream_sinks = session.get_or_emplace().streams; + ObjectHandle outgoing_audio {_os.registry(), _os.registry().create()}; + + auto new_asink = std::make_unique(_av, friend_number); + outgoing_audio.emplace(new_asink.get()); + outgoing_audio.emplace>(std::move(new_asink)); + outgoing_audio.emplace(Components::StreamSink::create("ToxAV Friend Call Outgoing Audio")); + + if ( + const auto* defaults = session.try_get(); + defaults != nullptr && defaults->outgoing_audio + ) { + outgoing_audio.emplace(); // depends on what was specified in enter() + } + + stream_sinks.push_back(outgoing_audio); + session.emplace(outgoing_audio); + // TODO: tie session to stream + + _os.throwEventConstruct(outgoing_audio); +} + +void ToxAVVoIPModel::destroySession(ObjectHandle session) { + if (!static_cast(session)) { + return; + } + + // remove lookup + if (session.all_of()) { + auto it_asrc = std::find_if( + _audio_sources.cbegin(), _audio_sources.cend(), + [o = session.get().o](const auto& it) { + return it.second == o; + } + ); + + if (it_asrc != _audio_sources.cend()) { + _audio_sources.erase(it_asrc); + } + } + + // destory sources + if (auto* ss = session.try_get(); ss != nullptr) { + for (const auto ssov : ss->streams) { + + _os.throwEventDestroy(ssov); + _os.registry().destroy(ssov); + } + } + + // destory sinks + if (auto* ss = session.try_get(); ss != nullptr) { + for (const auto ssov : ss->streams) { + + _os.throwEventDestroy(ssov); + _os.registry().destroy(ssov); + } + } + + // destory session + _os.throwEventDestroy(session); + _os.registry().destroy(session); +} + +ToxAVVoIPModel::ToxAVVoIPModel(ObjectStore2& os, ToxAVI& av, Contact3Registry& cr, ToxContactModel2& tcm) : + _os(os), _av(av), _cr(cr), _tcm(tcm) +{ + _av.subscribe(this, ToxAV_Event::friend_call); + _av.subscribe(this, ToxAV_Event::friend_call_state); + _av.subscribe(this, ToxAV_Event::friend_audio_bitrate); + _av.subscribe(this, ToxAV_Event::friend_video_bitrate); + _av.subscribe(this, ToxAV_Event::friend_audio_frame); + _av.subscribe(this, ToxAV_Event::friend_video_frame); + + // attach to all tox friend contacts + + for (const auto& [cv, _] : _cr.view().each()) { + _cr.emplace(cv, this); + } + // TODO: events +} + +ToxAVVoIPModel::~ToxAVVoIPModel(void) { + for (const auto& [ov, voipmodel] : _os.registry().view().each()) { + if (voipmodel == this) { + destroySession(_os.objectHandle(ov)); + } + } +} + +void ToxAVVoIPModel::tick(void) { + for (const auto& [oc, asink] : _os.registry().view().each()) { + if (!asink->_writer) { + continue; + } + + for (size_t i = 0; i < 100; i++) { + auto new_frame_opt = asink->_writer->pop(); + if (!new_frame_opt.has_value()) { + break; + } + const auto& new_frame = new_frame_opt.value(); + + //* @param sample_count Number of samples in this frame. Valid numbers here are + //* `((sample rate) * (audio length) / 1000)`, where audio length can be + //* 2.5, 5, 10, 20, 40 or 60 milliseconds. + + // we likely needs to subdivide/repackage + // frame size should be an option exposed to the user + // with 10ms as a default ? + // the larger the frame size, the less overhead but the more delay + + auto err = _av.toxavAudioSendFrame( + asink->_fid, + new_frame.getSpan().ptr, + new_frame.getSpan().size / new_frame.channels, + new_frame.channels, + new_frame.sample_rate + ); + if (err != TOXAV_ERR_SEND_FRAME_OK) { + std::cerr << "DTC: failed to send audio frame " << err << "\n"; + } + } + } +} + +ObjectHandle ToxAVVoIPModel::enter(const Contact3 c, const Components::VoIP::DefaultConfig& defaults) { + if (!_cr.all_of(c)) { + return {}; + } + + const auto friend_number = _cr.get(c).friend_number; + + const auto err = _av.toxavCall(friend_number, 0, 0); + if (err != TOXAV_ERR_CALL_OK) { + std::cerr << "TAVVOIP error: failed to start call: " << err << "\n"; + return {}; + } + + ObjectHandle new_session {_os.registry(), _os.registry().create()}; + + new_session.emplace(this); + new_session.emplace(); // ?? + new_session.emplace(c); + new_session.emplace().state = Components::VoIP::SessionState::State::RINGING; + new_session.emplace(defaults); + + _os.throwEventConstruct(new_session); + return new_session; +} + +bool ToxAVVoIPModel::accept(ObjectHandle session, const Components::VoIP::DefaultConfig& defaults) { + if (!static_cast(session)) { + return false; + } + + if (!session.all_of< + Components::VoIP::TagVoIPSession, + VoIPModelI*, + Components::VoIP::SessionContact, + Components::VoIP::Incoming + >()) { + return false; + } + + // check if self + if (session.get() != this) { + return false; + } + + const auto session_contact = session.get().c; + if (!_cr.all_of(session_contact)) { + return false; + } + + const auto friend_number = _cr.get(session_contact).friend_number; + auto err = _av.toxavAnswer(friend_number, 0, 0); + if (err != TOXAV_ERR_ANSWER_OK) { + std::cerr << "TOXAVVOIP error: ansering call failed: " << err << "\n"; + // we simply let it be for now, it apears we can try ansering later again + // we also get an error here when the call is already in progress (: + return false; + } + + session.emplace(defaults); + + // answer defaults to enabled receiving audio and video + // TODO: think about how we should handle this + // set to disabled? and enable on src connection? + // we already default disabled send and enabled on sink connection + //_av.toxavCallControl(friend_number, TOXAV_CALL_CONTROL_HIDE_VIDEO); + //_av.toxavCallControl(friend_number, TOXAV_CALL_CONTROL_MUTE_AUDIO); + + + // how do we know the other side is accepting audio + // bitrate cb or what? + assert(!session.all_of()); + addAudioSink(session, friend_number); + + if (const auto* i_av = session.try_get(); i_av != nullptr) { + // create audio src + if (i_av->incoming_audio) { + assert(!session.all_of()); + addAudioSource(session, friend_number); + } + + // create video src + if (i_av->incoming_video) { + } + } + + session.get_or_emplace().state = Components::VoIP::SessionState::State::CONNECTED; + _os.throwEventUpdate(session); + return true; +} + +bool ToxAVVoIPModel::leave(ObjectHandle session) { + // rename to end? + + if (!static_cast(session)) { + return false; + } + + if (!session.all_of< + Components::VoIP::TagVoIPSession, + VoIPModelI*, + Components::VoIP::SessionContact + >()) { + return false; + } + + // check if self + if (session.get() != this) { + return false; + } + + const auto session_contact = session.get().c; + if (!_cr.all_of(session_contact)) { + return false; + } + + const auto friend_number = _cr.get(session_contact).friend_number; + // check error? (we delete anyway) + _av.toxavCallControl(friend_number, Toxav_Call_Control::TOXAV_CALL_CONTROL_CANCEL); + + destroySession(session); + return true; +} + +bool ToxAVVoIPModel::onEvent(const Events::FriendCall& e) { + // new incoming call, create voip session, ready to be accepted + // (or rejected...) + + const auto session_contact = _tcm.getContactFriend(e.friend_number); + if (!_cr.valid(session_contact)) { + return false; + } + + ObjectHandle new_session {_os.registry(), _os.registry().create()}; + + new_session.emplace(this); + new_session.emplace(); // ?? + new_session.emplace(session_contact); // in 1on1 its always the same contact, might leave blank + new_session.emplace(session_contact); + new_session.emplace().state = Components::VoIP::SessionState::State::RINGING; + new_session.emplace(e.audio_enabled, e.video_enabled); + + _os.throwEventConstruct(new_session); + return true; +} + +bool ToxAVVoIPModel::onEvent(const Events::FriendCallState& e) { + const auto session_contact = _tcm.getContactFriend(e.friend_number); + if (!_cr.valid(session_contact)) { + return false; + } + + ToxAVFriendCallState s{e.state}; + + // find session(s?) + // TODO: keep lookup table + for (const auto& [ov, voipmodel] : _os.registry().view().each()) { + if (voipmodel == this) { + auto o = _os.objectHandle(ov); + + if (!o.all_of()) { + continue; + } + if (session_contact != o.get().c) { + continue; + } + + if (s.is_error() || s.is_finished()) { + // destroy call + destroySession(o); + } else { + // remote accepted our call, or av send/recv conditions changed? + o.get().state = Components::VoIP::SessionState::State::CONNECTED; // set to in call ?? + + if (s.is_accepting_a() && !o.all_of()) { + addAudioSink(o, e.friend_number); + } else if (!s.is_accepting_a() && o.all_of()) { + // remove asink? + } + + // video + + // add/update sources + // audio + if (s.is_sending_a() && !o.all_of()) { + addAudioSource(o, e.friend_number); + } else if (!s.is_sending_a() && o.all_of()) { + // remove asrc? + } + + // video + } + } + } + + return true; +} + +bool ToxAVVoIPModel::onEvent(const Events::FriendAudioBitrate&) { + return false; +} + +bool ToxAVVoIPModel::onEvent(const Events::FriendVideoBitrate&) { + return false; +} + +bool ToxAVVoIPModel::onEvent(const Events::FriendAudioFrame& e) { + auto asrc_it = _audio_sources.find(e.friend_number); + if (asrc_it == _audio_sources.cend()) { + // missing src from lookup table + return false; + } + + auto asrc = asrc_it->second; + + if (!static_cast(asrc)) { + // missing src to put frame into ?? + return false; + } + + assert(asrc.all_of*>()); + assert(asrc.all_of>()); + + asrc.get*>()->push(AudioFrame2{ + e.sampling_rate, + e.channels, + std::vector(e.pcm.begin(), e.pcm.end()) // copy + }); + + return true; +} + +bool ToxAVVoIPModel::onEvent(const Events::FriendVideoFrame&) { + return false; +} + diff --git a/src/tox_av_voip_model.hpp b/src/tox_av_voip_model.hpp new file mode 100644 index 00000000..743e5952 --- /dev/null +++ b/src/tox_av_voip_model.hpp @@ -0,0 +1,46 @@ +#pragma once + +#include +#include +#include +#include "./frame_streams/voip_model.hpp" +#include "./tox_av.hpp" + +#include + +class ToxAVVoIPModel : protected ToxAVEventI, public VoIPModelI { + ObjectStore2& _os; + ToxAVI& _av; + Contact3Registry& _cr; + ToxContactModel2& _tcm; + + // for faster lookup + std::unordered_map _audio_sources; + + // TODO: virtual? strategy? protected? + virtual void addAudioSource(ObjectHandle session, uint32_t friend_number); + virtual void addAudioSink(ObjectHandle session, uint32_t friend_number); + // TODO: video + + void destroySession(ObjectHandle session); + + public: + ToxAVVoIPModel(ObjectStore2& os, ToxAVI& av, Contact3Registry& cr, ToxContactModel2& tcm); + ~ToxAVVoIPModel(void); + + void tick(void); + + public: // voip model + ObjectHandle enter(const Contact3 c, const Components::VoIP::DefaultConfig& defaults) override; + bool accept(ObjectHandle session, const Components::VoIP::DefaultConfig& defaults) override; + bool leave(ObjectHandle session) override; + + protected: // toxav events + bool onEvent(const Events::FriendCall&) override; + bool onEvent(const Events::FriendCallState&) override; + bool onEvent(const Events::FriendAudioBitrate&) override; + bool onEvent(const Events::FriendVideoBitrate&) override; + bool onEvent(const Events::FriendAudioFrame&) override; + bool onEvent(const Events::FriendVideoFrame&) override; +}; +