Voip toxav

.github/workflows/cd.yml

@@ -22,10 +22,10 @@ jobs:
         submodules: recursive
 
     - name: Install Dependencies
-      run: sudo apt update && sudo apt -y install libsodium-dev cmake
+      run: sudo apt update && sudo apt -y install libsodium-dev cmake libvpx-dev libopus-dev
 
     - name: Configure CMake
-      run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
+      run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DTOMATO_TOX_AV=ON
 
     - name: Build
       run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} -j 4 -t tomato
@@ -101,7 +101,7 @@ jobs:
     - name: Configure CMake
       env:
         ANDROID_NDK_HOME: ${{steps.setup_ndk.outputs.ndk-path}}
-      run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_TOOLCHAIN_FILE=/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=${{matrix.platform.vcpkg_toolkit}} -DANDROID=1 -DANDROID_PLATFORM=23 -DANDROID_ABI=${{matrix.platform.ndk_abi}} -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=${{steps.setup_ndk.outputs.ndk-path}}/build/cmake/android.toolchain.cmake -DSDLIMAGE_JPG_SHARED=OFF -DSDLIMAGE_PNG_SHARED=OFF -DTOMATO_MAIN_SO=ON
+      run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_TOOLCHAIN_FILE=/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=${{matrix.platform.vcpkg_toolkit}} -DANDROID=1 -DANDROID_PLATFORM=23 -DANDROID_ABI=${{matrix.platform.ndk_abi}} -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=${{steps.setup_ndk.outputs.ndk-path}}/build/cmake/android.toolchain.cmake -DSDLIMAGE_JPG_SHARED=OFF -DSDLIMAGE_PNG_SHARED=OFF -DTOMATO_MAIN_SO=ON -DTOMATO_TOX_AV=ON
 
     - name: Build (tomato)
       run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} -j 4 -t tomato
@@ -164,7 +164,7 @@ jobs:
     #- uses: ilammy/setup-nasm@v1
 
     - name: Configure CMake
-      run: cmake -G Ninja -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=x64-windows-static -DSDLIMAGE_VENDORED=ON -DSDLIMAGE_DEPS_SHARED=ON -DSDLIMAGE_JXL=OFF -DSDLIMAGE_AVIF=OFF -DPKG_CONFIG_EXECUTABLE=C:/vcpkg/installed/x64-windows/tools/pkgconf/pkgconf.exe
+      run: cmake -G Ninja -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=x64-windows-static -DSDLIMAGE_VENDORED=ON -DSDLIMAGE_DEPS_SHARED=ON -DSDLIMAGE_JXL=OFF -DSDLIMAGE_AVIF=OFF -DPKG_CONFIG_EXECUTABLE=C:/vcpkg/installed/x64-windows/tools/pkgconf/pkgconf.exe -DTOMATO_TOX_AV=ON
 
     - name: Build
       run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} -t tomato
@@ -229,7 +229,7 @@ jobs:
     #- uses: ilammy/setup-nasm@v1
 
     - name: Configure CMake
-      run: cmake -G Ninja -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=x64-windows-static -DTOMATO_ASAN=ON -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded -DSDLIMAGE_VENDORED=ON -DSDLIMAGE_DEPS_SHARED=ON -DSDLIMAGE_JXL=OFF -DSDLIMAGE_AVIF=OFF -DPKG_CONFIG_EXECUTABLE=C:/vcpkg/installed/x64-windows/tools/pkgconf/pkgconf.exe
+      run: cmake -G Ninja -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=x64-windows-static -DTOMATO_ASAN=ON -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded -DSDLIMAGE_VENDORED=ON -DSDLIMAGE_DEPS_SHARED=ON -DSDLIMAGE_JXL=OFF -DSDLIMAGE_AVIF=OFF -DPKG_CONFIG_EXECUTABLE=C:/vcpkg/installed/x64-windows/tools/pkgconf/pkgconf.exe -DTOMATO_TOX_AV=ON
 
     - name: Build
       run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} -j 4 -t tomato

flake.nix

@@ -80,6 +80,7 @@
         ] ++ self.packages.${system}.default.dlopenBuildInputs;
 
         cmakeFlags = [
+          "-DTOMATO_TOX_AV=ON"
           "-DTOMATO_ASAN=OFF"
           "-DCMAKE_BUILD_TYPE=RelWithDebInfo"
           #"-DCMAKE_BUILD_TYPE=Debug"

src/CMakeLists.txt

@@ -107,6 +107,10 @@ target_sources(tomato PUBLIC
 	./frame_streams/audio_stream2.hpp
 	./frame_streams/stream_manager.hpp
 	./frame_streams/stream_manager.cpp
+	./frame_streams/locked_frame_stream.hpp
+	./frame_streams/multi_source.hpp
+
+	./frame_streams/voip_model.hpp
 
 	./frame_streams/sdl/sdl_audio2_frame_stream2.hpp
 	./frame_streams/sdl/sdl_audio2_frame_stream2.cpp
@@ -123,6 +127,9 @@ if (TOMATO_TOX_AV)
 	target_sources(tomato PUBLIC
 		./tox_av.hpp
 		./tox_av.cpp
+
+		./tox_av_voip_model.hpp
+		./tox_av_voip_model.cpp
 	)
 
 	target_compile_definitions(tomato PUBLIC TOMATO_TOX_AV)
@@ -162,3 +169,18 @@ target_link_libraries(tomato PUBLIC
 
 set_target_properties(tomato PROPERTIES POSITION_INDEPENDENT_CODE ON)
 
+########################################
+
+add_executable(test_frame_stream2_pop_reframer EXCLUDE_FROM_ALL
+	./frame_streams/frame_stream2.hpp
+	./frame_streams/audio_stream2.hpp
+	./frame_streams/locked_frame_stream.hpp
+	./frame_streams/multi_source.hpp
+
+	./frame_streams/test_pop_reframer.cpp
+)
+
+target_link_libraries(test_frame_stream2_pop_reframer
+	solanaceae_util
+)
+

src/chat_gui4.cpp

@@ -9,9 +9,13 @@
 #include <solanaceae/contact/components.hpp>
 #include <solanaceae/util/utils.hpp>
 
+#include "./frame_streams/voip_model.hpp"
+
 // HACK: remove them
 #include <solanaceae/tox_contacts/components.hpp>
 
+#include <entt/entity/entity.hpp>
+
 #include <imgui/imgui.h>
 #include <imgui/misc/cpp/imgui_stdlib.h>
 
@@ -30,6 +34,7 @@
 #include <fstream>
 #include <iomanip>
 #include <sstream>
+#include <string>
 #include <variant>
 
 namespace Components {
@@ -257,6 +262,97 @@ float ChatGui4::render(float time_delta) {
 
 			if (ImGui::BeginChild(chat_label.c_str(), {0, 0}, ImGuiChildFlags_Border, ImGuiWindowFlags_MenuBar)) {
 				if (ImGui::BeginMenuBar()) {
+					// check if contact has voip model
+					// use activesessioncomp instead?
+					if (_cr.all_of<VoIPModelI*>(*_selected_contact)) {
+						if (ImGui::BeginMenu("VoIP")) {
+							auto* voip_model = _cr.get<VoIPModelI*>(*_selected_contact);
+
+							std::vector<ObjectHandle> contact_sessions;
+							std::vector<ObjectHandle> acceptable_sessions;
+							for (const auto& [ov, o_vm, sc] : _os.registry().view<VoIPModelI*, Components::VoIP::SessionContact>().each()) {
+								if (o_vm != voip_model) {
+									continue;
+								}
+								if (sc.c != *_selected_contact) {
+									continue;
+								}
+
+								auto o = _os.objectHandle(ov);
+								contact_sessions.push_back(o);
+
+								if (!o.all_of<Components::VoIP::Incoming>()) {
+									continue; // not incoming
+								}
+
+								// state is ringing/not yet accepted
+								const auto* session_state = o.try_get<Components::VoIP::SessionState>();
+								if (session_state == nullptr) {
+									continue;
+								}
+
+								if (session_state->state != Components::VoIP::SessionState::State::RINGING) {
+									continue;
+								}
+								acceptable_sessions.push_back(o);
+							}
+
+							static Components::VoIP::DefaultConfig g_default_connections{};
+
+							if (ImGui::BeginMenu("default connections")) {
+								ImGui::MenuItem("incoming audio", nullptr, &g_default_connections.incoming_audio);
+								ImGui::MenuItem("incoming video", nullptr, &g_default_connections.incoming_video);
+								ImGui::Separator();
+								ImGui::MenuItem("outgoing audio", nullptr, &g_default_connections.outgoing_audio);
+								ImGui::MenuItem("outgoing video", nullptr, &g_default_connections.outgoing_video);
+								ImGui::EndMenu();
+							}
+
+							if (acceptable_sessions.size() < 2) {
+								if (ImGui::MenuItem("accept call", nullptr, false, !acceptable_sessions.empty())) {
+									voip_model->accept(acceptable_sessions.front(), g_default_connections);
+								}
+							} else {
+								if (ImGui::BeginMenu("accept call", !acceptable_sessions.empty())) {
+									for (const auto o : acceptable_sessions) {
+										std::string label = "accept #";
+										label += std::to_string(entt::to_integral(entt::to_entity(o.entity())));
+
+										if (ImGui::MenuItem(label.c_str())) {
+											voip_model->accept(o, g_default_connections);
+										}
+									}
+									ImGui::EndMenu();
+								}
+							}
+
+							// TODO: disable if already in call?
+							if (ImGui::Button(" call ")) {
+								voip_model->enter(*_selected_contact, g_default_connections);
+							}
+
+							if (contact_sessions.size() < 2) {
+								if (ImGui::MenuItem("leave/reject call", nullptr, false, !contact_sessions.empty())) {
+									voip_model->leave(contact_sessions.front());
+								}
+							} else {
+								if (ImGui::BeginMenu("leave/reject call")) {
+									// list
+									for (const auto o : contact_sessions) {
+										std::string label = "end #";
+										label += std::to_string(entt::to_integral(entt::to_entity(o.entity())));
+
+										if (ImGui::MenuItem(label.c_str())) {
+											voip_model->leave(o);
+										}
+									}
+									ImGui::EndMenu();
+								}
+							}
+
+							ImGui::EndMenu();
+						}
+					}
 					if (ImGui::BeginMenu("debug")) {
 						ImGui::Checkbox("show extra info", &_show_chat_extra_info);
 						ImGui::Checkbox("show avatar transfers", &_show_chat_avatar_tf);

src/frame_streams/audio_stream_pop_reframer.hpp

@@ -0,0 +1,103 @@
+#pragma once
+
+#include "./audio_stream2.hpp"
+
+// reframes audio frames to a specified size in ms
+// TODO: use absolute sample count instead??
+template<typename RealAudioStream>
+struct AudioStreamPopReFramer : public FrameStream2I<AudioFrame2> {
+	uint32_t _frame_length_ms {20};
+
+	// gotta be careful of the multithreaded nature
+	// and false(true) sharing
+	uint64_t _pad0{};
+	RealAudioStream _stream;
+	uint64_t _pad1{};
+
+	// dequeue?
+	std::vector<int16_t> _buffer;
+
+	uint32_t _sample_rate {48'000};
+	size_t _channels {0};
+
+	AudioStreamPopReFramer(uint32_t frame_length_ms = 20)
+	: _frame_length_ms(frame_length_ms) {
+	}
+
+	AudioStreamPopReFramer(uint32_t frame_length_ms, FrameStream2I<AudioFrame2>&& stream)
+	: _frame_length_ms(frame_length_ms), _stream(std::move(stream)) {
+	}
+
+	~AudioStreamPopReFramer(void) {}
+
+	size_t getDesiredSize(void) const {
+		return _frame_length_ms * _sample_rate * _channels / 1000;
+	}
+
+	int32_t size(void) override { return -1; }
+
+	std::optional<AudioFrame2> pop(void) override {
+		do {
+			auto new_in = _stream.pop();
+			if (new_in.has_value()) {
+				auto& new_value = new_in.value();
+
+				// changed
+				if (_sample_rate != new_value.sample_rate || _channels != new_value.channels) {
+					//if (_channels != 0) {
+					//    std::cerr << "ReFramer warning: reconfiguring, dropping buffer\n";
+					//}
+					_sample_rate = new_value.sample_rate;
+					_channels = new_value.channels;
+
+					// buffer does not exist or config changed and we discard
+					_buffer = {};
+				}
+
+				//std::cout << "new incoming frame is " << new_value.getSpan().size/new_value.channels*1000/new_value.sample_rate << "ms\n";
+
+				auto new_span = new_value.getSpan();
+
+				if (_buffer.empty()) {
+					_buffer = {new_span.cbegin(), new_span.cend()};
+				} else {
+					_buffer.insert(_buffer.cend(), new_span.cbegin(), new_span.cend());
+				}
+			} else if (_buffer.empty()) {
+				// first pop might result in invalid state
+				return std::nullopt;
+			} else {
+				// inner stream pop did not give a new value
+				break; // out of loop
+			}
+		} while (_buffer.size() < getDesiredSize());
+
+		const auto desired_size = getDesiredSize();
+
+		// > threshold?
+		if (_buffer.size() < desired_size) {
+			return std::nullopt;
+		}
+
+		// copy data
+		std::vector<int16_t> return_buffer(_buffer.cbegin(), _buffer.cbegin()+desired_size);
+
+		// now crop buffer (meh)
+		// move data from back to front
+		_buffer.erase(_buffer.cbegin(), _buffer.cbegin() + desired_size);
+
+		return AudioFrame2{
+			_sample_rate,
+			_channels,
+			std::move(return_buffer),
+		};
+	}
+
+	bool push(const AudioFrame2& value) override {
+		// might be worth it to instead do the work on push
+		//assert(false && "push reframing not implemented");
+		// passthrough
+		return _stream.push(value);
+	}
+};
+

src/frame_streams/locked_frame_stream.hpp

@@ -0,0 +1,46 @@
+#pragma once
+
+#include "./frame_stream2.hpp"
+
+#include <mutex>
+#include <deque>
+
+// threadsafe queue frame stream
+// protected by a simple mutex lock
+// prefer lockless queue implementations, when available
+template<typename FrameType>
+struct LockedFrameStream2 : public FrameStream2I<FrameType> {
+	std::mutex _lock;
+
+	std::deque<FrameType> _frames;
+
+	~LockedFrameStream2(void) {}
+
+	int32_t size(void) { return -1; }
+
+	std::optional<FrameType> pop(void) {
+		std::lock_guard lg{_lock};
+
+		if (_frames.empty()) {
+			return std::nullopt;
+		}
+
+		FrameType new_frame = std::move(_frames.front());
+		_frames.pop_front();
+
+		return std::move(new_frame);
+	}
+
+	bool push(const FrameType& value) {
+		std::lock_guard lg{_lock};
+
+		if (_frames.size() > 1024) {
+			return false; // hard limit
+		}
+
+		_frames.push_back(value);
+
+		return true;
+	}
+};
+