From 3fddc36a4e172e3b17c29a10db1561aca9ab49ee Mon Sep 17 00:00:00 2001 From: juliobecgom Date: Mon, 20 Feb 2023 19:33:08 +0100 Subject: [PATCH 1/2] OPENTOK-51009: Create sample app for Qt --- Qt-Sample-App/CMakeLists.txt | 43 ++++++ Qt-Sample-App/glvideowidget.cpp | 165 +++++++++++++++++++++++ Qt-Sample-App/glvideowidget.h | 57 ++++++++ Qt-Sample-App/main.cpp | 28 ++++ Qt-Sample-App/mainwindow.cpp | 76 +++++++++++ Qt-Sample-App/mainwindow.h | 85 ++++++++++++ Qt-Sample-App/mainwindow.ui | 107 +++++++++++++++ Qt-Sample-App/participant.h | 22 +++ Qt-Sample-App/qtpublisher.cpp | 1 + Qt-Sample-App/qtpublisher.h | 229 ++++++++++++++++++++++++++++++++ Qt-Sample-App/qtsession.cpp | 1 + Qt-Sample-App/qtsession.h | 183 +++++++++++++++++++++++++ Qt-Sample-App/qtsubscriber.cpp | 1 + Qt-Sample-App/qtsubscriber.h | 72 ++++++++++ 14 files changed, 1070 insertions(+) create mode 100644 Qt-Sample-App/CMakeLists.txt create mode 100644 Qt-Sample-App/glvideowidget.cpp create mode 100644 Qt-Sample-App/glvideowidget.h create mode 100644 Qt-Sample-App/main.cpp create mode 100644 Qt-Sample-App/mainwindow.cpp create mode 100644 Qt-Sample-App/mainwindow.h create mode 100644 Qt-Sample-App/mainwindow.ui create mode 100644 Qt-Sample-App/participant.h create mode 100644 Qt-Sample-App/qtpublisher.cpp create mode 100644 Qt-Sample-App/qtpublisher.h create mode 100644 Qt-Sample-App/qtsession.cpp create mode 100644 Qt-Sample-App/qtsession.h create mode 100644 Qt-Sample-App/qtsubscriber.cpp create mode 100644 Qt-Sample-App/qtsubscriber.h diff --git a/Qt-Sample-App/CMakeLists.txt b/Qt-Sample-App/CMakeLists.txt new file mode 100644 index 0000000..18b7cf4 --- /dev/null +++ b/Qt-Sample-App/CMakeLists.txt @@ -0,0 +1,43 @@ +cmake_minimum_required(VERSION 3.12) + +project(Qt-Sample-App VERSION 0.1 LANGUAGES CXX) + +set(CMAKE_INCLUDE_CURRENT_DIR ON) + +set(CMAKE_AUTOUIC ON) +set(CMAKE_AUTOMOC ON) +set(CMAKE_AUTORCC ON) + +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +find_package(Qt6 COMPONENTS OpenGL OpenGLWidgets Widgets Multimedia MultimediaWidgets REQUIRED) + +set(PROJECT_SOURCES + main.cpp + mainwindow.cpp + mainwindow.h + mainwindow.ui + glvideowidget.cpp + glvideowidget.h + participant.h + qtsession.h + qtsession.cpp + qtpublisher.h + qtpublisher.cpp + qtsubscriber.h + qtsubscriber.cpp +) + +add_executable(qt_sample_app + ${PROJECT_SOURCES} +) + +target_include_directories(qt_sample_app PRIVATE /home/parallels/opentok/include) +target_link_directories(qt_sample_app PRIVATE /home/parallels/opentok/lib) +find_library(OTK opentok PATHS /home/parallels/opentok/lib) +target_link_libraries(qt_sample_app Qt${QT_VERSION_MAJOR}::Widgets Qt${QT_VERSION_MAJOR}::OpenGLWidgets Qt${QT_VERSION_MAJOR}::OpenGL Qt${QT_VERSION_MAJOR}::Multimedia Qt${QT_VERSION_MAJOR}::MultimediaWidgets ${OTK}) + +if(QT_VERSION_MAJOR EQUAL 6) + qt_finalize_executable(Qt-Sample-App) +endif() diff --git a/Qt-Sample-App/glvideowidget.cpp b/Qt-Sample-App/glvideowidget.cpp new file mode 100644 index 0000000..9c69acd --- /dev/null +++ b/Qt-Sample-App/glvideowidget.cpp @@ -0,0 +1,165 @@ +#include "glvideowidget.h" +#include + +namespace { +QVector GetVertexData(float ratio) +{ + const auto x = std::min(1.0f, ratio); + const auto y = std::min(1.0f, 1 / ratio); + return {-x, y, 0.0, 0.0, x, y, 1.0, 0.0, x, -y, 1.0, 1.0, -x, -y, 0.0, 1.0}; +} +} // namespace + +constexpr const char *vertexSource = "#version 330\n" + "layout(location = 0) in vec2 position;\n" + "layout(location = 1) in vec2 texCoord;\n" + "out vec4 texc;\n" + "void main( void )\n" + "{\n" + " gl_Position = vec4(position, 0.0, 1.0);\n" + " texc = vec4(texCoord, 0.0, 1.0);\n" + "}\n"; + +constexpr const char *fragmentSource = "#version 330\n" + "uniform sampler2D tex;\n" + "in vec4 texc;\n" + "out vec4 fragColor;\n" + "void main( void )\n" + "{\n" + " fragColor = texture(tex, texc.st);\n" + "}\n"; + +GLVideoWidget::GLVideoWidget(QWidget *parent) : QOpenGLWidget(parent), texture(0) {} + +void GLVideoWidget::initializeGL() +{ + initializeOpenGLFunctions(); + glClearColor(0.0, 0.0, 0.0, 1.0); + + // shader + shaderProgram = new QOpenGLShaderProgram; + shaderProgram->addShaderFromSourceCode(QOpenGLShader::Vertex, vertexSource); + shaderProgram->addShaderFromSourceCode(QOpenGLShader::Fragment, fragmentSource); + + // bind location for the vertex shader + shaderProgram->bindAttributeLocation("position", 0); + shaderProgram->link(); + shaderProgram->bind(); + + // Vertex array + vaoQuad.create(); + vaoQuad.bind(); + + // Vertex buffer + vboQuad.create(); + vboQuad.setUsagePattern(QOpenGLBuffer::StaticDraw); + vboQuad.bind(); + const auto vertexData = GetVertexData(1.0f); + vboQuad.allocate(vertexData.constData(), vertexData.count() * sizeof(GLfloat)); + + // Connect inputs to the shader + shaderProgram->enableAttributeArray(0); + shaderProgram->enableAttributeArray(1); + shaderProgram->setAttributeBuffer(0, GL_FLOAT, 0, 2, 4 * sizeof(GLfloat)); + shaderProgram->setAttributeBuffer(1, GL_FLOAT, 2 * sizeof(GLfloat), 2, 4 * sizeof(GLfloat)); + + vaoQuad.release(); + vboQuad.release(); + shaderProgram->release(); +} + +void GLVideoWidget::resizeGL(int w, int h) +{ + if (lastImageHeight == 0 || lastImageWidth == 0 || w == 0 || h == 0 || vboQuad.size() == 0) + return; + // Force update vertex data when we resize the widget + vboQuad.bind(); + auto ptr = vboQuad.map(QOpenGLBuffer::WriteOnly); + const auto ratio = static_cast(lastImageWidth) / lastImageHeight * height() / width(); + + const auto vertexData = GetVertexData(ratio); + memcpy(ptr, vertexData.constData(), vertexData.count() * sizeof(GLfloat)); + vboQuad.unmap(); + vboQuad.release(); +} + +void GLVideoWidget::paintGL() +{ + glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); + if (texture == nullptr) + return; + if (!shaderProgram->bind()) + return; + vaoQuad.bind(); + if (texture->isCreated()) + texture->bind(); + glDrawArrays(GL_TRIANGLE_FAN, 0, 4); + vaoQuad.release(); + shaderProgram->release(); +} + +void GLVideoWidget::setTexture(const otc_video_frame *frame) +{ + if ((otc_video_frame_get_width(frame) != lastImageWidth + || otc_video_frame_get_height(frame) != lastImageHeight)) { + vboQuad.bind(); + auto ptr = vboQuad.map(QOpenGLBuffer::WriteOnly); + const auto ratio = static_cast(otc_video_frame_get_width(frame)) + / otc_video_frame_get_height(frame) * (float) height() / (float) width(); + + const auto vertexData = GetVertexData(ratio); + if (ptr != nullptr) + memcpy(ptr, vertexData.constData(), vertexData.count() * sizeof(GLfloat)); + vboQuad.unmap(); + vboQuad.release(); + + resizeTexture(frame); + lastImageWidth = otc_video_frame_get_width(frame); + lastImageHeight = otc_video_frame_get_height(frame); + } + setTextureData(frame); + update(); +} + +void GLVideoWidget::resizeTexture(const otc_video_frame *image) +{ + if (image == nullptr) { + return; + } + if (texture != nullptr) { + delete texture; + } + makeCurrent(); + texture = new QOpenGLTexture(QOpenGLTexture::Target2D); + QOpenGLContext *context = QOpenGLContext::currentContext(); + if (!context) { + qWarning("QOpenGLTexture::setData() requires a valid current context"); + return; + } + if (context->isOpenGLES() && context->format().majorVersion() < 3) + texture->setFormat(QOpenGLTexture::RGBAFormat); + else + texture->setFormat(QOpenGLTexture::RGBA8_UNorm); + texture->setSize(otc_video_frame_get_width(image), otc_video_frame_get_height(image)); + texture->setMipLevels(texture->maximumMipLevels()); + texture->allocateStorage(QOpenGLTexture::BGRA, QOpenGLTexture::UInt8); + doneCurrent(); +} + +void GLVideoWidget::setTextureData(const otc_video_frame *image) +{ + QOpenGLPixelTransferOptions uploadOptions; + uploadOptions.setAlignment(1); + texture->setData(0, + QOpenGLTexture::BGRA, + QOpenGLTexture::UInt8, + otc_video_frame_get_buffer(image), + &uploadOptions); +} + +void GLVideoWidget::frame(std::shared_ptr frame) +{ + if (frame != nullptr) { + setTexture(frame.get()); + } +} diff --git a/Qt-Sample-App/glvideowidget.h b/Qt-Sample-App/glvideowidget.h new file mode 100644 index 0000000..70e1e19 --- /dev/null +++ b/Qt-Sample-App/glvideowidget.h @@ -0,0 +1,57 @@ +#ifndef GLVIDEOWIDGET_H +#define GLVIDEOWIDGET_H + +#include "opentok.h" +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +class GLVideoWidget : public QOpenGLWidget, protected QOpenGLFunctions +{ + Q_OBJECT +public: + GLVideoWidget(QWidget *parent); + ~GLVideoWidget() + { + makeCurrent(); + + delete shaderProgram; + delete texture; + + vboQuad.destroy(); + vaoQuad.destroy(); + + doneCurrent(); + } + void setTexture(const otc_video_frame *frame); + +protected: + void paintGL() override; + void resizeGL(int w, int h) override; + void initializeGL() override; + +private: + QOpenGLVertexArrayObject vaoQuad; + QOpenGLBuffer vboQuad; + QOpenGLShaderProgram *shaderProgram; + + QOpenGLTexture *texture; + + void resizeTexture(const otc_video_frame *image); + void setTextureData(const otc_video_frame *image); + + int lastImageWidth = 0; + int lastImageHeight = 0; + +public slots: + void frame(std::shared_ptr frame); +}; + +#endif // GLVIDEOWIDGET_H diff --git a/Qt-Sample-App/main.cpp b/Qt-Sample-App/main.cpp new file mode 100644 index 0000000..568fc43 --- /dev/null +++ b/Qt-Sample-App/main.cpp @@ -0,0 +1,28 @@ +#include "mainwindow.h" + +#include + +#include + +int main(int argc, char *argv[]) +{ + QSurfaceFormat fmt; + fmt.setDepthBufferSize(24); + fmt.setVersion(3, 3); + fmt.setProfile(QSurfaceFormat::CoreProfile); + QSurfaceFormat::setDefaultFormat(fmt); + QCoreApplication::setAttribute(Qt::AA_ShareOpenGLContexts); + QApplication a(argc, argv); + + if (otc_init(nullptr) != OTC_SUCCESS) { + std::cout << "Could not init OpenTok library" << std::endl; + } + otc_log_enable(OTC_LOG_LEVEL_TRACE); + + MainWindow w; + w.show(); + return a.exec(); + otc_destroy(); +} + +#include "main.moc" diff --git a/Qt-Sample-App/mainwindow.cpp b/Qt-Sample-App/mainwindow.cpp new file mode 100644 index 0000000..7373b17 --- /dev/null +++ b/Qt-Sample-App/mainwindow.cpp @@ -0,0 +1,76 @@ +#include "mainwindow.h" +#include "./ui_mainwindow.h" + +#include "qtsession.h" + +constexpr const char *kApiKey = ""; +constexpr const char *kSession = ""; +constexpr const char *kToken = ""; + +MainWindow::MainWindow(QWidget *parent) : QMainWindow(parent), ui(new Ui::MainWindow) +{ + ui->setupUi(this); + ui->lineEdit->setText(kApiKey); + ui->lineEdit_2->setText(kSession); + ui->lineEdit_3->setText(kToken); + connect(ui->pushButton, &QPushButton::clicked, this, &MainWindow::connectSession); + connect(ui->pushButton_2, &QPushButton::clicked, this, &MainWindow::newSessionWindow); +} + +MainWindow::~MainWindow() +{ + const auto session = findChild(); + if (session != nullptr) + delete session; + delete ui; +} + +void MainWindow::closeEvent(QCloseEvent *event) +{ + const auto session = findChild(); + if (session != nullptr) + delete session; + QMainWindow::closeEvent(event); +} + +void MainWindow::connectSession() +{ + if (findChild() == nullptr) { + new opentok_qt::Session(this, ui->lineEdit->text(), ui->lineEdit_2->text(), ui->lineEdit_3->text()); + ui->pushButton->setDisabled(true); + } else { + destroySession(); + } +} + +void MainWindow::destroySession() +{ + const auto session = findChild(); + session->setParent(nullptr); + session->deleteLater(); +} + +void MainWindow::sessionConnected() +{ + ui->pushButton->setText("Disconnect"); + ui->pushButton->setEnabled(true); +} + +void MainWindow::sessionDisconnected() +{ + destroySession(); + ui->pushButton->setText("Connect"); + ui->pushButton->setEnabled(true); +} + +void MainWindow::newSessionWindow() +{ + auto newWindow = new MainWindow(nullptr); + newWindow->setAttribute(Qt::WA_DeleteOnClose, true); + newWindow->show(); +} + +void MainWindow::addMessage(QString message) +{ + ui->textEdit->append(message); +} diff --git a/Qt-Sample-App/mainwindow.h b/Qt-Sample-App/mainwindow.h new file mode 100644 index 0000000..66730d5 --- /dev/null +++ b/Qt-Sample-App/mainwindow.h @@ -0,0 +1,85 @@ +#ifndef MAINWINDOW_H +#define MAINWINDOW_H + +#include +#include +#include +#include +#include + +#include "./ui_mainwindow.h" +#include "glvideowidget.h" +#include "participant.h" + +QT_BEGIN_NAMESPACE +namespace Ui { class MainWindow; } +QT_END_NAMESPACE + +class MainWindow : public QMainWindow +{ + Q_OBJECT + +public: + MainWindow(QWidget *parent = nullptr); + ~MainWindow(); + + Ui::MainWindow *ui; + +public slots: + void addVideoWidget(opentok_qt::Participant *participant) + { + auto openGLWidget = new GLVideoWidget(ui->widget); + openGLWidget->setObjectName(QString::fromUtf8("openGLWidget")); + + // Connect this widget to the new participant frames + connect(participant, &opentok_qt::Participant::frame, openGLWidget, &GLVideoWidget::frame); + participants[participant->getId()] = openGLWidget; + + resetGrid(); + } + + void removeVideoWidget(QString id) + { + const auto it = participants.find(id); + if (it != participants.end()) { + it->second->deleteLater(); + participants.erase(it); + } + resetGrid(); + } + + void connectSession(); + + void destroySession(); + + void sessionConnected(); + + void sessionDisconnected(); + + void newSessionWindow(); + + void addMessage(QString message); + +protected: + void closeEvent(QCloseEvent *event) override; + +private: + std::unordered_map participants; + + void resetGrid() + { + delete ui->gridLayout_3; + ui->gridLayout_3 = new QGridLayout(ui->widget); + ui->gridLayout_3->setObjectName(QString::fromUtf8("gridLayout")); + ui->gridLayout_3->setContentsMargins(0, 0, 0, 0); + + const auto size = static_cast(std::ceil(std::sqrt(participants.size()))); + + std::size_t index = 0; + for (const auto &p : participants) { + ui->gridLayout_3->addWidget(p.second, index / size, index % size, 1, 1); + ++index; + } + } +}; +#endif // MAINWINDOW_H diff --git a/Qt-Sample-App/mainwindow.ui b/Qt-Sample-App/mainwindow.ui new file mode 100644 index 0000000..25863d1 --- /dev/null +++ b/Qt-Sample-App/mainwindow.ui @@ -0,0 +1,107 @@ + + + MainWindow + + + + 0 + 0 + 800 + 616 + + + + Session + + + + + + + Connect + + + + + + + New session + + + + + + + + + + + + 0 + + + + Session settings + + + + + + API Key + + + + + + + + + + Session ID + + + + + + + + + + Token + + + + + + + + + + + Events + + + + + + + + + + + + + + + 0 + 0 + 800 + 29 + + + + + + + + diff --git a/Qt-Sample-App/participant.h b/Qt-Sample-App/participant.h new file mode 100644 index 0000000..a829e18 --- /dev/null +++ b/Qt-Sample-App/participant.h @@ -0,0 +1,22 @@ +#ifndef PARTICIPANT_H +#define PARTICIPANT_H + +#include "opentok.h" +#include +#include +#include + +namespace opentok_qt { + +class Participant : public QObject +{ + Q_OBJECT +public: + Participant(QObject *parent) : QObject(parent) {} + virtual QString getId() = 0; +signals: + void frame(std::shared_ptr); + void connected(Participant *); +}; +} +#endif // PARTICIPANT_H diff --git a/Qt-Sample-App/qtpublisher.cpp b/Qt-Sample-App/qtpublisher.cpp new file mode 100644 index 0000000..873ae79 --- /dev/null +++ b/Qt-Sample-App/qtpublisher.cpp @@ -0,0 +1 @@ +#include "qtpublisher.h" diff --git a/Qt-Sample-App/qtpublisher.h b/Qt-Sample-App/qtpublisher.h new file mode 100644 index 0000000..3335454 --- /dev/null +++ b/Qt-Sample-App/qtpublisher.h @@ -0,0 +1,229 @@ +#ifndef QTPUBLISHER_H +#define QTPUBLISHER_H + +#include "opentok.h" +#include "participant.h" + +#include +#include +#include + +#include + +namespace { +otc_video_frame_format qtframe_to_otcframe_format(QVideoFrameFormat::PixelFormat qvideoformat) +{ + switch (qvideoformat) { + case QVideoFrameFormat::Format_YUV420P: + return OTC_VIDEO_FRAME_FORMAT_YUV420P; + case QVideoFrameFormat::Format_NV12: + return OTC_VIDEO_FRAME_FORMAT_NV12; + case QVideoFrameFormat::Format_NV21: + return OTC_VIDEO_FRAME_FORMAT_NV21; + case QVideoFrameFormat::Format_YUYV: + return OTC_VIDEO_FRAME_FORMAT_YUY2; + case QVideoFrameFormat::Format_UYVY: + return OTC_VIDEO_FRAME_FORMAT_UYVY; + case QVideoFrameFormat::Format_ARGB8888: + return OTC_VIDEO_FRAME_FORMAT_ARGB32; + case QVideoFrameFormat::Format_BGRA8888: + return OTC_VIDEO_FRAME_FORMAT_BGRA32; + case QVideoFrameFormat::Format_ABGR8888: + return OTC_VIDEO_FRAME_FORMAT_ABGR32; + case QVideoFrameFormat::Format_Jpeg: + return OTC_VIDEO_FRAME_FORMAT_MJPEG; + case QVideoFrameFormat::Format_RGBA8888: + return OTC_VIDEO_FRAME_FORMAT_RGBA32; + default: + return OTC_VIDEO_FRAME_FORMAT_UNKNOWN; + } +} +} // namespace + +namespace opentok_qt { +class QCameraCapturer : public QObject +{ + // Allways add the Q_OBJECT macro + Q_OBJECT +public: + QCameraCapturer(QObject *parent) : QObject(parent) + { + connect(&sink, &QVideoSink::videoFrameChanged, this, &QCameraCapturer::processVideoFrame); + } + static otc_bool init(const otc_video_capturer *capturer, void *user_data) + { + const auto self = static_cast(user_data); + self->otc_catpurer = capturer; + + const auto width = 1760; + const auto height = 1328; + const auto comparator = [width, height](const QCameraFormat &a, const QCameraFormat &b) { + return std::abs(a.resolution().width() * a.resolution().height() - width * height) + < std::abs(b.resolution().width() * b.resolution().height() - width * height); + }; + + std::set formats(comparator); + + const auto cameraDevice = self->camera.cameraDevice(); + qInfo() << "Supported camera resolutions:"; + for (auto &&f : cameraDevice.videoFormats()) { + qInfo() << f.resolution(); + formats.insert(f); + } + + qInfo() << "Setting camera resolution to:" << formats.begin()->resolution(); + + self->camera.setCameraFormat(*formats.begin()); + self->session.setCamera(&self->camera); + self->session.setVideoSink(&self->sink); + return OTC_TRUE; + } + static otc_bool destroy(const otc_video_capturer *capturer, void *user_data) + { + const auto self = static_cast(user_data); + return OTC_TRUE; + } + static otc_bool start(const otc_video_capturer *capturer, void *user_data) + { + const auto self = static_cast(user_data); + self->camera.start(); + return OTC_TRUE; + } + static otc_bool stop(const otc_video_capturer *capturer, void *user_data) + { + const auto self = static_cast(user_data); + self->camera.stop(); + return OTC_TRUE; + } + static otc_bool getCaptureSettings(const otc_video_capturer *capturer, + void *user_data, + struct otc_video_capturer_settings *settings) + { + const auto self = static_cast(user_data); + const auto format = self->camera.cameraFormat(); + settings->height = format.resolution().height(); + settings->width = format.resolution().width(); + + settings->expected_delay = 0; + settings->fps = format.maxFrameRate(); + settings->format = 0; + settings->mirror_on_local_render = OTC_FALSE; + return OTC_TRUE; + } + +private: + QCamera camera; + QMediaCaptureSession session; + QVideoSink sink; + const otc_video_capturer *otc_catpurer; + +private slots: + void processVideoFrame() + { + const uint8_t *planes[3]; + int strides[3]; + + QVideoFrame videoframe = sink.videoFrame(); + if (videoframe.map(QVideoFrame::ReadOnly)) { + for (std::size_t i = 0; i < videoframe.planeCount(); i++) { + planes[i] = videoframe.bits(i); + strides[i] = videoframe.bytesPerLine(i); + } + otc_video_frame *otc_frame + = otc_video_frame_new_from_planes(qtframe_to_otcframe_format( + videoframe.pixelFormat()), + videoframe.width(), + videoframe.height(), + planes, + strides); + + const auto ret = otc_video_capturer_provide_frame(otc_catpurer, 0, otc_frame); + otc_video_frame_delete(otc_frame); + } + } +}; + +class Publisher : public Participant +{ + Q_OBJECT + static void onPublisherStreamCreated(otc_publisher *publisher, + void *user_data, + const otc_stream *stream) + { + const auto self = static_cast(user_data); + emit self->connected(self); + } + + static void onPublisherRenderFrame(otc_publisher *publisher, + void *user_data, + const otc_video_frame *frame) + { + const auto self = static_cast(user_data); + std::shared_ptr + converted_frame(otc_video_frame_convert(OTC_VIDEO_FRAME_FORMAT_ARGB32, frame), + otc_video_frame_delete); + + otc_video_frame_set_timestamp(converted_frame.get(), otc_video_frame_get_timestamp(frame)); + emit self->frame(converted_frame); + } + + static void onPublisherStreamDestroyed(otc_publisher *publisher, + void *user_data, + const otc_stream *stream) + { + qDebug() << __FUNCTION__ << "callback function"; + } + + static void onPublisherError(otc_publisher *publisher, + void *user_data, + const char *error_string, + enum otc_publisher_error_code error_code) + { + qDebug() << "Publisher error. Error code:" << error_string; + } + + static void on_otc_log_message(const char *message) + { + qDebug() << __FUNCTION__ << ":" << message; + } + +public: + Publisher(otc_session *s, QObject *parent) + : Participant(parent), publisher(nullptr, nullptr), session(s) + { + otc_publisher_callbacks publisher_callbacks = {0}; + publisher_callbacks.user_data = this; + publisher_callbacks.on_stream_created = onPublisherStreamCreated; + publisher_callbacks.on_render_frame = onPublisherRenderFrame; + publisher_callbacks.on_stream_destroyed = onPublisherStreamDestroyed; + publisher_callbacks.on_error = onPublisherError; + + QCameraCapturer *capturer = new QCameraCapturer(this); + + otc_video_capturer_callbacks video_capturer_callbacks; + video_capturer_callbacks.destroy = QCameraCapturer::destroy; + video_capturer_callbacks.init = QCameraCapturer::init; + video_capturer_callbacks.start = QCameraCapturer::start; + video_capturer_callbacks.stop = QCameraCapturer::stop; + video_capturer_callbacks.get_capture_settings = QCameraCapturer::getCaptureSettings; + video_capturer_callbacks.user_data = capturer; + video_capturer_callbacks.reserved = nullptr; + + publisher = std::unique_ptr( + otc_publisher_new("opentok-qt", &video_capturer_callbacks, &publisher_callbacks), + otc_publisher_delete); + otc_publisher_set_publish_audio (publisher.get(), OTC_FALSE); + + otc_session_publish(session, publisher.get()); + } + + QString getId() override { return otc_publisher_get_publisher_id(publisher.get()); } + + ~Publisher() { otc_session_unpublish(session, publisher.get()); } + +private: + std::unique_ptr publisher; + otc_session *session = nullptr; +}; +} +#endif // QTPUBLISHER_H diff --git a/Qt-Sample-App/qtsession.cpp b/Qt-Sample-App/qtsession.cpp new file mode 100644 index 0000000..790652f --- /dev/null +++ b/Qt-Sample-App/qtsession.cpp @@ -0,0 +1 @@ +#include "qtsession.h" diff --git a/Qt-Sample-App/qtsession.h b/Qt-Sample-App/qtsession.h new file mode 100644 index 0000000..3e8634e --- /dev/null +++ b/Qt-Sample-App/qtsession.h @@ -0,0 +1,183 @@ +#ifndef QTSESSION_H +#define QTSESSION_H + +#include "mainwindow.h" +#include "opentok.h" +#include "participant.h" +#include "qtpublisher.h" +#include "qtsubscriber.h" + +#include + +#include + +namespace opentok_qt { +class Session : public QObject +{ + // Allways add the Q_OBJECT macro + Q_OBJECT +public: + Session(MainWindow *parent, + const QString &apikey, + const QString &sessionid, + const QString &token) + : QObject(parent), session_(nullptr, nullptr) + { + // Notify main window about new participants + connect(this, &Session::newPartipant, parent, &MainWindow::addVideoWidget); + + // Notify main window about destroyed participants + connect(this, &Session::removeParticipant, parent, &MainWindow::removeVideoWidget); + + // Notify main window when session is connected + connect(this, &Session::sessionConnected, parent, &MainWindow::sessionConnected); + + // Notify main window when session is disconnected + connect(this, &Session::sessionDisconnected, parent, &MainWindow::sessionDisconnected); + + // Logging messages + connect(this, &Session::message, parent, &MainWindow::addMessage); + + init_session(apikey, sessionid, token); + } + static void onSessionConnected(otc_session *session, void *user_data) + { + const auto self = static_cast(user_data); + emit self->message(QString("Session connected ") + otc_session_get_id(session)); + emit self->sessionConnected(); + + QMetaObject::invokeMethod(self, [self]() mutable { self->createPublisher(); }); + } + + static void onSessionConnectionCreated(otc_session *session, + void *user_data, + const otc_connection *connection) + { + const auto self = static_cast(user_data); + emit self->message(QString("Connection created ") + otc_connection_get_id(connection)); + } + + static void onSessionConnectionDropped(otc_session *session, + void *user_data, + const otc_connection *connection) + { + const auto self = static_cast(user_data); + emit self->message(QString("Connection dropped ") + otc_connection_get_id(connection)); + } + + static void onSessionStreamReceived(otc_session *session, + void *user_data, + const otc_stream *stream) + { + const auto self = static_cast(user_data); + emit self->message(QString("Stream received ") + otc_stream_get_id(stream)); + + std::shared_ptr stream_copy(otc_stream_copy(stream), otc_stream_delete); + + QMetaObject::invokeMethod(self, [self, stream_copy]() mutable { + self->streamReceived(stream_copy); + }); + } + + static void onSessionStreamDropped(otc_session *session, + void *user_data, + const otc_stream *stream) + { + const auto self = static_cast(user_data); + emit self->message(QString("Stream dropped ") + otc_stream_get_id(stream)); + + std::shared_ptr stream_copy(otc_stream_copy(stream), otc_stream_delete); + + QMetaObject::invokeMethod(self, [self, stream_copy]() mutable { + self->streamDropped(stream_copy); + }); + } + + static void onSessionDisconnected(otc_session *session, void *user_data) + { + const auto self = static_cast(user_data); + emit self->message(QString("Session disconnected ") + otc_session_get_id(session)); + emit self->sessionDisconnected(); + } + + static void onSessionError(otc_session *session, + void *user_data, + const char *error_string, + enum otc_session_error_code error) + { + const auto self = static_cast(user_data); + emit self->message(QString("Session error ") + error_string); + } + + void init_session(const QString &apikey, const QString &sessionid, const QString &token) + { + otc_session_callbacks session_callbacks = {0}; + session_callbacks.on_connected = onSessionConnected; + session_callbacks.on_connection_created = onSessionConnectionCreated; + session_callbacks.on_connection_dropped = onSessionConnectionDropped; + session_callbacks.on_stream_received = onSessionStreamReceived; + session_callbacks.on_stream_dropped = onSessionStreamDropped; + session_callbacks.on_disconnected = onSessionDisconnected; + session_callbacks.on_error = onSessionError; + session_callbacks.user_data = this; + + session_ = std::unique_ptr( + otc_session_new(apikey.toStdString().c_str(), + sessionid.toStdString().c_str(), + &session_callbacks), + otc_session_delete); + + if (session_ == nullptr) { + qCritical() << "Could not create OpenTok session successfully"; + } + + otc_session_connect(session_.get(), token.toStdString().c_str()); + } + +private: + std::unique_ptr session_; + std::multimap> participants; + +signals: + void newPartipant(Participant *); + void removeParticipant(QString id); + void sessionConnected(); + void sessionDisconnected(); + void message(QString message); + +private slots: + void participantConnected(Participant *part) + { + emit newPartipant(static_cast(part)); + } + + void streamReceived(std::shared_ptr stream) + { + auto newsubscriber = std::make_unique(session_.get(), stream.get(), nullptr); + // Wait until participant is connected + connect(newsubscriber.get(), &Participant::connected, this, &Session::participantConnected); + participants.insert({otc_stream_get_id(stream.get()), + std::unique_ptr(std::move(newsubscriber))}); + } + + void createPublisher() + { + auto newpublisher = std::make_unique(session_.get(), nullptr); + // Wait until participant is connected + connect(newpublisher.get(), &Participant::connected, this, &Session::participantConnected); + participants.insert({"PRIVATEID", std::unique_ptr(std::move(newpublisher))}); + } + + void streamDropped(std::shared_ptr stream) + { + for (auto its = participants.equal_range(otc_stream_get_id(stream.get())); + its.first != its.second;) { + const auto id = its.first->second->getId(); + its.first->second.reset(); + emit removeParticipant(id); + its.first = participants.erase(its.first); + } + } +}; +} +#endif // QTSESSION_H diff --git a/Qt-Sample-App/qtsubscriber.cpp b/Qt-Sample-App/qtsubscriber.cpp new file mode 100644 index 0000000..460bd2a --- /dev/null +++ b/Qt-Sample-App/qtsubscriber.cpp @@ -0,0 +1 @@ +#include "qtsubscriber.h" diff --git a/Qt-Sample-App/qtsubscriber.h b/Qt-Sample-App/qtsubscriber.h new file mode 100644 index 0000000..9f6addd --- /dev/null +++ b/Qt-Sample-App/qtsubscriber.h @@ -0,0 +1,72 @@ +#ifndef QTSUBSCRIBER_H +#define QTSUBSCRIBER_H + +#include "opentok.h" +#include "participant.h" + +#include + +namespace opentok_qt { +class Subscriber : public Participant +{ + Q_OBJECT + static void onConnected(otc_subscriber *subscriber, void *user_data, const otc_stream *stream) + { + const auto self = static_cast(user_data); + emit self->connected(self); + } + + static void onDisconnected(otc_subscriber *subscriber, void *user_data) {} + + static void onRenderFrame(otc_subscriber *subscriber, + void *user_data, + const otc_video_frame *frame) + { + const auto self = static_cast(user_data); + std::shared_ptr + converted_frame(otc_video_frame_convert(OTC_VIDEO_FRAME_FORMAT_ARGB32, frame), + otc_video_frame_delete); + emit self->frame(converted_frame); + } + + static void onError(otc_subscriber *subscriber, + void *user_data, + const char *error_string, + enum otc_subscriber_error_code error) + {} + +public: + Subscriber(otc_session *s, const otc_stream *stream, QObject *parent) + : Participant(parent), subscriber_(nullptr, nullptr), session(s) + { + otc_subscriber_callbacks subscriber_callbacks = {0}; + subscriber_callbacks.user_data = this; + subscriber_callbacks.on_connected = Subscriber::onConnected; + subscriber_callbacks.on_render_frame = Subscriber::onRenderFrame; + subscriber_callbacks.on_error = Subscriber::onError; + + subscriber_ = std::unique_ptr( + otc_subscriber_new(stream, &subscriber_callbacks), otc_subscriber_delete); + + if (subscriber_ == nullptr) { + qDebug() << "Could not create OpenTok subscriber successfully"; + return; + } + otc_session_subscribe(session, subscriber_.get()); + + // We will not reuse any subscriber, let's copy the initial ID and asume or subscriber + // wrapper has a constant ID (instead of changing after a stream dropped event) + id = otc_subscriber_get_subscriber_id(subscriber_.get()); + } + + QString getId() override { return id; } + + ~Subscriber() { otc_session_unsubscribe(session, subscriber_.get()); } + +private: + std::unique_ptr subscriber_; + QString id; + otc_session *session; +}; +} +#endif // QTSUBSCRIBER_H From 36a1d1ccfd7abdfff1f1609bd96267f24cfc255c Mon Sep 17 00:00:00 2001 From: juliobecgom Date: Tue, 21 Feb 2023 16:11:56 +0100 Subject: [PATCH 2/2] Fix connect button issue when session connection fails --- Qt-Sample-App/mainwindow.cpp | 17 +++++++++++++++-- Qt-Sample-App/mainwindow.h | 2 ++ Qt-Sample-App/participant.h | 2 +- Qt-Sample-App/qtsession.h | 9 +++++++-- 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/Qt-Sample-App/mainwindow.cpp b/Qt-Sample-App/mainwindow.cpp index 7373b17..d0b7956 100644 --- a/Qt-Sample-App/mainwindow.cpp +++ b/Qt-Sample-App/mainwindow.cpp @@ -6,6 +6,9 @@ constexpr const char *kApiKey = ""; constexpr const char *kSession = ""; constexpr const char *kToken = ""; +constexpr const char *kConnecting = "Connecting"; +constexpr const char *kConnect = "Connect"; +constexpr const char *kDisconnect = "Disconnect"; MainWindow::MainWindow(QWidget *parent) : QMainWindow(parent), ui(new Ui::MainWindow) { @@ -38,6 +41,7 @@ void MainWindow::connectSession() if (findChild() == nullptr) { new opentok_qt::Session(this, ui->lineEdit->text(), ui->lineEdit_2->text(), ui->lineEdit_3->text()); ui->pushButton->setDisabled(true); + ui->pushButton->setText(kConnecting); } else { destroySession(); } @@ -52,14 +56,23 @@ void MainWindow::destroySession() void MainWindow::sessionConnected() { - ui->pushButton->setText("Disconnect"); + ui->pushButton->setText(kDisconnect); ui->pushButton->setEnabled(true); } +void MainWindow::sessionError() +{ + if (ui->pushButton->text() == kConnecting) { + ui->pushButton->setText(kConnect); + ui->pushButton->setEnabled(true); + destroySession(); + } +} + void MainWindow::sessionDisconnected() { destroySession(); - ui->pushButton->setText("Connect"); + ui->pushButton->setText(kConnect); ui->pushButton->setEnabled(true); } diff --git a/Qt-Sample-App/mainwindow.h b/Qt-Sample-App/mainwindow.h index 66730d5..76ad629 100644 --- a/Qt-Sample-App/mainwindow.h +++ b/Qt-Sample-App/mainwindow.h @@ -56,6 +56,8 @@ public slots: void sessionDisconnected(); + void sessionError(); + void newSessionWindow(); void addMessage(QString message); diff --git a/Qt-Sample-App/participant.h b/Qt-Sample-App/participant.h index a829e18..2850183 100644 --- a/Qt-Sample-App/participant.h +++ b/Qt-Sample-App/participant.h @@ -16,7 +16,7 @@ class Participant : public QObject virtual QString getId() = 0; signals: void frame(std::shared_ptr); - void connected(Participant *); + void connected(opentok_qt::Participant *); }; } #endif // PARTICIPANT_H diff --git a/Qt-Sample-App/qtsession.h b/Qt-Sample-App/qtsession.h index 3e8634e..9a47a6b 100644 --- a/Qt-Sample-App/qtsession.h +++ b/Qt-Sample-App/qtsession.h @@ -35,6 +35,9 @@ class Session : public QObject // Notify main window when session is disconnected connect(this, &Session::sessionDisconnected, parent, &MainWindow::sessionDisconnected); + // Notify main window when there is an error + connect(this, &Session::sessionError, parent, &MainWindow::sessionError); + // Logging messages connect(this, &Session::message, parent, &MainWindow::addMessage); @@ -107,6 +110,7 @@ class Session : public QObject { const auto self = static_cast(user_data); emit self->message(QString("Session error ") + error_string); + emit self->sessionError(); } void init_session(const QString &apikey, const QString &sessionid, const QString &token) @@ -139,14 +143,15 @@ class Session : public QObject std::multimap> participants; signals: - void newPartipant(Participant *); + void newPartipant(opentok_qt::Participant *); void removeParticipant(QString id); void sessionConnected(); void sessionDisconnected(); void message(QString message); + void sessionError(); private slots: - void participantConnected(Participant *part) + void participantConnected(opentok_qt::Participant *part) { emit newPartipant(static_cast(part)); }