From a494ef1ddd615921413849949582134e8b7c37fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Micha=C3=ABl=20Celerier?= Date: Sun, 10 Dec 2023 23:07:10 -0500 Subject: [PATCH] [objc] Enable ARC, update Syphon & fix a lot of Syphon issues --- 3rdparty/Syphon-Framework | 2 +- src/plugins/score-plugin-gfx/CMakeLists.txt | 6 +++ .../score-plugin-gfx/Gfx/CameraDevice.avf.mm | 6 +-- .../score-plugin-gfx/Gfx/Graph/CommonUBOs.hpp | 9 ++++ .../score-plugin-gfx/Gfx/Graph/RenderList.hpp | 2 + .../Gfx/Graph/decoders/GPUVideoDecoder.hpp | 3 +- .../Gfx/Graph/decoders/RGBA.hpp | 2 +- .../Gfx/Syphon/SyphonInput.mm | 53 +++++++------------ .../Gfx/Syphon/SyphonOutput.mm | 14 +---- 9 files changed, 43 insertions(+), 54 deletions(-) diff --git a/3rdparty/Syphon-Framework b/3rdparty/Syphon-Framework index 2266d19e6e..12dd5e33f3 160000 --- a/3rdparty/Syphon-Framework +++ b/3rdparty/Syphon-Framework @@ -1 +1 @@ -Subproject commit 2266d19e6ec784a07ac9b8584c50eb345acbbdb4 +Subproject commit 12dd5e33f3d4fb122ae8bed739c8f992d1083484 diff --git a/src/plugins/score-plugin-gfx/CMakeLists.txt b/src/plugins/score-plugin-gfx/CMakeLists.txt index 202aba4d99..0ad4024191 100644 --- a/src/plugins/score-plugin-gfx/CMakeLists.txt +++ b/src/plugins/score-plugin-gfx/CMakeLists.txt @@ -279,6 +279,12 @@ if(APPLE) find_library(AudioToolbox_FK AudioToolbox) find_library(VideoToolbox_FK VideoToolbox) + target_compile_options(${PROJECT_NAME} + PRIVATE + $<$:-fobjc-arc> + $<$:-fobjc-arc-exceptions> + $<$:-fobjc-weak> + ) target_link_libraries(${PROJECT_NAME} PRIVATE ${QuartzCore_FK} ${AppKit_FK} diff --git a/src/plugins/score-plugin-gfx/Gfx/CameraDevice.avf.mm b/src/plugins/score-plugin-gfx/Gfx/CameraDevice.avf.mm index c02d3bb3bc..080c0382da 100644 --- a/src/plugins/score-plugin-gfx/Gfx/CameraDevice.avf.mm +++ b/src/plugins/score-plugin-gfx/Gfx/CameraDevice.avf.mm @@ -14,7 +14,7 @@ static void iterateCameraFormats(AVCaptureDevice* device, std::function func) { - NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init]; - { NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo]; for (AVCaptureDevice *device in devices) { @@ -46,8 +44,6 @@ void enumerateCameraDevices(std::function func) iterateCameraFormats(device, func); } } - - [pool release]; } } diff --git a/src/plugins/score-plugin-gfx/Gfx/Graph/CommonUBOs.hpp b/src/plugins/score-plugin-gfx/Gfx/Graph/CommonUBOs.hpp index 27961ab925..4408ddee41 100644 --- a/src/plugins/score-plugin-gfx/Gfx/Graph/CommonUBOs.hpp +++ b/src/plugins/score-plugin-gfx/Gfx/Graph/CommonUBOs.hpp @@ -51,5 +51,14 @@ struct OutputUBO float renderSize[2]{}; }; +/** + * @brief UBO shared across all video objects. + */ +struct VideoMaterialUBO +{ + float scale[2]{1.f, 1.f}; + float textureSize[2]{1.f, 1.f}; +}; + #pragma pack(pop) } diff --git a/src/plugins/score-plugin-gfx/Gfx/Graph/RenderList.hpp b/src/plugins/score-plugin-gfx/Gfx/Graph/RenderList.hpp index 29351db181..8094c882ff 100644 --- a/src/plugins/score-plugin-gfx/Gfx/Graph/RenderList.hpp +++ b/src/plugins/score-plugin-gfx/Gfx/Graph/RenderList.hpp @@ -130,6 +130,8 @@ class SCORE_PLUGIN_GFX_EXPORT RenderList int samples() const noexcept { return m_samples; } + bool canRender() const noexcept { return m_ready; } + private: OutputUBO m_outputUBOData; diff --git a/src/plugins/score-plugin-gfx/Gfx/Graph/decoders/GPUVideoDecoder.hpp b/src/plugins/score-plugin-gfx/Gfx/Graph/decoders/GPUVideoDecoder.hpp index 06bd15a37b..c9ec208d68 100644 --- a/src/plugins/score-plugin-gfx/Gfx/Graph/decoders/GPUVideoDecoder.hpp +++ b/src/plugins/score-plugin-gfx/Gfx/Graph/decoders/GPUVideoDecoder.hpp @@ -12,13 +12,12 @@ extern "C" { #define SCORE_GFX_VIDEO_UNIFORMS \ "layout(std140, binding = 0) uniform renderer_t {\n" \ " mat4 clipSpaceCorrMatrix;\n" \ -" \n" \ " vec2 renderSize;\n" \ "} renderer;\n" \ "\n" \ "layout(std140, binding = 2) uniform material_t {\n" \ " vec2 scale;\n" \ -" vec2 textureSize;\n" \ +" vec2 texSz;\n" \ "} mat;\n" namespace score::gfx diff --git a/src/plugins/score-plugin-gfx/Gfx/Graph/decoders/RGBA.hpp b/src/plugins/score-plugin-gfx/Gfx/Graph/decoders/RGBA.hpp index cf8e54009a..3573908609 100644 --- a/src/plugins/score-plugin-gfx/Gfx/Graph/decoders/RGBA.hpp +++ b/src/plugins/score-plugin-gfx/Gfx/Graph/decoders/RGBA.hpp @@ -219,7 +219,7 @@ out gl_PerVertex { vec4 gl_Position; }; void main() { - v_texcoord = texcoord * mat.textureSize; + v_texcoord = texcoord * mat.texSz.xy; gl_Position = renderer.clipSpaceCorrMatrix * vec4(position.x * mat.scale.x, position.y * mat.scale.y, 0.0, 1.); } )_"; diff --git a/src/plugins/score-plugin-gfx/Gfx/Syphon/SyphonInput.mm b/src/plugins/score-plugin-gfx/Gfx/Syphon/SyphonInput.mm index 750c0c572f..cd36b04829 100644 --- a/src/plugins/score-plugin-gfx/Gfx/Syphon/SyphonInput.mm +++ b/src/plugins/score-plugin-gfx/Gfx/Syphon/SyphonInput.mm @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -56,10 +57,7 @@ explicit Renderer(const SyphonInputNode& n): node{n} { } QRhiBuffer* m_processUBO{}; QRhiBuffer* m_materialUBO{}; - struct Material { - float scale_w{1.0f}, scale_h{1.0f}; - float width{1.f}, height{1.f}; - } material; + score::gfx::VideoMaterialUBO material; std::unique_ptr m_gpu{}; bool enabled{}; @@ -87,7 +85,6 @@ void openServer(QRhi& rhi) { enabled = false; // Need pool - NSAutoreleasePool* pool = [[NSAutoreleasePool alloc] init]; SyphonServerDirectory *ssd = [SyphonServerDirectory sharedDirectory]; NSArray *servers = [ssd serversMatchingName:NULL appName:NULL]; @@ -95,7 +92,7 @@ void openServer(QRhi& rhi) { if (NSDictionary *desc = findServer(servers, node.settings.path)) { - m_receiver = [[SYPHON_CLIENT_UNIQUE_CLASS_NAME alloc] + m_receiver = [[SyphonOpenGLClient alloc] initWithServerDescription:desc context: nativeContext(rhi) options:NULL @@ -104,7 +101,6 @@ void openServer(QRhi& rhi) } enabled = true; } - [pool drain]; } score::gfx::TextureRenderTarget renderTargetForInput(const score::gfx::Port& p) override { return { }; } @@ -112,7 +108,7 @@ void init(score::gfx::RenderList& renderer, QRhiResourceUpdateBatch& res) overri { // Initialize our rendering structures auto& rhi = *renderer.state.rhi; - const auto& mesh = renderer.defaultQuad(); + const auto& mesh = renderer.defaultTriangle(); if (!m_meshBuffer) { auto [mbuffer, ibuffer] = renderer.initMeshBuffer(mesh, res); @@ -125,40 +121,37 @@ void init(score::gfx::RenderList& renderer, QRhiResourceUpdateBatch& res) overri m_processUBO->create(); m_materialUBO = rhi.newBuffer( - QRhiBuffer::Dynamic, QRhiBuffer::UniformBuffer, sizeof(Material)); + QRhiBuffer::Dynamic, QRhiBuffer::UniformBuffer, sizeof(score::gfx::VideoMaterialUBO)); m_materialUBO->create(); // Initialize syphon openServer(rhi); int w = 16, h = 16; - NSAutoreleasePool* pool = [[NSAutoreleasePool alloc] init]; SyphonOpenGLImage* img{}; if(enabled) { - if(img = [m_receiver newFrameImage]) { + if((img = [m_receiver newFrameImage])) { NSSize sz = img.textureSize; w = sz.width; h = sz.height; + currentTex = img.textureName; } } metadata.width = std::max((int)1, w); metadata.height = std::max((int)1, h); - material.width = metadata.width; - material.height = metadata.height; + material.textureSize[0] = w; + material.textureSize[1] = h; + res.updateDynamicBuffer(m_materialUBO, 0, sizeof(score::gfx::VideoMaterialUBO), &material); m_gpu = std::make_unique(QRhiTexture::RGBA8, 4, metadata, QString{}); createPipelines(renderer); - m_pixels.resize(w * h * 4); if(img) { rebuildTexture(img); - [img release]; } - - [pool drain]; } void createPipelines(score::gfx::RenderList& r) @@ -171,7 +164,7 @@ void createPipelines(score::gfx::RenderList& r) m_p, this->node.output[0]->edges, r, - r.defaultQuad(), + r.defaultTriangle(), shaders.first, shaders.second, m_processUBO, @@ -180,7 +173,6 @@ void createPipelines(score::gfx::RenderList& r) } } -#include // clang-format: keep void rebuildTexture(SyphonOpenGLImage* img) { SCORE_ASSERT(!m_gpu->samplers.empty()); @@ -188,11 +180,13 @@ void rebuildTexture(SyphonOpenGLImage* img) QRhiTexture::NativeTexture tt; tt.layout = 0; - tt.object = reinterpret_cast(¤tTex); + tt.object = currentTex; tex->destroy(); NSSize sz = img.textureSize; tex->setPixelSize(QSize(sz.width, sz.height)); + tex->setFormat(QRhiTexture::Format::BGRA8); + tex->setFlags(tex->flags() | QRhiTexture::TextureRectangleGL); tex->createFrom(tt); // FIXME how to ensure this ? @@ -208,7 +202,6 @@ void rebuildTexture(SyphonOpenGLImage* img) for(auto& pass : m_p) pass.second.srb->create(); } -#include // clang-format: keep GLuint currentTex = 0; void update(score::gfx::RenderList& renderer, QRhiResourceUpdateBatch& res) override @@ -229,7 +222,6 @@ void update(score::gfx::RenderList& renderer, QRhiResourceUpdateBatch& res) over // Check the current status of the Syphon remote bool connected{}; - NSAutoreleasePool* pool = [[NSAutoreleasePool alloc] init]; auto img = [m_receiver newFrameImage]; if(!img) { return; @@ -240,18 +232,17 @@ void update(score::gfx::RenderList& renderer, QRhiResourceUpdateBatch& res) over { metadata.width = std::max(1., sz.width); metadata.height = std::max(1., sz.height); - material.width = metadata.width; - material.height = metadata.height; + material.scale[0] = 1.f; + material.scale[1] = 1.f; + material.textureSize[0] = metadata.width; + material.textureSize[1] = metadata.height; currentTex = img.textureName; rebuildTexture(img); } - [img release]; - [pool drain]; - res.updateDynamicBuffer(m_processUBO, 0, sizeof(score::gfx::ProcessUBO), &this->node.standardUBO); - res.updateDynamicBuffer(m_materialUBO, 0, sizeof(Material), &material); + res.updateDynamicBuffer(m_materialUBO, 0, sizeof(score::gfx::VideoMaterialUBO), &material); } void runRenderPass( @@ -260,7 +251,7 @@ void runRenderPass( score::gfx::Edge& edge) override { const auto& mesh = renderer.defaultTriangle(); - score::gfx::quadRenderPass(renderer, {.mesh = m_meshBuffer, .index = m_idxBuffer}, cb, edge, m_p); + score::gfx::defaultRenderPass(renderer, mesh, {.mesh = m_meshBuffer, .index = m_idxBuffer}, cb, edge, m_p); } void release(score::gfx::RenderList& r) override @@ -289,7 +280,6 @@ void release(score::gfx::RenderList& r) override } SyphonOpenGLClient* m_receiver{}; - std::vector m_pixels; }; #include // clang-format: keep @@ -381,15 +371,12 @@ void registerServer(NSDictionary* desc, std::function f) const override { - NSAutoreleasePool* pool = [[NSAutoreleasePool alloc] init]; - auto ssd = [SyphonServerDirectory sharedDirectory]; NSArray *servers = [ssd serversMatchingName:NULL appName:NULL]; for(int i = 0; i < servers.count; i++) { registerServer(servers[i], f); } - [pool drain]; } }; diff --git a/src/plugins/score-plugin-gfx/Gfx/Syphon/SyphonOutput.mm b/src/plugins/score-plugin-gfx/Gfx/Syphon/SyphonOutput.mm index 201919dae5..dc45e7afbe 100644 --- a/src/plugins/score-plugin-gfx/Gfx/Syphon/SyphonOutput.mm +++ b/src/plugins/score-plugin-gfx/Gfx/Syphon/SyphonOutput.mm @@ -46,7 +46,7 @@ void startRendering() override void onRendererChange() override { } bool canRender() const override { - return bool(m_syphon); + return bool(m_syphon); } @@ -54,7 +54,6 @@ void createSyphon(QRhi& rhi) { if(!m_created) { - NSAutoreleasePool* pool = [[NSAutoreleasePool alloc] init]; auto serverName = this->m_settings.path.toNSString(); m_syphon = [[SyphonOpenGLServer alloc] initWithName:serverName @@ -62,8 +61,6 @@ void createSyphon(QRhi& rhi) options:NULL ]; m_created = true; - - [pool drain]; } } @@ -73,7 +70,7 @@ void render() override m_update(); auto renderer = m_renderer.lock(); - if (renderer && m_renderState) + if (renderer && renderer->nodes.size() > 1 && m_renderState) { auto rhi = m_renderState->rhi; @@ -94,7 +91,6 @@ void render() override if (m_created) { auto t = static_cast(m_texture); - NSAutoreleasePool* pool = [[NSAutoreleasePool alloc] init]; [m_syphon publishFrameTexture:t->texture @@ -103,7 +99,6 @@ void render() override textureDimensions:NSMakeSize(m_settings.width, m_settings.height) flipped:false ]; - [pool drain]; } } } @@ -164,14 +159,9 @@ void createOutput( void destroyOutput() override { - NSAutoreleasePool* pool = [[NSAutoreleasePool alloc] init]; - [m_syphon stop]; - [m_syphon release]; m_syphon = nullptr; - - [pool drain]; } std::shared_ptr renderState() const override