From b2905754543813f731a598c8594e241313443ce0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Micha=C3=ABl=20Celerier?= <jeanmichael.celerier@gmail.com> Date: Tue, 10 Dec 2024 23:35:09 -0500 Subject: [PATCH] wip: gfx: support direct video access to read point clouds straight from devices --- .../Crousti/CpuAnalysisNode.hpp | 57 +++++++++++- .../score-plugin-gfx/Gfx/GfxExecNode.cpp | 7 +- .../score-plugin-gfx/Gfx/Graph/Node.cpp | 3 +- .../score-plugin-gfx/Gfx/Graph/RenderList.cpp | 22 +++++ .../score-plugin-gfx/Gfx/Graph/Uniforms.hpp | 1 + .../score-plugin-gfx/Gfx/Graph/VideoNode.cpp | 92 +++++++++++++++++++ .../score-plugin-gfx/Gfx/Graph/VideoNode.hpp | 21 ++++- 7 files changed, 195 insertions(+), 8 deletions(-) diff --git a/src/plugins/score-plugin-avnd/Crousti/CpuAnalysisNode.hpp b/src/plugins/score-plugin-avnd/Crousti/CpuAnalysisNode.hpp index 5a06876f04..fa1142816f 100644 --- a/src/plugins/score-plugin-avnd/Crousti/CpuAnalysisNode.hpp +++ b/src/plugins/score-plugin-avnd/Crousti/CpuAnalysisNode.hpp @@ -2,6 +2,7 @@ #if SCORE_PLUGIN_GFX #include <Crousti/GfxNode.hpp> +#include <Gfx/Graph/VideoNode.hpp> namespace oscr { @@ -38,7 +39,7 @@ struct GfxRenderer<Node_T> final : score::gfx::OutputNodeRenderer return it->second; } - template <typename Tex> + template <avnd::cpu_fixed_format_texture Tex> void createInput( score::gfx::RenderList& renderer, int k, const Tex& texture_spec, QSize size) { @@ -52,6 +53,20 @@ struct GfxRenderer<Node_T> final : score::gfx::OutputNodeRenderer = score::gfx::createRenderTarget(renderer.state, texture, renderer.samples()); } + template <avnd::cpu_raw_texture Tex> + void createInput( + score::gfx::RenderList& renderer, int k, const Tex& texture_spec, QSize size) + { + auto port = parent.input[k]; + static constexpr auto flags + = QRhiTexture::RenderTarget | QRhiTexture::UsedAsTransferSource; + auto texture + = renderer.state.rhi->newTexture(QRhiTexture::R32F, size, 1, flags); // FIXME + SCORE_ASSERT(texture->create()); + m_rts[port] + = score::gfx::createRenderTarget(renderer.state, texture, renderer.samples()); + } + QRhiTexture* texture(int k) const noexcept { auto port = parent.input[k]; @@ -61,7 +76,7 @@ struct GfxRenderer<Node_T> final : score::gfx::OutputNodeRenderer return it->second.texture; } - void loadInputTexture(avnd::cpu_texture auto& cpu_tex, int k) + void loadInputTexture(avnd::cpu_fixed_format_texture auto& cpu_tex, int k) { auto& buf = m_readbacks[k].data; if(buf.size() != 4 * cpu_tex.width * cpu_tex.height) @@ -75,6 +90,30 @@ struct GfxRenderer<Node_T> final : score::gfx::OutputNodeRenderer } } + std::vector<std::shared_ptr<score::gfx::RefcountedFrame>> m_texFrame; + void loadInputTexture(avnd::cpu_raw_texture auto& cpu_tex, int k) + { + SCORE_ASSERT(this->parent.input.size() > k); + if(m_texFrame.size() <= k) + m_texFrame.resize(k + 1); + for(score::gfx::Edge* edge : this->parent.input[k]->edges) + { + if(auto buf = dynamic_cast<score::gfx::BufferNode*>(edge->source->node)) + { + auto old_frame = m_texFrame[k]; + auto texFrame = buf->reader.currentFrame(); + if(texFrame && texFrame->frame) + { + cpu_tex.bytesize = texFrame->frame->linesize[0]; + cpu_tex.bytes = texFrame->frame->data[0]; + cpu_tex.changed = true; + } + if(old_frame) + old_frame->use_count--; + } + } + } + void init(score::gfx::RenderList& renderer, QRhiResourceUpdateBatch& res) override { if constexpr(requires { state.prepare(); }) @@ -89,6 +128,7 @@ struct GfxRenderer<Node_T> final : score::gfx::OutputNodeRenderer avnd::cpu_texture_input_introspection<Node_T>::for_all( avnd::get_inputs<Node_T>(state), [&]<typename F>(F& t) { QSize sz = renderer.state.renderSize; + using texture_type = std::remove_cvref_t<decltype(t.texture)>; if constexpr(requires { t.request_width; t.request_height; @@ -97,9 +137,14 @@ struct GfxRenderer<Node_T> final : score::gfx::OutputNodeRenderer sz.rwidth() = t.request_width; sz.rheight() = t.request_height; } + createInput(renderer, k, t.texture, sz); - t.texture.width = sz.width(); - t.texture.height = sz.height(); + + if constexpr(avnd::cpu_fixed_format_texture<texture_type>) + { + t.texture.width = sz.width(); + t.texture.height = sz.height(); + } k++; }); } @@ -148,6 +193,10 @@ struct GfxRenderer<Node_T> final : score::gfx::OutputNodeRenderer for(auto [port, rt] : m_rts) rt.release(); m_rts.clear(); + + for(auto texFrame : m_texFrame) + texFrame->use_count--; + m_texFrame.clear(); } void inputAboutToFinish( diff --git a/src/plugins/score-plugin-gfx/Gfx/GfxExecNode.cpp b/src/plugins/score-plugin-gfx/Gfx/GfxExecNode.cpp index f0d0e9e3f6..9c456f5f3d 100644 --- a/src/plugins/score-plugin-gfx/Gfx/GfxExecNode.cpp +++ b/src/plugins/score-plugin-gfx/Gfx/GfxExecNode.cpp @@ -1,4 +1,3 @@ -#include <Gfx/CameraDevice.hpp> #include <Gfx/GfxExecNode.hpp> #include <Gfx/GfxParameter.hpp> @@ -87,6 +86,12 @@ void gfx_exec_node::run( } case ossia::geometry_port::which: { + // if(auto in = inlet->address.target<ossia::net::parameter_base*>(); + // in && (*in)->get_type() == ossia::parameter_type::GEOMETRY) + // { + // auto cam = static_cast<ossia::gfx::geometry_parameter*>(*in); + // cam->pull_geometry({this->id, inlet_i}); + // } // TODO try to handle the case where it's generated on another GPU node // to prevent going through the CPU there auto& p = inlet->cast<ossia::geometry_port>(); diff --git a/src/plugins/score-plugin-gfx/Gfx/Graph/Node.cpp b/src/plugins/score-plugin-gfx/Gfx/Graph/Node.cpp index a59550c0c4..6787913bdb 100644 --- a/src/plugins/score-plugin-gfx/Gfx/Graph/Node.cpp +++ b/src/plugins/score-plugin-gfx/Gfx/Graph/Node.cpp @@ -323,8 +323,9 @@ void ProcessNode::process(int32_t port, const ossia::geometry_spec& v) { if(this->geometry != v || this->geometryChanged == 0) { + qDebug() << (typeid(*this)).name() << " geometry change "; this->geometry = v; - ++this->geometryChanged; + geometryChange(); } } diff --git a/src/plugins/score-plugin-gfx/Gfx/Graph/RenderList.cpp b/src/plugins/score-plugin-gfx/Gfx/Graph/RenderList.cpp index f7ea2f7ed9..1e84782297 100644 --- a/src/plugins/score-plugin-gfx/Gfx/Graph/RenderList.cpp +++ b/src/plugins/score-plugin-gfx/Gfx/Graph/RenderList.cpp @@ -290,6 +290,10 @@ void RenderList::render(QRhiCommandBuffer& commands, bool force) } update(*updateBatch); + // FIXME if we want to do more complicated things this is not correct + // We should do a proper topological sort, and group together all the nodes that are going + // to be part of the same render pass + // For each texture input port // For all previous node // Update @@ -382,6 +386,24 @@ void RenderList::render(QRhiCommandBuffer& commands, bool force) } node_was_rendered = true; } + else if(input->type == Types::Geometry) + { + for(auto edge : input->edges) + { + // TODO: PCL: we could sum the geometries technically + auto renderer = edge->source->node->renderedNodes[this]; + qDebug() << "geometry: " << typeid(*renderer).name(); + } + } + else if(input->type == Types::Buffer) + { + for(auto edge : input->edges) + { + // TODO: PCL: we could sum the geometries technically + auto renderer = edge->source->node->renderedNodes[this]; + qDebug() << "buffer: " << typeid(*renderer).name(); + } + } } if(!node_was_rendered) diff --git a/src/plugins/score-plugin-gfx/Gfx/Graph/Uniforms.hpp b/src/plugins/score-plugin-gfx/Gfx/Graph/Uniforms.hpp index 027e187bc9..011618ff74 100644 --- a/src/plugins/score-plugin-gfx/Gfx/Graph/Uniforms.hpp +++ b/src/plugins/score-plugin-gfx/Gfx/Graph/Uniforms.hpp @@ -19,5 +19,6 @@ enum class Types Audio, Camera, Geometry, + Buffer }; } diff --git a/src/plugins/score-plugin-gfx/Gfx/Graph/VideoNode.cpp b/src/plugins/score-plugin-gfx/Gfx/Graph/VideoNode.cpp index c1675cab54..6df3b8f81e 100644 --- a/src/plugins/score-plugin-gfx/Gfx/Graph/VideoNode.cpp +++ b/src/plugins/score-plugin-gfx/Gfx/Graph/VideoNode.cpp @@ -4,6 +4,7 @@ #include <Video/FrameQueue.hpp> #include <score/tools/Debug.hpp> +#include <score/tools/SafeCast.hpp> #include <ossia/detail/flat_set.hpp> @@ -375,6 +376,97 @@ AVFrame* VideoFrameReader::nextFrame( return nullptr; } +class BufferNodeRenderer : public NodeRenderer +{ +public: + const BufferNode& node; + VideoFrameShare& reader; + std::shared_ptr<RefcountedFrame> m_currentFrame{}; + int64_t m_currentFrameIdx{-1}; + explicit BufferNodeRenderer( + const BufferNode& node, RenderList& r, VideoFrameShare& frames) noexcept; + TextureRenderTarget renderTargetForInput(const Port& input) override { return {}; } + + void init(RenderList& renderer, QRhiResourceUpdateBatch& res) override { } + void runRenderPass(RenderList&, QRhiCommandBuffer& commands, Edge& edge) override { } + + void update(RenderList& renderer, QRhiResourceUpdateBatch& res) override + { + auto reader_frame = reader.m_currentFrameIdx; + if(reader_frame > this->m_currentFrameIdx) + { + auto old_frame = m_currentFrame; + + m_currentFrame = reader.currentFrame(); + // if(m_currentFrame && m_currentFrame->frame) + // { + // auto f = m_currentFrame->frame; + // qDebug() << " => " << f->data[0] << f->linesize[0] << f->format; + // } + + if(old_frame) + old_frame->use_count--; + // TODO else ? fill with zeroes ?... does not that give green with YUV? + + this->m_currentFrameIdx = reader_frame; + } + } + void release(RenderList& r) override + { + if(m_currentFrame) + { + m_currentFrame->use_count--; + m_currentFrame.reset(); + } + } +}; + +void BufferNode::renderedNodesChanged() +{ + if(this->renderedNodes.size() == 0) + { + reader.releaseAllFrames(); + if(must_stop.exchange(false)) + { + safe_cast<::Video::ExternalInput*>(reader.m_decoder.get())->stop(); + } + } +} + +void BufferNode::process(Message&& msg) +{ + if(this->renderedNodes.size() > 0) + { + if(auto frame = reader.m_decoder->dequeue_frame()) + { + reader.updateCurrentFrame(frame); + } + } + + reader.releaseFramesToFree(); +} + +BufferNode::BufferNode(std::shared_ptr<Video::ExternalInput> dec) + : Node{} +{ + this->reader.m_decoder = std::move(dec); + output.push_back(new Port{this, {}, Types::Buffer, {}}); +} + +NodeRenderer* BufferNode::createRenderer(RenderList& r) const noexcept +{ + auto& reader + = const_cast<VideoFrameShare&>(static_cast<const VideoFrameShare&>(this->reader)); + return new BufferNodeRenderer{*this, r, reader}; +}; + +BufferNodeRenderer::BufferNodeRenderer( + const BufferNode& node, RenderList& r, VideoFrameShare& frames) noexcept + : NodeRenderer{} + , node{node} + , reader{frames} +{ +} } #include <hap/source/hap.c> diff --git a/src/plugins/score-plugin-gfx/Gfx/Graph/VideoNode.hpp b/src/plugins/score-plugin-gfx/Gfx/Graph/VideoNode.hpp index 29eca10b1f..65aee96cfd 100644 --- a/src/plugins/score-plugin-gfx/Gfx/Graph/VideoNode.hpp +++ b/src/plugins/score-plugin-gfx/Gfx/Graph/VideoNode.hpp @@ -24,7 +24,7 @@ struct RefcountedFrame std::atomic_int use_count{}; }; -struct VideoFrameShare +struct SCORE_PLUGIN_GFX_EXPORT VideoFrameShare { VideoFrameShare(); ~VideoFrameShare(); @@ -45,7 +45,7 @@ struct VideoFrameShare std::vector<std::shared_ptr<RefcountedFrame>> m_framesInFlight; }; -struct VideoFrameReader : VideoFrameShare +struct SCORE_PLUGIN_GFX_EXPORT VideoFrameReader : VideoFrameShare { VideoFrameReader(); ~VideoFrameReader(); @@ -137,4 +137,21 @@ class SCORE_PLUGIN_GFX_EXPORT CameraNode : public VideoNodeBase friend VideoNodeRenderer; }; +/** + * @brief Model for getting more general streams of data e.g. point clouds + */ +class SCORE_PLUGIN_GFX_EXPORT BufferNode : public Node +{ +public: + explicit BufferNode(std::shared_ptr<Video::ExternalInput> dec); + ~BufferNode() { } + + NodeRenderer* createRenderer(RenderList& r) const noexcept override; + + void process(Message&& msg) override; + void renderedNodesChanged() override; + + VideoFrameShare reader; + std::atomic_bool must_stop{}; +}; }