wip: gfx: support direct video access to read point clouds straight f…

src/plugins/score-plugin-avnd/Crousti/CpuAnalysisNode.hpp

@@ -2,6 +2,7 @@
 
 #if SCORE_PLUGIN_GFX
 #include <Crousti/GfxNode.hpp>
+#include <Gfx/Graph/VideoNode.hpp>
 
 namespace oscr
 {
@@ -38,7 +39,7 @@ struct GfxRenderer<Node_T> final : score::gfx::OutputNodeRenderer
     return it->second;
   }
 
-  template <typename Tex>
+  template <avnd::cpu_fixed_format_texture Tex>
   void createInput(
       score::gfx::RenderList& renderer, int k, const Tex& texture_spec, QSize size)
   {
@@ -52,6 +53,20 @@ struct GfxRenderer<Node_T> final : score::gfx::OutputNodeRenderer
         = score::gfx::createRenderTarget(renderer.state, texture, renderer.samples());
   }
 
+  template <avnd::cpu_raw_texture Tex>
+  void createInput(
+      score::gfx::RenderList& renderer, int k, const Tex& texture_spec, QSize size)
+  {
+    auto port = parent.input[k];
+    static constexpr auto flags
+        = QRhiTexture::RenderTarget | QRhiTexture::UsedAsTransferSource;
+    auto texture
+        = renderer.state.rhi->newTexture(QRhiTexture::R32F, size, 1, flags); // FIXME
+    SCORE_ASSERT(texture->create());
+    m_rts[port]
+        = score::gfx::createRenderTarget(renderer.state, texture, renderer.samples());
+  }
+
   QRhiTexture* texture(int k) const noexcept
   {
     auto port = parent.input[k];
@@ -61,7 +76,7 @@ struct GfxRenderer<Node_T> final : score::gfx::OutputNodeRenderer
     return it->second.texture;
   }
 
-  void loadInputTexture(avnd::cpu_texture auto& cpu_tex, int k)
+  void loadInputTexture(avnd::cpu_fixed_format_texture auto& cpu_tex, int k)
   {
     auto& buf = m_readbacks[k].data;
     if(buf.size() != 4 * cpu_tex.width * cpu_tex.height)
@@ -75,6 +90,30 @@ struct GfxRenderer<Node_T> final : score::gfx::OutputNodeRenderer
     }
   }
 
+  std::vector<std::shared_ptr<score::gfx::RefcountedFrame>> m_texFrame;
+  void loadInputTexture(avnd::cpu_raw_texture auto& cpu_tex, int k)
+  {
+    SCORE_ASSERT(this->parent.input.size() > k);
+    if(m_texFrame.size() <= k)
+      m_texFrame.resize(k + 1);
+    for(score::gfx::Edge* edge : this->parent.input[k]->edges)
+    {
+      if(auto buf = dynamic_cast<score::gfx::BufferNode*>(edge->source->node))
+      {
+        auto old_frame = m_texFrame[k];
+        auto texFrame = buf->reader.currentFrame();
+        if(texFrame && texFrame->frame)
+        {
+          cpu_tex.bytesize = texFrame->frame->linesize[0];
+          cpu_tex.bytes = texFrame->frame->data[0];
+          cpu_tex.changed = true;
+        }
+        if(old_frame)
+          old_frame->use_count--;
+      }
+    }
+  }
+
   void init(score::gfx::RenderList& renderer, QRhiResourceUpdateBatch& res) override
   {
     if constexpr(requires { state.prepare(); })
@@ -89,6 +128,7 @@ struct GfxRenderer<Node_T> final : score::gfx::OutputNodeRenderer
     avnd::cpu_texture_input_introspection<Node_T>::for_all(
         avnd::get_inputs<Node_T>(state), [&]<typename F>(F& t) {
       QSize sz = renderer.state.renderSize;
+      using texture_type = std::remove_cvref_t<decltype(t.texture)>;
       if constexpr(requires {
                      t.request_width;
                      t.request_height;
@@ -97,9 +137,14 @@ struct GfxRenderer<Node_T> final : score::gfx::OutputNodeRenderer
         sz.rwidth() = t.request_width;
         sz.rheight() = t.request_height;
       }
+
       createInput(renderer, k, t.texture, sz);
-      t.texture.width = sz.width();
-      t.texture.height = sz.height();
+
+      if constexpr(avnd::cpu_fixed_format_texture<texture_type>)
+      {
+        t.texture.width = sz.width();
+        t.texture.height = sz.height();
+      }
       k++;
     });
   }
@@ -148,6 +193,10 @@ struct GfxRenderer<Node_T> final : score::gfx::OutputNodeRenderer
     for(auto [port, rt] : m_rts)
       rt.release();
     m_rts.clear();
+
+    for(auto texFrame : m_texFrame)
+      texFrame->use_count--;
+    m_texFrame.clear();
   }
 
   void inputAboutToFinish(

src/plugins/score-plugin-gfx/Gfx/GfxExecNode.cpp

@@ -1,4 +1,3 @@
-#include <Gfx/CameraDevice.hpp>
 #include <Gfx/GfxExecNode.hpp>
 #include <Gfx/GfxParameter.hpp>
 
@@ -87,6 +86,12 @@ void gfx_exec_node::run(
       }
 
       case ossia::geometry_port::which: {
+        // if(auto in = inlet->address.target<ossia::net::parameter_base*>();
+        //    in && (*in)->get_type() == ossia::parameter_type::GEOMETRY)
+        // {
+        //   auto cam = static_cast<ossia::gfx::geometry_parameter*>(*in);
+        //   cam->pull_geometry({this->id, inlet_i});
+        // }
         // TODO try to handle the case where it's generated on another GPU node
         // to prevent going through the CPU there
         auto& p = inlet->cast<ossia::geometry_port>();

src/plugins/score-plugin-gfx/Gfx/Graph/Node.cpp

@@ -323,8 +323,9 @@ void ProcessNode::process(int32_t port, const ossia::geometry_spec& v)
 {
   if(this->geometry != v || this->geometryChanged == 0)
   {
+    qDebug() << (typeid(*this)).name() << " geometry change ";
     this->geometry = v;
-    ++this->geometryChanged;
+    geometryChange();
   }
 }
 

src/plugins/score-plugin-gfx/Gfx/Graph/RenderList.cpp

@@ -290,6 +290,10 @@ void RenderList::render(QRhiCommandBuffer& commands, bool force)
   }
   update(*updateBatch);
 
+  // FIXME if we want to do more complicated things this is not correct
+  // We should do a proper topological sort, and group together all the nodes that are going
+  // to be part of the same render pass
+
   // For each texture input port
   //  For all previous node
   //   Update
@@ -382,6 +386,24 @@ void RenderList::render(QRhiCommandBuffer& commands, bool force)
         }
         node_was_rendered = true;
       }
+      else if(input->type == Types::Geometry)
+      {
+        for(auto edge : input->edges)
+        {
+          // TODO: PCL: we could sum the geometries technically
+          auto renderer = edge->source->node->renderedNodes[this];
+          qDebug() << "geometry: " << typeid(*renderer).name();
+        }
+      }
+      else if(input->type == Types::Buffer)
+      {
+        for(auto edge : input->edges)
+        {
+          // TODO: PCL: we could sum the geometries technically
+          auto renderer = edge->source->node->renderedNodes[this];
+          qDebug() << "buffer: " << typeid(*renderer).name();
+        }
+      }
     }
 
     if(!node_was_rendered)

src/plugins/score-plugin-gfx/Gfx/Graph/Uniforms.hpp

@@ -19,5 +19,6 @@ enum class Types
   Audio,
   Camera,
   Geometry,
+  Buffer
 };
 }

src/plugins/score-plugin-gfx/Gfx/Graph/VideoNode.cpp

@@ -4,6 +4,7 @@
 #include <Video/FrameQueue.hpp>
 
 #include <score/tools/Debug.hpp>
+#include <score/tools/SafeCast.hpp>
 
 #include <ossia/detail/flat_set.hpp>
 
@@ -375,6 +376,97 @@ AVFrame* VideoFrameReader::nextFrame(
   return nullptr;
 }
 
+class BufferNodeRenderer : public NodeRenderer
+{
+public:
+  const BufferNode& node;
+  VideoFrameShare& reader;
+  std::shared_ptr<RefcountedFrame> m_currentFrame{};
+  int64_t m_currentFrameIdx{-1};
+  explicit BufferNodeRenderer(
+      const BufferNode& node, RenderList& r, VideoFrameShare& frames) noexcept;
+  TextureRenderTarget renderTargetForInput(const Port& input) override { return {}; }
+
+  void init(RenderList& renderer, QRhiResourceUpdateBatch& res) override { }
+  void runRenderPass(RenderList&, QRhiCommandBuffer& commands, Edge& edge) override { }
+
+  void update(RenderList& renderer, QRhiResourceUpdateBatch& res) override
+  {
+    auto reader_frame = reader.m_currentFrameIdx;
+    if(reader_frame > this->m_currentFrameIdx)
+    {
+      auto old_frame = m_currentFrame;
+
+      m_currentFrame = reader.currentFrame();
+      // if(m_currentFrame && m_currentFrame->frame)
+      // {
+      //   auto f = m_currentFrame->frame;
+      //   qDebug() << " => " << f->data[0] << f->linesize[0] << f->format;
+      // }
+
+      if(old_frame)
+        old_frame->use_count--;
+      // TODO else ? fill with zeroes ?... does not that give green with YUV?
+
+      this->m_currentFrameIdx = reader_frame;
+    }
+  }
+  void release(RenderList& r) override
+  {
+    if(m_currentFrame)
+    {
+      m_currentFrame->use_count--;
+      m_currentFrame.reset();
+    }
+  }
+};
+
+void BufferNode::renderedNodesChanged()
+{
+  if(this->renderedNodes.size() == 0)
+  {
+    reader.releaseAllFrames();
+    if(must_stop.exchange(false))
+    {
+      safe_cast<::Video::ExternalInput*>(reader.m_decoder.get())->stop();
+    }
+  }
+}
+
+void BufferNode::process(Message&& msg)
+{
+  if(this->renderedNodes.size() > 0)
+  {
+    if(auto frame = reader.m_decoder->dequeue_frame())
+    {
+      reader.updateCurrentFrame(frame);
+    }
+  }
+
+  reader.releaseFramesToFree();
+}
+
+BufferNode::BufferNode(std::shared_ptr<Video::ExternalInput> dec)
+    : Node{}
+{
+  this->reader.m_decoder = std::move(dec);
+  output.push_back(new Port{this, {}, Types::Buffer, {}});
+}
+
+NodeRenderer* BufferNode::createRenderer(RenderList& r) const noexcept
+{
+  auto& reader
+      = const_cast<VideoFrameShare&>(static_cast<const VideoFrameShare&>(this->reader));
+  return new BufferNodeRenderer{*this, r, reader};
+};
+
+BufferNodeRenderer::BufferNodeRenderer(
+    const BufferNode& node, RenderList& r, VideoFrameShare& frames) noexcept
+    : NodeRenderer{}
+    , node{node}
+    , reader{frames}
+{
+}
 }
 
 #include <hap/source/hap.c>

src/plugins/score-plugin-gfx/Gfx/Graph/VideoNode.hpp

@@ -24,7 +24,7 @@ struct RefcountedFrame
   std::atomic_int use_count{};
 };
 
-struct VideoFrameShare
+struct SCORE_PLUGIN_GFX_EXPORT VideoFrameShare
 {
   VideoFrameShare();
   ~VideoFrameShare();
@@ -45,7 +45,7 @@ struct VideoFrameShare
   std::vector<std::shared_ptr<RefcountedFrame>> m_framesInFlight;
 };
 
-struct VideoFrameReader : VideoFrameShare
+struct SCORE_PLUGIN_GFX_EXPORT VideoFrameReader : VideoFrameShare
 {
   VideoFrameReader();
   ~VideoFrameReader();
@@ -137,4 +137,21 @@ class SCORE_PLUGIN_GFX_EXPORT CameraNode : public VideoNodeBase
   friend VideoNodeRenderer;
 };
 
+/**
+ * @brief Model for getting more general streams of data e.g. point clouds
+ */
+class SCORE_PLUGIN_GFX_EXPORT BufferNode : public Node
+{
+public:
+  explicit BufferNode(std::shared_ptr<Video::ExternalInput> dec);
+  ~BufferNode() { }
+
+  NodeRenderer* createRenderer(RenderList& r) const noexcept override;
+
+  void process(Message&& msg) override;
+  void renderedNodesChanged() override;
+
+  VideoFrameShare reader;
+  std::atomic_bool must_stop{};
+};
 }