From 2bb897e980852d489525aefc3b22520f07372b3f Mon Sep 17 00:00:00 2001 From: designateddolt Date: Mon, 17 Apr 2023 23:43:41 +1200 Subject: [PATCH 1/5] initial feature skeleton, lacks async capability and doesn't work :'( --- lib/ferrum/browser.rb | 1 + lib/ferrum/page.rb | 15 +++++++++++++- lib/ferrum/page/screencast.rb | 19 +++++++++++++++++ lib/ferrum/screencaster.rb | 39 +++++++++++++++++++++++++++++++++++ 4 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 lib/ferrum/page/screencast.rb create mode 100644 lib/ferrum/screencaster.rb diff --git a/lib/ferrum/browser.rb b/lib/ferrum/browser.rb index 9626ad1d..da81f6e0 100644 --- a/lib/ferrum/browser.rb +++ b/lib/ferrum/browser.rb @@ -22,6 +22,7 @@ class Browser body doctype content= headers cookies network downloads mouse keyboard + start_screencast stop_screencast screenshot pdf mhtml viewport_size device_pixel_ratio frames frame_by main_frame evaluate evaluate_on evaluate_async execute evaluate_func diff --git a/lib/ferrum/page.rb b/lib/ferrum/page.rb index c019b831..dc0ad87c 100644 --- a/lib/ferrum/page.rb +++ b/lib/ferrum/page.rb @@ -7,10 +7,12 @@ require "ferrum/headers" require "ferrum/cookies" require "ferrum/dialog" +require "ferrum/screencaster" require "ferrum/network" require "ferrum/downloads" require "ferrum/page/frames" require "ferrum/page/screenshot" +require "ferrum/page/screencast" require "ferrum/page/animation" require "ferrum/page/tracing" require "ferrum/page/stream" @@ -30,6 +32,7 @@ class Page include Screenshot include Frames include Stream + include Screencast attr_accessor :referrer attr_reader :context_id, :target_id, :event, :tracing @@ -69,12 +72,13 @@ class Page # @return [Downloads] attr_reader :downloads + attr_reader :screencaster + def initialize(client, context_id:, target_id:, proxy: nil) @client = client @context_id = context_id @target_id = target_id @options = client.options - @frames = Concurrent::Map.new @main_frame = Frame.new(nil, self) @event = Utils::Event.new.tap(&:set) @@ -87,6 +91,7 @@ def initialize(client, context_id:, target_id:, proxy: nil) @network = Network.new(self) @tracing = Tracing.new(self) @downloads = Downloads.new(self) + @screencaster = Screencaster.new(self) subscribe prepare_page @@ -388,6 +393,10 @@ def on(name, &block) request = Network::AuthRequest.new(self, params) block.call(request, index, total) end + when :screencastFrame + @client.on("Page.screencastFrame") do |params, index, total| + block.call(params, index, total) + end else client.on(name, &block) end @@ -441,6 +450,10 @@ def subscribe dialog.accept end end + + on(:screencastFrame) do |params, index, total| + @screencaster.add_frame(params) + end end def prepare_page diff --git a/lib/ferrum/page/screencast.rb b/lib/ferrum/page/screencast.rb new file mode 100644 index 00000000..9eca08ac --- /dev/null +++ b/lib/ferrum/page/screencast.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true +require "ferrum/screencaster" + + +module Ferrum + class Page + module Screencast + attr_reader :screencaster + + def start_screencast #(options) + @screencaster.await.start_screencast + end + + def stop_screencast + @screencaster.await.stop_screencast + end + end + end +end diff --git a/lib/ferrum/screencaster.rb b/lib/ferrum/screencaster.rb new file mode 100644 index 00000000..8a4727af --- /dev/null +++ b/lib/ferrum/screencaster.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +require 'open3' +require 'stringio' +#TODO these ^ need to be added to gemfile +require 'concurrent-ruby' + +module Ferrum + class Screencaster + # attr_reader :message, :default_prompt + include Concurrent::Async + + def initialize(page) + @page = page #might be wrong + @stdin = StringIO.new + @stdout = StringIO.new + @stderr = StringIO.new + @wait_thr = nil + end + + def add_frame(params) + @page.command("Page.screencastFrameAck") + img = params["data"] + img_decoded = Base64.decode64(img) + @stdin.write(img_decoded) + end + + def start_screencast #(options) + cmd = "ffmpeg -y -f rawvideo -pix_fmt rgb24 -s 640x480 -r 30 -i - -vcodec mp4v -c:v libx264 -preset slow -crf 22 output_video.mp4" + @stdin, @stdout, @stderr, @wait_thr = Open3.popen3(cmd) + @page.command("Page.startScreencast") #, **options) + end + + def stop_screencast + @page.command("Page.stopScreencast") + @stdin.close + end + end +end From e94d41824349e284a6f3d0ef2a9468c5721212a8 Mon Sep 17 00:00:00 2001 From: designateddolt Date: Tue, 18 Apr 2023 14:48:41 +1200 Subject: [PATCH 2/5] tweaks, methods can now generate very tiny mp4s --- lib/ferrum/page.rb | 1 + lib/ferrum/page/screencast.rb | 4 ++-- lib/ferrum/screencaster.rb | 12 +++++++++--- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/lib/ferrum/page.rb b/lib/ferrum/page.rb index dc0ad87c..b24b4a76 100644 --- a/lib/ferrum/page.rb +++ b/lib/ferrum/page.rb @@ -453,6 +453,7 @@ def subscribe on(:screencastFrame) do |params, index, total| @screencaster.add_frame(params) + warn "DEBUG: frame added in page.rb" end end diff --git a/lib/ferrum/page/screencast.rb b/lib/ferrum/page/screencast.rb index 9eca08ac..f77ba0e7 100644 --- a/lib/ferrum/page/screencast.rb +++ b/lib/ferrum/page/screencast.rb @@ -8,11 +8,11 @@ module Screencast attr_reader :screencaster def start_screencast #(options) - @screencaster.await.start_screencast + @screencaster.start_screencast end def stop_screencast - @screencaster.await.stop_screencast + @screencaster.stop_screencast end end end diff --git a/lib/ferrum/screencaster.rb b/lib/ferrum/screencaster.rb index 8a4727af..88e0354e 100644 --- a/lib/ferrum/screencaster.rb +++ b/lib/ferrum/screencaster.rb @@ -19,21 +19,27 @@ def initialize(page) end def add_frame(params) - @page.command("Page.screencastFrameAck") + warn 'frame' + @page.command("Page.screencastFrameAck", sessionId: params["sessionId"]) img = params["data"] img_decoded = Base64.decode64(img) @stdin.write(img_decoded) + end def start_screencast #(options) - cmd = "ffmpeg -y -f rawvideo -pix_fmt rgb24 -s 640x480 -r 30 -i - -vcodec mp4v -c:v libx264 -preset slow -crf 22 output_video.mp4" + cmd = "ffmpeg -y -f image2pipe -i - -c:v libx264 -preset slow -crf 22 -r 1 -an -f mp4 -movflags +faststart output_video2.mp4" @stdin, @stdout, @stderr, @wait_thr = Open3.popen3(cmd) - @page.command("Page.startScreencast") #, **options) + @page.command("Page.startScreencast", format: "jpeg") #, **options) end def stop_screencast + warn 'stopped' @page.command("Page.stopScreencast") @stdin.close + @stdout.close + @stderr.close + @wait_thr.join end end end From b421183d4ecd870f4842d7910fdef16bb410c12e Mon Sep 17 00:00:00 2001 From: Peter Singh Date: Thu, 2 Jan 2025 22:03:28 +0000 Subject: [PATCH 3/5] Only save images from screencast: Images can be combined later --- lib/ferrum/page.rb | 3 +++ lib/ferrum/page/screencast.rb | 1 - lib/ferrum/screencaster.rb | 29 +++++------------------------ 3 files changed, 8 insertions(+), 25 deletions(-) diff --git a/lib/ferrum/page.rb b/lib/ferrum/page.rb index b24b4a76..d820841c 100644 --- a/lib/ferrum/page.rb +++ b/lib/ferrum/page.rb @@ -72,6 +72,9 @@ class Page # @return [Downloads] attr_reader :downloads + # Control Screencasting + # + # @return [nil] attr_reader :screencaster def initialize(client, context_id:, target_id:, proxy: nil) diff --git a/lib/ferrum/page/screencast.rb b/lib/ferrum/page/screencast.rb index f77ba0e7..ae05abf7 100644 --- a/lib/ferrum/page/screencast.rb +++ b/lib/ferrum/page/screencast.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true require "ferrum/screencaster" - module Ferrum class Page module Screencast diff --git a/lib/ferrum/screencaster.rb b/lib/ferrum/screencaster.rb index 88e0354e..3cbe169d 100644 --- a/lib/ferrum/screencaster.rb +++ b/lib/ferrum/screencaster.rb @@ -1,45 +1,26 @@ # frozen_string_literal: true -require 'open3' -require 'stringio' -#TODO these ^ need to be added to gemfile -require 'concurrent-ruby' - module Ferrum class Screencaster - # attr_reader :message, :default_prompt - include Concurrent::Async - def initialize(page) - @page = page #might be wrong - @stdin = StringIO.new - @stdout = StringIO.new - @stderr = StringIO.new - @wait_thr = nil + @page = page end def add_frame(params) warn 'frame' @page.command("Page.screencastFrameAck", sessionId: params["sessionId"]) - img = params["data"] - img_decoded = Base64.decode64(img) - @stdin.write(img_decoded) + ts = (Time.now.to_f * 1000).to_i + File.binwrite("img_#{ts}.jpeg", Base64.decode64(params["data"])) end - def start_screencast #(options) - cmd = "ffmpeg -y -f image2pipe -i - -c:v libx264 -preset slow -crf 22 -r 1 -an -f mp4 -movflags +faststart output_video2.mp4" - @stdin, @stdout, @stderr, @wait_thr = Open3.popen3(cmd) - @page.command("Page.startScreencast", format: "jpeg") #, **options) + def start_screencast + @page.command("Page.startScreencast", format: "jpeg") end def stop_screencast warn 'stopped' @page.command("Page.stopScreencast") - @stdin.close - @stdout.close - @stderr.close - @wait_thr.join end end end From 6206429907b5fd7d254f6e1bd5598174287aee63 Mon Sep 17 00:00:00 2001 From: Peter Singh Date: Sat, 4 Jan 2025 14:16:43 +0000 Subject: [PATCH 4/5] Rubocop formatting --- lib/ferrum/page.rb | 2 +- lib/ferrum/page/screencast.rb | 3 ++- lib/ferrum/screencaster.rb | 32 ++++++++++++++++---------------- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/lib/ferrum/page.rb b/lib/ferrum/page.rb index d820841c..5ee121e3 100644 --- a/lib/ferrum/page.rb +++ b/lib/ferrum/page.rb @@ -454,7 +454,7 @@ def subscribe end end - on(:screencastFrame) do |params, index, total| + on(:screencastFrame) do |params, _index, _total| @screencaster.add_frame(params) warn "DEBUG: frame added in page.rb" end diff --git a/lib/ferrum/page/screencast.rb b/lib/ferrum/page/screencast.rb index ae05abf7..1ad3a02f 100644 --- a/lib/ferrum/page/screencast.rb +++ b/lib/ferrum/page/screencast.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true + require "ferrum/screencaster" module Ferrum @@ -6,7 +7,7 @@ class Page module Screencast attr_reader :screencaster - def start_screencast #(options) + def start_screencast @screencaster.start_screencast end diff --git a/lib/ferrum/screencaster.rb b/lib/ferrum/screencaster.rb index 3cbe169d..5482ed6d 100644 --- a/lib/ferrum/screencaster.rb +++ b/lib/ferrum/screencaster.rb @@ -2,25 +2,25 @@ module Ferrum class Screencaster - def initialize(page) - @page = page - end + def initialize(page) + @page = page + end - def add_frame(params) - warn 'frame' - @page.command("Page.screencastFrameAck", sessionId: params["sessionId"]) + def add_frame(params) + warn "frame" + @page.command("Page.screencastFrameAck", sessionId: params["sessionId"]) - ts = (Time.now.to_f * 1000).to_i - File.binwrite("img_#{ts}.jpeg", Base64.decode64(params["data"])) - end + ts = (Time.now.to_f * 1000).to_i + File.binwrite("img_#{ts}.jpeg", Base64.decode64(params["data"])) + end - def start_screencast - @page.command("Page.startScreencast", format: "jpeg") - end + def start_screencast + @page.command("Page.startScreencast", format: "jpeg") + end - def stop_screencast - warn 'stopped' - @page.command("Page.stopScreencast") - end + def stop_screencast + warn "stopped" + @page.command("Page.stopScreencast") + end end end From 7f0e35512d1451bb420565624c047f7b179a68e1 Mon Sep 17 00:00:00 2001 From: Peter Singh Date: Sat, 4 Jan 2025 16:28:38 +0000 Subject: [PATCH 5/5] Background writing frames to disk; put them in a dir; start from 0 to more easily use ffmpeg to create videos from them --- lib/ferrum/screencaster.rb | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/lib/ferrum/screencaster.rb b/lib/ferrum/screencaster.rb index 5482ed6d..c70ea716 100644 --- a/lib/ferrum/screencaster.rb +++ b/lib/ferrum/screencaster.rb @@ -1,24 +1,45 @@ # frozen_string_literal: true +require 'fileutils' +# Combine the resulting frames together into a video with: +# ffmpeg -y -framerate 30 -i 'frame-%d.jpeg' -c:v libx264 -r 30 -pix_fmt yuv420p try2.mp4 +# module Ferrum class Screencaster def initialize(page) @page = page + @frame_number = 0 + @threads = [] + @base_dir = "" end def add_frame(params) warn "frame" + @page.command("Page.screencastFrameAck", sessionId: params["sessionId"]) - ts = (Time.now.to_f * 1000).to_i - File.binwrite("img_#{ts}.jpeg", Base64.decode64(params["data"])) + t = Thread.new { File.binwrite("#{recordings_dir}/frame-#{@frame_number}.jpeg", Base64.decode64(params["data"])) } + @frame_number += 1 + @threads << t + true + end + + def recordings_dir + return @recordings_dir if defined? @recordings_dir + + session_id = @page.client.session_id + @recordings_dir = FileUtils.mkdir_p("#{@base_dir}/screencast_recordings/#{session_id}/").first + @recordings_dir end - def start_screencast + def start_screencast(base_dir = "") + @base_dir = base_dir @page.command("Page.startScreencast", format: "jpeg") end def stop_screencast + warn "joining threads" + @threads.each(&:join) warn "stopped" @page.command("Page.stopScreencast") end