Testing upstream

CHANGELOG.md

@@ -1,4 +1,20 @@
-## [Unreleased](https://github.com/rubycdp/ferrum/compare/v0.12...main) ##
+## [Unreleased](https://github.com/rubycdp/ferrum/compare/v0.13...main) ##
+
+### Added
+- `Ferrum::Network::Exchange#xhr?` determines if the exchange is XHR
+- `Ferrum::Network::Request#xhr?` determines if the request is XHR
+- `Ferrum::Network::Response#loaded?` returns true if the response is fully loaded
+- `Ferrum::Node#in_viewport?` checks if the element in viewport (optional argument `scope` as `Ferrum::Node`)
+- `Ferrum::Node#scroll_into_view` - scrolls to element if needed (when it's not in the viewport)
+
+### Changed
+
+### Fixed
+- `Ferrum::Network::Exchange#finished?` returns `true` only fully loaded responses
+
+### Removed
+
+## [0.13](https://github.com/rubycdp/ferrum/compare/v0.12...v0.13) - (Nov 12, 2022) ##
 
 ### Added
 

README.md

@@ -1128,6 +1128,8 @@ frame.at_css("//a[text() = 'Log in']") # => Node
 #### evaluate
 #### selected : `Array<Node>`
 #### select
+#### scroll_into_view
+#### in_viewport?(of: `Node | nil`) : `Boolean`
 
 (chainable) Selects options by passed attribute.
 

lib/ferrum/browser/client.rb

@@ -43,7 +43,7 @@ def command(method, params = {})
         @pendings.delete(message[:id])
 
         raise DeadBrowserError if data.nil? && @ws.messages.closed?
-        raise TimeoutError unless data
+        raise TimeoutError.new() unless data
 
         error, response = data.values_at("error", "result")
         raise_browser_error(error) if error

lib/ferrum/errors.rb

@@ -27,11 +27,17 @@ def initialize(url, pendings = [])
   end
 
   class TimeoutError < Error
-    def message
-      "Timed out waiting for response. It's possible that this happened " \
-        "because something took a very long time (for example a page load " \
-        "was slow). If so, setting the :timeout option to a higher value might " \
-        "help."
+    attr_reader :pending_connections_info
+
+    def initialize(pending_connections_info = [])
+      @pending_connections_info = pending_connections_info
+
+      message = "Timed out waiting for response. It's possible that this happened " \
+                "because something took a very long time (for example a page load " \
+                "was slow). If so, setting the :timeout option to a higher value might " \
+                "help."
+      message = "#{message}\nConnections still pending:\n #{pending_connections_info.join(', ')}" unless pending_connections_info.empty?
+      super(message)
     end
   end
 

lib/ferrum/frame.rb

@@ -113,6 +113,7 @@ def content=(html)
         document.close();
         arguments[1](true);
       ), @page.timeout, html)
+      @page.document_node_id
     end
     alias set_content content=
 

lib/ferrum/network.rb

@@ -11,9 +11,10 @@ module Ferrum
   class Network
     CLEAR_TYPE = %i[traffic cache].freeze
     AUTHORIZE_TYPE = %i[server proxy].freeze
-    RESOURCE_TYPES = %w[Document Stylesheet Image Media Font Script TextTrack
-                        XHR Fetch EventSource WebSocket Manifest
-                        SignedExchange Ping CSPViolationReport Other].freeze
+    REQUEST_STAGES = %i[Request Response].freeze
+    RESOURCE_TYPES = %i[Document Stylesheet Image Media Font Script TextTrack
+                        XHR Fetch Prefetch EventSource WebSocket Manifest
+                        SignedExchange Ping CSPViolationReport Preflight Other].freeze
     AUTHORIZE_BLOCK_MISSING = "Block is missing, call `authorize(...) { |r| r.continue } " \
                               "or subscribe to `on(:request)` events before calling it"
     AUTHORIZE_TYPE_WRONG = ":type should be in #{AUTHORIZE_TYPE}"
@@ -62,7 +63,15 @@ def wait_for_idle(connections: 0, duration: 0.05, timeout: @page.browser.timeout
       start = Utils::ElapsedTime.monotonic_time
 
       until idle?(connections)
-        raise TimeoutError if Utils::ElapsedTime.timeout?(start, timeout)
+        if Utils::ElapsedTime.timeout?(start, timeout)
+          if @page.browser.options.pending_connection_errors
+            pending_connections = traffic.select(&:pending?)
+            pending_connections_info = pending_connections.map(&:response).map do |connection|
+              {status: connection.status, status_text: connection.status_text, url: connection.url } unless connection.nil?
+            end
+          end
+          raise TimeoutError.new(pending_connections_info = pending_connections_info)
+        end
 
         sleep(duration)
       end
@@ -187,11 +196,20 @@ def whitelist=(patterns)
     #   end
     #   browser.go_to("https://google.com")
     #
-    def intercept(pattern: "*", resource_type: nil)
+    def intercept(pattern: "*", resource_type: nil, request_stage: nil, handle_auth_requests: true)
       pattern = { urlPattern: pattern }
-      pattern[:resourceType] = resource_type if resource_type && RESOURCE_TYPES.include?(resource_type.to_s)
 
-      @page.command("Fetch.enable", handleAuthRequests: true, patterns: [pattern])
+      if resource_type && RESOURCE_TYPES.none?(resource_type.to_sym)
+        raise ArgumentError, "Unknown resource type '#{resource_type}' must be #{RESOURCE_TYPES.join(' | ')}"
+      end
+
+      if request_stage && REQUEST_STAGES.none?(request_stage.to_sym)
+        raise ArgumentError, "Unknown request stage '#{request_stage}' must be #{REQUEST_STAGES.join(' | ')}"
+      end
+
+      pattern[:resourceType] = resource_type if resource_type
+      pattern[:requestStage] = request_stage if request_stage
+      @page.command("Fetch.enable", patterns: [pattern], handleAuthRequests: handle_auth_requests)
     end
 
     #
@@ -374,8 +392,12 @@ def subscribe_response_received
 
     def subscribe_loading_finished
       @page.on("Network.loadingFinished") do |params|
-        exchange = select(params["requestId"]).last
-        exchange.response.body_size = params["encodedDataLength"] if exchange&.response
+        response = select(params["requestId"]).last&.response
+
+        if response
+          response.loaded = true
+          response.body_size = params["encodedDataLength"]
+        end
       end
     end
 

lib/ferrum/network/exchange.rb

@@ -79,7 +79,7 @@ def blocked?
       # @return [Boolean]
       #
       def finished?
-        blocked? || !response.nil? || !error.nil?
+        blocked? || response&.loaded? || !error.nil?
       end
 
       #
@@ -100,6 +100,15 @@ def intercepted?
         !intercepted_request.nil?
       end
 
+      #
+      # Determines if the exchange is XHR.
+      #
+      # @return [Boolean]
+      #
+      def xhr?
+        !!request&.xhr?
+      end
+
       #
       # Returns request's URL.
       #

lib/ferrum/network/request.rb

@@ -50,6 +50,15 @@ def type?(value)
         type.downcase == value.to_s.downcase
       end
 
+      #
+      # Determines if the request is XHR.
+      #
+      # @return [Boolean]
+      #
+      def xhr?
+        type?("xhr")
+      end
+
       #
       # The frame ID of the request.
       #

lib/ferrum/network/response.rb

@@ -18,8 +18,13 @@ class Response
       # @return [Hash{String => Object}]
       attr_reader :params
 
+      # The response is fully loaded by the browser.
       #
-      # Initializes the respones object.
+      # @return [Boolean]
+      attr_writer :loaded
+
+      #
+      # Initializes the responses object.
       #
       # @param [Page] page
       #   The page associated with the network response.
@@ -121,9 +126,8 @@ def body_size=(size)
       #
       def body
         @body ||= begin
-          body, encoded = @page
-                          .command("Network.getResponseBody", requestId: id)
-                          .values_at("body", "base64Encoded")
+          body, encoded = @page.command("Network.getResponseBody", requestId: id)
+                               .values_at("body", "base64Encoded")
           encoded ? Base64.decode64(body) : body
         end
       end
@@ -135,6 +139,13 @@ def main?
         @page.network.response == self
       end
 
+      # The response is fully loaded by the browser or not.
+      #
+      # @return [Boolean]
+      def loaded?
+        @loaded
+      end
+
       #
       # Comapres the respones ID to another response's ID.
       #

lib/ferrum/node.rb

@@ -88,6 +88,26 @@ def hover
       raise NotImplementedError
     end
 
+    def scroll_into_view
+      tap { page.command("DOM.scrollIntoViewIfNeeded", nodeId: node_id) }
+    end
+
+    def in_viewport?(of: nil)
+      function = <<~JS
+        function(element, scope) {
+          const rect = element.getBoundingClientRect();
+          const [height, width] = scope
+            ? [scope.offsetHeight, scope.offsetWidth]
+            : [window.innerHeight, window.innerWidth];
+          return rect.top >= 0 &&
+           rect.left >= 0 &&
+           rect.bottom <= height &&
+           rect.right <= width;
+        }
+      JS
+      page.evaluate_func(function, self, of)
+    end
+
     def select_file(value)
       page.command("DOM.setFileInputFiles", slowmoable: true, nodeId: node_id, files: Array(value))
     end

lib/ferrum/page.rb

@@ -282,7 +282,7 @@ def command(method, wait: 0, slowmoable: false, **params)
         @event.wait(wait)
         if iteration != @event.iteration
           set = @event.wait(timeout)
-          raise TimeoutError unless set
+          raise TimeoutError.new() unless set
         end
       end
       result
@@ -325,6 +325,10 @@ def use_authorized_proxy?
       use_proxy? && @proxy_user && @proxy_password
     end
 
+    def document_node_id
+      command("DOM.getDocument", depth: 0).dig("root", "nodeId")
+    end
+
     private
 
     def subscribe
@@ -441,10 +445,6 @@ def combine_url!(url_or_path)
       (nil_or_relative ? @browser.base_url.join(url.to_s) : url).to_s
     end
 
-    def document_node_id
-      command("DOM.getDocument", depth: 0).dig("root", "nodeId")
-    end
-
     def ws_url
       "ws://#{@browser.process.host}:#{@browser.process.port}/devtools/page/#{@target_id}"
     end

spec/frame_spec.rb

@@ -181,9 +181,10 @@
   end
 
   it "can set page content" do
-    browser.content = "<html><head></head><body>Voila!</body></html>"
+    browser.content = "<html><head></head><body>Voila! <a href='#'>Link</a></body></html>"
 
     expect(browser.body).to include("Voila!")
+    expect(browser.at_css("a").text).to eq("Link")
   end
 
   it "gets page doctype" do

spec/network/response_spec.rb

@@ -81,7 +81,7 @@
         %r{/ferrum/jquery.min.js$} => File.size("#{PROJECT_ROOT}/spec/support/public/jquery-1.11.3.min.js"),
         %r{/ferrum/jquery-ui.min.js$} => File.size("#{PROJECT_ROOT}/spec/support/public/jquery-ui-1.11.4.min.js"),
         %r{/ferrum/test.js$} => File.size("#{PROJECT_ROOT}/spec/support/public/test.js"),
-        %r{/ferrum/with_js$} => 2343
+        %r{/ferrum/with_js$} => 2312
       }
 
       resources_size.each do |resource, size|

spec/network_spec.rb

@@ -302,6 +302,55 @@
   end
 
   describe "#intercept" do
+    it "supports :pattern argument" do
+      network.intercept(pattern: "*/ferrum/frame_child")
+      page.on(:request) do |request|
+        request.respond(body: "<h1>hello</h1>")
+      end
+
+      page.go_to("/ferrum/frame_parent")
+
+      expect(network.status).to eq(200)
+      frame = page.at_xpath("//iframe").frame
+      expect(frame.body).to include("hello")
+    end
+
+    context "with :resource_type argument" do
+      it "raises an error with wrong type" do
+        expect { network.intercept(resource_type: :BlaBla) }.to raise_error(ArgumentError)
+      end
+
+      it "intercepts only given type" do
+        network.intercept(resource_type: :Document)
+        page.on(:request) do |request|
+          request.respond(body: "<h1>hello</h1>")
+        end
+
+        page.go_to("/ferrum/non_existing")
+
+        expect(network.status).to eq(200)
+        expect(page.body).to include("hello")
+      end
+    end
+
+    context "with :request_stage argument" do
+      it "raises an error with wrong stage" do
+        expect { network.intercept(request_stage: :BlaBla) }.to raise_error(ArgumentError)
+      end
+
+      it "intercepts only given stage" do
+        network.intercept(request_stage: :Response)
+        page.on(:request) do |request|
+          request.respond(body: "<h1>hello</h1>")
+        end
+
+        page.go_to("/ferrum/index")
+
+        expect(network.status).to eq(200)
+        expect(page.body).to include("hello")
+      end
+    end
+
     it "supports custom responses" do
       network.intercept
       page.on(:request) do |request|