Merge pull request #372 from scrapinghub/issue-366-skip_fieldcoverage…

…monitor_no_item_returned Allow to skip execution of FieldCoverageMonitor if no item is returned
scrapinghub · Mar 3, 2023 · 234da8d · 234da8d
2 parents 558aace + a918c16
commit 234da8d
Show file tree

Hide file tree

Showing 2 changed files with 42 additions and 0 deletions.
diff --git a/spidermon/contrib/scrapy/monitors/monitors.py b/spidermon/contrib/scrapy/monitors/monitors.py
@@ -380,6 +380,9 @@ class FieldCoverageMonitor(BaseScrapyMonitor):
     You are not obligated to set rules for every field, just for the ones in which you are interested.
     Also, you can monitor nested fields if available in your returned items.
 
+    In case you have a job without items scraped, and you want to skip this test, you have to enable the
+    ``SPIDERMON_FIELD_COVERAGE_SKIP_IF_NO_ITEM`` setting to avoid the field coverage monitor error.
+
     .. warning::
 
        Rules for nested fields will be validated against the total number of items returned.
@@ -417,9 +420,17 @@ def run(self, result):
             raise NotConfigured(
                 "To enable field coverage monitor, set SPIDERMON_ADD_FIELD_COVERAGE=True in your project settings"
             )
+
         return super().run(result)
 
     def test_check_if_field_coverage_rules_are_met(self):
+        skip_no_items = self.crawler.settings.getbool(
+            "SPIDERMON_FIELD_COVERAGE_SKIP_IF_NO_ITEM", False
+        )
+        items_scraped = self.data.stats.get("item_scraped_count", 0)
+        if skip_no_items and int(items_scraped) == 0:
+            self.skipTest("No items were scraped.")
+
         failures = []
         field_coverage_rules = self.crawler.settings.getdict(
             "SPIDERMON_FIELD_COVERAGE_RULES"

diff --git a/tests/contrib/scrapy/test_monitors_field_coverage.py b/tests/contrib/scrapy/test_monitors_field_coverage.py
@@ -112,3 +112,34 @@ def test_monitor_pass_if_coverage_greater_than_expected(field_coverage_monitor_s
     monitor_runner.run(field_coverage_monitor_suite, **data)
 
     assert monitor_runner.result.wasSuccessful()
+
+
+def test_monitor_skip_if_no_items_set_true(field_coverage_monitor_suite):
+    settings = {
+        "SPIDERMON_ADD_FIELD_COVERAGE": True,
+        "SPIDERMON_FIELD_COVERAGE_RULES": {
+            "dict/field": 0.8,
+        },
+        "SPIDERMON_FIELD_COVERAGE_SKIP_IF_NO_ITEM": True,
+    }
+    stats = {"item_scraped_count": 0}
+    data = make_data_for_monitor(settings=settings, stats=stats)
+    monitor_runner = data.pop("runner")
+    monitor_runner.run(field_coverage_monitor_suite, **data)
+
+    assert monitor_runner.result.wasSuccessful()
+
+
+def test_monitor_skip_if_no_items_set_false(field_coverage_monitor_suite):
+    settings = {
+        "SPIDERMON_ADD_FIELD_COVERAGE": True,
+        "SPIDERMON_FIELD_COVERAGE_RULES": {
+            "dict/field": 0.8,
+        },
+    }
+    stats = {"item_scraped_count": 0}
+    data = make_data_for_monitor(settings=settings, stats=stats)
+    monitor_runner = data.pop("runner")
+    monitor_runner.run(field_coverage_monitor_suite, **data)
+
+    assert not monitor_runner.result.wasSuccessful()