diff --git a/zyte_common_items/log_formatters.py b/zyte_common_items/log_formatters.py new file mode 100644 index 0000000..863b5f3 --- /dev/null +++ b/zyte_common_items/log_formatters.py @@ -0,0 +1,17 @@ +from logging import INFO + +from scrapy.exceptions import DropItem +from scrapy.logformatter import LogFormatter + + +class InfoDropItem(DropItem): + """DropItem subclass for items that should be dropped with an INFO message + (instead of the default WARNING message).""" + + +class ZyteLogFormatter(LogFormatter): + def dropped(self, item, exception, response, spider): + data = super().dropped(item, exception, response, spider) + if isinstance(exception, InfoDropItem): + data["level"] = INFO + return data diff --git a/zyte_common_items/pipelines.py b/zyte_common_items/pipelines.py index ed564aa..64dc225 100644 --- a/zyte_common_items/pipelines.py +++ b/zyte_common_items/pipelines.py @@ -4,9 +4,8 @@ from copy import deepcopy from warnings import warn -from scrapy.exceptions import DropItem - -from zyte_common_items import ae +from .ae import ae +from .log_formatters import InfoDropItem class AEPipeline: @@ -61,11 +60,6 @@ def process_item(self, item, spider): return ae.downgrade(item) -class InfoDropItem(DropItem): - """DropItem subclass for items that should be dropped with an INFO message - (instead of the default WARNING message).""" - - class DropLowProbabilityItemPipeline: """This pipeline drops an item if its probability, defined in the settings, is less than the specified threshold.