Skip to content

Commit

Permalink
fix: partition evaluator thread safety
Browse files Browse the repository at this point in the history
  • Loading branch information
skellys committed Oct 30, 2023
1 parent d92bc18 commit 24f0939
Showing 1 changed file with 6 additions and 2 deletions.
8 changes: 6 additions & 2 deletions pyiceberg/table/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -828,8 +828,12 @@ def _build_partition_evaluator(self, spec_id: int) -> Callable[[DataFile], bool]
partition_schema = Schema(*partition_type.fields)
partition_expr = self.partition_filters[spec_id]

evaluator = visitors.expression_evaluator(partition_schema, partition_expr, self.case_sensitive)
return lambda data_file: evaluator(data_file.partition)
# The lambda created here is run in multiple threads.
# So we avoid creating _EvaluatorExpression methods bound to a single
# shared instance across multiple threads.
return lambda data_file: visitors.expression_evaluator(partition_schema, partition_expr, self.case_sensitive)(
data_file.partition
)

def _check_sequence_number(self, min_data_sequence_number: int, manifest: ManifestFile) -> bool:
"""Ensure that no manifests are loaded that contain deletes that are older than the data.
Expand Down

0 comments on commit 24f0939

Please sign in to comment.