Fix issues with postprocessed data in label repair and data discard s…

…cenarios.

datascope/importance/common.py

@@ -573,6 +573,9 @@ def _process_metric_score_inputs(
         y_test: Union[NDArray, Series, DataFrame],
         y_pred: Union[NDArray, Series, DataFrame],
         y_pred_proba: Optional[Union[NDArray, Series, DataFrame]] = None,
+        *,
+        metric_requires_probabilities: bool = False,
+        classes: Optional[List[Hashable]] = None,
     ) -> Tuple[NDArray, NDArray, Optional[NDArray]]:
 
         # If any of the prediction inputs are Series, then we assume they are either series of scalar values
@@ -597,6 +600,14 @@ def _process_metric_score_inputs(
         if y_test.ndim == 2:
             y_test = np.argmax(y_test, axis=1)
 
+        # If the metric required probabilities but they were not provided, we use one-hot encoding to obtain them.
+        if metric_requires_probabilities and y_pred_proba is None:
+            if classes is None:
+                assert isinstance(y_test, np.ndarray)
+                classes = np.unique(y_test)
+            assert isinstance(y_pred, np.ndarray)
+            y_pred_proba = _one_hot_encode_probabilities(y_pred, classes)
+
         # If the predions are given as a 2D array of probabilities and there are only two classes, then we need to
         # select the probabilities for the positive class.
         if y_pred_proba is not None and y_pred_proba.shape[1] == 2:
@@ -621,7 +632,7 @@ def _metric_score(
         if metric is None:
             raise ValueError("The metric was not provided.")
         y_test_processed, y_pred_processed, y_pred_proba_processed = self._process_metric_score_inputs(
-            y_test, y_pred, y_pred_proba
+            y_test, y_pred, y_pred_proba, metric_requires_probabilities=metric_requires_probabilities, classes=classes
         )
         if metric_requires_probabilities:
             if y_pred_proba_processed is None:

experiments/datascope/experiments/scenarios/data_discard.py

@@ -52,7 +52,7 @@
     DEFAULT_BIAS_METHOD,
     DEFAULT_CACHE_DIR,
 )
-from ..pipelines import Pipeline, FlattenPipeline, get_model, DistanceModelMixin
+from ..pipelines import Pipeline, FlattenPipeline, get_model, DistanceModelMixin, Postprocessor
 
 
 DEFAULT_MAX_REMOVE = 0.5
@@ -219,45 +219,46 @@ def _run(self, progress_bar: bool = True, **kwargs: Any) -> None:
         # pipeline.steps.append(("model", model))
         # if RepairMethod.is_pipe(self.method):
         #     model = model_pipeline
-        accuracy_utility = SklearnModelAccuracy(model, postprocessor=self.postprocessor)
-        roc_auc_utility = SklearnModelRocAuc(model, postprocessor=self.postprocessor)
+        postprocessor = None if self.postprocessor is None else Postprocessor.postprocessors[self.postprocessor]()
+        accuracy_utility = SklearnModelAccuracy(model, postprocessor=postprocessor)
+        roc_auc_utility = SklearnModelRocAuc(model, postprocessor=postprocessor)
         eqodds_utility: Optional[SklearnModelEqualizedOddsDifference] = None
         if self.repairgoal == RepairGoal.FAIRNESS:
             assert isinstance(dataset, BiasedMixin)
             eqodds_utility = SklearnModelEqualizedOddsDifference(
                 model,
                 sensitive_features=dataset.sensitive_feature,
                 groupings=groupings_test,
-                postprocessor=self.postprocessor,
+                postprocessor=postprocessor,
             )
 
         # target_model = model if RepairMethod.is_tmc_nonpipe(self.method) else pipeline
         target_utility: Utility
         if self.utility == UtilityType.ACCURACY:
-            target_utility = JointUtility(SklearnModelAccuracy(model, postprocessor=self.postprocessor), weights=[-1.0])
+            target_utility = JointUtility(SklearnModelAccuracy(model, postprocessor=postprocessor), weights=[-1.0])
             # target_utility = SklearnModelAccuracy(model)
         elif self.utility == UtilityType.EQODDS:
             assert self.repairgoal == RepairGoal.FAIRNESS and isinstance(dataset, BiasedMixin)
             target_utility = SklearnModelEqualizedOddsDifference(
                 model,
                 sensitive_features=dataset.sensitive_feature,
                 groupings=groupings_val,
-                postprocessor=self.postprocessor,
+                postprocessor=postprocessor,
             )
         elif self.utility == UtilityType.EQODDS_AND_ACCURACY:
             assert self.repairgoal == RepairGoal.FAIRNESS and isinstance(dataset, BiasedMixin)
             target_utility = JointUtility(
-                SklearnModelAccuracy(model, postprocessor=self.postprocessor),
+                SklearnModelAccuracy(model, postprocessor=postprocessor),
                 SklearnModelEqualizedOddsDifference(
                     model,
                     sensitive_features=dataset.sensitive_feature,
                     groupings=groupings_val,
-                    postprocessor=self.postprocessor,
+                    postprocessor=postprocessor,
                 ),
                 weights=[-0.5, 0.5],
             )
         elif self.utility == UtilityType.ROC_AUC:
-            target_utility = JointUtility(SklearnModelRocAuc(model, postprocessor=self.postprocessor), weights=[-1.0])
+            target_utility = JointUtility(SklearnModelRocAuc(model, postprocessor=postprocessor), weights=[-1.0])
         else:
             raise ValueError("Unknown utility type '%s'." % repr(self.utility))
 
@@ -304,7 +305,7 @@ def _run(self, progress_bar: bool = True, **kwargs: Any) -> None:
         importance_time_end = process_time_ns()
         importance_cputime = (importance_time_end - importance_time_start) / 1e9
         self.logger.debug("Importance computed in: %s", str(timedelta(seconds=importance_cputime)))
-        n_units = dataset.trainsize
+        n_units = dataset.provenance.num_units
         discarded_units = np.zeros(n_units, dtype=bool)
         argsorted_importances = (-np.array(importances)).argsort()
         # argsorted_importances = np.ma.array(importances, mask=discarded_units).argsort()
@@ -353,7 +354,7 @@ def _run(self, progress_bar: bool = True, **kwargs: Any) -> None:
                 0,
                 0.0,
                 dataset_bias,
-                dataset.y_train.mean(),
+                # dataset.y_train.mean(),
                 0.0,
             ]
         ]
@@ -468,7 +469,7 @@ def _run(self, progress_bar: bool = True, **kwargs: Any) -> None:
             discarded = discarded_units.sum(dtype=int)
             discarded_rel = discarded / float(n_units)
             dataset_bias = dataset_current.train_bias if isinstance(dataset_current, BiasedMixin) else None
-            mean_label = np.mean(dataset_current.y_train)
+            # mean_label = np.mean(dataset_current.y_train)
             evolution.append(
                 [
                     steps_rel,
@@ -487,7 +488,7 @@ def _run(self, progress_bar: bool = True, **kwargs: Any) -> None:
                     discarded,
                     discarded_rel,
                     dataset_bias,
-                    mean_label,
+                    # mean_label,
                     importance_cputime,
                 ]
             )
@@ -530,7 +531,7 @@ def _run(self, progress_bar: bool = True, **kwargs: Any) -> None:
                 "discarded",
                 "discarded_rel",
                 "dataset_bias",
-                "mean_label",
+                # "mean_label",
                 "importance_cputime",
             ],
         )

experiments/datascope/experiments/scenarios/label_repair.py

@@ -51,7 +51,7 @@
     DEFAULT_TESTSIZE,
     DEFAULT_CACHE_DIR,
 )
-from ..pipelines import Pipeline, FlattenPipeline, get_model, DistanceModelMixin
+from ..pipelines import Pipeline, FlattenPipeline, get_model, DistanceModelMixin, Postprocessor
 
 
 DEFAULT_DIRTY_RATIO = 0.5
@@ -253,51 +253,52 @@ def _run(self, progress_bar: bool = True, **kwargs: Any) -> None:
         # pipeline.steps.append(("model", model))
         # if RepairMethod.is_pipe(self.method):
         #     model = model_pipeline
-        accuracy_utility = SklearnModelAccuracy(model, postprocessor=self.postprocessor)
-        roc_auc_utility = SklearnModelRocAuc(model, postprocessor=self.postprocessor)
+        postprocessor = None if self.postprocessor is None else Postprocessor.postprocessors[self.postprocessor]()
+        accuracy_utility = SklearnModelAccuracy(model, postprocessor=postprocessor)
+        roc_auc_utility = SklearnModelRocAuc(model, postprocessor=postprocessor)
         eqodds_utility: Optional[SklearnModelEqualizedOddsDifference] = None
         if self.repairgoal == RepairGoal.FAIRNESS:
             assert isinstance(dataset, BiasedNoisyLabelDataset)
             eqodds_utility = SklearnModelEqualizedOddsDifference(
                 model,
                 sensitive_features=dataset.sensitive_feature,
                 groupings=groupings_test,
-                postprocessor=self.postprocessor,
+                postprocessor=postprocessor,
             )
 
         # target_model = model if RepairMethod.is_tmc_nonpipe(self.method) else pipeline
         target_utility: Utility
         if self.utility == UtilityType.ACCURACY:
-            target_utility = JointUtility(SklearnModelAccuracy(model, postprocessor=self.postprocessor), weights=[-1.0])
+            target_utility = JointUtility(SklearnModelAccuracy(model, postprocessor=postprocessor), weights=[-1.0])
             # target_utility = SklearnModelAccuracy(model)
         elif self.utility == UtilityType.EQODDS:
             assert self.repairgoal == RepairGoal.FAIRNESS and isinstance(dataset, BiasedNoisyLabelDataset)
             target_utility = SklearnModelEqualizedOddsDifference(
                 model,
                 sensitive_features=dataset.sensitive_feature,
                 groupings=groupings_val,
-                postprocessor=self.postprocessor,
+                postprocessor=postprocessor,
             )
         elif self.utility == UtilityType.EQODDS_AND_ACCURACY:
             assert self.repairgoal == RepairGoal.FAIRNESS and isinstance(dataset, BiasedNoisyLabelDataset)
             target_utility = JointUtility(
-                SklearnModelAccuracy(model, postprocessor=self.postprocessor),
+                SklearnModelAccuracy(model, postprocessor=postprocessor),
                 SklearnModelEqualizedOddsDifference(
                     model,
                     sensitive_features=dataset.sensitive_feature,
                     groupings=groupings_val,
-                    postprocessor=self.postprocessor,
+                    postprocessor=postprocessor,
                 ),
                 weights=[-0.5, 0.5],
             )
         elif self.utility == UtilityType.ROC_AUC:
-            target_utility = JointUtility(SklearnModelRocAuc(model, postprocessor=self.postprocessor), weights=[-1.0])
+            target_utility = JointUtility(SklearnModelRocAuc(model, postprocessor=postprocessor), weights=[-1.0])
         else:
             raise ValueError("Unknown utility type '%s'." % repr(self.utility))
 
         # Compute importance scores and time it.
         importance_time_start = process_time_ns()
-        n_units = dataset_dirty.units.shape[0]
+        n_units = dataset.provenance.num_units
         importance: Optional[Importance] = None
         importances: Optional[Iterable[float]] = None
         random = np.random.RandomState(seed=self._seed + self._iteration + 1)