v0.2.0 now with CLI and better output.

README.md

@@ -196,6 +196,76 @@ This method tunes hyperparameters to maximize or minimize the objective value wi
 
 - The concrete analysis, including baseline performance and the evaluation of the best parameters, is printed to the console.
 
+## Using the `cpsat-autotune` CLI
+
+The `cpsat-autotune` CLI is a command-line interface for tuning CP-SAT hyperparameters to optimize the performance of your models. Below are the instructions on how to use the CLI.
+
+### Commands
+
+The `cpsat-autotune` CLI provides two main commands: `time` and `quality`.
+
+#### `time` Command
+
+The `time` command tunes CP-SAT hyperparameters to minimize the time required to find an optimal solution.
+
+##### Usage
+
+```sh
+cpsat-autotune time [OPTIONS] MODEL_PATH
+```
+
+##### Options
+
+- `MODEL_PATH`: Path to the model file (required).
+- `--max-time`: Maximum time allowed for each solve operation in seconds (required).
+- `--relative-gap`: Relative optimality gap for considering a solution as optimal (default: 0.0).
+- `--n-trials`: Number of trials to execute in the tuning process (default: 100).
+- `--n-samples-trial`: Number of samples to take in each trial (default: 10).
+- `--n-samples-verification`: Number of samples for verifying parameters (default: 30).
+
+##### Example
+
+```sh
+cpsat-autotune time --max-time 60 --relative-gap 0.01 --n-trials 50 --n-samples-trial 5 --n-samples-verification 20 path/to/model/file
+```
+
+#### `quality` Command
+
+The `quality` command tunes CP-SAT hyperparameters to maximize or minimize solution quality within a given time limit.
+
+##### Usage
+
+```sh
+cpsat-autotune quality [OPTIONS] MODEL_PATH
+```
+
+##### Options
+
+- `MODEL_PATH`: Path to the model file (required).
+- `--max-time`: Time limit for each solve operation in seconds (required).
+- `--obj-for-timeout`: Objective value to return if the solver times out (required).
+- `--direction`: Direction to optimize the objective value (`maximize` or `minimize`, required).
+- `--n-trials`: Number of trials to execute in the tuning process (default: 100).
+- `--n-samples-trial`: Number of samples to take in each trial (default: 10).
+- `--n-samples-verification`: Number of samples for verifying parameters (default: 30).
+
+##### Example
+
+```sh
+cpsat-autotune quality --max-time 60 --obj-for-timeout 100 --direction maximize --n-trials 50 --n-samples-trial 5 --n-samples-verification 20 path/to/model/file
+```
+
+### Help
+
+For more information on each command and its options, you can use the `--help` flag:
+
+```sh
+cpsat-autotune time --help
+cpsat-autotune quality --help
+```
+
+This will display detailed descriptions and usage instructions for each command.
+
 ## The Importance of Avoiding Overfitting
 
 While tuning hyperparameters can improve solver performance for specific

pyproject.toml

@@ -7,7 +7,7 @@ where = ["src"]
 
 [project]
 name = "cpsat-autotune"
-version = "0.1.1"
+version = "0.2.0"
 authors = [
     { name = "Dominik Krupke", email = "[email protected]" },
 ]
@@ -20,5 +20,8 @@ classifiers = [
     "Development Status :: 3 - Alpha",
 ]
 dependencies = [
-    "optuna", "ortools", "numpy", "scipy", "rich"
+    "optuna", "ortools", "numpy", "scipy", "rich", "click"
 ]
+
+[project.scripts]
+cpsat-autotune = "cpsat_autotune.cli:cli"

src/cpsat_autotune/__main__.py

@@ -0,0 +1,4 @@
+from .cli import cli
+
+if __name__ == "__main__":
+    cli()

src/cpsat_autotune/cli.py

@@ -0,0 +1,177 @@
+import click
+from .model_loading import import_model
+from .tune import tune_time_to_optimal, tune_for_quality_within_timelimit
+
+import logging
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(levelname)s - %(message)s",
+    handlers=[logging.StreamHandler()],
+)
+logger = logging.getLogger(__name__)
+
+
+@click.group()
+def cli():
+    """CLI for CP-SAT hyperparameter tuning."""
+    pass
+
+
+def _estimate_time(max_time, n_trials, n_samples):
+    expected_time = max_time * n_samples * n_trials
+    # convert to hours and minutes
+    hours = int(expected_time // 3600)
+    minutes = int((expected_time % 3600) // 60)
+    if hours > 0:
+        logging.info(
+            f"The expected time for the tuning process is {hours} hours and {minutes} minutes."
+        )
+    else:
+        logging.info(f"The expected time for the tuning process is {minutes} minutes.")
+    logging.info(
+        "The tuning algorithm will try to take shortcuts whenever possible, potentially reducing the time drastically."
+    )
+    logging.info(
+        "To reduce the expected time, you can try to reduce the number of trials or samples per trial, as well as the maximum time allowed for each solve operation. However, this may affect the reliability of the tuning process."
+    )
+
+@click.command(
+    help="""
+    Tune CP-SAT hyperparameters to minimize the time required to find an optimal solution.
+
+    This command tunes the hyperparameters of a CP-SAT model to minimize the time required to find an optimal solution. 
+    You need to provide the path to the model file and specify the maximum time allowed for each solve operation, 
+    the relative optimality gap, and the number of trials and samples for the tuning process.
+    """
+)
+@click.argument("model_path", type=click.Path(exists=True))
+@click.option(
+    "--max-time",
+    type=float,
+    required=True,
+    help="Maximum time allowed for each solve operation in seconds."
+)
+@click.option(
+    "--relative-gap",
+    type=float,
+    default=0.0,
+    help="Relative optimality gap for considering a solution as optimal."
+)
+@click.option(
+    "--n-trials",
+    type=int,
+    default=100,
+    help="Number of trials to execute in the tuning process."
+)
+@click.option(
+    "--n-samples-trial",
+    type=int,
+    default=10,
+    help="Number of samples to take in each trial."
+)
+@click.option(
+    "--n-samples-verification",
+    type=int,
+    default=30,
+    help="Number of samples for verifying parameters."
+)
+def time(
+    model_path,
+    max_time,
+    relative_gap,
+    n_trials,
+    n_samples_trial,
+    n_samples_verification,
+):
+    """Tune CP-SAT hyperparameters to minimize the time required to find an optimal solution."""
+    _estimate_time(max_time, n_trials, n_samples_trial)
+    model = import_model(model_path)
+    best_params = tune_time_to_optimal(
+        model=model,
+        max_time_in_seconds=max_time,
+        relative_gap_limit=relative_gap,
+        n_samples_for_trial=n_samples_trial,
+        n_samples_for_verification=n_samples_verification,
+        n_trials=n_trials,
+    )
+    click.echo(f"Best parameters: {best_params}")
+
+
+@click.command(
+    help="""
+    Tune CP-SAT hyperparameters to maximize or minimize solution quality within a given time limit.
+
+    This command tunes the hyperparameters of a CP-SAT model to optimize the solution quality within a specified time limit. 
+    You need to provide the path to the model file, the maximum time allowed for each solve operation, the objective value 
+    to return if the solver times out, and the direction to optimize the objective value. Additionally, you can specify 
+    the number of trials and samples for the tuning process.
+    """
+)
+@click.argument("model_path", type=click.Path(exists=True))
+@click.option(
+    "--max-time",
+    type=float,
+    required=True,
+    help="Time limit for each solve operation in seconds."
+)
+@click.option(
+    "--obj-for-timeout",
+    type=int,
+    required=True,
+    help="Objective value to return if the solver times out."
+)
+@click.option(
+    "--direction",
+    type=click.Choice(["maximize", "minimize"]),
+    required=True,
+    help="Direction to optimize the objective value."
+)
+@click.option(
+    "--n-trials",
+    type=int,
+    default=100,
+    help="Number of trials to execute in the tuning process."
+)
+@click.option(
+    "--n-samples-trial",
+    type=int,
+    default=10,
+    help="Number of samples to take in each trial."
+)
+@click.option(
+    "--n-samples-verification",
+    type=int,
+    default=30,
+    help="Number of samples for verifying parameters."
+)
+def quality(
+    model_path,
+    max_time,
+    obj_for_timeout,
+    direction,
+    n_trials,
+    n_samples_trial,
+    n_samples_verification,
+):
+    """Tune CP-SAT hyperparameters to maximize or minimize solution quality within a given time limit."""
+    _estimate_time(max_time, n_trials, n_samples_trial)
+    model = import_model(model_path)
+    best_params = tune_for_quality_within_timelimit(
+        model=model,
+        max_time_in_seconds=max_time,
+        obj_for_timeout=obj_for_timeout,
+        direction=direction,
+        n_samples_for_trial=n_samples_trial,
+        n_samples_for_verification=n_samples_verification,
+        n_trials=n_trials,
+    )
+    click.echo(f"Best parameters: {best_params}")
+
+
+cli.add_command(time)
+cli.add_command(quality)
+
+if __name__ == "__main__":
+    cli()

src/cpsat_autotune/metrics.py

@@ -89,6 +89,16 @@ def comp(self, a: T, b: T, key: Callable[[T], Any] = lambda x: x) -> Comparison:
     def knockout_score(self) -> float:
         pass
 
+    def unit(self) -> str|None:
+        """
+        Returns the unit of the metric.
+        """
+        return None
+
+    @abstractmethod
+    def objective_name(self) -> str:
+        pass
+
 
 class MaxObjective(Metric):
     """
@@ -130,6 +140,9 @@ def __call__(
 
     def knockout_score(self) -> float:
         return self.obj_for_timeout
+
+    def objective_name(self) -> str:
+        return "Objective [MAX]"
 
 
 class MinObjective(Metric):
@@ -168,6 +181,9 @@ def __call__(
 
     def knockout_score(self) -> float:
         return self.obj_for_timeout
+
+    def objective_name(self) -> str:
+        return "Objective [MIN]"
 
 
 class MinTimeToOptimal(Metric):
@@ -222,3 +238,6 @@ def __call__(
 
     def knockout_score(self) -> float:
         return self.max_time_in_seconds * self.par_multiplier
+
+    def objective_name(self) -> str:
+        return "Time in seconds"

src/cpsat_autotune/print_result.py

@@ -4,6 +4,8 @@
 from rich.table import Table
 from rich.markdown import Markdown
 from rich.rule import Rule
+
+from .metrics import Metric
 from .caching_solver import MultiResult
 
 from .cpsat_parameters import get_parameter_by_name
@@ -13,7 +15,7 @@
 
 
 def print_results(
-    result, default_score: MultiResult, fn: Callable = console.print
+    result, default_score: MultiResult, metric: Metric,  fn: Callable = console.print
 ) -> None:
     """
     Prints the evaluation results in a professional format.
@@ -67,14 +69,14 @@ def print_results(
     metrics_table.add_column("#Samples", justify="right")
 
     metrics_table.add_row(
-        "Default Metric Value",
+        f"{metric.objective_name()} with Default Parameters",
         str(round(default_score.mean(), 2)),
         str(round(default_score.min(), 2)),
         str(round(default_score.max(), 2)),
         str(len(default_score)),
     )
     metrics_table.add_row(
-        "Optimized Metric Value",
+        f"{metric.objective_name()} with Optimized Parameters",
         str(round(result.optimized_score.mean(), 2)),
         str(round(result.optimized_score.min(), 2)),
         str(round(result.optimized_score.max(), 2)),

src/cpsat_autotune/tune.py

@@ -86,7 +86,7 @@ def _tune(
         n_samples_for_trial=n_samples_for_trial,
     )
     result = evaluator.evaluate()
-    print_results(result, default_baseline)
+    print_results(result, default_score=default_baseline, metric=metric)
 
     logger.info("Hyperparameter tuning completed.")
     return best_params