Merge pull request #516 from shubhbapna/build-sequence-cache-wheel-se…

…rver add `--cache-wheel-server-url` and `--force` options to build-sequence
python-wheel-build · Dec 6, 2024 · 2f76a26 · 2f76a26
2 parents a58d2de + 95d2311
commit 2f76a26
Show file tree

Hide file tree

Showing 2 changed files with 45 additions and 55 deletions.
diff --git a/e2e/test_build_order.sh b/e2e/test_build_order.sh
@@ -21,15 +21,15 @@ fromager \
 # Save the build order file but remove everything else.
 cp "$OUTDIR/work-dir/build-order.json" "$OUTDIR/"
 
-# Rebuild everything
+# Rebuild everything even if it already exists
 log="$OUTDIR/build-logs/${DIST}-build.log"
 fromager \
     --log-file "$log" \
     --work-dir "$OUTDIR/work-dir" \
     --sdists-repo "$OUTDIR/sdists-repo" \
     --wheels-repo "$OUTDIR/wheels-repo" \
     --settings-dir="$SCRIPTDIR/changelog_settings" \
-    build-sequence "$OUTDIR/build-order.json"
+    build-sequence --force "$OUTDIR/build-order.json"
 
 find "$OUTDIR/wheels-repo/"
 
@@ -63,15 +63,15 @@ done
 
 $pass
 
-# Rebuild everything with the skip flag and verify we reuse the existing wheels
+# Rebuild everything while reusing existing local wheels
 log="$OUTDIR/build-logs/${DIST}-build-skip.log"
 fromager \
     --log-file "$log" \
     --work-dir "$OUTDIR/work-dir" \
     --sdists-repo "$OUTDIR/sdists-repo" \
     --wheels-repo "$OUTDIR/wheels-repo" \
     --settings-dir="$SCRIPTDIR/changelog_settings" \
-    build-sequence --skip-existing "$OUTDIR/build-order.json"
+    build-sequence "$OUTDIR/build-order.json"
 
 find "$OUTDIR/wheels-repo/"
 
@@ -86,16 +86,15 @@ fi
 
 $pass
 
-# Rebuild everything with the skip env var and verify we reuse the existing wheels
-export FROMAGER_BUILD_SEQUENCE_SKIP_EXISTING=true
+# Rebuild everything while reusing wheels from external server
+rm -rf $OUTDIR/wheels-repo
 log="$OUTDIR/build-logs/${DIST}-build-skip-env.log"
 fromager \
     --log-file "$log" \
     --work-dir "$OUTDIR/work-dir" \
     --sdists-repo "$OUTDIR/sdists-repo" \
     --wheels-repo "$OUTDIR/wheels-repo" \
-    --settings-dir="$SCRIPTDIR/changelog_settings" \
-    build-sequence "$OUTDIR/build-order.json"
+    build-sequence --cache-wheel-server-url="https://pypi.org/simple" "$OUTDIR/build-order.json"
 
 find "$OUTDIR/wheels-repo/"
 
@@ -109,38 +108,3 @@ if ! grep -q "skipping building wheels for stevedore" "$log"; then
 fi
 
 $pass
-
-# bootstrap stevedore with 2 changelog.
-log="$OUTDIR/build-logs/${DIST}-build-changelog.log"
-fromager \
-    --log-file "$log" \
-    --work-dir "$OUTDIR/work-dir" \
-    --sdists-repo "$OUTDIR/sdists-repo" \
-    --wheels-repo "$OUTDIR/wheels-repo" \
-    --settings-dir="$SCRIPTDIR/changelog_settings-2" \
-    build-sequence --skip-existing "$OUTDIR/build-order.json"
-
-find "$OUTDIR/wheels-repo/"
-
-if grep -q "skipping building wheels for stevedore" "$log"; then
-  echo "Found message indicating build of stevedore was skipped" 1>&2
-  pass=false
-fi
-
-find "$OUTDIR/wheels-repo/"
-
-EXPECTED_FILES="
-$OUTDIR/wheels-repo/downloads/stevedore-5.2.0-2*.whl
-
-$OUTDIR/sdists-repo/downloads/stevedore-*.tar.gz
-"
-
-pass=true
-for pattern in $EXPECTED_FILES; do
-  if [ ! -f "${pattern}" ]; then
-    echo "Did not find $pattern" 1>&2
-    pass=false
-  fi
-done
-
-$pass
diff --git a/src/fromager/commands/build.py b/src/fromager/commands/build.py
@@ -106,17 +106,26 @@ def build(
 
 
 @click.command()
-@click.argument("build_order_file")
 @click.option(
-    "--skip-existing",
-    default=False,
+    "-f",
+    "--force",
     is_flag=True,
+    default=False,
+    help="rebuild wheels even if they have already been built",
 )
+@click.option(
+    "-c",
+    "--cache-wheel-server-url",
+    "cache_wheel_server_url",
+    help="url to a wheel server from where fromager can check if it had already built the wheel",
+)
+@click.argument("build_order_file")
 @click.pass_obj
 def build_sequence(
     wkctx: context.WorkContext,
     build_order_file: str,
-    skip_existing: bool,
+    force: bool,
+    cache_wheel_server_url: str | None,
 ) -> None:
     """Build a sequence of wheels in order
 
@@ -129,10 +138,16 @@ def build_sequence(
 
     """
     server.start_wheel_server(wkctx)
-    if skip_existing:
+    wheel_server_urls = [wkctx.wheel_server_url]
+    if cache_wheel_server_url:
+        # put after local server so we always check local server first
+        wheel_server_urls.append(cache_wheel_server_url)
+
+    if force:
+        logger.info(f"rebuilding all wheels even if they exist in {wheel_server_urls}")
+    else:
         logger.info(
-            "skipping builds for versions of packages available at %s",
-            wkctx.wheel_server_url,
+            f"skipping builds for versions of packages available at {wheel_server_urls}"
         )
 
     entries: list[BuildSequenceEntry] = []
@@ -147,9 +162,9 @@ def build_sequence(
 
             req = Requirement(f"{dist_name}=={resolved_version}")
 
-            if skip_existing:
+            if not force:
                 is_built, wheel_filename = _is_wheel_built(
-                    wkctx, dist_name, resolved_version
+                    wkctx, dist_name, resolved_version, wheel_server_urls
                 )
                 if is_built:
                     logger.info(
@@ -375,14 +390,19 @@ def _build(
 
 
 def _is_wheel_built(
-    wkctx: context.WorkContext, dist_name: str, resolved_version: Version
+    wkctx: context.WorkContext,
+    dist_name: str,
+    resolved_version: Version,
+    wheel_server_urls: list[str],
 ) -> tuple[True, pathlib.Path] | tuple[False, None]:
     req = Requirement(f"{dist_name}=={resolved_version}")
 
     try:
         logger.info(f"{req.name}: checking if {req} was already built")
         url, _ = wheels.resolve_prebuilt_wheel(
-            ctx=wkctx, req=req, wheel_server_urls=[wkctx.wheel_server_url]
+            ctx=wkctx,
+            req=req,
+            wheel_server_urls=wheel_server_urls,
         )
         pbi = wkctx.package_build_info(req)
         build_tag_from_settings = pbi.build_tag(resolved_version)
@@ -397,7 +417,13 @@ def _is_wheel_built(
             raise ValueError(
                 f"{dist_name}: changelog for version {resolved_version} is inconsistent. Found build tag {existing_build_tag} but expected {build_tag}"
             )
-        return existing_build_tag == build_tag, pathlib.Path(wheel_filename)
+        is_built = existing_build_tag == build_tag
+        if is_built and wkctx.wheel_server_url not in url:
+            # if the found wheel was on an external server, then download it
+            wheels.download_wheel(req, url, wkctx.wheels_downloads)
+            server.update_wheel_mirror(wkctx)
+
+        return is_built, pathlib.Path(wheel_filename)
     except Exception:
         logger.info(f"{req.name}: could not locate prebuilt wheel. Will build {req}")
         return False, None