Bug 1878386 - Rewrite ui-test.sh into Python

taskcluster/ci/ui-test-apk/kind.yml

@@ -256,7 +256,7 @@ tasks:
             commands:
                 - [wget, {artifact-reference: '<signed-apk-debug-apk/public/build/target.arm64-v8a.apk>'}, '-O', app.apk]
                 - [wget, {artifact-reference: '<signed-apk-android-test/public/build/target.noarch.apk>'}, '-O', android-test.apk]
-                - [automation/taskcluster/androidTest/ui-test.sh, arm64-v8a, app.apk, android-test.apk, '100']
+                - [python3, ../taskcluster/scripts/tests/test-lab.py, arm64-v8a, app.apk, android-test.apk]
         treeherder:
             platform: 'fenix-ui-test/opt'
             symbol: fenix-debug(ui-test-arm)

taskcluster/scripts/tests/test-lab.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+# Firebase Test Lab (Flank) test runner script for Taskcluster
+# This script is used to run UI tests on Firebase Test Lab using Flank
+# It requires a service account key file to authenticate with Firebase Test Lab
+# The service account key file is stored in the `secrets` section of the task definition
+
+import argparse
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+# Define global constants
+JAVA_BIN = "/usr/bin/java"
+FLANK_BIN = "/builds/worker/test-tools/flank.jar"
+PATH_TEST = "./automation/taskcluster/androidTest"
+RESULTS_DIR = "/builds/worker/artifacts/results"
+ARTIFACTS_DIR = "/builds/worker/artifacts"
+
+
+def run_command(command, log_path=None):
+    """Execute a command, log its output, and check for errors."""
+    with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) as process:
+        if log_path:
+            with open(log_path, 'a') as log_file:
+                for line in process.stdout:
+                    sys.stdout.write(line)
+                    log_file.write(line)
+        else:
+            for line in process.stdout:
+                sys.stdout.write(line)
+        process.wait()
+        sys.stdout.flush()
+        if process.returncode != 0:
+            error_message = f"Command {' '.join(command)} failed with exit code {process.returncode}"
+            sys.exit(error_message)
+
+
+def setup_environment():
+    """Configure Google Cloud and authenticate with the service account."""
+    project_id = os.getenv('GOOGLE_PROJECT')
+    credentials_file = os.getenv('GOOGLE_APPLICATION_CREDENTIALS')
+    if not project_id or not credentials_file:
+        print("Error: GOOGLE_PROJECT and GOOGLE_APPLICATION_CREDENTIALS environment variables must be set.")
+        sys.exit(1)
+
+    print("Setting project and activating Google service account...")
+    run_command(['gcloud', 'config', 'set', 'project', project_id])
+    run_command(['gcloud', 'auth', 'activate-service-account', '--key-file', credentials_file])
+
+
+def execute_tests(flank_config, apk_app, apk_test):
+    """Run UI tests on Firebase Test Lab using Flank."""
+
+    print("\nFlank Version")
+    run_command([JAVA_BIN, '-jar', FLANK_BIN, '--version'])
+
+    print("\nExecuting test(s)...")
+    flank_command = [
+        JAVA_BIN, '-jar', FLANK_BIN, 'android', 'run',
+        '--config', f"{PATH_TEST}/flank-{flank_config}.yml",
+        '--app', str(apk_app),
+        '--test', str(apk_test),
+        '--local-result-dir', RESULTS_DIR,
+        '--project', os.environ.get('GOOGLE_PROJECT'),
+        '--client-details', f'matrixLabel={os.environ.get("PULL_REQUEST_NUMBER", "None")}']
+
+    run_command(flank_command, 'flank.log')
+    print("All UI test(s) have passed!")
+
+
+def process_results(path_test, results_dir, artifact_dir, flank_config, exitcode):
+    """Process test results and handle errors."""
+    # Ensure directories exist and scripts are executable
+    github_dir = os.path.join(artifact_dir, "github")
+    os.makedirs(github_dir, exist_ok=True)
+
+    parse_ui_test_script = os.path.join(path_test, "parse-ui-test.py")
+    parse_ui_test_fromfile_script = os.path.join(path_test, "parse-ui-test-fromfile.py")
+
+    os.chmod(parse_ui_test_script, 0o755)
+    os.chmod(parse_ui_test_fromfile_script, 0o755)
+
+    # Run parsing scripts and check for errors
+    run_command([parse_ui_test_script,
+                 '--exit-code', str(exitcode),
+                 '--log', 'flank.log',
+                 '--results', results_dir,
+                 '--output-md', os.path.join(github_dir, 'customCheckRunText.md'),
+                 '--device-type', flank_config], 'flank.log')
+
+    run_command([parse_ui_test_fromfile_script,
+                 '--results', results_dir], 'flank.log')
+
+
+def main():
+    """Parse command line arguments and execute the test runner."""
+    parser = argparse.ArgumentParser(description='Run UI tests on Firebase Test Lab using Flank as a testrunner')
+    parser.add_argument('flank_config', help='The configuration for Flank to use e.g, automation/taskcluster/androidTest/flank-<config>.yml')
+    parser.add_argument('apk_app', help='Absolute path to the APK application package')
+    parser.add_argument('apk_test', help='Absolute path to the APK androidTest package')
+    args = parser.parse_args()
+
+    setup_environment()
+    execute_tests(args.flank_config, Path(args.apk_app).resolve(), Path(args.apk_test).resolve())
+    process_results(path_test=PATH_TEST, results_dir=RESULTS_DIR, artifact_dir=ARTIFACTS_DIR, flank_config=args.flank_config, exitcode=0)
+
+
+if __name__ == "__main__":
+    main()