CI: Add CI for package index sync

.github/workflows/package-index-sync.yaml

@@ -0,0 +1,106 @@
+name: Sync Package Index
+
+# Below is the configuration and variables that you need to set up.
+# 1. A environment named `ruyi-sync` is required.
+# 2. Some variables are required to be set in the environments:
+# - GHO_TOKEN: is described in [docs](https://github.com/ruyisdk/support-matrix/blob/main/assets/docs/ruyi_index_updator.md) as GITHUB_TOKEN (rename due to a same name default var in CI env)
+# - PACKAGE_INDEX_OWNER: decide where the pr goes: blank defaults to `ruyisdk`
+# - SSH_PRIVATE: A private SSH key of the bot account
+
+on:
+  pull_request:
+  workflow_dispatch:
+    inputs:
+      makepr:
+        description: 'Make a PR to sync package index'
+        required: false
+        default: 'false'
+      debuginfo:
+        description: 'Output Debug'
+        required: false
+        default: 'false'
+  # push: # Uncomment this line to enable push event, but it is not recommended as it may cause unnecessary PRs. Use workflow_dispatch instead.
+
+jobs:
+  build:
+    name: Generate and Upload
+    runs-on: ubuntu-latest
+    environment: ruyi-sync
+    env:
+      GITHUB_TOKEN: ${{ secrets.GHO_TOKEN }}
+      CACHE_DIR: ~/cache
+      PACKAGE_INDEX_OWNER: ${{ vars.PACKAGE_INDEX_OWNER }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+          cache: 'pip'
+          cache-dependency-path: '**/requirements*.txt'
+      - uses: webfactory/[email protected]
+        with:
+          ssh-private-key: ${{ secrets.SSH_PRIVATE }}
+      - name: Load Cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/cache
+          key: ${{ runner.os }}-package-index-sync
+      - name: Install Dependencies
+        run: |
+          sudo apt-get update
+          pip install -r assets/requirements_ruyinv.txt
+      - name: Run tool to generate and upload
+        if: ${{ ( github.event_name == 'workflow_dispatch' && inputs.makepr == false ) || github.event_name != 'pull_request' || ( github.event_name == 'pull_request' && github.event.pull_request.merged == false ) }}
+        run: |
+          echo "Generate Only" > $RUNNER_TEMP/type.txt
+          export CI_RUN_ID=${{ github.run_id }}
+          export CI_RUN_URL=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          python assets/renew_ruyi_index.py -c assets/config.toml -p . -i $RUNNER_TEMP/cache --log $RUNNER_TEMP/log.txt --warn $RUNNER_TEMP/warn.txt
+      - name: Run tool to generate and upload and PR
+        if: ${{ ( github.event_name == 'workflow_dispatch' && inputs.makepr == true ) || ( github.event_name == 'pull_request' && github.event.pull_request.merged == true ) }}
+        run: |
+          echo "Generate and PR" > $RUNNER_TEMP/type.txt
+          export CI_RUN_ID=${{ github.run_id }}
+          export CI_RUN_URL=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          python assets/renew_ruyi_index.py -c assets/config.toml -p . -i $RUNNER_TEMP/cache --log $RUNNER_TEMP/log.txt --warn $RUNNER_TEMP/warn.txt --pr
+      - name: Output Summary
+        run: |
+          echo "# Package Index Sync Report" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "[CI #${{ github.run_number }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) powered by [${{ github.repository }}](${{ github.server_url }}/${{ github.repository }})" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "Type: $(cat $RUNNER_TEMP/type.txt)" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "CI has built on commit [${{ steps.truecommit.outputs.shortid }}](${{ github.server_url }}/${{ github.repository }}/commit/${{ steps.truecommit.outputs.longid }}). The commit message is shown below." >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "\`\`\`text" >> $GITHUB_STEP_SUMMARY
+          echo "$(git log -1)" >> $GITHUB_STEP_SUMMARY
+          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "## Warn Log" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "\`\`\`text" >> $GITHUB_STEP_SUMMARY
+          cat $RUNNER_TEMP/warn.txt >> $GITHUB_STEP_SUMMARY
+          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "## Build Log" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "<details><summary>Click to expand</summary>" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "\`\`\`text" >> $GITHUB_STEP_SUMMARY
+          cat $RUNNER_TEMP/log.txt >> $GITHUB_STEP_SUMMARY
+          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "</details>" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+
+      - name: Output Debug Info
+        if: ${{ github.event_name == 'workflow_dispatch' && inputs.debuginfo }}
+        run: |
+          echo "## Debug Info" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "\`\`\`json" >> $GITHUB_STEP_SUMMARY
+          echo "${{ toJSON( github ) }}" >> $GITHUB_STEP_SUMMARY
+          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY

assets/renew_ruyi_index.py

@@ -2,6 +2,7 @@
 Automatically check the index of ruyi and renew it.
 """
 
+import os
 import shutil
 import argparse
 import tempfile
@@ -11,14 +12,6 @@
 from src.matrix_parser import Systems
 from src.ruyi_index_updator import RuyiDiff, RuyiGitRepo
 
-handler = colorlog.StreamHandler()
-handler.setFormatter(colorlog.ColoredFormatter(
-    '%(log_color)s[%(relativeCreated)d %(levelname)s]%(reset)s: %(message)s'))
-
-logging.basicConfig(level=logging.INFO, handlers=[handler])
-
-logger = logging.getLogger(__name__)
-
 
 def main():
     """
@@ -36,8 +29,40 @@ def main():
     arg.add_argument(
         '--pr', help='create a PR for the update', action='store_true'
     )
+    arg.add_argument(
+        '--log', help='output the log to the file', default=None
+    )
+    arg.add_argument(
+        '--warn', help='output the warn to the file', default=None
+    )
     args = arg.parse_args()
 
+    handler = colorlog.StreamHandler()
+    handler.setFormatter(colorlog.ColoredFormatter(
+        '%(log_color)s[%(relativeCreated)d %(levelname)s]%(reset)s: %(message)s'))
+
+    handlers = [handler]
+
+    if args.log is not None:
+        p = os.path.abspath(args.log)
+        log_handler = logging.FileHandler(p)
+        log_handler.setFormatter(logging.Formatter(
+            '%(relativeCreated)d %(levelname)s: %(message)s'))
+        log_handler.setLevel(logging.INFO)
+        handlers.append(log_handler)
+
+    if args.warn is not None:
+        p = os.path.abspath(args.warn)
+        warn_handler = logging.FileHandler(p)
+        warn_handler.setFormatter(logging.Formatter(
+            '%(relativeCreated)d %(levelname)s: %(message)s'))
+        warn_handler.setLevel(logging.WARNING)
+        handlers.append(warn_handler)
+
+    logging.basicConfig(level=logging.INFO, handlers=handlers)
+
+    logger = logging.getLogger()
+
     index_path = args.index
     if index_path is None:
         index_path = tempfile.mkdtemp()
@@ -51,15 +76,7 @@ def main():
         if pr is None:
             continue
         if not args.pr:
-            logger.info("""\
-PR info:
-    Title: %s
-
-    Body: 
-%s
-    <Body End>
-    Branch: %s -> upstream/%s
-""", pr.title, pr.body, pr.self_branch, pr.upstream_branch)
+            logger.info("%s", repr(pr))
             continue
         repo.create_wrapped_pr(pr)
 

assets/src/matrix_parser/matrix_parser.py

@@ -284,6 +284,7 @@ class Systems:
         '.git',
         '.vscode',
         '__pycache__',
+        '~', # ?
     ]
 
     def should_exclude(self, path):

assets/src/ruyi_index_parser/parse_board_img.py

@@ -171,10 +171,8 @@ class BoardImages:
     @property
     def version(self) -> str:
         """
-        return the version, following bot identifier
+        return the version
         """
-        if self.is_bot_created:
-            return f"{self.raw_version}-matrix.bot"
         return self.raw_version
 
     @version.setter

assets/src/ruyi_index_updator/index_handler.py

@@ -11,9 +11,13 @@
 
 logger = logging.getLogger(__name__)
 
-PACKAGE_INDEX_OWNER = "ruyisdk"
+PACKAGE_INDEX_OWNER = os.getenv("PACKAGE_INDEX_OWNER", "ruyisdk")
+if len(PACKAGE_INDEX_OWNER) == 0 or PACKAGE_INDEX_OWNER is None:
+    PACKAGE_INDEX_OWNER = "ruyisdk"
 PACKAGE_INDEX_REPO = "packages-index"
 
+CI_RUN_ID = os.getenv("CI_RUN_ID", None)
+CI_RUN_URL = os.getenv("CI_RUN_URL", None)
 
 class PrWrapper:
     """
@@ -33,6 +37,7 @@ def __repr__(self) -> str:
     Body: 
 {self.body} 
 
+<Body End>
     From: {self.self_branch}
     To: {self.upstream_branch}
 """
@@ -68,13 +73,33 @@ def check_pr_exist(self, identifer: str) -> bool:
                 return True
         return False
 
+    def check_pr_updated(self, head: str, base: str) -> bool:
+        """
+        Check if the PR with head and base exist.
+        Different from check_pr_exist, this senerio is for the PR is already exist,
+        but identifier is not the same.
+        Maybe the PR is manually created, or something went wrong.
+        Neverthless, manual interraction is needed.
+        """
+        prs = self.upstream.get_pulls(state="open")
+        for pr in prs:
+            if pr.head.ref == head and pr.base.ref == base:
+                return True
+        return False
+
     def __create_pr(self, wrapper: PrWrapper):
         """
         Create a pull request.
         """
         head = f"{self.user.login}:{wrapper.self_branch}"
+        base = wrapper.upstream_branch
+        if self.check_pr_updated(head, base):
+            logger.error(
+                "PR already exist for %s -> %s, please check manually", head, base)
+            logger.error("New PR info: %s", repr(wrapper))
+            return
         pr = self.upstream.create_pull(
-            title=wrapper.title, body=wrapper.body, head=head, base=wrapper.upstream_branch)
+            title=wrapper.title, body=wrapper.body, head=head, base=base)
         logger.info("PR created: %s at %s", pr.title, pr.html_url)
 
     def create_wrapped_pr(self, wrapper: PrWrapper):
@@ -91,11 +116,38 @@ def create_pr(self, title: str, body: str, self_branch: str, upstream_branch: st
             title, body, self_branch, upstream_branch))
 
     def __reset_to_upstream(self):
-        self.local_repo.remote().set_url(self.upstream.ssh_url)
-        self.local_repo.remote().fetch()
+        # self.local_repo.remote().set_url(self.upstream.ssh_url)
+        # self.local_repo.remote().fetch()
+
+        # self.local_repo.remote().refs['main'].checkout()
+        # self.local_repo.remote().set_url(self.repo.ssh_url)
+        flag = False
+        for ref in self.local_repo.remotes:
+            if ref.name == "upstream":
+                flag = True
+                break
+        if not flag:
+            self.local_repo.git.execute(
+                ["git", "remote", "add", "upstream", self.upstream.ssh_url]
+            )
 
-        self.local_repo.remote().refs['main'].checkout()
-        self.local_repo.remote().set_url(self.repo.ssh_url)
+        self.local_repo.git.execute(
+            ["git", "remote", "set-url", "upstream", self.upstream.ssh_url]
+        )
+        self.local_repo.git.execute(
+            ["git", "fetch", "upstream"]
+        )
+        self.local_repo.git.execute(
+            ["git", "reset", "--hard", "upstream/main"]
+        )
+        self.local_repo.git.execute(
+            ["git", "clean", "-xdf"]
+        )
+
+    def __clean(self):
+        self.local_repo.git.execute(
+            ["git", "clean", "-xdf"]
+        )
 
     def __init__(self, repo_dir: str) -> None:
         github_token = os.getenv('GITHUB_TOKEN', None)
@@ -133,6 +185,7 @@ def local_checkout(self, branch: str):
         self.__reset_to_upstream()
         head = self.local_repo.create_head(branch)
         head.checkout()
+        self.__clean()
         logger.info("Checkout to %s", branch)
 
     def local_commit(self, message: str):
@@ -158,17 +211,28 @@ def add_image(self, image: BoardImageWrapper):
         """
         Add a image index to the repo.
         """
-        index_name = image.new_index_name()
+        file_name = image.new_index_name()
         index_file = os.path.join(
             self.local_repo.working_dir,
             "manifests",
             "board-image",
-            index_name
+            image.index_name,
+            file_name
         )
         with open(index_file, "w", encoding="utf-8") as f:
             f.write(image.new_index_toml())
-        self.local_repo.index.add([index_file])
-        logger.info("Add %s", index_name)
+            if image.index.is_bot_created and CI_RUN_ID is None:
+                f.write("\n# This file is created by program renew_ruyi_index in support-matrix\n")
+                f.write("# Run: In local\n")
+            elif image.index.is_bot_created:
+                f.write("\n# This file is created by CI Sync Package Index inside support-matrix\n")
+                f.write(f"# Run ID: {CI_RUN_ID}\n")
+                f.write(f"# Run URL: {CI_RUN_URL}\n")
+        # self.local_repo.index.add([index_file])
+        self.local_repo.git.execute(
+            ["git", "add", index_file]
+        )
+        logger.info("Add %s", file_name)
 
     def upload_image(self, image: BoardImageWrapper):
         """

assets/src/ruyi_index_updator/upload_plugin_base.py

@@ -57,7 +57,8 @@ def handle_version(self, vinfo: VInfo) -> str:
         raise NotImplementedError
 
     @abstractmethod
-    def handle_report(self, vinfo: VInfo, index: str, last_index: list[BoardImages]) -> BoardImages | None:
+    def handle_report(self, vinfo: VInfo,
+                      index: str, last_index: list[BoardImages]) -> BoardImages | None:
         """
         Handle the report data from the system.
         """
@@ -131,6 +132,7 @@ def gen_distfile(self, file: str, url: str) -> BoardIndexDistfiles:
             }
         })
 
+
 def register() -> UploadPluginBase | None:
     """
     Register the plugin to the system

assets/src/ruyi_index_updator/version_diff.py

@@ -84,7 +84,7 @@ def gen_hash(self) -> str:
         Generate the hash of the new index, used to identify the pr
         """
         h = hashlib.sha224(
-            self.new_index_toml().encode("utf-8")
+            self.new_index_toml().encode("utf-8") + self.new_index_name().encode("utf-8")
         )
         return h.hexdigest()