read-sources.yml

name: Read Sources

on:
  workflow_dispatch:
  schedule:
    - cron: '0 17 * * *'
jobs:
  readsources:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v3

    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: '3.10'

    - name: Install dependencies
      run: |
        pip install poetry
        poetry install

    - name: Checkout remote repository (dnsReaper)
      uses: actions/checkout@v3
      with:
        repository: punk-security/dnsReaper
        path: dnsReaper 

    - name: Checkout remote repository (nuclei-templates)
      uses: actions/checkout@v3
      with:
        repository: projectdiscovery/nuclei-templates
        path: nuclei-templates


    - name: Read Sources
      run: |
        poetry run python3 baddns/scripts/readsources.py

    - name: Stage, commit, and create PR for changed files
      env:
        GH_TOKEN: ${{ secrets.DNSBOT_TOKEN }}
      run: |
        echo "starting Read Sources run" >> readsources_action.log
        git config --local user.email "action@github.com"
        git config --local user.name "GitHub Action"

        echo "Current directory and contents:" >> readsources_action.log
        pwd >> readsources_action.log
        echo "Contents of signatures_to_test directory:" >> readsources_action.log
        ls -la signatures_to_test >> readsources_action.log

        # Check for existence of previous hashes file and read its content
        if [[ -f "baddns/signatures/signature_history.txt" ]]; then
          prev_hashes=$(cat baddns/signatures/signature_history.txt)
          echo "signature_history.txt was present and read into prev_hashes..." >> readsources_action.log
        else
          echo "signature_history.txt was NOT present" >> readsources_action.log
          prev_hashes=""
        fi

        # Get only the files in the signatures_to_test directory
        files=$(basename -a signatures_to_test/*.yml)

        for file in $files; do
          echo "attempting to parse signature file: $file" >> readsources_action.log

          if [[ ! -s "signatures_to_test/$file" ]]; then
            echo "File $file is empty or doesn't exist, skipping" >> readsources_action.log
            continue
          fi

          # Check if the file exists in the signatures directory and if it has been changed
          if [[ ! -f "baddns/signatures/$file" ]] || ! cmp -s "baddns/signatures/$file" "signatures_to_test/$file"; then

            # Calculate hash of the new file and check if we've seen it before
            new_hash=$(sha256sum "signatures_to_test/$file" | cut -d ' ' -f 1)
            if echo "$prev_hashes" | grep -q "$new_hash"; then
              echo "File $file has been seen before, skipping" >> readsources_action.log
              continue
            fi

            PR_BLOCKED=false

            # Check for existing PRs with the same title

            sleep 5
            PR_TITLE="[SignatureBot] Add or update signature $file"
            SEARCH_QUERY="repo:${{ github.repository }} type:pr state:open in:title \"$PR_TITLE\""

            # Encode the search query
            ENCODED_QUERY=$(echo "$SEARCH_QUERY" | jq -sRr @uri)

            EXISTING_OPEN_PRS=$(curl -H "Authorization: token $GH_TOKEN" -H "Accept: application/vnd.github.v3+json" "https://api.github.com/search/issues?q=$ENCODED_QUERY" | jq '.items')

            if [[ -n "$EXISTING_OPEN_PRS" && "$EXISTING_OPEN_PRS" != "[]" ]]; then
              echo "Found an open PR with title '$PR_TITLE', skipping this signature" >> readsources_action.log
              continue
            fi

            sleep 5
            # Encode the PR title for use in a URL
            ENCODED_PR_TITLE=$(echo "$PR_TITLE" | jq -sRr @uri)

            # Check for existing closed PRs with the same title and label 'signature-blocked'
            SEARCH_RESULT=$(curl -H "Authorization: token $GH_TOKEN" -H "Accept: application/vnd.github.v3+json" "https://api.github.com/search/issues?q=repo:${{ github.repository }}+type:pr+state:closed+label:signature-blocked+in:title+$ENCODED_PR_TITLE")

            # Use jq to determine if any matching PRs are found. If so, skip processing this signature.
            MATCHING_CLOSED_BLOCKED_PR_COUNT=$(echo "$SEARCH_RESULT" | jq '.total_count')

            if [[ $MATCHING_CLOSED_BLOCKED_PR_COUNT -gt 0 ]]; then
                echo "Found a closed PR with title '$PR_TITLE' and label 'signature-blocked', skipping this signature" >> readsources_action.log
                continue
            fi

            # Add the new hash to the history file
            echo "adding hash $new_hash to signature file" >> readsources_action.log
            echo "$new_hash #$file" >> "baddns/signatures/signature_history.txt"

            # Copy the file to the signatures directory
            cp "signatures_to_test/$file" "baddns/signatures/$file"

            BRANCH_NAME="new-signature-$file"

            # Fetch the latest remote references
            git fetch

            # Check if the branch exists
            if git ls-remote --heads origin $BRANCH_NAME | grep $BRANCH_NAME; then
                # Check for existing PRs with the same title
                PR_TITLE="[SignatureBot] Add or update signature $file"

                # Construct the search query
                SEARCH_QUERY="repo:${{ github.repository }} type:pr state:open in:title \"$PR_TITLE\""

                # Encode the search query for the URL
                ENCODED_QUERY=$(echo "$SEARCH_QUERY" | jq -sRr @uri)

                # Short delay, then check for existing PRs with the Search API
                sleep 5
                EXISTING_OPEN_PRS=$(curl -H "Authorization: token $GH_TOKEN" -H "Accept: application/vnd.github.v3+json" "https://api.github.com/search/issues?q=$ENCODED_QUERY" | jq '.items')

                if [[ -n "$EXISTING_OPEN_PRS" && "$EXISTING_OPEN_PRS" != "[]" ]]; then
                    echo "Found an open PR with title '$PR_TITLE', skipping branch deletion for safety" >> readsources_action.log
                    continue
                fi

                # No open PR, so safe to delete the branch
                sleep 5
                git push origin --delete $BRANCH_NAME
                echo "Found and deleted stale branch $BRANCH_NAME remotely" >> readsources_action.log

                # If needed, delete the branch locally too. This step might not be necessary if the action doesn't check out all branches.
                if git show-ref --verify --quiet refs/heads/$BRANCH_NAME; then
                    git branch -D $BRANCH_NAME
                    echo "Found and deleted stale branch $BRANCH_NAME locally" >> readsources_action.log
                fi
            else
                echo "No stale branch $BRANCH_NAME found remotely" >> readsources_action.log
            fi

            echo "checking out new branch $BRANCH_NAME" >> readsources_action.log
            git checkout -b $BRANCH_NAME
            git add "baddns/signatures/$file"
            git add "baddns/signatures/signature_history.txt"
            echo "about to commit with the following files changed:" >> readsources_action.log
            git status -s >> readsources_action.log

            echo "adding commit and pushing branch..." >> readsources_action.log
            # If the commit operation is successful, then push the changes and create a PR
            git commit -m "[SignatureBot] Add or update signature $file and update signature history" && {
              git push origin new-signature-$file
              echo "## Add or update signature: $file" > pr_message
              echo "This PR adds or updates the follow signature:" >> pr_message
              echo '```' >> pr_message
              echo "Attempting to read baddns/signatures/$file" >> readsources_action.log
              cat baddns/signatures/$file >> pr_message
              echo '```' >> pr_message
              echo "creating PR: '[SignatureBot] Add or update signature $file'" >> readsources_action.log
              sleep 5
              gh pr create --title "[SignatureBot] Add or update signature $file" --body-file pr_message --head new-signature-$file --repo ${{ github.repository }} 2>&1 | tee -a readsources_action.log
            }
            git checkout main
          else
            echo "Skipping signature file: $file - no changes detected" >> readsources_action.log
          fi
        done
        echo "completed read sources" >> readsources_action.log


    - name: Upload readsources.py log
      uses: actions/upload-artifact@v4
      if: always()
      with:
        name: readsources.py-log
        path: readsources.log

    - name: Upload read-sources action log
      uses: actions/upload-artifact@v4
      if: always()
      with:
        name: read-sources-action-log
        path: readsources_action.log