From b81bb86dfbed4cb86717303f0b178dc8b6444f02 Mon Sep 17 00:00:00 2001
From: Evan Tahler <evantahler@gmail.com>
Date: Wed, 16 Jul 2025 16:53:26 -0700
Subject: [PATCH] Skip tests that require secrets in forks (#495)

We know that forks won't have our secrets, so we separate those toolkits
out to new a new test group and skip them if you aren't in this main
repo. We know if a test suite has a secret via a magic comment

---------

Co-authored-by: Eric Gustin <34000337+EricGustin@users.noreply.github.com>
---
 .github/workflows/test-toolkits.yml | 56 +++++++++++++++++++++++++----
 1 file changed, 50 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/test-toolkits.yml b/.github/workflows/test-toolkits.yml
index 3cefacf0..1f506a55 100644
--- a/.github/workflows/test-toolkits.yml
+++ b/.github/workflows/test-toolkits.yml
@@ -11,25 +11,31 @@ jobs:
   setup:
     runs-on: ubuntu-latest
     outputs:
-      tool_matrix: ${{ steps.get-toolkits.outputs.toolkits }}
+      toolkits_with_gha_secrets: ${{ steps.load_toolkits.outputs.toolkits_with_gha_secrets }}
+      toolkits_without_gha_secrets: ${{ steps.load_toolkits.outputs.toolkits_without_gha_secrets }}
     steps:
       - name: Check out
         uses: actions/checkout@v4
 
-      - name: Get toolkits
-        id: get-toolkits
+      - name: determine toolkits with and without GHA secrets
+        id: load_toolkits
         run: |
           # Find all directories in toolkits/ that have a pyproject.toml
           TOOLKITS=$(find toolkits -maxdepth 1 -type d -not -name "toolkits" -exec test -f {}/pyproject.toml \; -exec basename {} \; | jq -R -s -c 'split("\n")[:-1]')
+          TOOLKITS_WITH_GHA_SECRETS='["postgres"]'
+          TOOLKITS_WITHOUT_GHA_SECRETS=$(echo "$TOOLKITS" | jq -c --argjson with "$TOOLKITS_WITH_GHA_SECRETS" '[.[] | select(. as $t | $with | index($t) | not)]')
           echo "Found toolkits: $TOOLKITS"
-          echo "toolkits=$TOOLKITS" >> $GITHUB_OUTPUT
+          echo "Found toolkits without GHA secrets: $TOOLKITS_WITHOUT_GHA_SECRETS"
+          echo "Found toolkits with GHA secrets: $TOOLKITS_WITH_GHA_SECRETS"
+          echo "toolkits_without_gha_secrets=$TOOLKITS_WITHOUT_GHA_SECRETS" >> $GITHUB_OUTPUT
+          echo "toolkits_with_gha_secrets=$TOOLKITS_WITH_GHA_SECRETS" >> $GITHUB_OUTPUT
 
   test-toolkits:
     needs: setup
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        toolkit: ${{ fromJson(needs.setup.outputs.tool_matrix) }}
+        toolkit: ${{ fromJson(needs.setup.outputs.toolkits_without_gha_secrets) }}
       fail-fast: true
     steps:
       - name: Check out
@@ -48,7 +54,45 @@ jobs:
           uv run --active pre-commit run -a
           uv run --active mypy --config-file=pyproject.toml
 
-      - name: Test toolkit
+      - name: Test stand-alone toolkits (no secrets)
+        working-directory: toolkits/${{ matrix.toolkit }}
+        run: |
+          # Run pytest and capture exit code
+          uv run --active pytest -W ignore -v --cov=arcade_${{ matrix.toolkit }} --cov-report=xml || EXIT_CODE=$?
+
+          if [ "${EXIT_CODE:-0}" -eq 5 ]; then
+            echo "No tests found for toolkit ${{ matrix.toolkit }}, skipping..."
+            exit 0
+          elif [ "${EXIT_CODE:-0}" -ne 0 ]; then
+            exit ${EXIT_CODE}
+          fi
+
+  test-toolkits-with-gha-secrets:
+    needs: setup
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        toolkit: ${{ fromJson(needs.setup.outputs.toolkits_with_gha_secrets) }}
+      fail-fast: true
+    steps:
+      - name: Check out
+        uses: actions/checkout@v4
+
+      - name: Set up the environment
+        uses: ./.github/actions/setup-uv-env
+
+      - name: Install toolkit dependencies
+        working-directory: toolkits/${{ matrix.toolkit }}
+        run: uv pip install -e ".[dev]"
+
+      - name: Check toolkit
+        working-directory: toolkits/${{ matrix.toolkit }}
+        run: |
+          uv run --active pre-commit run -a
+          uv run --active mypy --config-file=pyproject.toml
+
+      - name: Test stand-alone toolkits (with secrets)
+        if: github.repository == 'ArcadeAI/arcade-ai'
         working-directory: toolkits/${{ matrix.toolkit }}
         env:
           TEST_POSTGRES_DATABASE_CONNECTION_STRING: ${{ secrets.TEST_POSTGRES_DATABASE_CONNECTION_STRING }} # TODO: dynamically only load the `TEST_${{ matrix.toolkit }}_DATABASE_CONNECTION_STRING secret`