admin-valentin/arcade-mcp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add PyPI release workflow (#429)

Browse source 
This is the first of a few PRs. Deploy to staging will fail until we
have `arcade-core`, `arcade-serve`, and `arcade-ai` released to PyPI.
This PR will release `arcade-core` to PyPI.


### PR Description
* Adds workflow that checks for changes in any pyproject.toml, and if
its version has changed, then tests, builds wheel, then publishes to
PyPI
* Updates the Dockerfile for our new structure
* Updates porter yamls
* Updates `make full-dist`
* Removes a couple unused workflows

Check out https://github.com/ArcadeAI/arcade-ai/actions/runs/15622059209
to see how the new workflow works (note that it failed publishing to
PyPI on purpose)
This commit is contained in:
Eric Gustin 2025-06-13 11:22:31 -07:00 committed by GitHub
parent 7ac6147733
commit 86cde2d9bd
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 254 additions and 256 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

49
.github/scripts/check-version-changes.sh vendored Executable file
View file

@ -0,0 +1,49 @@
#!/bin/bash
set -e
# Get changed files from command line argument or environment variable
CHANGED_FILES="${1:-$CHANGED_FILES}"
if [ -z "$CHANGED_FILES" ]; then
echo "No changed files provided"
echo "packages=" >> $GITHUB_OUTPUT
exit 0
fi
echo "Changed pyproject.toml files:"
echo "$CHANGED_FILES"
# Initialize array to store packages to release
packages_to_release=()
# Check each changed pyproject.toml
for file in $CHANGED_FILES; do
echo "Checking $file..."
# Get the full directory path (relative to repo root)
package_dir=$(dirname "$file")
# Check if this is a new file (added in this commit)
if git diff HEAD^ HEAD --name-status -- "$file" | grep -E "^A\s+$file$" > /dev/null; then
echo "New package detected: $file"
packages_to_release+=("$package_dir")
# Otherwise check for version changes
elif git diff HEAD^ HEAD -- "$file" | grep -E '^\+version = ".*"$' > /dev/null; then
echo "Version changed in $file"
packages_to_release+=("$package_dir")
else
echo "No version change in $file"
fi
done
# Output the packages to release
if [ ${#packages_to_release[@]} -eq 0 ]; then
echo "No packages to release found."
echo "packages=" >> $GITHUB_OUTPUT
else
echo "Packages to release: ${packages_to_release[@]}"
# Convert array to JSON format for matrix
packages_json=$(printf '%s\n' "${packages_to_release[@]}" | jq -R . | jq -s -c .)
echo "packages=$packages_json" >> $GITHUB_OUTPUT
fi

12
.github/scripts/get_toolkits.sh vendored
View file

@ -1,12 +0,0 @@
#!/bin/bash
# Get all directories and format as JSON array
echo -n '['
ls -d toolkits/*/ | cut -d'/' -f2 | sort -u | awk '{
if (NR==1) {
printf "\"%s\"", $0
} else {
printf ", \"%s\"", $0
}
}'
echo ']'

68
.github/workflows/check-toolkits.yml vendored
View file

@ -1,68 +0,0 @@
name: Check Toolkits
on:
push:
branches:
- main
jobs:
check-toolkits:
runs-on: ubuntu-latest
permissions:
contents: write
packages: write
steps:
- name: Checkout code
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Get changed files
id: changed-files
run: |
echo "Determining the base commit from the event payload..."
# Attempt to retrieve the 'before' commit from the event JSON.
if [ -f "$GITHUB_EVENT_PATH" ]; then
BASE=$(jq -r '.before' "$GITHUB_EVENT_PATH")
fi
# If not available or if it's the all-zero SHA (i.e. first commit), fallback to HEAD^.
if [ -z "$BASE" ] || [ "$BASE" = "0000000000000000000000000000000000000000" ]; then
BASE=HEAD^
fi
echo "Using commit range: $BASE...$GITHUB_SHA"
# List all files changed between BASE and the current commit.
CHANGED_FILES=$(git diff --name-only "$BASE" "$GITHUB_SHA")
echo "Changed files (raw):"
echo "$CHANGED_FILES"
# Filter only files under the toolkits/ directory.
matched=""
for file in $CHANGED_FILES; do
if [[ "$file" == toolkits/* ]]; then
matched="$matched$file "
fi
done
# Trim any extra whitespace.
matched=$(echo "$matched" | xargs)
echo "Matched changed files: $matched"
# Make the list available to subsequent steps as an output.
echo "all_changed_files=$matched" >> $GITHUB_OUTPUT
- name: List all added files
env:
CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
GITHUB_TOKEN: ${{ secrets.PAT }}
run: |
dirs=$(echo "${CHANGED_FILES}" | tr ' ' '\n' | grep "toolkits/" | cut -d'/' -f2 | sort -u)
if [ -n "$dirs" ]; then
echo "$dirs" | while read -r dir; do
echo "Publishing toolkit: $dir"
gh workflow -R ArcadeAI/arcade-ai run "Publish Toolkit" -f toolkit="${dir}"
done
else
echo "No toolkit directories were changed"
fi

9
.github/workflows/porter_app_worker_4828.yml vendored
View file

@ -18,18 +18,9 @@ jobs:
- name: Setup porter
uses: porter-dev/setup-porter@v0.1.0
- name: Install Poetry
uses: snok/install-poetry@v1
with:
version: 1.8.5
- name: Build Dist
run: make full-dist
- name: Export Requirements
run: |
cd arcade && poetry export --output ../dist/requirements.txt
- name: Deploy stack
timeout-minutes: 30
run: exec porter apply -f porter/staging.yaml

9
.github/workflows/porter_app_worker_4879.yml vendored
View file

@ -18,18 +18,9 @@ jobs:
- name: Setup porter
uses: porter-dev/setup-porter@v0.1.0
- name: Install Poetry
uses: snok/install-poetry@v1
with:
version: 1.8.5
- name: Build Dist
run: make full-dist
- name: Export Requirements
run: |
cd arcade && poetry export --output ../dist/requirements.txt
- name: Deploy stack
timeout-minutes: 30
run: exec porter apply -f ./porter/prod.yaml

9
.github/workflows/promote.yml vendored
View file

@ -8,6 +8,15 @@ on:
type: string
required: false
default: ''
worker_container_increment:
description: 'Worker container version increment. Defaults to patch.'
type: choice
options:
- major
- minor
- patch
required: false
default: 'patch'
jobs:
promote:

93
.github/workflows/publish-toolkit.yml vendored
View file

@ -1,93 +0,0 @@
name: Publish Toolkit
on:
workflow_dispatch:
inputs:
toolkit:
description: 'The directory of the toolkit to publish'
required: true
jobs:
release-and-publish:
runs-on: ubuntu-latest
permissions:
contents: write
packages: write
steps:
- name: Set toolkit input
id: set-toolkit
run: |
if [ "${{ github.event_name }}" = "push" ]; then
# Extract toolkit name from changed files
TOOLKIT=this_is_for_testing
echo "toolkit=$TOOLKIT" >> $GITHUB_OUTPUT
else
echo "toolkit=${{ inputs.toolkit }}" >> $GITHUB_OUTPUT
fi
- name: Checkout code
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Install Poetry
uses: snok/install-poetry@v1
with:
version: 1.8.5
- uses: actions/setup-python@v5
with:
python-version: '3.12'
cache: 'pip'
- name: Test Toolkit
id: Test_Toolkit
working-directory: toolkits/${{ steps.set-toolkit.outputs.toolkit }}
run: |
make install
make check
make test
- name: Publish Toolkit
# Publish the toolkit to PyPI if the version is not already published
id: Publish_Toolkit
working-directory: toolkits/${{ steps.set-toolkit.outputs.toolkit }}
run: |
poetry build
# Extract version from pyproject.toml using poetry and save it
VERSION=$(poetry version -s)
echo "version=$VERSION" >> $GITHUB_OUTPUT
poetry config pypi-token.pypi ${{ secrets.PYPI_TOKEN }}
# Run the publish command in an if-statement to capture output and status.
if PUBLISH_OUTPUT=$(poetry publish --skip-existing 2>&1); then
PUBLISH_STATUS=0
else
PUBLISH_STATUS=$?
fi
# If the output indicates that the version already exists, mark it as a skip.
if echo "$PUBLISH_OUTPUT" | grep -q "File exists. Skipping"; then
echo "Version already exists on PyPI. Skipping publish."
echo "skip_publish=true" >> $GITHUB_OUTPUT
elif [ $PUBLISH_STATUS -ne 0 ]; then
echo "Failed to publish package:"
echo "$PUBLISH_OUTPUT"
echo "skip_publish=false" >> $GITHUB_OUTPUT
exit $PUBLISH_STATUS
else
echo "skip_publish=false" >> $GITHUB_OUTPUT
fi
- name: Send status to Slack
if: always() && steps.Publish_Toolkit.outputs.skip_publish != 'true'
uses: slackapi/slack-github-action@v2.0.0
with:
webhook: ${{ secrets.SLACK_WEBHOOK_URL }}
webhook-type: webhook-trigger
payload: |
status: "${{ (steps.Test_Toolkit.outcome == 'failure' || steps.Publish_Toolkit.outcome == 'failure') && 'Failed' || 'Success' }}"
toolkit: ${{ steps.set-toolkit.outputs.toolkit }}
version: ${{ steps.Publish_Toolkit.outputs.version }}
url: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"

141
.github/workflows/release-on-version-change.yml vendored Normal file
View file

@ -0,0 +1,141 @@
# This workflow is used to release packages to PyPI when its
# pyproject.toml version is changed or a new package is added.
name: Release on Version Change
on:
push:
branches:
- main
jobs:
detect-version-changes:
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.set-matrix.outputs.matrix }}
has-changes: ${{ steps.set-matrix.outputs.has-changes }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Get changed files
id: changed-files
uses: tj-actions/changed-files@ed68ef82c095e0d48ec87eccea555d944a631a4c # v46
with:
files: |
**/pyproject.toml
- name: Check for version changes or new packages
id: check-versions
if: steps.changed-files.outputs.any_changed == 'true'
run: |
./.github/scripts/check-version-changes.sh "${{ steps.changed-files.outputs.all_changed_files }}"
- name: Set matrix
id: set-matrix
run: |
packages='${{ steps.check-versions.outputs.packages }}'
if [ -z "$packages" ] || [ "$packages" = "[]" ]; then
echo "has-changes=false" >> $GITHUB_OUTPUT
echo "matrix={\"include\":[]}" >> $GITHUB_OUTPUT
else
echo "has-changes=true" >> $GITHUB_OUTPUT
# Create matrix with package directories
matrix=$(echo "$packages" | jq -c '{include: [.[] | {package: .}]}')
echo "matrix=$matrix" >> $GITHUB_OUTPUT
echo "Matrix: $matrix"
fi
build-and-test:
needs: detect-version-changes
if: needs.detect-version-changes.outputs.has-changes == 'true'
runs-on: ubuntu-latest
strategy:
matrix: ${{ fromJson(needs.detect-version-changes.outputs.matrix) }}
permissions:
contents: write
packages: write
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Set up the environment
uses: ./.github/actions/setup-uv-env
with:
python-version: "3.10"
- name: Extract package name and version
working-directory: ${{ matrix.package }}
run: |
# Extract package name
PACKAGE_NAME=$(grep -m1 '^name = ' pyproject.toml | cut -d'"' -f2)
echo "PACKAGE_NAME=$PACKAGE_NAME" >> $GITHUB_ENV
# Extract version
VERSION=$(grep -m1 '^version = ' pyproject.toml | cut -d'"' -f2)
echo "VERSION=$VERSION" >> $GITHUB_ENV
echo "Building $PACKAGE_NAME version $VERSION"
- name: Run tests
working-directory: ${{ matrix.package }}
run: |
# Run tests if they exist
if [ -f "Makefile" ] && grep -q "^test:" Makefile; then
make test
elif [ -f "../Makefile" ] && grep -q "^test:" ../Makefile; then
cd .. && make test
else
echo "No tests found, skipping test step"
fi
- name: Build release distributions
working-directory: ${{ matrix.package }}
run: |
uv build --out-dir dist | tee build.log
# Verify build artifacts
ls -la dist/
echo "Built artifacts for ${{ env.PACKAGE_NAME }} v${{ env.VERSION }}"
- name: Upload release distributions
uses: actions/upload-artifact@v4
with:
name: release-dists-${{ env.PACKAGE_NAME }}-${{ env.VERSION }}
path: ${{ matrix.package }}/dist/
pypi-publish:
needs: [detect-version-changes, build-and-test]
if: needs.detect-version-changes.outputs.has-changes == 'true'
runs-on: ubuntu-latest
strategy:
matrix: ${{ fromJson(needs.detect-version-changes.outputs.matrix) }}
permissions:
id-token: write
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Extract package name and version
working-directory: ${{ matrix.package }}
run: |
PACKAGE_NAME=$(grep -m1 '^name = ' pyproject.toml | cut -d'"' -f2)
echo "PACKAGE_NAME=$PACKAGE_NAME" >> $GITHUB_ENV
VERSION=$(grep -m1 '^version = ' pyproject.toml | cut -d'"' -f2)
echo "VERSION=$VERSION" >> $GITHUB_ENV
- name: Retrieve release distributions
uses: actions/download-artifact@v4
with:
name: release-dists-${{ env.PACKAGE_NAME }}-${{ env.VERSION }}
path: dist/
- name: Publish release distributions to PyPI
uses: pypa/gh-action-pypi-publish@release/v1

38
Makefile
View file

@ -1,7 +1,3 @@
CLI_VERSION ?= "2.0.0"
TDK_VERSION ?= "2.0.0"
SERVE_VERSION ?= "2.0.0"
CORE_VERSION ?= "2.0.0"
.PHONY: install
install: ## Install the uv environment and all packages with dependencies
@ -101,28 +97,6 @@ coverage: ## Generate coverage report
@echo "Generating coverage report"
@uv run coverage html
.PHONY: set-version
set-version: ## Set the version in all lib pyproject.toml files
@echo "🚀 Setting versions in all lib packages"
@echo "Setting arcade-ai version to $(CLI_VERSION)"
@sed -i.bak '/^\[project\]/,/^\[/ s/^version = .*/version = $(CLI_VERSION)/' pyproject.toml && rm pyproject.toml.bak
@echo "Setting libs/arcade-tdk version to $(TDK_VERSION)"
@cd libs/arcade-tdk && sed -i.bak '/^\[project\]/,/^\[/ s/^version = .*/version = $(TDK_VERSION)/' pyproject.toml && rm pyproject.toml.bak
@echo "Setting libs/arcade-serve version to $(SERVE_VERSION)"
@cd libs/arcade-serve && sed -i.bak '/^\[project\]/,/^\[/ s/^version = .*/version = $(SERVE_VERSION)/' pyproject.toml && rm pyproject.toml.bak
@echo "Setting libs/arcade-core version to $(CORE_VERSION)"
@cd libs/arcade-core && sed -i.bak '/^\[project\]/,/^\[/ s/^version = .*/version = $(CORE_VERSION)/' pyproject.toml && rm pyproject.toml.bak
.PHONY: unset-version
unset-version: ## Reset version to 0.1.0 in all lib pyproject.toml files
@echo "🚀 Resetting version to 0.1.0 in all lib packages"
@for lib in libs/arcade*/ ; do \
if [ -f "$$lib/pyproject.toml" ]; then \
echo "Resetting version in $$lib"; \
(cd $$lib && sed -i.bak 's/version = "[^"]*"/version = "0.1.0"/' pyproject.toml && rm pyproject.toml.bak); \
fi; \
done
.PHONY: build
build: clean-build ## Build wheel files using uv
@echo "🚀 Creating wheel files for all lib packages"
@ -219,19 +193,18 @@ publish-ghcr: ## Publish to the GHCR
full-dist: clean-dist ## Build all projects and copy wheels to ./dist
@echo "🛠️ Building a full distribution with lib packages and toolkits"
@echo "Setting version to $(CLI_VERSION)"
@make set-version
@echo "🛠️ Building all lib packages and copying wheels to ./dist"
@mkdir -p dist
# Build all lib packages in dependency order
@for lib in arcade-core arcade-tdk arcade-serve ; do \
echo "🛠️ Building libs/$$lib wheel..."; \
(cd libs/$$lib && uv build); \
cp libs/$$lib/dist/*.whl dist/; \
done
@echo "🛠️ Building arcade-ai package and copying wheel to ./dist"
@uv build
@rm -f dist/*.tar.gz
@echo "🛠️ Building all toolkit packages and copying wheels to ./dist"
@for dir in toolkits/*/ ; do \
if [ -d "$$dir" ] && [ -f "$$dir/pyproject.toml" ]; then \
@ -242,9 +215,6 @@ full-dist: clean-dist ## Build all projects and copy wheels to ./dist
fi; \
done
@echo "Reset version to default (0.1.0)"
@make unset-version
.PHONY: clean-dist
clean-dist: ## Clean all built distributions
@echo "🗑️ Cleaning dist directory"

59
docker/Dockerfile
View file

@ -3,7 +3,6 @@ FROM python:3.11-slim
# Define build arguments with default values
ARG PORT=8001
ARG HOST=0.0.0.0
ARG VERSION=${VERSION:-0.1.0.dev0}
ARG INSTALL_TOOLKITS=true
# Set environment variables using the build arguments
@ -20,41 +19,53 @@ RUN apt-get update && apt-get upgrade -y && apt-get install -y \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /app/arcade
WORKDIR /app
# Copy the parent directory contents into the container
COPY ./dist ./arcade /app/arcade/
# Copy the dist directory contents into the container
COPY ./dist /app/dist/
# Copy the toolkits.txt file into the container
COPY ./docker/toolkits.txt /app/arcade/
COPY ./docker/toolkits.txt /app/
# Expose the port
EXPOSE $PORT
# List files for debugging purposes
RUN ls -la /app/arcade/
# List wheel files for debugging purposes
RUN ls -la /app/dist/
# Conditional installation based on version
RUN if [ ! "$(echo ${VERSION} | grep -E '\.dev0$')" ]; then \
echo "Installing wheel file" && \
python -m pip install ./arcade_ai-${VERSION}-py3-none-any.whl && \
python -m pip install -r ./requirements.txt; \
else \
echo "Installing from source" && \
cd /app/arcade && \
pip install poetry && \
poetry lock && \
poetry version 0.1.0 && \
pip install -r requirements.txt && \
pip install .; \
fi
# Install the core Arcade packages (each has its own version)
RUN python -m pip install \
/app/dist/arcade_core-*.whl \
/app/dist/arcade_serve-*.whl \
/app/dist/arcade_ai-*.whl
# Conditionally install toolkits.txt dependencies
# Conditionally install toolkit wheels from dist directory if INSTALL_TOOLKITS is true and the toolkit is in toolkits.txt
RUN if [ "$INSTALL_TOOLKITS" = "true" ] ; then \
python -m pip install -r ./toolkits.txt ; \
while IFS= read -r toolkit; do \
# Skip empty lines and comments (lines starting with #)
if [ -n "$toolkit" ] && [ "${toolkit#\#}" = "$toolkit" ]; then \
# Convert toolkit name to match wheel filename format (replace - with _)
wheel_name=$(echo "$toolkit" | sed 's/-/_/g'); \
wheel_file="/app/dist/${wheel_name}-"*.whl; \
# Check if this is not a core package and if the wheel file exists
if [ "$wheel_name" != "arcade_core" ] && \
[ "$wheel_name" != "arcade_serve" ] && \
[ "$wheel_name" != "arcade_ai" ] && \
[ "$wheel_name" != "arcade_tdk" ]; then \
if ls $wheel_file 1> /dev/null 2>&1; then \
echo "Installing $toolkit from $wheel_file"; \
python -m pip install $wheel_file; \
else \
echo "Warning: Wheel file not found for $toolkit (looked for $wheel_file)"; \
fi; \
else \
echo "Skipping core package: $toolkit"; \
fi; \
fi; \
done < /app/toolkits.txt ; \
fi
# Run the arcade workerup (hidden cli command)
# Run the arcade worker
COPY docker/start.sh /app/start.sh
RUN chmod +x /app/start.sh
CMD ["/app/start.sh"]

16
docker/README.md
View file

@ -17,14 +17,22 @@ Begin by cloning the Arcade repository:
git clone https://github.com/ArcadeAI/arcade-ai.git
```
### 2. Build package wheels
From the root of the arcade-ai repository:
```bash
make full-dist
```
### 3. Copy and Configure Environment Variables
Change to the `docker` directory:
```bash
cd arcade-ai/docker
```
### 2. Copy and Configure Environment Variables
Copy the example environment file to `.env`:
```bash
@ -45,7 +53,7 @@ If you plan to use other Large Language Model (LLM) providers, add their API key
ANTHROPIC_API_KEY=your_anthropic_api_key_here
```
### 3. Run Docker Compose
### 4. Run Docker Compose
Start the Arcade services using Docker Compose:
@ -55,7 +63,7 @@ docker compose up
This command will build and start all the services defined in the `docker-compose.yml` file and make their ports available to your host machine.
### 4. Verify the Engine is Running
### 5. Verify the Engine is Running
In a separate terminal window, check if the engine is running:

2
libs/arcade-core/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "arcade-core"
version = "1.0.0"
version = "2.0.0"
description = "Arcade Core - Core library for Arcade platform"
readme = "README.md"
license = {text = "MIT"}

2
libs/arcade-serve/pyproject.toml
View file

@ -19,7 +19,7 @@ classifiers = [
]
requires-python = ">=3.10"
dependencies = [
"arcade-core>=0.1.0",
"arcade-core>=2.0.0,<3.0.0",
"fastapi>=0.115.3",
"uvicorn>=0.30.0",
"watchfiles>=1.0.5",

3
libs/arcade-tdk/pyproject.toml
View file

@ -15,10 +15,11 @@ classifiers = [
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
"Programming Language :: Python :: 3.13",
]
requires-python = ">=3.10"
dependencies = [
"arcade-core>=0.1.0",
"arcade-core>=2.0.0,<3.0.0",
"pydantic>=2.7.0",
]