From 093ac56040ae3fd1c5b773a62ae525f71c36be76 Mon Sep 17 00:00:00 2001 From: Eric Gustin <34000337+EricGustin@users.noreply.github.com> Date: Tue, 17 Jun 2025 10:21:08 -0700 Subject: [PATCH] Update release containers GH Action (#439) 1. Patch, Minor, or Major is selected during workflow dispatch (defaults to patch) 2. Get most recent release version 3. Calculate new version based on findings from step 2 4. Build wheels 5. Build worker and base-worker 6. Push to GHCR 7. Create new release with release notes Also, I removed everything related to ECR --- .github/workflows/promote.yml | 232 +++++++++++++++++++ .github/workflows/release-containers.yml | 277 ----------------------- Makefile | 7 - docker/Dockerfile | 2 +- docker/Makefile | 41 +--- 5 files changed, 234 insertions(+), 325 deletions(-) delete mode 100644 .github/workflows/release-containers.yml diff --git a/.github/workflows/promote.yml b/.github/workflows/promote.yml index 3de1a1ce..4b2c6928 100644 --- a/.github/workflows/promote.yml +++ b/.github/workflows/promote.yml @@ -170,3 +170,235 @@ jobs: echo "### Branches" >> $GITHUB_STEP_SUMMARY echo "- **Source:** main" >> $GITHUB_STEP_SUMMARY echo "- **Target:** production" >> $GITHUB_STEP_SUMMARY + + calculate-version: + runs-on: ubuntu-latest + needs: promote + outputs: + version: ${{ steps.calculate_version.outputs.version }} + previous_version: ${{ steps.calculate_version.outputs.previous_version }} + steps: + - name: Checkout production branch + uses: actions/checkout@v4 + with: + ref: production + fetch-depth: 0 + + - name: Get latest release and calculate next version + id: calculate_version + run: | + # Get the latest release version from GitHub + LATEST_RELEASE=$(gh release view --json tagName --jq .tagName 2>/dev/null || echo "") + + if [ -z "$LATEST_RELEASE" ]; then + echo "No previous releases found, starting at 0.0.0" + CURRENT_VERSION="0.0.0" + else + # Remove 'v' prefix if present + CURRENT_VERSION=${LATEST_RELEASE#v} + echo "Latest release: $LATEST_RELEASE (version: $CURRENT_VERSION)" + fi + + # Split version into components + IFS='.' read -r MAJOR MINOR PATCH <<< "$CURRENT_VERSION" + + # Increment based on input + case "${{ inputs.worker_container_increment }}" in + major) + MAJOR=$((MAJOR + 1)) + MINOR=0 + PATCH=0 + ;; + minor) + MINOR=$((MINOR + 1)) + PATCH=0 + ;; + patch) + PATCH=$((PATCH + 1)) + ;; + esac + + NEW_VERSION="$MAJOR.$MINOR.$PATCH" + echo "New version: $NEW_VERSION" + + echo "version=$NEW_VERSION" >> $GITHUB_OUTPUT + echo "previous_version=$CURRENT_VERSION" >> $GITHUB_OUTPUT + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + build-and-push: + needs: calculate-version + strategy: + matrix: + include: + - arch: amd64 + os: ubuntu-latest + - arch: arm64 + os: linux-arm64 + runs-on: ${{ matrix.os }} + permissions: + contents: write + packages: write + env: + VERSION: ${{ needs.calculate-version.outputs.version }} + REGISTRY: ghcr.io + + steps: + - name: Checkout production branch + uses: actions/checkout@v4 + with: + ref: production + fetch-depth: 0 + + - name: Install uv + uses: astral-sh/setup-uv@v5 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build distributions + run: | + echo "Building full distribution for production release..." + make full-dist + + - name: Build and push Worker images + run: | + # Build both worker and worker-base images + make docker VERSION=${{ env.VERSION }} ARCH=${{ matrix.arch }} + make docker-base VERSION=${{ env.VERSION }} ARCH=${{ matrix.arch }} + + # Push to GHCR only + make publish-ghcr VERSION=${{ env.VERSION }} ARCH=${{ matrix.arch }} + + push-manifest: + runs-on: ubuntu-latest + needs: [calculate-version, build-and-push] + permissions: + contents: write + packages: write + env: + VERSION: ${{ needs.calculate-version.outputs.version }} + REGISTRY: ghcr.io + steps: + - name: Checkout production branch + uses: actions/checkout@v4 + with: + ref: production + fetch-depth: 0 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Push manifest to GHCR + working-directory: ./docker + run: | + make ghcr-manifest VERSION=${{ env.VERSION }} + make ghcr-manifest INSTALL_TOOLKITS=false VERSION=${{ env.VERSION }} + + create-release: + runs-on: ubuntu-latest + needs: [calculate-version, push-manifest] + permissions: + contents: write + env: + VERSION: ${{ needs.calculate-version.outputs.version }} + PREVIOUS_VERSION: ${{ needs.calculate-version.outputs.previous_version }} + steps: + - name: Checkout production branch + uses: actions/checkout@v4 + with: + ref: production + fetch-depth: 0 + token: ${{ secrets.PROMOTE_PAT_TOKEN }} + + - name: Generate release notes + id: generate_release_notes + run: | + echo "# Release v${{ env.VERSION }}" > release_notes.md + echo "" >> release_notes.md + echo "## Container Images" >> release_notes.md + echo "" >> release_notes.md + echo "### GitHub Container Registry (GHCR)" >> release_notes.md + echo "\`\`\`bash" >> release_notes.md + echo "docker pull ghcr.io/arcadeai/worker:${{ env.VERSION }}" >> release_notes.md + echo "docker pull ghcr.io/arcadeai/worker-base:${{ env.VERSION }}" >> release_notes.md + echo "\`\`\`" >> release_notes.md + echo "" >> release_notes.md + echo "## What's Changed" >> release_notes.md + echo "" >> release_notes.md + + if [ "${{ inputs.commit_sha }}" != "" ]; then + # Case 1: Specific commit SHA to cherry-pick was provided + echo "### Cherry-picked commit:" >> release_notes.md + echo "" >> release_notes.md + git log -1 --pretty=format:"- **%h** %s (%an)" "${{ inputs.commit_sha }}" >> release_notes.md + echo "" >> release_notes.md + echo "_This release contains a single cherry-picked commit from main._" >> release_notes.md + else + # Case 2: Bulk promotion (production == main) + echo "### Commits in this release:" >> release_notes.md + echo "" >> release_notes.md + + # Get the list of previous releases to find the last release + PREV_RELEASE_DATE=$(gh release list --limit 5 --json publishedAt,tagName --jq '.[1].publishedAt // empty' 2>/dev/null || echo "") + + if [ -z "$PREV_RELEASE_DATE" ]; then + # First promotion ever + echo "**Initial release** - Recent commits from main branch:" >> release_notes.md + echo "" >> release_notes.md + git log -20 --pretty=format:"- **%h** %s (%an)" --reverse >> release_notes.md + else + # Show commits since the previous promotion + echo "Changes since previous release ($(echo $PREV_RELEASE_DATE | cut -d'T' -f1)):" >> release_notes.md + echo "" >> release_notes.md + git log --since="$PREV_RELEASE_DATE" --pretty=format:"- **%h** %s (%an)" --reverse >> release_notes.md + echo "" >> release_notes.md + + # Fallback if no commits since last release (shouldn't happen, but just in case) + if [ $(git log --since="$PREV_RELEASE_DATE" --oneline | wc -l) -eq 0 ]; then + echo "_No commits found since previous release. Showing recent commits:_" >> release_notes.md + echo "" >> release_notes.md + git log -10 --pretty=format:"- **%h** %s (%an)" --reverse >> release_notes.md + fi + fi + echo "" >> release_notes.md + echo "_This release includes all changes from the main branch._" >> release_notes.md + fi + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Create GitHub Release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: v${{ env.VERSION }} + release_name: Release v${{ env.VERSION }} + body_path: release_notes.md + draft: false + prerelease: false + + - name: Update workflow summary + run: | + echo "" >> $GITHUB_STEP_SUMMARY + echo "## šŸš€ Container Release" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Version Information" >> $GITHUB_STEP_SUMMARY + echo "- **Previous Version:** v${{ env.PREVIOUS_VERSION }}" >> $GITHUB_STEP_SUMMARY + echo "- **New Version:** v${{ env.VERSION }}" >> $GITHUB_STEP_SUMMARY + echo "- **Increment Type:** ${{ inputs.worker_container_increment }}" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Container Images Published to GHCR" >> $GITHUB_STEP_SUMMARY + echo "- āœ… ghcr.io/arcadeai/worker:${{ env.VERSION }}" >> $GITHUB_STEP_SUMMARY + echo "- āœ… ghcr.io/arcadeai/worker-base:${{ env.VERSION }}" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/release-containers.yml b/.github/workflows/release-containers.yml deleted file mode 100644 index 165e0fc5..00000000 --- a/.github/workflows/release-containers.yml +++ /dev/null @@ -1,277 +0,0 @@ -name: Release and Publish Containers - -on: - push: - branches: - - main - tags: - - '*' - workflow_dispatch: - inputs: - version: - description: 'Version to release (e.g., 0.1.0)' - required: true - default: '0.0.1' - -env: - REGISTRY: ghcr.io - -jobs: - set-version: - runs-on: ubuntu-latest - outputs: - version: ${{ steps.set_version.outputs.version }} - steps: - - name: Wait for tests to succeed - uses: lewagon/wait-on-check-action@v1.3.4 - with: - ref: ${{ github.ref }} - running-workflow-name: 'Main' - repo-token: ${{ secrets.PAT }} - wait-interval: 10 - ignore-checks: "set-version,release-and-publish" - - - name: Checkout code - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Set version - id: set_version - run: | - if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then - echo "version=${{ github.event.inputs.version }}" >> $GITHUB_OUTPUT - elif [[ $GITHUB_REF == refs/tags/* ]]; then - echo "version=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT - else - echo "version=$(date +'%Y.%-m.%-d').dev0" >> $GITHUB_OUTPUT - fi - - build-and-push: - needs: set-version - strategy: - matrix: - include: - - arch: amd64 - os: ubuntu-latest - - arch: arm64 - os: linux-arm64 - runs-on: ${{ matrix.os }} - permissions: - contents: write - packages: write - - steps: - - name: Checkout code - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Install Poetry - uses: snok/install-poetry@v1 - with: - version: 1.8.5 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-1 - - - name: Login to ECR - uses: docker/login-action@v3 - with: - registry: 471112909428.dkr.ecr.us-east-1.amazonaws.com - username: ${{ secrets.AWS_ACCESS_KEY_ID }} - password: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - - name: Log in to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push Worker image - id: build - run: | - make docker VERSION=${{ needs.set-version.outputs.version }} ARCH=${{ matrix.arch }} - make docker-base VERSION=${{ needs.set-version.outputs.version }} ARCH=${{ matrix.arch }} - make publish-ecr VERSION=${{ needs.set-version.outputs.version }} ARCH=${{ matrix.arch }} - - - name: Push GHCR - if: github.event_name != 'push' || startsWith(github.ref, 'refs/tags/') - run: | - make publish-ghcr VERSION=${{ needs.set-version.outputs.version }} ARCH=${{ matrix.arch }} - - - push-manifest: - runs-on: ubuntu-latest - needs: [set-version, build-and-push] - permissions: - contents: write - packages: write - steps: - - name: Checkout code - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Log in to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-1 - - - name: Login to ECR - uses: docker/login-action@v3 - with: - registry: 471112909428.dkr.ecr.us-east-1.amazonaws.com - username: ${{ secrets.AWS_ACCESS_KEY_ID }} - password: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - - name: Push manifest to ECR - working-directory: ./docker - run: | - make ecr-manifest VERSION=${{ needs.set-version.outputs.version }} - make ecr-manifest INSTALL_TOOLKITS=false VERSION=${{ needs.set-version.outputs.version }} - - - name: Push manifest to GHCR - if: github.event_name != 'push' || startsWith(github.ref, 'refs/tags/') - working-directory: ./docker - run: | - make ghcr-manifest VERSION=${{ needs.set-version.outputs.version }} - make ghcr-manifest INSTALL_TOOLKITS=false VERSION=${{ needs.set-version.outputs.version }} - - # Currently broken: workflow deploy requires all versions - # deploy: - # if: github.event_name == 'push' - # runs-on: ubuntu-latest - # needs: [set-version, push-manifest] - # steps: - # - name: Checkout code - # uses: actions/checkout@v3 - # with: - # fetch-depth: 0 - - # - name: Set image - # run: | - # echo "image=471112909428.dkr.ecr.us-east-1.amazonaws.com/arcadeai/arcade-ai:${{ needs.set-version.outputs.version }}" >> $GITHUB_OUTPUT - - # - name: Deploy to Amazon ECS - # env: - # GITHUB_TOKEN: ${{ secrets.PAT }} - # run: gh workflow -R ArcadeAI/Team run Deploy -f worker-version=${{ needs.set-version.outputs.version }} - - - release: - permissions: - contents: write - packages: write - needs: [set-version, build-and-push] - runs-on: ubuntu-latest - if: github.event_name != 'push' || startsWith(github.ref, 'refs/tags/') - steps: - - name: Checkout code - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Install Poetry - uses: snok/install-poetry@v1 - with: - version: 1.8.5 - - - name: Make dist - run: make full-dist VERSION=${{ needs.set-version.outputs.version }} - - - name: Set TAR and WHL names - run: | - export PKG=$(ls arcade/dist/ | grep tar) - set -- $PKG - echo "TAR_NAME=$1" >> $GITHUB_ENV - export PKG=$(ls arcade/dist/ | grep whl) - set -- $PKG - echo "WHL_NAME=$1" >> $GITHUB_ENV - - - name: Generate release notes - id: generate_release_notes - run: | - echo "Release notes for version ${{ needs.set-version.outputs.version }}" > release_notes.md - echo "" >> release_notes.md - echo "Changes in this release:" >> release_notes.md - git log $(git describe --tags --abbrev=0)..HEAD --pretty=format:"- %s" >> release_notes.md - - - name: Create Release - id: create_release - uses: actions/create-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: ${{ needs.set-version.outputs.version }} - release_name: Release ${{ needs.set-version.outputs.version }} - body_path: release_notes.md - draft: false - prerelease: false - - - name: Upload Release Asset - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: ./release_notes.md - asset_name: release_notes.md - asset_content_type: text/markdown - - - name: Upload Python Tar Asset - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: ./arcade/dist/${{ env.TAR_NAME }} - asset_name: ${{ env.TAR_NAME }} - asset_content_type: application/zip - - - name: Upload Python Whl Asset - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: ./arcade/dist/${{ env.WHL_NAME }} - asset_name: ${{ env.WHL_NAME }} - asset_content_type: application/zip - - - name: Zip Full Dist - run: tar -cvf python-package-distributions.tar.gz ./dist - - - - name: Upload Full Dist - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: python-package-distributions.tar.gz - asset_name: python-package-distributions.tar.gz - asset_content_type: application/zip - - - name: Upload the distribution packages - uses: actions/upload-artifact@v4 - with: - name: python-package-distributions - path: dist/ diff --git a/Makefile b/Makefile index e9efbaed..dea27130 100644 --- a/Makefile +++ b/Makefile @@ -175,13 +175,6 @@ docker-base: ## Build and run the Docker container @cd docker && INSTALL_TOOLKITS=false make docker-build @cd docker && INSTALL_TOOLKITS=false make docker-run -.PHONY: publish-ecr -publish-ecr: ## Publish to the ECR - # Publish the base image - /arcadeai/worker-base - @cd docker && INSTALL_TOOLKITS=false make publish-ecr - # Publish the image with toolkits - /arcadeai/worker - @cd docker && INSTALL_TOOLKITS=true make publish-ecr - .PHONY: publish-ghcr publish-ghcr: ## Publish to the GHCR # Publish the base image - ghcr.io/arcadeai/worker-base diff --git a/docker/Dockerfile b/docker/Dockerfile index 75abe0b3..85abf2e0 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -33,7 +33,7 @@ EXPOSE $PORT # List wheel files for debugging purposes RUN ls -la /app/dist/ -# Install the CLI and worker package +# Install the worker and CLI package RUN python -m pip install \ /app/dist/arcade_serve-*.whl \ /app/dist/arcade_ai-*.whl diff --git a/docker/Makefile b/docker/Makefile index 29067854..5d71c173 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -4,7 +4,6 @@ SOURCE ?= https://github.com/ArcadeAI/arcade-ai LICENSE ?= MIT DESCRIPTION ?= "Arcade Worker for LLM Tool Serving" REPOSITORY ?= arcadeai/worker -ECR_ENDPOINT ?= 471112909428.dkr.ecr.us-east-1.amazonaws.com ARCH ?= $(shell uname -m) VERSION ?= 0.1.0.dev0 @@ -45,29 +44,6 @@ docker-build: ## Build the Docker container --label=org.opencontainers.image.description=$(DESCRIPTION) \ .. - -ecr-manifest: ## Make a manifest file for the image - @echo "šŸ› ļø Build manifest file for $(REPOSITORY):$(VERSION).." - @echo "- Commit: $(COMMIT)" - @echo "- Build Date: $(BUILD_DATE)" - @export DOCKER_CLI_EXPERIMENTAL=enabled - @echo "- Creating manifest $(ECR_ENDPOINT)/$(REPOSITORY):$(VERSION)" - @docker manifest create $(ECR_ENDPOINT)/$(REPOSITORY):$(VERSION) \ - --amend $(ECR_ENDPOINT)/$(REPOSITORY):$(VERSION)-arm64 \ - --amend $(ECR_ENDPOINT)/$(REPOSITORY):$(VERSION)-amd64 - @echo "- Creating manifest $(ECR_ENDPOINT)/$(REPOSITORY):latest" - @docker manifest create $(ECR_ENDPOINT)/$(REPOSITORY):latest \ - --amend $(ECR_ENDPOINT)/$(REPOSITORY):$(VERSION)-arm64 \ - --amend $(ECR_ENDPOINT)/$(REPOSITORY):$(VERSION)-amd64 - @echo "- Inspecting manifest $(ECR_ENDPOINT)/$(REPOSITORY):$(VERSION)" - @docker manifest inspect $(ECR_ENDPOINT)/$(REPOSITORY):$(VERSION) - @echo "- Inspecting manifest $(ECR_ENDPOINT)/$(REPOSITORY):latest" - @docker manifest inspect $(ECR_ENDPOINT)/$(REPOSITORY):latest - @echo "- Pushing manifest $(ECR_ENDPOINT)/$(REPOSITORY):$(VERSION)" - @docker manifest push $(ECR_ENDPOINT)/$(REPOSITORY):$(VERSION) - @echo "- Pushing manifest $(ECR_ENDPOINT)/$(REPOSITORY):latest" - @docker manifest push $(ECR_ENDPOINT)/$(REPOSITORY):latest - ghcr-manifest: ## Make a manifest file for the image @echo "šŸ› ļø Build manifest file for $(REPOSITORY):$(VERSION).." @echo "- Commit: $(COMMIT)" @@ -90,22 +66,12 @@ ghcr-manifest: ## Make a manifest file for the image @echo "- Pushing manifest ghcr.io/$(REPOSITORY):latest" @docker manifest push ghcr.io/$(REPOSITORY):latest + .PHONY: docker-run docker-run: ## Run the Docker container @echo "\nšŸš€ Run the container with the following ..." @echo ">>> docker run -d -p $(PORT):$(PORT) $(REPOSITORY):$(VERSION_TAG)" -.PHONY: publish-ecr -publish-ecr: - @echo "šŸšš Pushing the Agent image to ECR.." - @docker tag $(REPOSITORY):$(VERSION_TAG) $(ECR_ENDPOINT)/$(REPOSITORY):$(VERSION_TAG) - @echo "- pushing $(ECR_ENDPOINT)/$(REPOSITORY):$(VERSION_TAG)" - @docker push $(ECR_ENDPOINT)/$(REPOSITORY):$(VERSION_TAG) - @echo $(VERSION_TAG) | grep -q $(RC_PART) || { \ - docker tag $(REPOSITORY):$(VERSION_TAG) $(ECR_ENDPOINT)/$(REPOSITORY):latest-$(ARCH); \ - echo "- pushing $(ECR_ENDPOINT)/$(REPOSITORY):latest-$(ARCH)"; \ - docker push $(ECR_ENDPOINT)/$(REPOSITORY):latest-$(ARCH); \ - } .PHONY: publish-ghcr publish-ghcr: @@ -120,11 +86,6 @@ publish-ghcr: } -.PHONY: ecr-login -ecr-login: ## Login to ECR - @aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin $(ECR_ENDPOINT) - - .PHONY: gh-login gh-login: ## Login to GHCR @echo "šŸšš Logging in to GHCR..."