ci: samcli with qemu+vz (runfinch#354)

Some fixes to workflows following issues in emulation for cross-arch images and updates in SAMcli.

Signed-off-by: ayush-panta <ayushkp@amazon.com>

Author: ayush-panta

.github/workflows/finch-vm-test.yaml

@@ -21,6 +21,14 @@ jobs:
       - name: Clean macOS runner workspace
         run: |
           rm -rf ${{ github.workspace }}/*
+          # Clean up any leftover Finch VM state
+          su ec2-user -c 'finch vm remove -f' || true
+          sudo pkill -f socket_vmnet || true
+          sudo rm -rf /private/var/run/finch-lima/*.sock || true
+          sudo rm -rf /Applications/Finch/lima/data/finch/_cache || true
+          # Clean up containers and images via Finch CLI
+          su ec2-user -c 'finch system prune -f' || true
+
       - name: Configure Git for ec2-user
         run: |
           git config --global --add safe.directory "*"
@@ -77,12 +85,18 @@ jobs:
       - name: Checkout finch-daemon PR
         uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
-          ref: ${{ github.head_ref }}
+          ref: ${{ github.event.pull_request.head.sha || 'main' }}
+          repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
           fetch-depth: 0
           persist-credentials: false
           submodules: recursive
           path: finch-daemon-pr
 
+      - name: Check Finch configuration
+        run: |
+          echo "Config (finch.yaml):"
+          su ec2-user -c 'cat ~/.finch/finch.yaml || echo "No config file found"'
+
       - name: Build and setup Finch VM
         run: ./finch-daemon-pr/scripts/build-and-setup-finch-vm.sh
 

.github/workflows/samcli-direct.yaml

@@ -34,7 +34,7 @@ jobs:
           - name: start-api
     env:
       AWS_DEFAULT_REGION: "${{ secrets.REGION }}"
-      DOCKER_HOST: unix:///run/finch.sock
+      DOCKER_HOST: unix:///var/run/finch.sock
       DOCKER_CONFIG: $HOME/.finch
       BY_CANARY: true # allows full testing
       SAM_CLI_DEV: 1
@@ -63,6 +63,7 @@ jobs:
       - name: Checkout finch-daemon
         uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
+          ref: ${{ github.head_ref }}
           fetch-depth: 0
           persist-credentials: false
           submodules: recursive
@@ -93,12 +94,19 @@ jobs:
           sudo bin/finch-daemon --debug --socket-owner $UID 2>&1 | tee finch-daemon.log &
           sleep 10
 
+      - name: Get latest SAM CLI tag
+        id: sam-tag
+        run: |
+          TAG=$(curl -s https://api.github.com/repos/aws/aws-sam-cli/releases/latest | jq -r .tag_name)
+          echo "tag=$TAG" >> $GITHUB_OUTPUT
+
       - name: Checkout SAM CLI
         uses: actions/checkout@v4
         with:
           repository: aws/aws-sam-cli
           submodules: recursive
           path: aws-sam-cli
+          ref: ${{ steps.sam-tag.outputs.tag }}
 
       - name: Set up SAM CLI from source
         working-directory: aws-sam-cli

.github/workflows/samcli-vm.yaml

@@ -34,8 +34,7 @@ jobs:
           sudo rm -rf /private/var/run/finch-lima/*.sock || true
           sudo rm -rf /Applications/Finch/lima/data/finch/_cache || true
           # Clean up containers and images via Finch CLI
-          su ec2-user -c 'finch container prune -f' || true
-          su ec2-user -c 'finch image prune -a -f' || true
+          su ec2-user -c 'finch system prune -f' || true
 
       - name: Configure Git for ec2-user
         run: |
@@ -111,9 +110,44 @@ jobs:
           submodules: recursive
           path: finch-daemon-pr
 
+      - name: Configure Finch
+        run: |
+          echo "Original config:"
+          su ec2-user -c 'cat ~/.finch/finch.yaml || echo "No config file found"'
+          echo "Configuring Finch to use QEMU+VZ"
+          su ec2-user -c 'yq eval ".vmType = \"vz\"" -i ~/.finch/finch.yaml'
+          su ec2-user -c 'yq eval ".rosetta = false" -i ~/.finch/finch.yaml'
+          echo "Updated config:"
+          su ec2-user -c 'cat ~/.finch/finch.yaml'
+
       - name: Build and setup Finch VM
         run: ./finch-daemon-pr/scripts/build-and-setup-finch-vm.sh
 
+      - name: Verify Finch socket
+        run: |
+          # Test socket connectivity
+          if su ec2-user -c 'curl -s --unix-socket /Applications/Finch/lima/data/finch/sock/finch.sock http://localhost/version' > /dev/null; then
+            echo "✓ Finch daemon is accessible"
+          else
+            echo "✗ Finch daemon connection failed"
+            ls -la /Applications/Finch/lima/data/finch/sock/ || echo "Socket directory not found"
+            exit 1
+          fi
+
+      - name: Ensure Docker is not available (force Finch usage)
+        run: |
+          echo "Ensuring Docker is not accessible to force SAM CLI to use Finch..."
+          # Remove docker binaries from PATH
+          sudo rm -f /usr/local/bin/docker /opt/homebrew/bin/docker || true
+          # Verify docker is not accessible
+          if su ec2-user -c 'which docker' > /dev/null 2>&1; then
+            echo "WARNING: Docker is still accessible"
+            su ec2-user -c 'which docker'
+          else
+            echo "SUCCESS: Docker is not accessible - SAM CLI will use Finch"
+          fi
+        shell: bash
+
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1
         with:
@@ -122,23 +156,19 @@ jobs:
           aws-region: ${{ secrets.REGION }}
           role-duration-seconds: 14400
 
-      - name: Install Docker CLI for SAM CLI compatibility
+      - name: Get latest SAM CLI tag
+        id: sam-tag
         run: |
-          echo "Checking Docker CLI installation..."
-          if ! su ec2-user -c 'which docker' > /dev/null 2>&1; then
-            echo "Installing Docker CLI..."
-            su ec2-user -c 'source /Users/ec2-user/.brewrc && brew install --formula docker'
-          else
-            echo "Docker CLI already installed"
-          fi
-        shell: bash
+          TAG=$(curl -s https://api.github.com/repos/aws/aws-sam-cli/releases/latest | jq -r .tag_name)
+          echo "tag=$TAG" >> $GITHUB_OUTPUT
 
       - name: Checkout SAM CLI
         uses: actions/checkout@v4
         with:
           repository: aws/aws-sam-cli
           submodules: recursive
           path: aws-sam-cli
+          ref: ${{ steps.sam-tag.outputs.tag }}
 
       - name: Set up SAM CLI from source
         run: |
@@ -154,33 +184,26 @@ jobs:
         shell: bash
 
       - name: Run unit tests
-        timeout-minutes: 30
         run: ./finch-daemon-pr/scripts/samcli-vm/run-unit-tests.sh
 
-      - name: Patch SAM CLI for Docker image cleanup
-        run: |
-          # Apply patch to handle ImageNotFound exceptions for all Docker tests
-          su ec2-user -c 'cd /Users/ec2-user/aws-sam-cli && patch -p1 tests/integration/local/invoke/test_integrations_cli.py < ${{ github.workspace }}/finch-daemon-pr/scripts/samcli-vm/invoke-teardown.patch'
-        shell: bash
-
       - name: Run invoke tests
         timeout-minutes: 40
         run: ./finch-daemon-pr/scripts/samcli-vm/run-invoke-tests.sh
 
       - name: Run start-api tests
-        timeout-minutes: 70
+        timeout-minutes: 60
         run: ./finch-daemon-pr/scripts/samcli-vm/run-start-api-tests.sh
 
       - name: Run sync tests
         timeout-minutes: 20
         run: ./finch-daemon-pr/scripts/samcli-vm/run-sync-tests.sh
 
       - name: Run package tests
-        timeout-minutes: 10
+        timeout-minutes: 5
         run: ./finch-daemon-pr/scripts/samcli-vm/run-package-tests.sh
 
       - name: Run start-lambda tests
-        timeout-minutes: 30
+        timeout-minutes: 20
         run: ./finch-daemon-pr/scripts/samcli-vm/run-start-lambda-tests.sh
 
   # ensuring resources are clean post-test

scripts/samcli-direct/run-invoke-tests.sh

@@ -3,7 +3,7 @@ set -e
 
 cd aws-sam-cli
 
-python -m pytest tests/integration/local/invoke -k 'not Terraform' -v --tb=short > invoke_output.txt 2>&1 || true
+python -m pytest tests/integration/local/invoke -k 'not Terraform' -v --tb=short 2>&1 | tee invoke_output.txt || true
 
 # test_invoke_with_error_during_image_build: Build error message differs from expected.
 # test_invoke_with_timeout_set_X_TimeoutFunction: Returns timeout message instead of empty string,

scripts/samcli-direct/run-start-api-tests.sh

@@ -4,7 +4,7 @@ set -e
 cd aws-sam-cli
 
 ulimit -n 65536
-python -m pytest tests/integration/local/start_api -k 'not Terraform' -v --tb=short > start_api_output.txt 2>&1 || true
+python -m pytest tests/integration/local/start_api -k 'not Terraform' -v --tb=short 2>&1 | tee start_api_output.txt || true
 
 # test_can_invoke_lambda_layer_successfully: Uses random port, fails occasionally.
 #         Only 1 test of 386 total, acceptable failure rate.

scripts/samcli-direct/run-unit-tests.sh

@@ -4,7 +4,7 @@ set -e
 cd aws-sam-cli
 
 ulimit -n 65536
-make test > unit_test_output.txt 2>&1 || true
+make test 2>&1 | tee unit_test_output.txt || true
 
 echo ""
 echo "=== PASSES ==="

scripts/samcli-vm/run-package-tests.sh

@@ -3,7 +3,7 @@ set -e
 
 echo "=== PACKAGE TESTS - Started at $(date) ==="
 touch /tmp/package_test_output.txt
-chown ec2-user:staff /tmp/package_output.txt
+chown ec2-user:staff /tmp/package_test_output.txt
 su ec2-user -c "
   cd /Users/ec2-user/aws-sam-cli && \
   export PATH='/Users/ec2-user/Library/Python/$PYTHON_VERSION/bin:$PATH' && \

scripts/samcli-vm/run-start-api-tests.sh

@@ -3,20 +3,9 @@ set -e
 
 echo "=== START-API TESTS - Started at $(date) ==="
 touch /tmp/start_api_test_output.txt
-chown ec2-user:staff /tmp/start_api_output.txt
+chown ec2-user:staff /tmp/start_api_test_output.txt
 
-# Start background monitor to show progress
-(
-  while true; do
-    sleep 30
-    echo "[$(date)] Progress check - Last 20 lines:"
-    tail -20 /tmp/start_api_test_output.txt 2>/dev/null || echo "No output yet"
-    echo "---"
-  done
-) &
-MONITOR_PID=$!
-
-# Run tests (output to file only, show progress via monitor)
+# Run tests with live output
 su ec2-user -c "
   cd /Users/ec2-user/aws-sam-cli && \
   export PATH='/Users/ec2-user/Library/Python/$PYTHON_VERSION/bin:$PATH' && \
@@ -27,10 +16,7 @@ su ec2-user -c "
   SAM_CLI_DEV='$SAM_CLI_DEV' \
   SAM_CLI_TELEMETRY='$SAM_CLI_TELEMETRY' \
   '$PYTHON_BINARY' -m pytest tests/integration/local/start_api -k 'not Terraform' -v --tb=short
-" > /tmp/start_api_test_output.txt 2>&1 || true
-
-# Stop monitor
-kill $MONITOR_PID 2>/dev/null || true
+" 2>&1 | tee /tmp/start_api_test_output.txt || true
 echo "=== START-API TESTS - Finished at $(date) ==="
 
 # test_can_invoke_lambda_layer_successfully: Uses random port, fails occasionally.

scripts/samcli-vm/run-start-lambda-tests.sh

@@ -3,7 +3,7 @@ set -e
 
 echo "=== START-LAMBDA TESTS - Started at $(date) ==="
 touch /tmp/start_lambda_test_output.txt
-chown ec2-user:staff /tmp/start_lambda_output.txt
+chown ec2-user:staff /tmp/start_lambda_test_output.txt
 su ec2-user -c "
   cd /Users/ec2-user/aws-sam-cli && \
   export PATH='/Users/ec2-user/Library/Python/$PYTHON_VERSION/bin:$PATH' && \

scripts/samcli-vm/run-unit-tests.sh

@@ -16,7 +16,7 @@ su ec2-user -c "
   SAM_CLI_DEV='$SAM_CLI_DEV' \
   SAM_CLI_TELEMETRY='$SAM_CLI_TELEMETRY' \
   make test
-" > /tmp/unit_test_output.txt 2>&1 || true
+" 2>&1 | tee /tmp/unit_test_output.txt || true
 
 echo ""
 echo "=== PASSES ==="