Repository navigation
feat(): Update DUCKDB_VERSION to 1.5.4 #112
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build AWS Lambda Layers for DuckDB | |
| ################################ | |
| #### TRIGGER | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| paths: | |
| - ".github/.duckdb-version" | |
| - ".github/workflows/build-layer.yml" | |
| - "docker/**" | |
| ################################ | |
| #### ENV VARIABLES | |
| env: | |
| ARCHITECTURES: "x86_64,arm64" | |
| PYTHON_VERSIONS: "3.10,3.11,3.12,3.13,3.14" | |
| AWS_REGIONS: ${{ vars.AWS_REGIONS }} | |
| AWS_ACCOUNT_ID: ${{ vars.AWS_ACCOUNT_ID }} | |
| ################################ | |
| #### JOBS | |
| jobs: | |
| setup: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| architectures: ${{ steps.matrix.outputs.architectures }} | |
| python-versions: ${{ steps.matrix.outputs.python-versions }} | |
| duckdb-version: ${{ steps.version.outputs.duckdb-version }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v6 | |
| - name: Read DuckDB version | |
| id: version | |
| run: echo "duckdb-version=$(cat .github/.duckdb-version | tr -d '[:space:]')" >> $GITHUB_OUTPUT | |
| - id: matrix | |
| run: | | |
| # Convert comma-separated architectures to JSON array | |
| ARCH_ARRAY=$(echo "${{ env.ARCHITECTURES }}" | sed 's/,/", "/g' | sed 's/^/["/' | sed 's/$/"]/') | |
| echo "architectures=$ARCH_ARRAY" >> $GITHUB_OUTPUT | |
| # Convert comma-separated Python versions to JSON array | |
| PY_ARRAY=$(echo "${{ env.PYTHON_VERSIONS }}" | sed 's/,/", "/g' | sed 's/^/["/' | sed 's/$/"]/') | |
| echo "python-versions=$PY_ARRAY" >> $GITHUB_OUTPUT | |
| ################################ | |
| #### PACKAGE DUCKDB | |
| package-duckdb: | |
| name: Build DuckDB | |
| needs: setup | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| architecture: ${{ fromJSON(needs.setup.outputs.architectures) }} | |
| python-version: ${{ fromJSON(needs.setup.outputs.python-versions) }} | |
| steps: | |
| - name: Checkout code | |
| id: checkout | |
| uses: actions/checkout@v6 | |
| - name: Set up QEMU | |
| id: setup-qemu | |
| uses: docker/setup-qemu-action@v4 | |
| with: | |
| platforms: ${{ matrix.architecture == 'x86_64' && 'linux/amd64' || 'linux/arm64' }} | |
| - name: Build DuckDB | |
| id: build | |
| run: | | |
| docker build --platform ${{ matrix.architecture == 'x86_64' && 'linux/amd64' || 'linux/arm64' }} --build-arg DUCKDB_VERSION=${{ needs.setup.outputs.duckdb-version }} --build-arg PYTHON_VERSION=${{ matrix.python-version }} --build-arg ARCHITECTURE=${{ matrix.architecture }} -f ./docker/Dockerfile -t layer-duckdb:latest . | |
| - name: Extract zip from Docker container | |
| id: extract-zip | |
| run: | | |
| # Create a temporary container | |
| container_id=$(docker create --platform ${{ matrix.architecture == 'x86_64' && 'linux/amd64' || 'linux/arm64' }} layer-duckdb:latest) | |
| # Copy the zip file from the container | |
| docker cp $container_id:/tmp/duckdb.zip ./duckdb-${{ needs.setup.outputs.duckdb-version }}-python-${{ matrix.python-version }}-${{ matrix.architecture }}.zip | |
| # Clean up the container | |
| docker rm $container_id | |
| - name: Upload artifact | |
| id: upload | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: duckdb-${{ needs.setup.outputs.duckdb-version }}-python-${{ matrix.python-version }}-${{ matrix.architecture }} | |
| path: duckdb-${{ needs.setup.outputs.duckdb-version }}-python-${{ matrix.python-version }}-${{ matrix.architecture }}.zip | |
| retention-days: 1 | |
| ################################ | |
| #### PUBLISH LAYERS | |
| publish-layer: | |
| name: Publish Layer to all AWS Regions | |
| needs: [setup, package-duckdb] | |
| if: github.event_name != 'pull_request' | |
| runs-on: ubuntu-latest | |
| environment: ${{ github.ref == 'refs/heads/main' && 'production' || 'test' }} | |
| outputs: | |
| layer-version: ${{ steps.publish.outputs.layer-version }} | |
| strategy: | |
| matrix: | |
| architecture: ${{ fromJSON(needs.setup.outputs.architectures) }} | |
| python-version: ${{ fromJSON(needs.setup.outputs.python-versions) }} | |
| steps: | |
| - name: Download artifact | |
| id: download | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: duckdb-${{ needs.setup.outputs.duckdb-version }}-python-${{ matrix.python-version }}-${{ matrix.architecture }} | |
| # Configure AWS credentials | |
| - name: Configure AWS credentials | |
| id: configure-aws-credentials | |
| uses: aws-actions/configure-aws-credentials@v6 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: us-east-1 # Default region for initial setup | |
| - name: Publish Lambda Layer to All Regions | |
| id: publish | |
| run: | | |
| PYTHON_VERSION_NODOT=$(echo "${{ matrix.python-version }}" | sed 's/\.//g') | |
| IFS=',' read -ra REGIONS <<< "${{ env.AWS_REGIONS }}" | |
| mkdir -p results | |
| RESULT_FILE="results/arn-${{ needs.setup.outputs.duckdb-version }}-${{ matrix.python-version }}-${{ matrix.architecture }}.json" | |
| # Initiate JSON structure for this combination | |
| cat > $RESULT_FILE <<EOF | |
| { | |
| "${{ matrix.python-version }}": { | |
| "${{ matrix.architecture }}": { | |
| } | |
| } | |
| } | |
| EOF | |
| for region in "${REGIONS[@]}"; do | |
| echo "Publishing to region: $region" | |
| if ARN=$(aws lambda publish-layer-version \ | |
| --region $region \ | |
| --layer-name duckdb-python${PYTHON_VERSION_NODOT}-${{ matrix.architecture }} \ | |
| --description "DuckDB ${{ needs.setup.outputs.duckdb-version }} Python ${{ matrix.python-version }} for AWS Lambda (${{ matrix.architecture }})" \ | |
| --zip-file fileb://duckdb-${{ needs.setup.outputs.duckdb-version }}-python-${{ matrix.python-version }}-${{ matrix.architecture }}.zip \ | |
| --compatible-runtimes python${{ matrix.python-version }} \ | |
| --compatible-architectures ${{ matrix.architecture }} \ | |
| --query 'LayerVersionArn' \ | |
| --output text); then | |
| echo "Published: $ARN" | |
| # Add region:arn pair to JSON structure | |
| tmp=$(mktemp) | |
| jq --arg region "$region" --arg arn "$ARN" \ | |
| '.["'${{ matrix.python-version }}'"]["'${{ matrix.architecture }}'"] += {($region): $arn}' \ | |
| $RESULT_FILE > $tmp | |
| mv $tmp $RESULT_FILE | |
| # Rendre la layer publique | |
| aws lambda add-layer-version-permission \ | |
| --region $region \ | |
| --layer-name duckdb-python${PYTHON_VERSION_NODOT}-${{ matrix.architecture }} \ | |
| --version-number $(echo $ARN | awk -F: '{print $NF}') \ | |
| --statement-id public-access \ | |
| --action lambda:GetLayerVersion \ | |
| --principal "*" || true | |
| fi | |
| sleep 1 | |
| done | |
| echo "Results saved in $RESULT_FILE" | |
| - name: Upload publish results | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: arns-${{ needs.setup.outputs.duckdb-version }}-${{ matrix.python-version }}-${{ matrix.architecture }} | |
| path: results/ | |
| retention-days: 1 | |
| ################################ | |
| #### UPDATE DOCUMENTATION | |
| update-documentation: | |
| name: Update Documentation | |
| needs: [setup, publish-layer] | |
| if: github.event_name != 'pull_request' | |
| runs-on: ubuntu-latest | |
| environment: ${{ github.ref == 'refs/heads/main' && 'production' || 'test' }} | |
| permissions: | |
| contents: write | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v6 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.x" | |
| cache: "pip" | |
| - name: Download all publish results | |
| uses: actions/download-artifact@v8 | |
| with: | |
| pattern: arns-* | |
| path: arn-results | |
| merge-multiple: true | |
| - name: Merge JSON results into data/arns.json | |
| run: | | |
| mkdir -p data | |
| # Ensure data/arns.json exists | |
| if [ ! -f data/arns.json ]; then | |
| echo "{}" > data/arns.json | |
| fi | |
| # Merge all results into a single block | |
| jq -s 'reduce .[] as $item ({}; . * $item)' arn-results/*.json > arn-results/all.json | |
| echo "Merged all publish results:" | |
| cat arn-results/all.json | |
| # Inject all.json under the DUCKDB_VERSION key in data/arns.json | |
| tmp=$(mktemp) | |
| jq --arg version "${{ needs.setup.outputs.duckdb-version }}" \ | |
| --slurpfile new arn-results/all.json \ | |
| '.[$version] = $new[0]' data/arns.json > "$tmp" && mv "$tmp" data/arns.json | |
| echo "data/arns.json updated." | |
| - name: Update README mappings table | |
| run: | | |
| echo "Updating README table..." | |
| pip install -r scripts/requirements.txt | |
| python scripts/update_readme.py | |
| - uses: stefanzweifel/git-auto-commit-action@v7 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| commit_message: "docs(): update documentation for DuckDB ${{ needs.setup.outputs.duckdb-version }}" | |
| branch: "main" | |
| commit_user_name: "bengeois" | |
| commit_user_email: "me@bbourgeois.dev" | |
| commit_author: bengeois 🤖 <me@bbourgeois.dev> |