diff --git a/.github/workflows/ci-and-release-tag.yml b/.github/workflows/ci-and-release-tag.yml index 3cd52bf..a1bf39d 100644 --- a/.github/workflows/ci-and-release-tag.yml +++ b/.github/workflows/ci-and-release-tag.yml @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright the Vortex contributors -name: CI and Release Tag +name: CI and Release on: pull_request: @@ -50,6 +50,10 @@ jobs: runs-on: ubuntu-latest permissions: contents: write + outputs: + tag: ${{ steps.release.outputs.tag }} + commit: ${{ steps.release.outputs.commit }} + publish: ${{ steps.release.outputs.publish }} steps: - name: Checkout uses: actions/checkout@v6 @@ -123,3 +127,137 @@ jobs: git tag "${TAG}" git push origin "refs/tags/${TAG}" + + - name: Select binary release + id: release + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + + tag="$(python3 scripts/version_pair.py tag)" + commit="$(git rev-list -n 1 "refs/tags/${tag}")" + releases="$(gh api --paginate "repos/${GITHUB_REPOSITORY}/releases")" + published="$(jq -s --arg tag "${tag}" 'any(.[][]; .tag_name == $tag and .draft == false)' <<< "${releases}")" + { + echo "tag=${tag}" + echo "commit=${commit}" + echo "publish=$([[ "${published}" == false ]] && echo true || echo false)" + } >> "$GITHUB_OUTPUT" + + release-build: + name: Release build (${{ matrix.target }}) + needs: release-tag + # Tag pushes made with GITHUB_TOKEN do not trigger another workflow. + # Exercise the same packaging on PRs without publishing a release. + if: >- + always() && !cancelled() && + (github.event_name == 'pull_request' || needs.release-tag.outputs.publish == 'true') + strategy: + fail-fast: false + matrix: + include: + - target: x86_64-unknown-linux-gnu + runner: ubuntu-22.04 + - target: aarch64-apple-darwin + runner: macos-14 + runs-on: ${{ matrix.runner }} + env: + TARGET: ${{ matrix.target }} + MACOSX_DEPLOYMENT_TARGET: "14.0" + CARGO_PROFILE_RELEASE_STRIP: symbols + steps: + - name: Checkout release source + uses: actions/checkout@v6 + with: + ref: ${{ needs.release-tag.outputs.commit || github.sha }} + persist-credentials: false + + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} + + - name: Cache Rust dependencies + uses: Swatinem/rust-cache@v2 + with: + key: release-${{ matrix.target }} + + - name: Build release binary + run: cargo build --release --locked --bin vortex-datafusion-cli --target "${TARGET}" + + - name: Package binary + shell: bash + run: | + set -euo pipefail + + archive="vortex-datafusion-cli-${TARGET}" + mkdir -p "dist/${archive}" + cp "target/${TARGET}/release/vortex-datafusion-cli" "dist/${archive}/" + cp LICENSE NOTICE "dist/${archive}/" + cp -R LICENSES "dist/${archive}/" + tar -czf "dist/${archive}.tar.gz" -C dist "${archive}" + cd dist + shasum -a 256 "${archive}.tar.gz" > "${archive}.tar.gz.sha256" + + - name: Smoke test packaged binary + shell: bash + run: | + set -euo pipefail + + archive="vortex-datafusion-cli-${TARGET}" + cd dist + shasum -a 256 -c "${archive}.tar.gz.sha256" + smoke_dir="$(mktemp -d)" + tar -xzf "${archive}.tar.gz" -C "${smoke_dir}" + "${smoke_dir}/${archive}/vortex-datafusion-cli" --version + "${smoke_dir}/${archive}/vortex-datafusion-cli" --quiet --command 'SELECT 1;' + + - name: Upload release artifacts + uses: actions/upload-artifact@v4 + with: + name: release-${{ matrix.target }} + path: | + dist/*.tar.gz + dist/*.sha256 + if-no-files-found: error + + release-publish: + name: Publish binary release + needs: [release-tag, release-build] + if: needs.release-tag.outputs.publish == 'true' + runs-on: ubuntu-latest + concurrency: + group: release-${{ needs.release-tag.outputs.tag }} + cancel-in-progress: false + permissions: + contents: write + steps: + - name: Download release artifacts + uses: actions/download-artifact@v4 + with: + pattern: release-* + merge-multiple: true + path: dist + + - name: Publish release + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + TAG: ${{ needs.release-tag.outputs.tag }} + run: | + set -euo pipefail + + releases="$(gh api --paginate "repos/${GH_REPO}/releases")" + state="$(jq -sr --arg tag "${TAG}" '.[][] | select(.tag_name == $tag) | .draft' <<< "${releases}")" + if [[ "${state}" == false ]]; then + echo "Release ${TAG} is already published." + exit 0 + fi + + if [[ -z "${state}" ]]; then + gh release create "${TAG}" --verify-tag --draft --title "${TAG}" --generate-notes + fi + # Keep incomplete uploads private and allow a failed publication to be retried. + gh release upload "${TAG}" dist/*.tar.gz dist/*.sha256 --clobber + gh release edit "${TAG}" --draft=false --latest diff --git a/README.md b/README.md index 2c7e929..5d4b7f9 100644 --- a/README.md +++ b/README.md @@ -9,10 +9,43 @@ This is a CLI tool to run use Vortex through the Apache DataFusion CLI tool, mos In the future we hope we can minimize the amount of identical code, but for the time being this is a pragmatic solution. +## Download + +Download a prebuilt binary from the [latest release](https://github.com/vortex-data/vortex-datafusion-cli/releases/latest). No Rust toolchain is needed. + +| Platform | Archive | +| --- | --- | +| Linux x86-64 (Ubuntu 22.04 or newer) | `vortex-datafusion-cli-x86_64-unknown-linux-gnu.tar.gz` | +| macOS Apple Silicon (macOS 14 or newer) | `vortex-datafusion-cli-aarch64-apple-darwin.tar.gz` | + +The Linux binary is dynamically linked and targets the Ubuntu 22.04 runtime libraries (including glibc 2.35). Other Linux distributions need compatible runtime libraries; Alpine/musl is not supported by this build. + +For example, on Linux x86-64: + +```sh +target=x86_64-unknown-linux-gnu +archive="vortex-datafusion-cli-${target}" +base=https://github.com/vortex-data/vortex-datafusion-cli/releases/latest/download +curl -fLO "${base}/${archive}.tar.gz" +curl -fLO "${base}/${archive}.tar.gz.sha256" +sha256sum -c "${archive}.tar.gz.sha256" +tar -xzf "${archive}.tar.gz" +mkdir -p "$HOME/.local/bin" +install -m 755 "${archive}/vortex-datafusion-cli" "$HOME/.local/bin/vortex-datafusion-cli" +export PATH="$HOME/.local/bin:$PATH" +vortex-datafusion-cli --version +``` + +On macOS, set `target=aarch64-apple-darwin` and use `shasum -a 256 -c` instead of `sha256sum -c`. Add `$HOME/.local/bin` to your shell's startup configuration to keep the command on `PATH` in new sessions. + +For reproducible benchmarks, download a specific [release](https://github.com/vortex-data/vortex-datafusion-cli/releases) by replacing `releases/latest/download` in `base` with `releases/download/`. + ## Versioning Release tags use the format `-`, where `vortex-version` is the resolved `vortex-datafusion` crate version and `df-version` is the resolved `datafusion`/`datafusion-cli` crate version. Pushes to `main` create the current tag if it does not already exist. Later merges to `main` create a new tag only when `Cargo.lock` changes either of those resolved versions. Code, docs, or unrelated dependency changes still run CI, but do not create release tags once the current tag exists. +After CI passes on `main`, the workflow builds binaries from the current tag's commit and publishes a GitHub Release with both archives and their SHA-256 checksums. An existing tag without a published release is also packaged, so the first release does not require a dependency update. Published releases are left unchanged. Failed builds or draft uploads can be retried by rerunning the workflow; publication waits for both platforms to succeed. Pull requests build and smoke-test the same archives without publishing them. + [Renovate](https://docs.renovatebot.com/) updates Cargo manifests, `Cargo.lock`, and GitHub Actions workflow action versions. Vortex, DataFusion, and DataFusion CLI updates are grouped together, including major upgrades, so their integration uses compatible DataFusion versions.