diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index e588a9d279d..d27ed6515dc 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -108,7 +108,7 @@ jobs: # actions/cache@v2.1.4 uses: actions/cache@26968a09c0ea4f3e233fdddbafd1166051a095f6 with: - key: ${{ runner.os }}-clickhouse-binary-${{ hashFiles('tools/clickhouse_checksums') }} + key: ${{ runner.os }}-clickhouse-binary-${{ hashFiles('tools/clickhouse_versions') }} path: "clickhouse" - name: Download ClickHouse if: steps.cache-clickhouse.outputs.cache-hit != 'true' diff --git a/tools/ci_download_clickhouse b/tools/ci_download_clickhouse index c1099216941..215b7724221 100755 --- a/tools/ci_download_clickhouse +++ b/tools/ci_download_clickhouse @@ -13,14 +13,30 @@ set -o errexit SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" ARG0="$(basename ${BASH_SOURCE[0]})" -# If you change this, you must also update the md5sums below -CIDL_VERSION="v21.7" -source "$SOURCE_DIR/clickhouse_checksums" +CIDL_VERSION_FILE="$SOURCE_DIR/clickhouse_versions" +source "$CIDL_VERSION_FILE" CIDL_ASSEMBLE_DIR="./clickhouse" # Download from manually-populated S3 bucket for now CIDL_URL_BASE="https://oxide-clickhouse-build.s3.us-west-2.amazonaws.com" +function per_platform +{ + # + # We should use "${1^^}" here instead of tr(1), except that Apple, in a + # thoroughly characteristic fit of user hostility, ship a flagrantly + # obsolete version of bash that has no features. + # + local pp_key="$(tr '[:lower:]' '[:upper:]' <<< "$1")" + local pp_platform="$(tr '[:lower:]' '[:upper:]' <<< "$2")" + local pp_varname="CIDL_${pp_key}_${pp_platform}" + local pp_val="${!pp_varname}" + if [[ -z "$pp_val" ]]; then + fail "missing \"$pp_varname\" in \"$CIDL_VERSION_FILE\"" + fi + printf '%s' "$pp_val" +} + function main { # @@ -44,18 +60,42 @@ function main configure_os "$CIDL_OS" CIDL_URL="$CIDL_URL_BASE/$CIDL_FILE" - # Download the file. echo "URL: $CIDL_URL" echo "Local file: $CIDL_FILE" - do_download_curl "$CIDL_URL" "$CIDL_FILE" || \ - fail "failed to download file" - - # Verify the md5sum. - calculated_md5="$($CIDL_MD5FUNC "$CIDL_FILE")" || \ - fail "failed to calculate md5sum" - if [[ "$calculated_md5" != "$CIDL_MD5" ]]; then - fail "md5sum mismatch \ - (expected $CIDL_MD5, found $calculated_md5)" + + # + # If the file exists already, check to see if the hash matches so that + # we can avoid a second download. + # + if [[ -f "$CIDL_FILE" ]]; then + calculated_md5="$($CIDL_MD5FUNC "$CIDL_FILE")" || \ + fail "failed to calculate md5sum" + if [[ "$calculated_md5" == "$CIDL_MD5" ]]; then + download=no + else + rm -f "$CIDL_FILE" + download=yes + fi + else + download=yes + fi + + if [[ "$download" == yes ]]; then + # + # Download the file. + # + do_download_curl "$CIDL_URL" "$CIDL_FILE" || \ + fail "failed to download file" + + # + # Verify the md5sum. + # + calculated_md5="$($CIDL_MD5FUNC "$CIDL_FILE")" || \ + fail "failed to calculate md5sum" + if [[ "$calculated_md5" != "$CIDL_MD5" ]]; then + fail "md5sum mismatch \ + (expected $CIDL_MD5, found $calculated_md5)" + fi fi # Unpack the tarball into a local directory @@ -76,27 +116,29 @@ function configure_os echo "current directory: $PWD" echo "configuring based on OS: \"$1\"" case "$1" in - darwin*) - CIDL_PLATFORM="macos" - CIDL_MD5="$CIDL_MD5_DARWIN" - CIDL_MD5FUNC="do_md5" - ;; - linux-gnu*) - CIDL_PLATFORM="linux" - CIDL_MD5="$CIDL_MD5_LINUX" - CIDL_MD5FUNC="do_md5sum" - ;; - solaris*) - CIDL_PLATFORM="illumos" - CIDL_MD5="$CIDL_MD5_ILLUMOS" - CIDL_MD5FUNC="do_md5sum" - ;; - *) - fail "unsupported OS: $1" - ;; + darwin*) + CIDL_PLATFORM="macos" + CIDL_MD5FUNC="do_md5" + ;; + linux-gnu*) + CIDL_PLATFORM="linux" + CIDL_MD5FUNC="do_md5sum" + ;; + solaris*) + CIDL_PLATFORM="illumos" + CIDL_MD5FUNC="do_md5sum" + ;; + *) + fail "unsupported OS: $1" + ;; esac - CIDL_DIR="clickhouse-$CIDL_VERSION" + if ! CIDL_VERSION=$(per_platform version "$CIDL_PLATFORM") || + ! CIDL_MD5=$(per_platform md5 "$CIDL_PLATFORM"); then + fail "could not configure platform $CIDL_PLATFORM" + fi + + CIDL_DIR="clickhouse-v$CIDL_VERSION" CIDL_FILE="$CIDL_DIR.$CIDL_PLATFORM.tar.gz" } diff --git a/tools/clickhouse_checksums b/tools/clickhouse_checksums deleted file mode 100644 index 2eb9636de5e..00000000000 --- a/tools/clickhouse_checksums +++ /dev/null @@ -1,3 +0,0 @@ -CIDL_MD5_DARWIN="7e22bd0b0108dee55605dd563a5698df" -CIDL_MD5_LINUX="f821613422cda8bb3c7f281eeb87f145" -CIDL_MD5_ILLUMOS="2870157143dc78b1ebe1e3dfabe5fd61" diff --git a/tools/clickhouse_versions b/tools/clickhouse_versions new file mode 100644 index 00000000000..6efc5848426 --- /dev/null +++ b/tools/clickhouse_versions @@ -0,0 +1,8 @@ +CIDL_VERSION_ILLUMOS="21.10.4.26" +CIDL_MD5_ILLUMOS="d2ff9c8d15d017a76a605a33618b9da4" + +CIDL_VERSION_LINUX="21.10.4.26" +CIDL_MD5_LINUX="7aa1e2eb55d7540ab71fde2e4abb4c81" + +CIDL_VERSION_MACOS="21.7" +CIDL_MD5_MACOS="7e22bd0b0108dee55605dd563a5698df"