diff --git a/.github/grype/parse_vulnerabilities_grype.py b/.github/grype/parse_vulnerabilities_grype.py new file mode 100644 index 0000000000000..fec2ef3bfac7a --- /dev/null +++ b/.github/grype/parse_vulnerabilities_grype.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +import json + +from testflows.core import * + +xfails = {} + + +@Name("docker vulnerabilities") +@XFails(xfails) +@TestModule +def docker_vulnerabilities(self): + with Given("I gather grype scan results"): + with open("./result.json", "r") as f: + results = json.load(f) + + for vulnerability in results["matches"]: + with Test( + f"{vulnerability['vulnerability']['id']}@{vulnerability['vulnerability']['namespace']},{vulnerability['vulnerability']['severity']}", + flags=TE, + ): + note(vulnerability) + critical_levels = set(["HIGH", "CRITICAL"]) + if vulnerability['vulnerability']["severity"].upper() in critical_levels: + with Then( + f"Found vulnerability of {vulnerability['vulnerability']['severity']} severity" + ): + result(Fail) + + +if main(): + docker_vulnerabilities() diff --git a/.github/grype/run_grant_scan.sh b/.github/grype/run_grant_scan.sh new file mode 100755 index 0000000000000..a364dc0698753 --- /dev/null +++ b/.github/grype/run_grant_scan.sh @@ -0,0 +1,16 @@ +set -x +set -e + +IMAGE=$1 + +GRANT_VERSION=${GRANT_VERSION:-"v0.6.8"} +SAFE_IMAGE=$(echo "$IMAGE" | sed 's/[\/:]/_/g') + +docker pull anchore/grant:${GRANT_VERSION} + +docker run --rm \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + --name Grant anchore/grant:${GRANT_VERSION} \ + list "$IMAGE" --output json > "${SAFE_IMAGE}-licenses.json" + +ls -sh diff --git a/.github/grype/run_grype_scan.sh b/.github/grype/run_grype_scan.sh new file mode 100755 index 0000000000000..9cf99e9f9f9cb --- /dev/null +++ b/.github/grype/run_grype_scan.sh @@ -0,0 +1,18 @@ +set -x +set -e + +IMAGE=$1 + +GRYPE_VERSION=${GRYPE_VERSION:-"v0.115.0"} + +docker pull $IMAGE +docker pull anchore/grype:${GRYPE_VERSION} + +docker run \ + --rm --volume /var/run/docker.sock:/var/run/docker.sock \ + --name Grype anchore/grype:${GRYPE_VERSION} \ + --scope all-layers \ + -o json \ + $IMAGE > result.json + +ls -sh diff --git a/.github/grype/run_syft_scan.sh b/.github/grype/run_syft_scan.sh new file mode 100755 index 0000000000000..78ad6b6a57809 --- /dev/null +++ b/.github/grype/run_syft_scan.sh @@ -0,0 +1,21 @@ +set -x +set -e + +IMAGE=$1 +VERSION=$2 + +SYFT_VERSION=${SYFT_VERSION:-"v1.51.1"} + +docker pull anchore/syft:${SYFT_VERSION} + +docker run --rm \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + --volume "$PWD:/out" --workdir /out \ + --name Syft anchore/syft:${SYFT_VERSION} \ + --scope all-layers \ + -o table \ + -o "spdx-json=${VERSION}.spdx.json" \ + -o "cyclonedx-json=${VERSION}.cdx.json" \ + "$IMAGE" + +ls -sh diff --git a/.github/grype/transform_and_upload_results_s3.sh b/.github/grype/transform_and_upload_results_s3.sh new file mode 100755 index 0000000000000..e53cd284f90e6 --- /dev/null +++ b/.github/grype/transform_and_upload_results_s3.sh @@ -0,0 +1,31 @@ +SANITIZED_IMAGE=$(echo "$DOCKER_IMAGE" | sed 's/[\/:]/_/g') + +if [ "$PR_NUMBER" -eq 0 ]; then + BASE_PREFIX="REFs/$GITHUB_REF_NAME/$COMMIT_SHA" +else + BASE_PREFIX="PRs/$PR_NUMBER/$COMMIT_SHA" +fi + +GRYPE_PREFIX="$BASE_PREFIX/grype/$SANITIZED_IMAGE" +SYFT_PREFIX="$BASE_PREFIX/scan/$SANITIZED_IMAGE/syft" +GRANT_PREFIX="$BASE_PREFIX/scan/$SANITIZED_IMAGE/grant" + +S3_GRYPE_PATH="s3://$S3_BUCKET/$GRYPE_PREFIX" +S3_SYFT_PATH="s3://$S3_BUCKET/$SYFT_PREFIX" +S3_GRANT_PATH="s3://$S3_BUCKET/$GRANT_PREFIX" +HTTPS_RESULTS_PATH="https://$S3_BUCKET.s3.amazonaws.com/index.html#$GRYPE_PREFIX/" +HTTPS_REPORT_PATH="https://s3.amazonaws.com/$S3_BUCKET/$GRYPE_PREFIX/results.html" +echo "https_report_path=$HTTPS_REPORT_PATH" >> $GITHUB_OUTPUT + +tfs --no-colors transform nice raw.log nice.log.txt +tfs --no-colors report results -a $HTTPS_RESULTS_PATH raw.log - --copyright "Altinity LTD" | tfs --no-colors document convert > results.html + +aws s3 cp --no-progress nice.log.txt $S3_GRYPE_PATH/nice.log.txt --content-type "text/plain; charset=utf-8" || echo "nice log file not found". +aws s3 cp --no-progress results.html $S3_GRYPE_PATH/results.html || echo "results file not found". +aws s3 cp --no-progress raw.log $S3_GRYPE_PATH/raw.log || echo "raw.log file not found". +aws s3 cp --no-progress result.json $S3_GRYPE_PATH/result.json --content-type "text/plain; charset=utf-8" || echo "result.json not found". + +aws s3 cp --no-progress "${VERSION}.spdx.json" "$S3_SYFT_PATH/${VERSION}.spdx.json" --content-type "application/json" || echo "${VERSION}.spdx.json not found". +aws s3 cp --no-progress "${VERSION}.cdx.json" "$S3_SYFT_PATH/${VERSION}.cdx.json" --content-type "application/json" || echo "${VERSION}.cdx.json not found". + +aws s3 cp --no-progress "${SANITIZED_IMAGE}-licenses.json" "$S3_GRANT_PATH/${SANITIZED_IMAGE}-licenses.json" --content-type "application/json" || echo "${SANITIZED_IMAGE}-licenses.json not found". diff --git a/.github/workflows/grype_scan.yml b/.github/workflows/grype_scan.yml new file mode 100644 index 0000000000000..8483750eff81d --- /dev/null +++ b/.github/workflows/grype_scan.yml @@ -0,0 +1,178 @@ +name: Grype Scan +run-name: Grype Scan ${{ inputs.docker_image }} + +on: + workflow_dispatch: + # Inputs for manual run + inputs: + docker_image: + description: 'Docker image. If no tag, it will be determined by version_helper.py' + required: true + workflow_call: + # Inputs for workflow call + inputs: + docker_image: + description: 'Docker image. If no tag, it will be determined by version_helper.py' + required: true + type: string + version: + description: 'Version tag. If no version, it will be determined by version_helper.py' + required: false + type: string + default: "" + tag-suffix: + description: 'Tag suffix. To be appended the version from version_helper.py' + required: false + type: string + default: "" +env: + PYTHONUNBUFFERED: 1 + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + GRYPE_VERSION: "v0.121.0" + SYFT_VERSION: "v1.51.1" + GRANT_VERSION: "v0.6.8" + +jobs: + grype_scan: + name: Grype Scan + runs-on: [self-hosted, altinity-on-demand, altinity-style-checker] + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set up Docker + uses: docker/setup-buildx-action@v3 + + - name: Set up Python + run: | + export TESTFLOWS_VERSION="2.4.19" + sudo apt-get update + sudo apt-get install -y python3-pip python3-venv + python3 -m venv venv + source venv/bin/activate + pip install --upgrade requests chardet urllib3 unidiff 'boto3==1.43.33' PyGithub + pip install testflows==$TESTFLOWS_VERSION awscli==1.33.28 + echo PATH=$PATH >>$GITHUB_ENV + + - name: Set image tag and version + id: set_version + env: + TAG_SUFFIX: ${{ inputs.tag-suffix }} + SPECIFIED_VERSION: ${{ inputs.version }} + INPUT_DOCKER_IMAGE: ${{ inputs.docker_image }} + PR_NUMBER: ${{ github.event.pull_request.number || 0 }} + run: | + python3 ./tests/ci/version_helper.py | grep = | tee /tmp/version_info + source /tmp/version_info + if [ -z "$SPECIFIED_VERSION" ]; then + VERSION=$CLICKHOUSE_VERSION_STRING + else + VERSION=$SPECIFIED_VERSION + fi + if [[ "$INPUT_DOCKER_IMAGE" == *:* ]]; then + DOCKER_IMAGE=$INPUT_DOCKER_IMAGE + else + DOCKER_IMAGE="${INPUT_DOCKER_IMAGE}:${PR_NUMBER}-${VERSION}${TAG_SUFFIX}" + fi + echo "docker_image=$DOCKER_IMAGE" >> $GITHUB_OUTPUT + echo "version=$VERSION" >> $GITHUB_OUTPUT + + - name: Run Grype Scan + run: | + ./.github/grype/run_grype_scan.sh ${{ steps.set_version.outputs.docker_image }} + + - name: Run Syft Scan + if: always() + run: | + ./.github/grype/run_syft_scan.sh ${{ steps.set_version.outputs.docker_image }} ${{ steps.set_version.outputs.version }} + + - name: Run Grant Scan + if: always() + run: | + ./.github/grype/run_grant_scan.sh ${{ steps.set_version.outputs.docker_image }} + + - name: Parse grype results + if: always() + id: parse_grype + run: | + python3 -u ./.github/grype/parse_vulnerabilities_grype.py -o nice --no-colors --log raw.log --test-to-end + + - name: Transform and Upload Scan Results + if: always() + id: upload_results + env: + S3_BUCKET: "altinity-build-artifacts" + COMMIT_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + PR_NUMBER: ${{ env.PR_NUMBER || github.event.pull_request.number || 0 }} + DOCKER_IMAGE: ${{ steps.set_version.outputs.docker_image }} + VERSION: ${{ steps.set_version.outputs.version }} + run: | + echo "PR_NUMBER=$PR_NUMBER" + ./.github/grype/transform_and_upload_results_s3.sh + + - name: Create step summary + if: always() + id: create_summary + run: | + jq -r '"**Image**: \(.source.target.userInput)"' result.json >> $GITHUB_STEP_SUMMARY + jq -r '.distro | "**Distro**: \(.name):\(.version)"' result.json >> $GITHUB_STEP_SUMMARY + if jq -e '.matches | length == 0' result.json > /dev/null; then + echo "No CVEs" >> $GITHUB_STEP_SUMMARY + else + echo "| Severity | Count |" >> $GITHUB_STEP_SUMMARY + echo "|------------|-------|" >> $GITHUB_STEP_SUMMARY + jq -r ' + .matches | + map(.vulnerability.severity) | + group_by(.) | + map({severity: .[0], count: length}) | + sort_by(.severity) | + map("| \(.severity) | \(.count) |") | + .[] + ' result.json >> $GITHUB_STEP_SUMMARY + fi + + HIGH_COUNT=$(jq -r '.matches | map(.vulnerability) | unique_by(.id) | map(.severity) | map(select(. == "High")) | length' result.json) + CRITICAL_COUNT=$(jq -r '.matches | map(.vulnerability) | unique_by(.id) | map(.severity) | map(select(. == "Critical")) | length' result.json) + TOTAL_HIGH_CRITICAL=$((HIGH_COUNT + CRITICAL_COUNT)) + echo "total_high_critical=$TOTAL_HIGH_CRITICAL" >> $GITHUB_OUTPUT + + if [ $TOTAL_HIGH_CRITICAL -gt 0 ]; then + echo '## High and Critical vulnerabilities found' >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + cat raw.log | tfs --no-colors show tests | grep -Pi 'High|Critical' >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + fi + + - name: Set commit status + if: always() + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const totalHighCritical = '${{ steps.create_summary.outputs.total_high_critical }}'; + const hasError = totalHighCritical === ''; + const hasVulnerabilities = parseInt(totalHighCritical) > 0; + github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: '${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}', + state: hasError ? 'error' : '${{ steps.parse_grype.outcome == 'success' && 'success' || 'failure' }}', + target_url: '${{ steps.upload_results.outputs.https_report_path }}', + description: hasError ? 'An error occurred' : `Grype Scan Completed with ${totalHighCritical} high/critical vulnerabilities`, + context: 'Grype Scan ${{ steps.set_version.outputs.docker_image }}' + }); + + - name: Upload artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: grype-results-${{ hashFiles('raw.log') }} + path: | + result.json + nice.log.txt + *.spdx.json + *.cdx.json + *-licenses.json diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index e040cbe20804c..158086f465972 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -30,6 +30,8 @@ on: # yamllint disable-line rule:truthy push: branches: - 'releases/23.8**' + tags: + - 'v23.8**' workflow_dispatch: jobs: @@ -259,6 +261,26 @@ jobs: docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" + GrypeScanServer: + needs: [DockerServerImages] + strategy: + fail-fast: false + matrix: + suffix: ['', '-alpine'] + uses: ./.github/workflows/grype_scan.yml + secrets: inherit + with: + docker_image: altinityinfra/clickhouse-server + tag-suffix: ${{ matrix.suffix }} + + GrypeScanKeeper: + needs: [DockerServerImages] + uses: ./.github/workflows/grype_scan.yml + secrets: inherit + with: + docker_image: altinityinfra/clickhouse-keeper + + ############################################################################################ ##################################### BUILD REPORTER ####################################### ############################################################################################ diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 8697087101812..0b5c75485cbb0 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -8,9 +8,9 @@ SET(VERSION_MINOR 8) SET(VERSION_PATCH 16) SET(VERSION_GITHASH 060ff8e813a4a16a540063127f8c91e2108d9adf) -SET(VERSION_TWEAK 43) +SET(VERSION_TWEAK 10044) SET(VERSION_FLAVOUR altinitystable) -SET(VERSION_DESCRIBE v23.8.16.43.altinitystable) -SET(VERSION_STRING 23.8.16.43.altinitystable) +SET(VERSION_DESCRIBE v23.8.16.10044.altinitystable) +SET(VERSION_STRING 23.8.16.10044.altinitystable) # end of autochange diff --git a/docker/keeper/Dockerfile b/docker/keeper/Dockerfile index c090a50439ddd..3fd91f0197fc0 100644 --- a/docker/keeper/Dockerfile +++ b/docker/keeper/Dockerfile @@ -1,7 +1,7 @@ # The Dockerfile.ubuntu exists for the tests/ci/docker_server.py script # If the image is built from Dockerfile.alpine, then the `-alpine` suffix is added automatically, # so the only purpose of Dockerfile.ubuntu is to push `latest`, `head` and so on w/o suffixes -FROM ubuntu:20.04 AS glibc-donor +FROM ubuntu:22.04 AS glibc-donor ARG TARGETARCH RUN arch=${TARGETARCH:-amd64} \ @@ -9,8 +9,11 @@ RUN arch=${TARGETARCH:-amd64} \ amd64) rarch=x86_64 ;; \ arm64) rarch=aarch64 ;; \ esac \ - && ln -s "${rarch}-linux-gnu" /lib/linux-gnu - + && ln -s "${rarch}-linux-gnu" /lib/linux-gnu \ + && case $arch in \ + amd64) ln /lib/linux-gnu/ld-linux-x86-64.so.2 /lib/linux-gnu/ld-2.35.so ;; \ + arm64) ln /lib/linux-gnu/ld-linux-aarch64.so.1 /lib/linux-gnu/ld-2.35.so ;; \ + esac FROM alpine @@ -20,7 +23,7 @@ ENV LANG=en_US.UTF-8 \ TZ=UTC \ CLICKHOUSE_CONFIG=/etc/clickhouse-server/config.xml -COPY --from=glibc-donor /lib/linux-gnu/libc.so.6 /lib/linux-gnu/libdl.so.2 /lib/linux-gnu/libm.so.6 /lib/linux-gnu/libpthread.so.0 /lib/linux-gnu/librt.so.1 /lib/linux-gnu/libnss_dns.so.2 /lib/linux-gnu/libnss_files.so.2 /lib/linux-gnu/libresolv.so.2 /lib/linux-gnu/ld-2.31.so /lib/ +COPY --from=glibc-donor /lib/linux-gnu/libc.so.6 /lib/linux-gnu/libdl.so.2 /lib/linux-gnu/libm.so.6 /lib/linux-gnu/libpthread.so.0 /lib/linux-gnu/librt.so.1 /lib/linux-gnu/libnss_dns.so.2 /lib/linux-gnu/libnss_files.so.2 /lib/linux-gnu/libresolv.so.2 /lib/linux-gnu/ld-2.35.so /lib/ COPY --from=glibc-donor /etc/nsswitch.conf /etc/ COPY entrypoint.sh /entrypoint.sh @@ -75,6 +78,8 @@ RUN mkdir -p "${DEFAULT_DATA_DIR}" "${DEFAULT_LOG_DIR}" "${DEFAULT_CONFIG_DIR}" && chown root:clickhouse "${DEFAULT_LOG_DIR}" \ && chmod ugo+Xrw -R "${DEFAULT_DATA_DIR}" "${DEFAULT_LOG_DIR}" "${DEFAULT_CONFIG_DIR}" +RUN apk update && apk upgrade --no-cache + # /var/lib/clickhouse is necessary due to the current default configuration for Keeper VOLUME "${DEFAULT_DATA_DIR}" /var/lib/clickhouse EXPOSE 2181 10181 44444 9181 diff --git a/docker/server/Dockerfile.alpine b/docker/server/Dockerfile.alpine index d06b1fb73bb11..bff9324b2e071 100644 --- a/docker/server/Dockerfile.alpine +++ b/docker/server/Dockerfile.alpine @@ -1,4 +1,4 @@ -FROM ubuntu:20.04 AS glibc-donor +FROM ubuntu:22.04 AS glibc-donor ARG TARGETARCH RUN arch=${TARGETARCH:-amd64} \ @@ -6,8 +6,11 @@ RUN arch=${TARGETARCH:-amd64} \ amd64) rarch=x86_64 ;; \ arm64) rarch=aarch64 ;; \ esac \ - && ln -s "${rarch}-linux-gnu" /lib/linux-gnu - + && ln -s "${rarch}-linux-gnu" /lib/linux-gnu \ + && case $arch in \ + amd64) ln /lib/linux-gnu/ld-linux-x86-64.so.2 /lib/linux-gnu/ld-2.35.so ;; \ + arm64) ln /lib/linux-gnu/ld-linux-aarch64.so.1 /lib/linux-gnu/ld-2.35.so ;; \ + esac FROM alpine @@ -17,7 +20,7 @@ ENV LANG=en_US.UTF-8 \ TZ=UTC \ CLICKHOUSE_CONFIG=/etc/clickhouse-server/config.xml -COPY --from=glibc-donor /lib/linux-gnu/libc.so.6 /lib/linux-gnu/libdl.so.2 /lib/linux-gnu/libm.so.6 /lib/linux-gnu/libpthread.so.0 /lib/linux-gnu/librt.so.1 /lib/linux-gnu/libnss_dns.so.2 /lib/linux-gnu/libnss_files.so.2 /lib/linux-gnu/libresolv.so.2 /lib/linux-gnu/ld-2.31.so /lib/ +COPY --from=glibc-donor /lib/linux-gnu/libc.so.6 /lib/linux-gnu/libdl.so.2 /lib/linux-gnu/libm.so.6 /lib/linux-gnu/libpthread.so.0 /lib/linux-gnu/librt.so.1 /lib/linux-gnu/libnss_dns.so.2 /lib/linux-gnu/libnss_files.so.2 /lib/linux-gnu/libresolv.so.2 /lib/linux-gnu/ld-2.35.so /lib/ COPY --from=glibc-donor /etc/nsswitch.conf /etc/ COPY docker_related_config.xml /etc/clickhouse-server/config.d/ COPY entrypoint.sh /entrypoint.sh @@ -81,6 +84,8 @@ RUN mkdir -p \ && chown root:clickhouse "${DEFAULT_LOG_DIR}" \ && chmod ugo+Xrw -R "${DEFAULT_DATA_DIR}" "${DEFAULT_LOG_DIR}" "${DEFAULT_CLIENT_CONFIG_DIR}" "${DEFAULT_SERVER_CONFIG_DIR}" +RUN apk update && apk upgrade --no-cache + VOLUME "${DEFAULT_DATA_DIR}" EXPOSE 9000 8123 9009 diff --git a/docker/server/Dockerfile.ubuntu b/docker/server/Dockerfile.ubuntu index 11fa26dc88891..a9644d98c0451 100644 --- a/docker/server/Dockerfile.ubuntu +++ b/docker/server/Dockerfile.ubuntu @@ -1,4 +1,4 @@ -FROM ubuntu:20.04 +FROM ubuntu:22.04 # see https://github.com/moby/moby/issues/4032#issuecomment-192327844 # It could be removed after we move on a version 23:04+ @@ -20,10 +20,13 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list && useradd -r -g clickhouse --uid=101 --home-dir=/var/lib/clickhouse --shell=/bin/bash clickhouse \ && apt-get update \ && apt-get install --yes --no-install-recommends \ + busybox \ ca-certificates \ locales \ tzdata \ wget \ + && apt-get upgrade --yes --no-install-recommends \ + && busybox --install -s \ && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/* ARG REPO_CHANNEL="stable"