Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 32 additions & 0 deletions .github/grype/parse_vulnerabilities_grype.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
#!/usr/bin/env python3
import json

from testflows.core import *

xfails = {}


@Name("docker vulnerabilities")
@XFails(xfails)
@TestModule
def docker_vulnerabilities(self):
with Given("I gather grype scan results"):
with open("./result.json", "r") as f:
results = json.load(f)

for vulnerability in results["matches"]:
with Test(
f"{vulnerability['vulnerability']['id']}@{vulnerability['vulnerability']['namespace']},{vulnerability['vulnerability']['severity']}",
flags=TE,
):
note(vulnerability)
critical_levels = set(["HIGH", "CRITICAL"])
if vulnerability['vulnerability']["severity"].upper() in critical_levels:
with Then(
f"Found vulnerability of {vulnerability['vulnerability']['severity']} severity"
):
result(Fail)


if main():
docker_vulnerabilities()
16 changes: 16 additions & 0 deletions .github/grype/run_grant_scan.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
set -x
set -e

IMAGE=$1

GRANT_VERSION=${GRANT_VERSION:-"v0.6.8"}
SAFE_IMAGE=$(echo "$IMAGE" | sed 's/[\/:]/_/g')

docker pull anchore/grant:${GRANT_VERSION}

docker run --rm \
--volume /var/run/docker.sock:/var/run/docker.sock \
--name Grant anchore/grant:${GRANT_VERSION} \
list "$IMAGE" --output json > "${SAFE_IMAGE}-licenses.json"

ls -sh
18 changes: 18 additions & 0 deletions .github/grype/run_grype_scan.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
set -x
set -e

IMAGE=$1

GRYPE_VERSION=${GRYPE_VERSION:-"v0.115.0"}

docker pull $IMAGE
docker pull anchore/grype:${GRYPE_VERSION}

docker run \
--rm --volume /var/run/docker.sock:/var/run/docker.sock \
--name Grype anchore/grype:${GRYPE_VERSION} \
--scope all-layers \
-o json \
$IMAGE > result.json

ls -sh
21 changes: 21 additions & 0 deletions .github/grype/run_syft_scan.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
set -x
set -e

IMAGE=$1
VERSION=$2

SYFT_VERSION=${SYFT_VERSION:-"v1.51.1"}

docker pull anchore/syft:${SYFT_VERSION}

docker run --rm \
--volume /var/run/docker.sock:/var/run/docker.sock \
--volume "$PWD:/out" --workdir /out \
--name Syft anchore/syft:${SYFT_VERSION} \
--scope all-layers \
-o table \
-o "spdx-json=${VERSION}.spdx.json" \
-o "cyclonedx-json=${VERSION}.cdx.json" \
"$IMAGE"

ls -sh
31 changes: 31 additions & 0 deletions .github/grype/transform_and_upload_results_s3.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
SANITIZED_IMAGE=$(echo "$DOCKER_IMAGE" | sed 's/[\/:]/_/g')

if [ "$PR_NUMBER" -eq 0 ]; then
BASE_PREFIX="REFs/$GITHUB_REF_NAME/$COMMIT_SHA"
else
BASE_PREFIX="PRs/$PR_NUMBER/$COMMIT_SHA"
fi

GRYPE_PREFIX="$BASE_PREFIX/grype/$SANITIZED_IMAGE"
SYFT_PREFIX="$BASE_PREFIX/scan/$SANITIZED_IMAGE/syft"
GRANT_PREFIX="$BASE_PREFIX/scan/$SANITIZED_IMAGE/grant"

S3_GRYPE_PATH="s3://$S3_BUCKET/$GRYPE_PREFIX"
S3_SYFT_PATH="s3://$S3_BUCKET/$SYFT_PREFIX"
S3_GRANT_PATH="s3://$S3_BUCKET/$GRANT_PREFIX"
HTTPS_RESULTS_PATH="https://$S3_BUCKET.s3.amazonaws.com/index.html#$GRYPE_PREFIX/"
HTTPS_REPORT_PATH="https://s3.amazonaws.com/$S3_BUCKET/$GRYPE_PREFIX/results.html"
echo "https_report_path=$HTTPS_REPORT_PATH" >> $GITHUB_OUTPUT

tfs --no-colors transform nice raw.log nice.log.txt
tfs --no-colors report results -a $HTTPS_RESULTS_PATH raw.log - --copyright "Altinity LTD" | tfs --no-colors document convert > results.html

aws s3 cp --no-progress nice.log.txt $S3_GRYPE_PATH/nice.log.txt --content-type "text/plain; charset=utf-8" || echo "nice log file not found".
aws s3 cp --no-progress results.html $S3_GRYPE_PATH/results.html || echo "results file not found".
aws s3 cp --no-progress raw.log $S3_GRYPE_PATH/raw.log || echo "raw.log file not found".
aws s3 cp --no-progress result.json $S3_GRYPE_PATH/result.json --content-type "text/plain; charset=utf-8" || echo "result.json not found".

aws s3 cp --no-progress "${VERSION}.spdx.json" "$S3_SYFT_PATH/${VERSION}.spdx.json" --content-type "application/json" || echo "${VERSION}.spdx.json not found".
aws s3 cp --no-progress "${VERSION}.cdx.json" "$S3_SYFT_PATH/${VERSION}.cdx.json" --content-type "application/json" || echo "${VERSION}.cdx.json not found".

aws s3 cp --no-progress "${SANITIZED_IMAGE}-licenses.json" "$S3_GRANT_PATH/${SANITIZED_IMAGE}-licenses.json" --content-type "application/json" || echo "${SANITIZED_IMAGE}-licenses.json not found".
177 changes: 177 additions & 0 deletions .github/workflows/grype_scan.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,177 @@
name: Grype Scan
run-name: Grype Scan ${{ inputs.docker_image }}

on:
workflow_dispatch:
# Inputs for manual run
inputs:
docker_image:
description: 'Docker image. If no tag, it will be determined by version_helper.py'
required: true
workflow_call:
# Inputs for workflow call
inputs:
docker_image:
description: 'Docker image. If no tag, it will be determined by version_helper.py'
required: true
type: string
version:
description: 'Version tag. If no version, it will be determined by version_helper.py'
required: false
type: string
default: ""
tag-suffix:
description: 'Tag suffix. To be appended the version from version_helper.py'
required: false
type: string
default: ""
env:
PYTHONUNBUFFERED: 1
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
GRYPE_VERSION: "v0.121.0"
SYFT_VERSION: "v1.51.1"
GRANT_VERSION: "v0.6.8"

jobs:
grype_scan:
name: Grype Scan
runs-on: [self-hosted, altinity-on-demand, altinity-style-checker]
steps:
- name: Checkout repository
uses: actions/checkout@v6

- name: Set up Docker
uses: docker/setup-buildx-action@v3

- name: Set up Python
run: |
export TESTFLOWS_VERSION="2.4.19"
sudo apt-get update
sudo apt-get install -y python3-pip python3-venv
python3 -m venv venv
source venv/bin/activate
pip install --upgrade requests chardet urllib3 unidiff 'boto3==1.43.33' PyGithub
pip install testflows==$TESTFLOWS_VERSION awscli==1.33.28
echo PATH=$PATH >>$GITHUB_ENV

- name: Set image tag and version
id: set_version
env:
TAG_SUFFIX: ${{ inputs.tag-suffix }}
SPECIFIED_VERSION: ${{ inputs.version }}
INPUT_DOCKER_IMAGE: ${{ inputs.docker_image }}
run: |
python3 ./tests/ci/version_helper.py | grep = | tee /tmp/version_info
source /tmp/version_info
if [ -z "$SPECIFIED_VERSION" ]; then
VERSION=$CLICKHOUSE_VERSION_STRING
else
VERSION=$SPECIFIED_VERSION
fi
if [[ "$INPUT_DOCKER_IMAGE" == *:* ]]; then
DOCKER_IMAGE=$INPUT_DOCKER_IMAGE
else
DOCKER_IMAGE="${INPUT_DOCKER_IMAGE}:$PR_NUMBER-$VERSION$TAG_SUFFIX"
fi
echo "docker_image=$DOCKER_IMAGE" >> $GITHUB_OUTPUT
echo "version=$VERSION" >> $GITHUB_OUTPUT

- name: Run Grype Scan
run: |
./.github/grype/run_grype_scan.sh ${{ steps.set_version.outputs.docker_image }}

- name: Run Syft Scan
if: always()
run: |
./.github/grype/run_syft_scan.sh ${{ steps.set_version.outputs.docker_image }} ${{ steps.set_version.outputs.version }}

- name: Run Grant Scan
if: always()
run: |
./.github/grype/run_grant_scan.sh ${{ steps.set_version.outputs.docker_image }}

- name: Parse grype results
if: always()
id: parse_grype
run: |
python3 -u ./.github/grype/parse_vulnerabilities_grype.py -o nice --no-colors --log raw.log --test-to-end

- name: Transform and Upload Scan Results
if: always()
id: upload_results
env:
S3_BUCKET: "altinity-build-artifacts"
COMMIT_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
PR_NUMBER: ${{ env.PR_NUMBER || github.event.pull_request.number || 0 }}
DOCKER_IMAGE: ${{ steps.set_version.outputs.docker_image }}
VERSION: ${{ steps.set_version.outputs.version }}
run: |
echo "PR_NUMBER=$PR_NUMBER"
./.github/grype/transform_and_upload_results_s3.sh

- name: Create step summary
if: always()
id: create_summary
run: |
jq -r '"**Image**: \(.source.target.userInput)"' result.json >> $GITHUB_STEP_SUMMARY
jq -r '.distro | "**Distro**: \(.name):\(.version)"' result.json >> $GITHUB_STEP_SUMMARY
if jq -e '.matches | length == 0' result.json > /dev/null; then
echo "No CVEs" >> $GITHUB_STEP_SUMMARY
else
echo "| Severity | Count |" >> $GITHUB_STEP_SUMMARY
echo "|------------|-------|" >> $GITHUB_STEP_SUMMARY
jq -r '
.matches |
map(.vulnerability.severity) |
group_by(.) |
map({severity: .[0], count: length}) |
sort_by(.severity) |
map("| \(.severity) | \(.count) |") |
.[]
' result.json >> $GITHUB_STEP_SUMMARY
fi

HIGH_COUNT=$(jq -r '.matches | map(.vulnerability) | unique_by(.id) | map(.severity) | map(select(. == "High")) | length' result.json)
CRITICAL_COUNT=$(jq -r '.matches | map(.vulnerability) | unique_by(.id) | map(.severity) | map(select(. == "Critical")) | length' result.json)
TOTAL_HIGH_CRITICAL=$((HIGH_COUNT + CRITICAL_COUNT))
echo "total_high_critical=$TOTAL_HIGH_CRITICAL" >> $GITHUB_OUTPUT

if [ $TOTAL_HIGH_CRITICAL -gt 0 ]; then
echo '## High and Critical vulnerabilities found' >> $GITHUB_STEP_SUMMARY
echo '```' >> $GITHUB_STEP_SUMMARY
cat raw.log | tfs --no-colors show tests | grep -Pi 'High|Critical' >> $GITHUB_STEP_SUMMARY
echo '```' >> $GITHUB_STEP_SUMMARY
fi

- name: Set commit status
if: always()
uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const totalHighCritical = '${{ steps.create_summary.outputs.total_high_critical }}';
const hasError = totalHighCritical === '';
const hasVulnerabilities = parseInt(totalHighCritical) > 0;
github.rest.repos.createCommitStatus({
owner: context.repo.owner,
repo: context.repo.repo,
sha: '${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}',
state: hasError ? 'error' : '${{ steps.parse_grype.outcome == 'success' && 'success' || 'failure' }}',
target_url: '${{ steps.upload_results.outputs.https_report_path }}',
description: hasError ? 'An error occurred' : `Grype Scan Completed with ${totalHighCritical} high/critical vulnerabilities`,
context: 'Grype Scan ${{ steps.set_version.outputs.docker_image }}'
});

- name: Upload artifacts
if: always()
uses: actions/upload-artifact@v4
with:
name: grype-results-${{ hashFiles('raw.log') }}
path: |
result.json
nice.log.txt
*.spdx.json
*.cdx.json
*-licenses.json
20 changes: 20 additions & 0 deletions .github/workflows/release_branches.yml
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,26 @@ jobs:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"

GrypeScanServer:
needs: [DockerServerImages]
strategy:
fail-fast: false
matrix:
suffix: ['', '-alpine']
uses: ./.github/workflows/grype_scan.yml
secrets: inherit
with:
docker_image: altinityinfra/clickhouse-server
tag-suffix: ${{ matrix.suffix }}

GrypeScanKeeper:
needs: [DockerServerImages]
uses: ./.github/workflows/grype_scan.yml
secrets: inherit
with:
docker_image: altinityinfra/clickhouse-keeper


############################################################################################
##################################### BUILD REPORTER #######################################
############################################################################################
Expand Down
6 changes: 3 additions & 3 deletions cmake/autogenerated_versions.txt
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@ SET(VERSION_MINOR 8)
SET(VERSION_PATCH 16)

SET(VERSION_GITHASH 060ff8e813a4a16a540063127f8c91e2108d9adf)
SET(VERSION_TWEAK 43)
SET(VERSION_TWEAK 10044)
SET(VERSION_FLAVOUR altinitystable)

SET(VERSION_DESCRIBE v23.8.16.43.altinitystable)
SET(VERSION_STRING 23.8.16.43.altinitystable)
SET(VERSION_DESCRIBE v23.8.16.10044.altinitystable)
SET(VERSION_STRING 23.8.16.10044.altinitystable)
# end of autochange
13 changes: 9 additions & 4 deletions docker/keeper/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,16 +1,19 @@
# The Dockerfile.ubuntu exists for the tests/ci/docker_server.py script
# If the image is built from Dockerfile.alpine, then the `-alpine` suffix is added automatically,
# so the only purpose of Dockerfile.ubuntu is to push `latest`, `head` and so on w/o suffixes
FROM ubuntu:20.04 AS glibc-donor
FROM ubuntu:22.04 AS glibc-donor
ARG TARGETARCH

RUN arch=${TARGETARCH:-amd64} \
&& case $arch in \
amd64) rarch=x86_64 ;; \
arm64) rarch=aarch64 ;; \
esac \
&& ln -s "${rarch}-linux-gnu" /lib/linux-gnu

&& ln -s "${rarch}-linux-gnu" /lib/linux-gnu \
&& case $arch in \
amd64) ln /lib/linux-gnu/ld-linux-x86-64.so.2 /lib/linux-gnu/ld-2.35.so ;; \
arm64) ln /lib/linux-gnu/ld-linux-aarch64.so.1 /lib/linux-gnu/ld-2.35.so ;; \
esac

FROM alpine

Expand All @@ -20,7 +23,7 @@ ENV LANG=en_US.UTF-8 \
TZ=UTC \
CLICKHOUSE_CONFIG=/etc/clickhouse-server/config.xml

COPY --from=glibc-donor /lib/linux-gnu/libc.so.6 /lib/linux-gnu/libdl.so.2 /lib/linux-gnu/libm.so.6 /lib/linux-gnu/libpthread.so.0 /lib/linux-gnu/librt.so.1 /lib/linux-gnu/libnss_dns.so.2 /lib/linux-gnu/libnss_files.so.2 /lib/linux-gnu/libresolv.so.2 /lib/linux-gnu/ld-2.31.so /lib/
COPY --from=glibc-donor /lib/linux-gnu/libc.so.6 /lib/linux-gnu/libdl.so.2 /lib/linux-gnu/libm.so.6 /lib/linux-gnu/libpthread.so.0 /lib/linux-gnu/librt.so.1 /lib/linux-gnu/libnss_dns.so.2 /lib/linux-gnu/libnss_files.so.2 /lib/linux-gnu/libresolv.so.2 /lib/linux-gnu/ld-2.35.so /lib/
COPY --from=glibc-donor /etc/nsswitch.conf /etc/
COPY entrypoint.sh /entrypoint.sh

Expand Down Expand Up @@ -75,6 +78,8 @@ RUN mkdir -p "${DEFAULT_DATA_DIR}" "${DEFAULT_LOG_DIR}" "${DEFAULT_CONFIG_DIR}"
&& chown root:clickhouse "${DEFAULT_LOG_DIR}" \
&& chmod ugo+Xrw -R "${DEFAULT_DATA_DIR}" "${DEFAULT_LOG_DIR}" "${DEFAULT_CONFIG_DIR}"

RUN apk update && apk upgrade --no-cache

# /var/lib/clickhouse is necessary due to the current default configuration for Keeper
VOLUME "${DEFAULT_DATA_DIR}" /var/lib/clickhouse
EXPOSE 2181 10181 44444 9181
Expand Down
Loading
Loading