From 01518e9e315140e377fd45c4619277db23841262 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Thu, 17 Sep 2026 14:57:37 -0400 Subject: [PATCH 01/19] Merge pull request #2401 from Altinity/ci/minio-mirror-stable-25.3 Stable 25.3 - CI: use the clickhouse/minio-* mirror images --- tests/integration/compose/docker_compose_glue_catalog.yml | 4 ++-- .../compose/docker_compose_iceberg_rest_catalog.yml | 4 ++-- tests/integration/compose/docker_compose_minio.yml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/integration/compose/docker_compose_glue_catalog.yml b/tests/integration/compose/docker_compose_glue_catalog.yml index 9fba475bc02a3..b99fc3db12c16 100644 --- a/tests/integration/compose/docker_compose_glue_catalog.yml +++ b/tests/integration/compose/docker_compose_glue_catalog.yml @@ -13,7 +13,7 @@ services: start_period: 30s # TODO: can we simply use with_minio=True instead? minio: - image: minio/minio:RELEASE.2024-07-31T05-46-26Z + image: clickhouse/minio-minio:RELEASE.2024-07-31T05-46-26Z environment: - MINIO_ROOT_USER=minio - MINIO_ROOT_PASSWORD=ClickHouse_Minio_P@ssw0rd @@ -31,7 +31,7 @@ services: depends_on: - minio # Stick to version with "mc config" - image: minio/mc:RELEASE.2025-04-16T18-13-26Z + image: clickhouse/minio-mc:RELEASE.2025-04-16T18-13-26Z environment: - AWS_ACCESS_KEY_ID=minio - AWS_SECRET_ACCESS_KEY=ClickHouse_Minio_P@ssw0rd diff --git a/tests/integration/compose/docker_compose_iceberg_rest_catalog.yml b/tests/integration/compose/docker_compose_iceberg_rest_catalog.yml index 89663ce5799c1..082c0dd8bfe04 100644 --- a/tests/integration/compose/docker_compose_iceberg_rest_catalog.yml +++ b/tests/integration/compose/docker_compose_iceberg_rest_catalog.yml @@ -34,7 +34,7 @@ services: start_period: 30s # TODO: can we simply use with_minio=True instead? minio: - image: minio/minio:RELEASE.2024-07-31T05-46-26Z + image: clickhouse/minio-minio:RELEASE.2024-07-31T05-46-26Z environment: - MINIO_ROOT_USER=minio - MINIO_ROOT_PASSWORD=ClickHouse_Minio_P@ssw0rd @@ -52,7 +52,7 @@ services: depends_on: - minio # Stick to version with "mc config" - image: minio/mc:RELEASE.2025-04-16T18-13-26Z + image: clickhouse/minio-mc:RELEASE.2025-04-16T18-13-26Z environment: - AWS_ACCESS_KEY_ID=minio - AWS_SECRET_ACCESS_KEY=ClickHouse_Minio_P@ssw0rd diff --git a/tests/integration/compose/docker_compose_minio.yml b/tests/integration/compose/docker_compose_minio.yml index 39016633f132e..d1d618a3db1da 100644 --- a/tests/integration/compose/docker_compose_minio.yml +++ b/tests/integration/compose/docker_compose_minio.yml @@ -1,6 +1,6 @@ services: minio1: - image: minio/minio:RELEASE.2024-07-31T05-46-26Z + image: clickhouse/minio-minio:RELEASE.2024-07-31T05-46-26Z volumes: - data1-1:/data1 - ${MINIO_CERTS_DIR:-}:/certs From c628f944895e88b69d573a6d7f735ff9e02db22c Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Fri, 3 Jul 2026 10:32:37 -0400 Subject: [PATCH 02/19] Merge pull request #1969 from Altinity/ci/grype-CVE-fixes-stable-25.3 Stable 25.3: Fix Grype scan failures and Ubuntu package CVEs --- .github/workflows/grype_scan.yml | 2 +- docker/server/Dockerfile.ubuntu | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/grype_scan.yml b/.github/workflows/grype_scan.yml index e75043f03fe67..c060cfcaf22ae 100644 --- a/.github/workflows/grype_scan.yml +++ b/.github/workflows/grype_scan.yml @@ -50,7 +50,7 @@ jobs: sudo apt-get install -y python3-pip python3-venv python3 -m venv venv source venv/bin/activate - pip install --upgrade requests chardet urllib3 unidiff boto3 PyGithub + pip install --upgrade requests chardet urllib3 unidiff 'boto3==1.43.33' PyGithub pip install testflows==$TESTFLOWS_VERSION awscli==1.33.28 echo PATH=$PATH >>$GITHUB_ENV diff --git a/docker/server/Dockerfile.ubuntu b/docker/server/Dockerfile.ubuntu index bb6218b1a4a30..882af8c11bdb1 100644 --- a/docker/server/Dockerfile.ubuntu +++ b/docker/server/Dockerfile.ubuntu @@ -7,12 +7,13 @@ ARG DEBIAN_FRONTEND=noninteractive # ARG for quick switch to a given ubuntu mirror ARG apt_archive="http://archive.ubuntu.com" -# We shouldn't use `apt upgrade` to not change the upstream image. It's updated biweekly +# Upgrade already installed Ubuntu packages to apply available security fixes +# without installing recommended packages. # user/group precreated explicitly with fixed uid/gid on purpose. # It is especially important for rootless containers: in that case entrypoint # can't do chown and owners of mounted volumes should be configured externally. -# We do that in advance at the begining of Dockerfile before any packages will be +# We do that in advance at the beginning of Dockerfile before any packages will be # installed to prevent picking those uid / gid by some unrelated software. # The same uid / gid (101) is used both for alpine and ubuntu. RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list \ @@ -24,6 +25,9 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list locales \ tzdata \ wget \ + && apt-get upgrade --yes --no-install-recommends \ + && busybox --install -s \ + && apt-get clean \ && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/* #docker-official-library:off From 7d76790470e6a742dbceb8d617a48b3a463bf2fb Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Fri, 17 Jul 2026 12:59:07 -0400 Subject: [PATCH 03/19] Merge pull request #2063 from Altinity/ci/bump-grype-stable-25.3 Stable 25.3 - Fix Grype Scan: bump grype to v0.115.0 --- .github/grype/run_grype_scan.sh | 2 +- .github/workflows/grype_scan.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/grype/run_grype_scan.sh b/.github/grype/run_grype_scan.sh index af428e37d669f..9cf99e9f9f9cb 100755 --- a/.github/grype/run_grype_scan.sh +++ b/.github/grype/run_grype_scan.sh @@ -3,7 +3,7 @@ set -e IMAGE=$1 -GRYPE_VERSION=${GRYPE_VERSION:-"v0.92.2"} +GRYPE_VERSION=${GRYPE_VERSION:-"v0.115.0"} docker pull $IMAGE docker pull anchore/grype:${GRYPE_VERSION} diff --git a/.github/workflows/grype_scan.yml b/.github/workflows/grype_scan.yml index c060cfcaf22ae..4bd5cea72cc3c 100644 --- a/.github/workflows/grype_scan.yml +++ b/.github/workflows/grype_scan.yml @@ -30,7 +30,7 @@ env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - GRYPE_VERSION: "v0.92.2-arm64v8" + GRYPE_VERSION: "v0.115.0-arm64v8" jobs: grype_scan: From b0e3c0d3d2d2578897686eecb833b95b4cd05da3 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Thu, 10 Sep 2026 11:40:19 -0400 Subject: [PATCH 04/19] replace azure with azurite --- ci/jobs/functional_stateless_tests.py | 15 ++++----------- ci/jobs/scripts/clickhouse_proc.py | 16 ++++++++++++++++ tests/config/config.d/azure_storage_conf.xml | 6 ++---- tests/docker_scripts/stress_runner.sh | 1 + 4 files changed, 23 insertions(+), 15 deletions(-) diff --git a/ci/jobs/functional_stateless_tests.py b/ci/jobs/functional_stateless_tests.py index 4c14e3fa1e431..15aa0ad459225 100644 --- a/ci/jobs/functional_stateless_tests.py +++ b/ci/jobs/functional_stateless_tests.py @@ -1,5 +1,4 @@ import argparse -import os import time from pathlib import Path @@ -62,15 +61,6 @@ def main(): if "/" in to: batch_num, total_batches = map(int, to.split("/")) - # TODO: find a way to work with Azure secret so it's ok for local tests as well, for now keep azure disabled - os.environ["AZURE_CONNECTION_STRING"] = Shell.get_output( - f"aws ssm get-parameter --region us-east-1 --name azure_connection_string --with-decryption --output text --query Parameter.Value", - verbose=True, - ) - no_azure = False - if not os.environ["AZURE_CONNECTION_STRING"]: - no_azure = True - ch_path = args.ch_path assert ( Path(ch_path + "/clickhouse").is_file() @@ -109,7 +99,7 @@ def main(): f"ln -sf {ch_path}/clickhouse {ch_path}/clickhouse-format", f"rm -rf {temp_dir}/etc/ && mkdir -p {temp_dir}/etc/clickhouse-client {temp_dir}/etc/clickhouse-server", f"cp programs/server/config.xml programs/server/users.xml {temp_dir}/etc/clickhouse-server/", - f"./tests/config/install.sh {temp_dir}/etc/clickhouse-server {temp_dir}/etc/clickhouse-client --s3-storage {'--no-azure' if no_azure else ''}", + f"./tests/config/install.sh {temp_dir}/etc/clickhouse-server {temp_dir}/etc/clickhouse-client --s3-storage", # clickhouse benchmark segfaults with --config-path, so provide client config by its default location f"cp {temp_dir}/etc/clickhouse-client/* /etc/clickhouse-client/", # update_path_ch_config, @@ -140,6 +130,9 @@ def main(): res = res and Shell.check( "aws s3 ls s3://test --endpoint-url http://localhost:11111/", verbose=True ) + azurite_log = f"{temp_dir}/azurite.log" + res = res and CH.start_azurite(log_file_path=azurite_log) + logs_to_attach += [azurite_log] res = res and CH.log_cluster_config() res = res and CH.start() res = res and CH.wait_ready() diff --git a/ci/jobs/scripts/clickhouse_proc.py b/ci/jobs/scripts/clickhouse_proc.py index 08ea848e5ce41..3634b08e2a309 100644 --- a/ci/jobs/scripts/clickhouse_proc.py +++ b/ci/jobs/scripts/clickhouse_proc.py @@ -59,6 +59,22 @@ def start_minio(self, test_type, log_file_path): print(f"Started setup_minio.sh asynchronously with PID {process.pid}") return True + def start_azurite(self, log_file_path): + command = f"cd {temp_dir} && azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --silent --inMemoryPersistence" + with open(log_file_path, "w") as log_file: + process = subprocess.Popen( + command, stdout=log_file, stderr=subprocess.STDOUT, shell=True + ) + print(f"Started azurite asynchronously with PID {process.pid}") + if Shell.check( + "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:10000/ | grep -qE '400|200'", + verbose=False, + retries=6, + ): + return True + print("Failed to start azurite") + return False + @staticmethod def log_cluster_config(): return Shell.check( diff --git a/tests/config/config.d/azure_storage_conf.xml b/tests/config/config.d/azure_storage_conf.xml index 2624656f7e88d..8b90ee3654f7c 100644 --- a/tests/config/config.d/azure_storage_conf.xml +++ b/tests/config/config.d/azure_storage_conf.xml @@ -6,10 +6,8 @@ azure false 33554432 - - - - + clickhouse-tests + DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1; cache diff --git a/tests/docker_scripts/stress_runner.sh b/tests/docker_scripts/stress_runner.sh index 682cf00418045..7f8377e4bebd3 100755 --- a/tests/docker_scripts/stress_runner.sh +++ b/tests/docker_scripts/stress_runner.sh @@ -54,6 +54,7 @@ export ZOOKEEPER_FAULT_INJECTION=1 configure /repo/tests/docker_scripts/setup_minio.sh stateless # to have a proper environment +azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --silent --inMemoryPersistence & config_logs_export_cluster /etc/clickhouse-server/config.d/system_logs_export.yaml From a79ce1b97367612be2b8136df6556b439de19a23 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Thu, 18 Sep 2025 21:05:12 -0400 Subject: [PATCH 05/19] fix layout of report meta tags --- .../actions/create_workflow_report/ci_run_report.html.jinja | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/actions/create_workflow_report/ci_run_report.html.jinja b/.github/actions/create_workflow_report/ci_run_report.html.jinja index a92c1aa34e3a0..935d9f9d3847e 100644 --- a/.github/actions/create_workflow_report/ci_run_report.html.jinja +++ b/.github/actions/create_workflow_report/ci_run_report.html.jinja @@ -5,6 +5,9 @@ + {%- if is_preview %} + + {%- endif %}