diff --git a/.github/actions/runner_setup/action.yml b/.github/actions/runner_setup/action.yml index e978d8f57a741..a9f9d0d341f41 100644 --- a/.github/actions/runner_setup/action.yml +++ b/.github/actions/runner_setup/action.yml @@ -25,7 +25,7 @@ runs: ZRAM_SIZE=$(($MemTotal / 1024 / 1024 * $Percent / 100)) # Convert to GiB .github/retry.sh 30 2 sudo zramctl --size ${ZRAM_SIZE}GiB --algorithm zstd /dev/zram0 sudo mkswap /dev/zram0 && sudo swapon -p 100 /dev/zram0 - sudo sysctl vm.swappiness=200 + sudo sysctl vm.swappiness=80 - name: Install awscli shell: bash run: | diff --git a/.github/workflows/fast_builds.yml b/.github/workflows/fast_builds.yml index eaabcc3a63286..26a61911e0462 100644 --- a/.github/workflows/fast_builds.yml +++ b/.github/workflows/fast_builds.yml @@ -45,6 +45,8 @@ env: AZURE_CONTAINER_NAME: ${{ secrets.AZURE_CONTAINER_NAME }} AZURE_STORAGE_ACCOUNT_URL: "https://${{ secrets.AZURE_ACCOUNT_NAME }}.blob.core.windows.net/" ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }} + HETZNER_S3_ACCESS_KEY: ${{ secrets.HETZNER_S3_ACCESS_KEY }} + HETZNER_S3_SECRET_KEY: ${{ secrets.HETZNER_S3_SECRET_KEY }} diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 62f6d64627657..90d8b9e15d717 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -60,6 +60,8 @@ env: AZURE_CONTAINER_NAME: ${{ secrets.AZURE_CONTAINER_NAME }} AZURE_STORAGE_ACCOUNT_URL: "https://${{ secrets.AZURE_ACCOUNT_NAME }}.blob.core.windows.net/" ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }} + HETZNER_S3_ACCESS_KEY: ${{ secrets.HETZNER_S3_ACCESS_KEY }} + HETZNER_S3_SECRET_KEY: ${{ secrets.HETZNER_S3_SECRET_KEY }} diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index c638bc99d1e54..e16505d822493 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -35,6 +35,8 @@ env: AZURE_CONTAINER_NAME: ${{ secrets.AZURE_CONTAINER_NAME }} AZURE_STORAGE_ACCOUNT_URL: "https://${{ secrets.AZURE_ACCOUNT_NAME }}.blob.core.windows.net/" ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }} + HETZNER_S3_ACCESS_KEY: ${{ secrets.HETZNER_S3_ACCESS_KEY }} + HETZNER_S3_SECRET_KEY: ${{ secrets.HETZNER_S3_SECRET_KEY }} diff --git a/CMakeLists.txt b/CMakeLists.txt index 68b194ad629c3..ffcd02c5d0696 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -493,6 +493,24 @@ if (ENABLE_THINLTO AND NOT ENABLE_TESTS AND NOT SANITIZE) set(CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO} -Wl,--lto-whole-program-visibility") endif() + # Persistent ThinLTO cache. lld stores the optimized native object of every + # module keyed by the module hash, its import set and the LTO flags, and + # reuses it on the next link when nothing in that key changed. Only the + # backend codegen of the link is cached; the thin-link and the final native + # link still run in full. Entries are content-addressed, so concurrent + # writers cannot corrupt each other. + set (THINLTO_CACHE_DIR "" CACHE PATH "Directory for the ThinLTO cache (lld --thinlto-cache-dir); empty disables the cache") + # lld prunes after each link by default. That is off here, since CI keeps + # exactly the entries a build used; set a policy for a long-lived cache dir. + set (THINLTO_CACHE_POLICY "prune_after=0s:cache_size=0%:cache_size_bytes=0:cache_size_files=0" CACHE STRING "Pruning policy for the ThinLTO cache (lld --thinlto-cache-policy)") + if (THINLTO_CACHE_DIR) + if (NOT LINKER_NAME MATCHES "lld") + message (FATAL_ERROR "THINLTO_CACHE_DIR requires lld, but LINKER_NAME is '${LINKER_NAME}'") + endif () + message (STATUS "Using ThinLTO cache in ${THINLTO_CACHE_DIR} with policy '${THINLTO_CACHE_POLICY}'") + set (CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO} -Wl,--thinlto-cache-dir=${THINLTO_CACHE_DIR} -Wl,--thinlto-cache-policy=${THINLTO_CACHE_POLICY}") + endif () + if (ENABLE_CFI) # CFI checks: virtual dispatch type safety and bad downcasts via static_cast. # cfi-icall is omitted because C function pointers in contrib libraries (compiled without diff --git a/ci/defs/job_configs.py b/ci/defs/job_configs.py index eaec79a77d782..b79745981beb2 100644 --- a/ci/defs/job_configs.py +++ b/ci/defs/job_configs.py @@ -30,6 +30,8 @@ f"+--memory-reservation={Utils.physical_memory() * 9 // 10}" f"+--volume=.:/ClickHouse" '+--env=AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID"+--env=AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY"' + # Writes to the ThinLTO cache in Hetzner Object Storage; jobs without them read anonymously + '+--env=HETZNER_S3_ACCESS_KEY="$HETZNER_S3_ACCESS_KEY"+--env=HETZNER_S3_SECRET_KEY="$HETZNER_S3_SECRET_KEY"' ) BINARY_DOCKER_COMMAND_SANITIZER = ( BINARY_DOCKER_COMMAND + "+--security-opt seccomp=unconfined" @@ -443,7 +445,7 @@ class JobConfigs: ArtifactNames.RPM_ARM_RELEASE, ArtifactNames.TGZ_ARM_RELEASE, ], - runs_on=RunnerLabels.BUILDER_AMD, + runs_on=RunnerLabels.BUILDER_ARM, ), ) release_build_jobs_with_examples = [ diff --git a/ci/jobs/build_clickhouse.py b/ci/jobs/build_clickhouse.py index 23eec7551006d..f8207263d0b04 100644 --- a/ci/jobs/build_clickhouse.py +++ b/ci/jobs/build_clickhouse.py @@ -6,8 +6,10 @@ from ci.jobs.scripts.clickhouse_version import CHVersion from ci.praktika.info import Info from ci.praktika.result import Result +from ci.praktika.s3 import S3 from ci.praktika.settings import Settings from ci.praktika.utils import MetaClasses, Shell, Utils +from ci.settings.altinity_overrides import CI_CACHE_BUCKET current_directory = Utils.cwd() build_dir = f"{current_directory}/ci/tmp/build" @@ -15,6 +17,18 @@ repo_path_normalized = "/ClickHouse" build_dir_normalized = f"{repo_path_normalized}/ci/tmp/build" +# ThinLTO cache: lld stores the optimized native object of every module it +# links, and the next link of an unchanged module reuses it instead of running +# the backend again. The cache travels between builders as one archive per +# branch, build type and debug-info flavour. +THINLTO_CACHE_S3_PATH = f"{CI_CACHE_BUCKET}/ci_ch_cache/thinlto" +thinlto_cache_dir = f"{temp_dir}/thinlto_cache" +thinlto_cache_dir_normalized = f"{repo_path_normalized}/ci/tmp/thinlto_cache" +thinlto_cache_archive = f"{temp_dir}/thinlto_cache.tar.zst" +# An archive holds the entries one build used, about 15 GB for a `-g` build of +# `clickhouse` and `clickhouse-keeper`. Much more means pruning went wrong. +THINLTO_CACHE_MAX_BYTES = {"g": 20 << 30, "g0": 5 << 30} + BUILD_TYPE_TO_CMAKE = { BuildTypes.AMD_DEBUG: f" cmake --debug-trycompile -DCMAKE_VERBOSE_MAKEFILE=1 -LA -DCMAKE_BUILD_TYPE=Debug -DENABLE_THINLTO=0 -DSANITIZE= -DENABLE_CHECK_HEAVY_BUILDS=1 -DBUILD_STRIPPED_BINARY=1 -DENABLE_CLICKHOUSE_SELF_EXTRACTING=1 -DCMAKE_C_COMPILER={ToolSet.COMPILER_C} -DCMAKE_CXX_COMPILER={ToolSet.COMPILER_CPP} -DCOMPILER_CACHE={ToolSet.COMPILER_CACHE} -DCMAKE_TOOLCHAIN_FILE={repo_path_normalized}/cmake/linux/toolchain-x86_64.cmake -DENABLE_BUILD_PROFILING=1 -DENABLE_TESTS=1 -DENABLE_LEXER_TEST=1 -DENABLE_UTILS=0 -DCMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY=ON -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_INSTALL_LOCALSTATEDIR=/var -DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=ON -DENABLE_BUZZHOUSE=1", BuildTypes.AMD_RELEASE: f" cmake --debug-trycompile -DCMAKE_VERBOSE_MAKEFILE=1 -LA -DCMAKE_BUILD_TYPE=None -DENABLE_THINLTO=1 -DSANITIZE= -DENABLE_CHECK_HEAVY_BUILDS=1 -DBUILD_STRIPPED_BINARY=1 -DENABLE_CLICKHOUSE_SELF_EXTRACTING=1 -DCMAKE_C_COMPILER={ToolSet.COMPILER_C} -DCMAKE_CXX_COMPILER={ToolSet.COMPILER_CPP} -DCOMPILER_CACHE={ToolSet.COMPILER_CACHE} -DCMAKE_TOOLCHAIN_FILE={repo_path_normalized}/cmake/linux/toolchain-x86_64.cmake -DENABLE_BUILD_PROFILING=1 -DENABLE_TESTS=0 -DENABLE_LEXER_TEST=0 -DENABLE_UTILS=0 -DCMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY=ON -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_INSTALL_LOCALSTATEDIR=/var -DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=ON -DSPLIT_DEBUG_SYMBOLS=ON -DBUILD_STANDALONE_KEEPER=1", @@ -103,8 +117,114 @@ def parse_args(): return parser.parse_args() +def thinlto_cache_flavour(cmake_cmd): + return "g0" if "-DDISABLE_ALL_DEBUG_SYMBOLS=1" in cmake_cmd else "g" + + +def thinlto_cache_s3_path(info, build_type, cmake_cmd): + """PR builds share the cache of their base branch, other builds use their own branch.""" + branch = info.base_branch if info.pr_number > 0 else info.git_branch + if not branch: + return None + return f"{THINLTO_CACHE_S3_PATH}/{branch}/{build_type}-{thinlto_cache_flavour(cmake_cmd)}.tar.zst" + + +def restore_thinlto_cache(s3_path): + shutil.rmtree(thinlto_cache_dir, ignore_errors=True) + os.makedirs(thinlto_cache_dir) + if not S3.head_object(s3_path): + print(f"ThinLTO cache miss: {s3_path}") + return True + try: + S3.copy_file_from_s3(s3_path=s3_path, local_path=thinlto_cache_archive) + Shell.check( + f"set -o pipefail; zstd -d --long=27 -c {thinlto_cache_archive} | tar -xf - -C {thinlto_cache_dir}", + verbose=True, + strict=True, + ) + except BaseException: + # A partly extracted entry would be linked as is: start cold instead. + shutil.rmtree(thinlto_cache_dir, ignore_errors=True) + os.makedirs(thinlto_cache_dir) + raise + finally: + if os.path.exists(thinlto_cache_archive): + os.remove(thinlto_cache_archive) + # With an access time of zero, the kernel records the next read of an + # entry even under `relatime`, which is how the upload step tells the + # entries this build used from the ones it did not. + count = 0 + size = 0 + with os.scandir(thinlto_cache_dir) as entries: + for entry in entries: + stat = entry.stat(follow_symlinks=False) + os.utime(entry.path, (0, stat.st_mtime)) + count += 1 + size += stat.st_size + print(f"ThinLTO cache restored: {count} entries, {size / (1 << 30):.1f} GiB") + return True + + +def thinlto_cache_tracks_reads(): + probe = f"{thinlto_cache_dir}/.atime-probe" + with open(probe, "w") as f: + f.write("probe") + os.utime(probe, (0, os.stat(probe).st_mtime)) + with open(probe) as f: + f.read() + tracked = os.stat(probe).st_atime > 0 + os.remove(probe) + return tracked + + +def upload_thinlto_cache(s3_path, flavour): + """Keep the entries this build read or wrote, then upload them.""" + assert thinlto_cache_tracks_reads(), ( + f"{thinlto_cache_dir} does not record file reads (mounted with noatime?), " + "so the entries this build used are unknown" + ) + kept = 0 + removed = 0 + size = 0 + with os.scandir(thinlto_cache_dir) as entries: + for entry in entries: + stat = entry.stat(follow_symlinks=False) + # Anything else is a temporary file of an interrupted link. + used = entry.name.startswith("llvmcache-") and entry.is_file( + follow_symlinks=False + ) and stat.st_atime > 0 + if used: + kept += 1 + size += stat.st_size + else: + os.remove(entry.path) + removed += 1 + print( + f"ThinLTO cache: kept {kept} entries ({size / (1 << 30):.1f} GiB), " + f"removed {removed} unused" + ) + assert size <= THINLTO_CACHE_MAX_BYTES[flavour], ( + f"ThinLTO cache of {size} bytes exceeds the {THINLTO_CACHE_MAX_BYTES[flavour]} " + f"bytes expected for a `-{flavour}` build, not uploading it" + ) + Shell.check( + f"set -o pipefail; tar -cf - -C {thinlto_cache_dir} . | zstd -T0 -3 --long=27 -c > {thinlto_cache_archive}", + verbose=True, + strict=True, + ) + print( + f"ThinLTO cache archive: {os.path.getsize(thinlto_cache_archive) / (1 << 30):.1f} GiB" + ) + S3.copy_file_to_s3( + s3_path=s3_path, local_path=thinlto_cache_archive, with_rename=True + ) + os.remove(thinlto_cache_archive) + return True + + def run_shell(name, command, **kwargs): print(f"\n>>>> {name}\n") + kwargs.setdefault("verbose", True) Shell.check(command, **kwargs) print(f"\n<<<< {name}\n") @@ -214,6 +334,26 @@ def main(): ): cmake_cmd += " -DDISABLE_ALL_DEBUG_SYMBOLS=1" + # ThinLTO cache for every build that links with ThinLTO. The cache-warmup + # builds only compile. Everyone reads the archive (community PRs anonymously); + # pushes and internal PRs write it, like the shared compiler cache. + thinlto_cache_s3 = None + if "ENABLE_THINLTO=1" in cmake_cmd and not cache_warmup: + thinlto_cache_s3 = thinlto_cache_s3_path(info, build_type, cmake_cmd) + thinlto_cache_writable = not info.is_local_run and not info.is_community_pr + if thinlto_cache_s3: + print(f"ThinLTO cache: {thinlto_cache_s3}, writable: {thinlto_cache_writable}") + cmake_cmd += f" -DTHINLTO_CACHE_DIR={thinlto_cache_dir_normalized}" + + # Link `clickhouse` and `clickhouse-keeper` concurrently. Each ThinLTO link + # spends most of its time in a single-threaded thin-link phase, so the two + # overlap almost entirely: measured on a 16-core, 62 GB builder with zram, + # 114.5 -> 83 min cold and 65.5 -> 48.5 min warm. The memory-derived + # default in limit_jobs.cmake budgets ~100 GB per `-g` link and would + # allow only one; the real peak of both links together was ~48 GB. + if "ENABLE_THINLTO=1" in cmake_cmd: + cmake_cmd += " -DPARALLEL_LINK_JOBS=2" + # PGO/BOLT profile integration for release builds. The sccache-warmup builds # are deliberately excluded: they exist only to populate the shared compiler # cache, and PGO/BOLT belong to the real release builds. @@ -422,6 +562,15 @@ def do_checkout(): else: build_command = f"command time -v ninja {targets}" + if thinlto_cache_s3: + results.append( + Result.from_commands_run( + name="Restore ThinLTO cache", + command=restore_thinlto_cache, + command_args=[thinlto_cache_s3], + ) + ) + build_result_index = len(results) results.append( Result.from_commands_run( @@ -462,6 +611,19 @@ def do_checkout(): else: results.append(retry_cmake) + if ( + thinlto_cache_s3 + and thinlto_cache_writable + and results[build_result_index].is_ok() + ): + results.append( + Result.from_commands_run( + name="Upload ThinLTO cache", + command=upload_thinlto_cache, + command_args=[thinlto_cache_s3, thinlto_cache_flavour(cmake_cmd)], + ) + ) + run_shell("sccache stats", "sccache --show-stats") if build_type in (BuildTypes.AMD_TIDY, BuildTypes.ARM_TIDY): run_shell("clang-tidy-cache stats", "clang-tidy-cache.py --show-stats") diff --git a/ci/praktika/cache.py b/ci/praktika/cache.py index 148f4edfbc521..6c1545eef742e 100644 --- a/ci/praktika/cache.py +++ b/ci/praktika/cache.py @@ -39,6 +39,11 @@ def from_fs(cls, path): def from_dict(cls, obj): return Cache.CacheRecord(**obj) + @staticmethod + def submodule_archive_s3_path(cache_hash): + base = Settings.SUBMODULE_CACHE_S3_PATH or Settings.CACHE_S3_PATH + return f"{base}/submodules/{cache_hash}.tar.zst" + def __init__(self): self.digest = Digest() self.success = {} # type Dict[str, Any] diff --git a/ci/praktika/native_jobs.py b/ci/praktika/native_jobs.py index e6c9a673f43cd..d907dc574795b 100644 --- a/ci/praktika/native_jobs.py +++ b/ci/praktika/native_jobs.py @@ -10,6 +10,7 @@ from . import Job, Workflow from ._environment import _Environment +from .cache import Cache from .cidb import CIDB from .digest import Digest from .docker import Docker @@ -252,13 +253,22 @@ def _prepare_submodule_cache(workflow, workflow_config: RunConfig) -> Result: ) cache_hash = hashlib.sha256(submodule_shas.encode()).hexdigest()[:16] - s3_path = f"{Settings.CACHE_S3_PATH}/submodules/{cache_hash}.tar.zst" + s3_path = Cache.submodule_archive_s3_path(cache_hash) print(f"Submodule cache hash: {cache_hash}") # Check if cache already exists in S3 if S3.head_object(s3_path): print(f"Submodule cache hit: {s3_path}") info = f"cache hit: {cache_hash}" + elif Info().is_community_pr: + # Like sccache, community PRs only read; jobs clone without a hash. + print(f"Submodule cache miss, community PR will not create it: {s3_path}") + return Result.create_from( + name="Submodule Cache", + status=Result.Status.OK, + stopwatch=stop_watch, + info=f"cache miss, not creating (read-only): {cache_hash}", + ) else: print(f"Submodule cache miss, creating: {s3_path}") Shell.check("git submodule sync", verbose=True, strict=True) @@ -270,6 +280,12 @@ def _prepare_submodule_cache(workflow, workflow_config: RunConfig) -> Result: retries=3, env=_submodule_auth_env(workflow), ) + # The archive is public: never upload it with the clone token inside. + leaked = Shell.get_output( + "grep -rIlF x-access-token .git/config .git/modules --exclude-dir=objects" + ) + if leaked: + raise RuntimeError(f"submodule cache would contain a GitHub token: {leaked}") archive_path = f"{Settings.TEMP_DIR}/submodules_{cache_hash}.tar.zst" Shell.check( f"tar -cf - .git/modules | zstd -c -T0 > {archive_path}", @@ -892,7 +908,11 @@ def check_affected_jobs(): ) ) - if results[-1].is_ok() and workflow.enable_cache and Settings.ENABLE_SUBMODULE_CACHE: + if ( + results[-1].is_ok() + and (workflow.enable_cache or Info().is_community_pr) + and Settings.ENABLE_SUBMODULE_CACHE + ): result = _prepare_submodule_cache(workflow, workflow_config) results.append(result) diff --git a/ci/praktika/runner.py b/ci/praktika/runner.py index 6d105b87468ac..a09b09c3471e3 100644 --- a/ci/praktika/runner.py +++ b/ci/praktika/runner.py @@ -11,6 +11,7 @@ from pathlib import Path from ._environment import _Environment +from .cache import Cache from .artifact import Artifact from .cidb import CIDB from .digest import Digest @@ -362,7 +363,7 @@ def _restore_submodule_cache(): if not cache_hash: print("NOTE: No submodule cache hash in workflow config, skipping restore") return - s3_path = f"{Settings.CACHE_S3_PATH}/submodules/{cache_hash}.tar.zst" + s3_path = Cache.submodule_archive_s3_path(cache_hash) local_archive = f"{Settings.TEMP_DIR}/submodules_{cache_hash}.tar.zst" print(f"Restoring submodule cache: {s3_path}") S3.copy_file_from_s3(s3_path=s3_path, local_path=local_archive, no_strict=True) diff --git a/ci/praktika/s3.py b/ci/praktika/s3.py index c064f513e635b..863718e8b915d 100644 --- a/ci/praktika/s3.py +++ b/ci/praktika/s3.py @@ -15,7 +15,10 @@ try: import boto3 + from botocore import UNSIGNED + from botocore.config import Config from botocore.exceptions import ClientError, NoCredentialsError + from boto3.s3.transfer import TransferConfig BOTO3_AVAILABLE = True except ImportError: @@ -58,9 +61,64 @@ def clean_line(line): class S3: _boto3_client = None + _external_clients = {} @classmethod - def _get_boto3_client(cls): + def _external_bucket_config(cls, bucket): + return (Settings.S3_EXTERNAL_BUCKETS or {}).get(bucket) + + @classmethod + def _reads_anonymously(cls, bucket): + """Reads of a public_read bucket are unsigned unless this job has the bucket's keys.""" + config = cls._external_bucket_config(bucket) + return bool( + config + and config.get("public_read") + and not ( + os.environ.get(config["access_key_env"]) + and os.environ.get(config["secret_key_env"]) + ) + ) + + @classmethod + def _get_external_client(cls, bucket, config, anonymous): + assert BOTO3_AVAILABLE, f"boto3 is required for S3 bucket [{bucket}]" + client = cls._external_clients.get((bucket, anonymous)) + if client is None: + client_config = { + "max_pool_connections": config.get("download_concurrency", 10) + } + if anonymous: + credentials = { + "config": Config(signature_version=UNSIGNED, **client_config) + } + else: + access_key = os.environ.get(config["access_key_env"], "") + secret_key = os.environ.get(config["secret_key_env"], "") + # boto3 would otherwise send the ambient AWS credentials to this endpoint + assert ( + access_key and secret_key + ), f"S3 bucket [{bucket}] needs [{config['access_key_env']}] and [{config['secret_key_env']}] set" + credentials = { + "aws_access_key_id": access_key, + "aws_secret_access_key": secret_key, + "config": Config(**client_config), + } + client = boto3.client( + "s3", + endpoint_url=config["endpoint_url"], + region_name=config["region"], + **credentials, + ) + cls._external_clients[(bucket, anonymous)] = client + return client + + @classmethod + def _get_boto3_client(cls, bucket=None, read=False): + config = cls._external_bucket_config(bucket) + if config: + anonymous = read and cls._reads_anonymously(bucket) + return cls._get_external_client(bucket, config, anonymous) if not BOTO3_AVAILABLE: return None if cls._boto3_client is None: @@ -188,7 +246,7 @@ def copy_file_to_s3( extra_args["Tagging"] = urlencode(tags) def _upload(): - client = cls._get_boto3_client() + client = cls._get_boto3_client(bucket) if extra_args: client.upload_file( str(local_path), bucket, key, ExtraArgs=extra_args @@ -260,24 +318,69 @@ def put( s3_full_path = str(s3_full_path).removeprefix("s3://") bucket, key = s3_full_path.split("/", maxsplit=1) - command = ( - f"aws s3api put-object --bucket {bucket} --key {key} --body {local_path}" - ) - if if_none_matched: - command += ' --if-none-match "*"' - if if_match: - command += f' --if-match "{if_match}"' - if metadata: - for k, v in metadata.items(): - command += f" --metadata {k}={v}" - - if text: - command += ' --content-type "text/plain; charset=utf-8"' - res = cls.run_command_with_retries(command, no_strict=no_strict) + if cls._external_bucket_config(bucket): + res = cls._put_object_boto3( + bucket, + key, + local_path, + text, + metadata, + if_none_matched, + if_match, + no_strict, + ) + else: + command = f"aws s3api put-object --bucket {bucket} --key {key} --body {local_path}" + if if_none_matched: + command += ' --if-none-match "*"' + if if_match: + command += f' --if-match "{if_match}"' + if metadata: + for k, v in metadata.items(): + command += f" --metadata {k}={v}" + + if text: + command += ' --content-type "text/plain; charset=utf-8"' + res = cls.run_command_with_retries(command, no_strict=no_strict) if res: StorageUsage.add_uploaded(local_path) return res + @classmethod + def _put_object_boto3( + cls, + bucket, + key, + local_path, + text, + metadata, + if_none_matched, + if_match, + no_strict, + ): + args = {"Bucket": bucket, "Key": key} + if metadata: + args["Metadata"] = metadata + if text: + args["ContentType"] = "text/plain; charset=utf-8" + if if_none_matched: + args["IfNoneMatch"] = "*" + if if_match: + args["IfMatch"] = if_match + try: + with open(local_path, "rb") as f: + cls._get_boto3_client(bucket).put_object(Body=f, **args) + except ClientError as e: + error_code = e.response.get("Error", {}).get("Code", "") + if error_code in ("PreconditionFailed", "ConditionalRequestConflict"): + print(f"S3 conditional request failed (concurrent write detected): {error_code}") + return False + print(f"ERROR: Failed to put S3 object [{bucket}/{key}]: {error_code}") + if not no_strict: + raise + return False + return True + @classmethod def run_command_with_retries( cls, @@ -354,8 +457,18 @@ def copy_file_from_s3( local_file.parent.mkdir(parents=True, exist_ok=True) def _download(): - client = cls._get_boto3_client() - client.download_file(bucket, key, str(local_file)) + client = cls._get_boto3_client(bucket, read=True) + concurrency = (cls._external_bucket_config(bucket) or {}).get( + "download_concurrency" + ) + client.download_file( + bucket, + key, + str(local_file), + Config=TransferConfig(max_concurrency=concurrency) + if concurrency + else None, + ) # Retry on transient credential failures cls._retry_on_no_credentials(_download) @@ -426,18 +539,35 @@ def copy_file_from_s3_matching_pattern( def head_object(cls, s3_path): s3_path = str(s3_path).removeprefix("s3://") bucket, key = s3_path.split("/", maxsplit=1) - output = Shell.get_output( - f"aws s3api head-object --bucket {bucket} --key {key}", verbose=True - ) - if not output: - return None - data = json.loads(output) + if cls._external_bucket_config(bucket): + try: + data = cls._get_boto3_client(bucket, read=True).head_object( + Bucket=bucket, Key=key + ) + except ClientError as e: + if e.response.get("Error", {}).get("Code", "") in ( + "404", + "NoSuchKey", + "NotFound", + ): + return None + # A public bucket answers a missing key with 403 to callers who may not list it + if e.response.get("Error", {}).get("Code", "") == "403" and cls._reads_anonymously(bucket): + return None + raise + else: + output = Shell.get_output( + f"aws s3api head-object --bucket {bucket} --key {key}", verbose=True + ) + if not output: + return None + data = json.loads(output) # HeadObject omits optional fields (Expiration) unless a lifecycle Expire # rule applies, and emits extra keys Object does not declare. return cls.Object( AcceptRanges=data.get("AcceptRanges", ""), Expiration=data.get("Expiration", ""), - LastModified=data.get("LastModified", ""), + LastModified=str(data.get("LastModified", "")), ContentLength=data.get("ContentLength", 0), ETag=data.get("ETag", ""), ContentType=data.get("ContentType", ""), @@ -456,7 +586,7 @@ def assert_read_access(cls, s3_path): """ s3_path = str(s3_path).removeprefix("s3://") bucket, key = s3_path.split("/", maxsplit=1) - client = cls._get_boto3_client() + client = cls._get_boto3_client(bucket, read=True) assert client, "boto3 client is required for S3 read-access probe" def _probe(): diff --git a/ci/praktika/settings.py b/ci/praktika/settings.py index 760a0caa90748..3385cd6b57368 100644 --- a/ci/praktika/settings.py +++ b/ci/praktika/settings.py @@ -1,7 +1,7 @@ import dataclasses import importlib.util from pathlib import Path -from typing import Dict, Iterable, List, Optional +from typing import Any, Dict, Iterable, List, Optional @dataclasses.dataclass @@ -111,6 +111,8 @@ class _Settings: # If enabled, submodule clones authenticate with the App installation token # (required for private submodules); otherwise they run anonymously. ENABLE_SUBMODULE_CLONE_AUTH: bool = False + # bucket/prefix of the submodule archive, defaults to CACHE_S3_PATH + SUBMODULE_CACHE_S3_PATH: str = "" # v2: records carry the producing workflow event, used as the reuse trust # signal instead of the branch (see CacheRunnerHooks.configure). @@ -127,6 +129,10 @@ class _Settings: S3_UPSTREAM_REPORT_BUCKET: str = "" HTML_PAGE_FILE: str = "./ci/praktika/json.html" S3_BUCKET_TO_HTTP_ENDPOINT: Optional[Dict[str, str]] = None + # Non-AWS buckets: {bucket: {endpoint_url, region, access_key_env, secret_key_env, public_read, download_concurrency}} + # The keys are read from the named env vars; with public_read, jobs without them read unsigned. + # download_concurrency (optional): parallel ranged GETs, for links with a low rate per connection. + S3_EXTERNAL_BUCKETS: Optional[Dict[str, Dict[str, Any]]] = None TEXT_CONTENT_EXTENSIONS: Iterable[str] = frozenset([".txt", ".log"]) # Compress if text file size exceeds this threshold (in MB, 0 - disable compression) COMPRESS_THRESHOLD_MB: int = 0 @@ -177,6 +183,8 @@ class _Settings: "EVENT_FEED_S3_PATH", "AWS_REGION", "S3_BUCKET_TO_HTTP_ENDPOINT", + "S3_EXTERNAL_BUCKETS", + "SUBMODULE_CACHE_S3_PATH", "TEXT_CONTENT_EXTENSIONS", "TEMP_DIR", "OUTPUT_DIR", diff --git a/ci/praktika/yaml_additional_templates.py b/ci/praktika/yaml_additional_templates.py index 8731d5e5d5c17..2869279da9962 100644 --- a/ci/praktika/yaml_additional_templates.py +++ b/ci/praktika/yaml_additional_templates.py @@ -8,6 +8,8 @@ class AltinityWorkflowTemplates: AZURE_CONTAINER_NAME: ${{{{ secrets.AZURE_CONTAINER_NAME }}}} AZURE_STORAGE_ACCOUNT_URL: "https://${{{{ secrets.AZURE_ACCOUNT_NAME }}}}.blob.core.windows.net/" ROBOT_TOKEN: ${{{{ secrets.ROBOT_TOKEN }}}} + HETZNER_S3_ACCESS_KEY: ${{{{ secrets.HETZNER_S3_ACCESS_KEY }}}} + HETZNER_S3_SECRET_KEY: ${{{{ secrets.HETZNER_S3_SECRET_KEY }}}} """ # Additional pre steps for all jobs JOB_SETUP_STEPS = """ diff --git a/ci/settings/altinity_overrides.py b/ci/settings/altinity_overrides.py index 9e965980b8ef5..aac84ca4835dd 100644 --- a/ci/settings/altinity_overrides.py +++ b/ci/settings/altinity_overrides.py @@ -29,6 +29,8 @@ class RunnerLabels: S3_BUCKET_TO_HTTP_ENDPOINT = { S3_BUCKET_NAME: S3_BUCKET_HTTP_ENDPOINT, S3_REPORT_BUCKET_NAME: S3_REPORT_BUCKET_HTTP_ENDPOINT, + # S3.copy_file_to_s3 returns a public link to every upload + "altinity-builds-cache": "altinity-builds-cache.hel1.your-objectstorage.com", } ENABLE_ARTIFACTS_REPORT = True @@ -63,6 +65,21 @@ class RunnerLabels: # after the burst of ~150 clones; send the ambient gh token instead. ENABLE_SUBMODULE_CLONE_AUTH = True +# Most runners are in Hetzner (hel1): cache there to avoid AWS egress. +CI_CACHE_BUCKET = "altinity-builds-cache" +SUBMODULE_CACHE_S3_PATH = f"{CI_CACHE_BUCKET}/ci_ch_cache" +S3_EXTERNAL_BUCKETS = { + CI_CACHE_BUCKET: { + "endpoint_url": "https://hel1.your-objectstorage.com", + "region": "hel1", + "access_key_env": "HETZNER_S3_ACCESS_KEY", + "secret_key_env": "HETZNER_S3_SECRET_KEY", + "public_read": True, + # netcup runners get ~2.5 MB/s per connection from hel1 + "download_concurrency": 32, + } +} + DISABLED_WORKFLOWS = [ "backport_branches.py", "custom_build_praktika.py", diff --git a/ci/tests/test_s3_external_bucket.py b/ci/tests/test_s3_external_bucket.py new file mode 100644 index 0000000000000..9746cbeb7a115 --- /dev/null +++ b/ci/tests/test_s3_external_bucket.py @@ -0,0 +1,197 @@ +""" +Buckets listed in Settings.S3_EXTERNAL_BUCKETS live on an S3-compatible store other than AWS. +S3.put and S3.head_object must reach them through a boto3 client with their own endpoint and credentials, +and must keep using the AWS CLI for every other bucket. +""" + +import datetime +import os +import sys + +import pytest + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../..")) + +pytest.importorskip("boto3") +from botocore.exceptions import ClientError + +from ci.praktika import s3 as s3_module +from ci.praktika.cache import Cache +from ci.praktika.s3 import S3 +from ci.praktika.settings import Settings + +BUCKET = "external-bucket" +EXTERNAL = { + BUCKET: { + "endpoint_url": "https://s3.example.test", + "region": "example-1", + "access_key_env": "EXTERNAL_S3_ACCESS_KEY", + "secret_key_env": "EXTERNAL_S3_SECRET_KEY", + "public_read": True, + } +} + + +class _FakeClient: + def __init__(self, error_code=None): + self.error_code = error_code + self.put_args = None + + def _maybe_fail(self, operation): + if self.error_code: + raise ClientError({"Error": {"Code": self.error_code}}, operation) + + def put_object(self, Body, **kwargs): + self._maybe_fail("PutObject") + self.put_args = kwargs + + def head_object(self, Bucket, Key): + self._maybe_fail("HeadObject") + return { + "ETag": '"etag"', + "ContentLength": 3, + "LastModified": datetime.datetime(2026, 1, 2, 3, 4, 5), + } + + +@pytest.fixture(autouse=True) +def external_bucket(monkeypatch): + monkeypatch.setattr(Settings, "S3_EXTERNAL_BUCKETS", EXTERNAL, raising=False) + monkeypatch.setattr(S3, "_external_clients", {}) + + def no_cli(*args, **kwargs): + raise AssertionError("the AWS CLI must not be used for an external bucket") + + monkeypatch.setattr(s3_module.Shell, "get_output", no_cli) + monkeypatch.setattr(s3_module.Shell, "get_res_stdout_stderr", no_cli) + + +def _use_client(monkeypatch, client): + """Serve the same fake for signed and anonymous clients.""" + monkeypatch.setattr( + S3, "_external_clients", {(BUCKET, False): client, (BUCKET, True): client} + ) + + +def _put(tmp_path, **kwargs): + local_file = tmp_path / "archive" + local_file.write_text("abc") + return S3.put(f"{BUCKET}/prefix/archive", str(local_file), **kwargs) + + +def test_put_sends_if_none_match(tmp_path, monkeypatch): + client = _FakeClient() + _use_client(monkeypatch, client) + assert _put(tmp_path, if_none_matched=True) is True + assert client.put_args == { + "Bucket": BUCKET, + "Key": "prefix/archive", + "IfNoneMatch": "*", + } + + +@pytest.mark.parametrize("code", ["PreconditionFailed", "ConditionalRequestConflict"]) +def test_put_lost_race_returns_false(tmp_path, monkeypatch, code): + _use_client(monkeypatch, _FakeClient(code)) + assert _put(tmp_path, if_none_matched=True) is False + + +def test_put_unexpected_error(tmp_path, monkeypatch): + _use_client(monkeypatch, _FakeClient("AccessDenied")) + with pytest.raises(ClientError): + _put(tmp_path) + assert _put(tmp_path, no_strict=True) is False + + +def test_head_object(monkeypatch): + _use_client(monkeypatch, _FakeClient()) + obj = S3.head_object(f"{BUCKET}/prefix/archive") + assert obj.ContentLength == 3 + assert obj.LastModified == "2026-01-02 03:04:05" + + +@pytest.mark.parametrize("code", ["404", "NoSuchKey", "NotFound"]) +def test_head_object_missing_is_none(monkeypatch, code): + _use_client(monkeypatch, _FakeClient(code)) + assert S3.head_object(f"{BUCKET}/prefix/archive") is None + + +def test_head_object_other_error_raises(monkeypatch): + _use_client(monkeypatch, _FakeClient("AccessDenied")) + with pytest.raises(ClientError): + S3.head_object(f"{BUCKET}/prefix/archive") + + +@pytest.fixture +def no_credentials(monkeypatch): + monkeypatch.delenv("EXTERNAL_S3_ACCESS_KEY", raising=False) + monkeypatch.delenv("EXTERNAL_S3_SECRET_KEY", raising=False) + + +def test_writes_need_credentials(tmp_path, no_credentials): + with pytest.raises(AssertionError, match="EXTERNAL_S3_ACCESS_KEY"): + _put(tmp_path) + + +def test_reads_of_a_public_bucket_need_no_credentials(no_credentials): + client = S3._get_boto3_client(BUCKET, read=True) + assert client.meta.config.signature_version is s3_module.UNSIGNED + assert client.meta.endpoint_url == "https://s3.example.test" + + +def test_reads_of_a_private_bucket_need_credentials(monkeypatch, no_credentials): + private = {BUCKET: {**EXTERNAL[BUCKET], "public_read": False}} + monkeypatch.setattr(Settings, "S3_EXTERNAL_BUCKETS", private, raising=False) + with pytest.raises(AssertionError, match="EXTERNAL_S3_ACCESS_KEY"): + S3.head_object(f"{BUCKET}/prefix/archive") + + +def test_signed_client_uses_the_configured_endpoint(monkeypatch): + monkeypatch.setenv("EXTERNAL_S3_ACCESS_KEY", "key") + monkeypatch.setenv("EXTERNAL_S3_SECRET_KEY", "secret") + client = S3._get_boto3_client(BUCKET) + assert client.meta.endpoint_url == "https://s3.example.test" + assert client.meta.region_name == "example-1" + assert S3._get_boto3_client(BUCKET) is client + assert S3._get_boto3_client(BUCKET, read=True) is client + + +def test_forbidden_is_a_miss_only_for_anonymous_reads(monkeypatch, no_credentials): + _use_client(monkeypatch, _FakeClient("403")) + assert S3.head_object(f"{BUCKET}/prefix/archive") is None + monkeypatch.setenv("EXTERNAL_S3_ACCESS_KEY", "key") + monkeypatch.setenv("EXTERNAL_S3_SECRET_KEY", "secret") + with pytest.raises(ClientError): + S3.head_object(f"{BUCKET}/prefix/archive") + + +def test_other_buckets_still_use_the_aws_cli(monkeypatch): + calls = [] + monkeypatch.setattr( + s3_module.Shell, "get_output", lambda cmd, **kw: calls.append(cmd) or "" + ) + assert S3.head_object("aws-bucket/key") is None + assert calls == ["aws s3api head-object --bucket aws-bucket --key key"] + + +def test_submodule_archive_path(monkeypatch): + monkeypatch.setattr(Settings, "CACHE_S3_PATH", "bucket/cache") + monkeypatch.setattr(Settings, "SUBMODULE_CACHE_S3_PATH", "") + assert Cache.submodule_archive_s3_path("abc") == "bucket/cache/submodules/abc.tar.zst" + monkeypatch.setattr(Settings, "SUBMODULE_CACHE_S3_PATH", f"{BUCKET}/cache") + assert Cache.submodule_archive_s3_path("abc") == f"{BUCKET}/cache/submodules/abc.tar.zst" + + +def test_download_concurrency(monkeypatch, tmp_path, no_credentials): + seen = {} + + class Client: + def download_file(self, bucket, key, filename, Config=None): + seen["concurrency"] = Config.max_concurrency + + monkeypatch.setattr(S3, "_external_clients", {(BUCKET, True): Client()}) + monkeypatch.setitem(EXTERNAL[BUCKET], "download_concurrency", 32) + assert S3.copy_file_from_s3(f"{BUCKET}/prefix/archive", str(tmp_path / "out")) + assert seen["concurrency"] == 32 + monkeypatch.setattr(S3, "_external_clients", {}) + assert S3._get_boto3_client(BUCKET, read=True).meta.config.max_pool_connections == 32 diff --git a/ci/tests/test_selected_tests_run.py b/ci/tests/test_selected_tests_run.py index 28c4d2d809da0..489823d625327 100644 --- a/ci/tests/test_selected_tests_run.py +++ b/ci/tests/test_selected_tests_run.py @@ -342,10 +342,10 @@ def test_pr_workflow_keeps_full_suite_msan_wasmedge_functional_tests(): if "amd_msan, WasmEdge" in name: full_suite_msan_wasmedge_jobs.append(name) continue - # The flaky, targeted and azure jobs run their own selection or config. + # The flaky, targeted, azure and cas jobs run their own selection or config. assert any( marker in name - for marker in ("selected tests", "flaky check", "targeted", "azure") + for marker in ("selected tests", "flaky check", "targeted", "azure", "cas ") ), f"full-suite sanitizer functional test job in the PR workflow: {name}" assert len(full_suite_msan_wasmedge_jobs) == 10 diff --git a/cmake/limit_jobs.cmake b/cmake/limit_jobs.cmake index 8be747c6f27f0..366e3f71a8ba4 100644 --- a/cmake/limit_jobs.cmake +++ b/cmake/limit_jobs.cmake @@ -28,6 +28,10 @@ if (NOT PARALLEL_COMPILE_JOBS AND MAX_COMPILER_MEMORY) endif() endif() +# Remember whether the link-job limit was set explicitly: the ThinLTO memory +# cap below only applies to the auto-calculated value. +set (PARALLEL_LINK_JOBS_EXPLICIT ${PARALLEL_LINK_JOBS}) + if (NOT PARALLEL_LINK_JOBS AND MAX_LINKER_MEMORY) math(EXPR PARALLEL_LINK_JOBS ${TOTAL_PHYSICAL_MEMORY}/${MAX_LINKER_MEMORY}) @@ -54,7 +58,7 @@ endif () # links than physically fit in memory - otherwise the linker is OOM-killed. # With debug info disabled (-g0, DISABLE_ALL_DEBUG_SYMBOLS) a link needs only # ~50 GB, so two still fit and we keep them parallel. -if (CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" AND ENABLE_THINLTO AND PARALLEL_LINK_JOBS GREATER 1) +if (CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" AND ENABLE_THINLTO AND PARALLEL_LINK_JOBS GREATER 1 AND NOT PARALLEL_LINK_JOBS_EXPLICIT) if (DISABLE_ALL_DEBUG_SYMBOLS) set (MAX_THINLTO_LINK_MEMORY 51200) # megabytes per concurrent ThinLTO link, measured ~49 GB with -g0 else ()