Antalya 26.8 - ThinLTO link cache

.github/actions/runner_setup/action.yml

@@ -25,7 +25,7 @@ runs:
           ZRAM_SIZE=$(($MemTotal / 1024 / 1024 * $Percent / 100)) # Convert to GiB
           .github/retry.sh 30 2 sudo zramctl --size ${ZRAM_SIZE}GiB --algorithm zstd /dev/zram0
           sudo mkswap /dev/zram0 && sudo swapon -p 100 /dev/zram0
-          sudo sysctl vm.swappiness=200
+          sudo sysctl vm.swappiness=80
     - name: Install awscli
       shell: bash
       run: |

.github/workflows/fast_builds.yml

@@ -45,6 +45,8 @@ env:
   AZURE_CONTAINER_NAME: ${{ secrets.AZURE_CONTAINER_NAME }}
   AZURE_STORAGE_ACCOUNT_URL: "https://${{ secrets.AZURE_ACCOUNT_NAME }}.blob.core.windows.net/"
   ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }}
+  HETZNER_S3_ACCESS_KEY: ${{ secrets.HETZNER_S3_ACCESS_KEY }}
+  HETZNER_S3_SECRET_KEY: ${{ secrets.HETZNER_S3_SECRET_KEY }}
 
 
 

.github/workflows/master.yml

@@ -60,6 +60,8 @@ env:
   AZURE_CONTAINER_NAME: ${{ secrets.AZURE_CONTAINER_NAME }}
   AZURE_STORAGE_ACCOUNT_URL: "https://${{ secrets.AZURE_ACCOUNT_NAME }}.blob.core.windows.net/"
   ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }}
+  HETZNER_S3_ACCESS_KEY: ${{ secrets.HETZNER_S3_ACCESS_KEY }}
+  HETZNER_S3_SECRET_KEY: ${{ secrets.HETZNER_S3_SECRET_KEY }}
 
 
 

.github/workflows/pull_request.yml

@@ -35,6 +35,8 @@ env:
   AZURE_CONTAINER_NAME: ${{ secrets.AZURE_CONTAINER_NAME }}
   AZURE_STORAGE_ACCOUNT_URL: "https://${{ secrets.AZURE_ACCOUNT_NAME }}.blob.core.windows.net/"
   ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }}
+  HETZNER_S3_ACCESS_KEY: ${{ secrets.HETZNER_S3_ACCESS_KEY }}
+  HETZNER_S3_SECRET_KEY: ${{ secrets.HETZNER_S3_SECRET_KEY }}
 
 
 

CMakeLists.txt

@@ -493,6 +493,24 @@ if (ENABLE_THINLTO AND NOT ENABLE_TESTS AND NOT SANITIZE)
         set(CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO} -Wl,--lto-whole-program-visibility")
     endif()
 
+    # Persistent ThinLTO cache. lld stores the optimized native object of every
+    # module keyed by the module hash, its import set and the LTO flags, and
+    # reuses it on the next link when nothing in that key changed. Only the
+    # backend codegen of the link is cached; the thin-link and the final native
+    # link still run in full. Entries are content-addressed, so concurrent
+    # writers cannot corrupt each other.
+    set (THINLTO_CACHE_DIR "" CACHE PATH "Directory for the ThinLTO cache (lld --thinlto-cache-dir); empty disables the cache")
+    # lld prunes after each link by default. That is off here, since CI keeps
+    # exactly the entries a build used; set a policy for a long-lived cache dir.
+    set (THINLTO_CACHE_POLICY "prune_after=0s:cache_size=0%:cache_size_bytes=0:cache_size_files=0" CACHE STRING "Pruning policy for the ThinLTO cache (lld --thinlto-cache-policy)")
+    if (THINLTO_CACHE_DIR)
+        if (NOT LINKER_NAME MATCHES "lld")
+            message (FATAL_ERROR "THINLTO_CACHE_DIR requires lld, but LINKER_NAME is '${LINKER_NAME}'")
+        endif ()
+        message (STATUS "Using ThinLTO cache in ${THINLTO_CACHE_DIR} with policy '${THINLTO_CACHE_POLICY}'")
+        set (CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO} -Wl,--thinlto-cache-dir=${THINLTO_CACHE_DIR} -Wl,--thinlto-cache-policy=${THINLTO_CACHE_POLICY}")
+    endif ()
+
     if (ENABLE_CFI)
         # CFI checks: virtual dispatch type safety and bad downcasts via static_cast.
         # cfi-icall is omitted because C function pointers in contrib libraries (compiled without

ci/defs/job_configs.py

@@ -30,6 +30,8 @@
     f"+--memory-reservation={Utils.physical_memory() * 9 // 10}"
     f"+--volume=.:/ClickHouse"
     '+--env=AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID"+--env=AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY"'
+    # Writes to the ThinLTO cache in Hetzner Object Storage; jobs without them read anonymously
+    '+--env=HETZNER_S3_ACCESS_KEY="$HETZNER_S3_ACCESS_KEY"+--env=HETZNER_S3_SECRET_KEY="$HETZNER_S3_SECRET_KEY"'
 )
 BINARY_DOCKER_COMMAND_SANITIZER = (
     BINARY_DOCKER_COMMAND + "+--security-opt seccomp=unconfined"
@@ -443,7 +445,7 @@ class JobConfigs:
                 ArtifactNames.RPM_ARM_RELEASE,
                 ArtifactNames.TGZ_ARM_RELEASE,
             ],
-            runs_on=RunnerLabels.BUILDER_AMD,
+            runs_on=RunnerLabels.BUILDER_ARM,
         ),
     )
     release_build_jobs_with_examples = [

ci/jobs/build_clickhouse.py

@@ -6,15 +6,29 @@
 from ci.jobs.scripts.clickhouse_version import CHVersion
 from ci.praktika.info import Info
 from ci.praktika.result import Result
+from ci.praktika.s3 import S3
 from ci.praktika.settings import Settings
 from ci.praktika.utils import MetaClasses, Shell, Utils
+from ci.settings.altinity_overrides import CI_CACHE_BUCKET
 
 current_directory = Utils.cwd()
 build_dir = f"{current_directory}/ci/tmp/build"
 temp_dir = f"{current_directory}/ci/tmp"
 repo_path_normalized = "/ClickHouse"
 build_dir_normalized = f"{repo_path_normalized}/ci/tmp/build"
 
+# ThinLTO cache: lld stores the optimized native object of every module it
+# links, and the next link of an unchanged module reuses it instead of running
+# the backend again. The cache travels between builders as one archive per
+# branch, build type and debug-info flavour.
+THINLTO_CACHE_S3_PATH = f"{CI_CACHE_BUCKET}/ci_ch_cache/thinlto"
+thinlto_cache_dir = f"{temp_dir}/thinlto_cache"
+thinlto_cache_dir_normalized = f"{repo_path_normalized}/ci/tmp/thinlto_cache"
+thinlto_cache_archive = f"{temp_dir}/thinlto_cache.tar.zst"
+# An archive holds the entries one build used, about 15 GB for a `-g` build of
+# `clickhouse` and `clickhouse-keeper`. Much more means pruning went wrong.
+THINLTO_CACHE_MAX_BYTES = {"g": 20 << 30, "g0": 5 << 30}
+
 BUILD_TYPE_TO_CMAKE = {
     BuildTypes.AMD_DEBUG: f"    cmake --debug-trycompile -DCMAKE_VERBOSE_MAKEFILE=1 -LA -DCMAKE_BUILD_TYPE=Debug -DENABLE_THINLTO=0 -DSANITIZE=          -DENABLE_CHECK_HEAVY_BUILDS=1 -DBUILD_STRIPPED_BINARY=1 -DENABLE_CLICKHOUSE_SELF_EXTRACTING=1 -DCMAKE_C_COMPILER={ToolSet.COMPILER_C} -DCMAKE_CXX_COMPILER={ToolSet.COMPILER_CPP} -DCOMPILER_CACHE={ToolSet.COMPILER_CACHE}        -DCMAKE_TOOLCHAIN_FILE={repo_path_normalized}/cmake/linux/toolchain-x86_64.cmake -DENABLE_BUILD_PROFILING=1 -DENABLE_TESTS=1 -DENABLE_LEXER_TEST=1 -DENABLE_UTILS=0 -DCMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY=ON -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_INSTALL_LOCALSTATEDIR=/var -DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=ON -DENABLE_BUZZHOUSE=1",
     BuildTypes.AMD_RELEASE: f"  cmake --debug-trycompile -DCMAKE_VERBOSE_MAKEFILE=1 -LA -DCMAKE_BUILD_TYPE=None  -DENABLE_THINLTO=1 -DSANITIZE=          -DENABLE_CHECK_HEAVY_BUILDS=1 -DBUILD_STRIPPED_BINARY=1 -DENABLE_CLICKHOUSE_SELF_EXTRACTING=1 -DCMAKE_C_COMPILER={ToolSet.COMPILER_C} -DCMAKE_CXX_COMPILER={ToolSet.COMPILER_CPP} -DCOMPILER_CACHE={ToolSet.COMPILER_CACHE}        -DCMAKE_TOOLCHAIN_FILE={repo_path_normalized}/cmake/linux/toolchain-x86_64.cmake -DENABLE_BUILD_PROFILING=1 -DENABLE_TESTS=0 -DENABLE_LEXER_TEST=0 -DENABLE_UTILS=0 -DCMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY=ON -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_INSTALL_LOCALSTATEDIR=/var -DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=ON -DSPLIT_DEBUG_SYMBOLS=ON -DBUILD_STANDALONE_KEEPER=1",
@@ -103,8 +117,114 @@ def parse_args():
     return parser.parse_args()
 
 
+def thinlto_cache_flavour(cmake_cmd):
+    return "g0" if "-DDISABLE_ALL_DEBUG_SYMBOLS=1" in cmake_cmd else "g"
+
+
+def thinlto_cache_s3_path(info, build_type, cmake_cmd):
+    """PR builds share the cache of their base branch, other builds use their own branch."""
+    branch = info.base_branch if info.pr_number > 0 else info.git_branch
+    if not branch:
+        return None
+    return f"{THINLTO_CACHE_S3_PATH}/{branch}/{build_type}-{thinlto_cache_flavour(cmake_cmd)}.tar.zst"
+
+
+def restore_thinlto_cache(s3_path):
+    shutil.rmtree(thinlto_cache_dir, ignore_errors=True)
+    os.makedirs(thinlto_cache_dir)
+    if not S3.head_object(s3_path):
+        print(f"ThinLTO cache miss: {s3_path}")
+        return True
+    try:
+        S3.copy_file_from_s3(s3_path=s3_path, local_path=thinlto_cache_archive)
+        Shell.check(
+            f"set -o pipefail; zstd -d --long=27 -c {thinlto_cache_archive} | tar -xf - -C {thinlto_cache_dir}",
+            verbose=True,
+            strict=True,
+        )
+    except BaseException:
+        # A partly extracted entry would be linked as is: start cold instead.
+        shutil.rmtree(thinlto_cache_dir, ignore_errors=True)
+        os.makedirs(thinlto_cache_dir)
+        raise
+    finally:
+        if os.path.exists(thinlto_cache_archive):
+            os.remove(thinlto_cache_archive)
+    # With an access time of zero, the kernel records the next read of an
+    # entry even under `relatime`, which is how the upload step tells the
+    # entries this build used from the ones it did not.
+    count = 0
+    size = 0
+    with os.scandir(thinlto_cache_dir) as entries:
+        for entry in entries:
+            stat = entry.stat(follow_symlinks=False)
+            os.utime(entry.path, (0, stat.st_mtime))
+            count += 1
+            size += stat.st_size
+    print(f"ThinLTO cache restored: {count} entries, {size / (1 << 30):.1f} GiB")
+    return True
+
+
+def thinlto_cache_tracks_reads():
+    probe = f"{thinlto_cache_dir}/.atime-probe"
+    with open(probe, "w") as f:
+        f.write("probe")
+    os.utime(probe, (0, os.stat(probe).st_mtime))
+    with open(probe) as f:
+        f.read()
+    tracked = os.stat(probe).st_atime > 0
+    os.remove(probe)
+    return tracked
+
+
+def upload_thinlto_cache(s3_path, flavour):
+    """Keep the entries this build read or wrote, then upload them."""
+    assert thinlto_cache_tracks_reads(), (
+        f"{thinlto_cache_dir} does not record file reads (mounted with noatime?), "
+        "so the entries this build used are unknown"
+    )
+    kept = 0
+    removed = 0
+    size = 0
+    with os.scandir(thinlto_cache_dir) as entries:
+        for entry in entries:
+            stat = entry.stat(follow_symlinks=False)
+            # Anything else is a temporary file of an interrupted link.
+            used = entry.name.startswith("llvmcache-") and entry.is_file(
+                follow_symlinks=False
+            ) and stat.st_atime > 0
+            if used:
+                kept += 1
+                size += stat.st_size
+            else:
+                os.remove(entry.path)
+                removed += 1
+    print(
+        f"ThinLTO cache: kept {kept} entries ({size / (1 << 30):.1f} GiB), "
+        f"removed {removed} unused"
+    )
+    assert size <= THINLTO_CACHE_MAX_BYTES[flavour], (
+        f"ThinLTO cache of {size} bytes exceeds the {THINLTO_CACHE_MAX_BYTES[flavour]} "
+        f"bytes expected for a `-{flavour}` build, not uploading it"
+    )
+    Shell.check(
+        f"set -o pipefail; tar -cf - -C {thinlto_cache_dir} . | zstd -T0 -3 --long=27 -c > {thinlto_cache_archive}",
+        verbose=True,
+        strict=True,
+    )
+    print(
+        f"ThinLTO cache archive: {os.path.getsize(thinlto_cache_archive) / (1 << 30):.1f} GiB"
+    )
+    S3.copy_file_to_s3(
+        s3_path=s3_path, local_path=thinlto_cache_archive, with_rename=True
+    )
+    os.remove(thinlto_cache_archive)
+    return True
+
+
 def run_shell(name, command, **kwargs):
     print(f"\n>>>> {name}\n")
+    kwargs.setdefault("verbose", True)
     Shell.check(command, **kwargs)
     print(f"\n<<<< {name}\n")
 
@@ -214,6 +334,26 @@ def main():
     ):
         cmake_cmd += " -DDISABLE_ALL_DEBUG_SYMBOLS=1"
 
+    # ThinLTO cache for every build that links with ThinLTO. The cache-warmup
+    # builds only compile. Everyone reads the archive (community PRs anonymously);
+    # pushes and internal PRs write it, like the shared compiler cache.
+    thinlto_cache_s3 = None
+    if "ENABLE_THINLTO=1" in cmake_cmd and not cache_warmup:
+        thinlto_cache_s3 = thinlto_cache_s3_path(info, build_type, cmake_cmd)
+    thinlto_cache_writable = not info.is_local_run and not info.is_community_pr
+    if thinlto_cache_s3:
+        print(f"ThinLTO cache: {thinlto_cache_s3}, writable: {thinlto_cache_writable}")
+        cmake_cmd += f" -DTHINLTO_CACHE_DIR={thinlto_cache_dir_normalized}"
+
+    # Link `clickhouse` and `clickhouse-keeper` concurrently. Each ThinLTO link
+    # spends most of its time in a single-threaded thin-link phase, so the two
+    # overlap almost entirely: measured on a 16-core, 62 GB builder with zram,
+    # 114.5 -> 83 min cold and 65.5 -> 48.5 min warm. The memory-derived
+    # default in limit_jobs.cmake budgets ~100 GB per `-g` link and would
+    # allow only one; the real peak of both links together was ~48 GB.
+    if "ENABLE_THINLTO=1" in cmake_cmd:
+        cmake_cmd += " -DPARALLEL_LINK_JOBS=2"
+
     # PGO/BOLT profile integration for release builds. The sccache-warmup builds
     # are deliberately excluded: they exist only to populate the shared compiler
     # cache, and PGO/BOLT belong to the real release builds.
@@ -422,6 +562,15 @@ def do_checkout():
         else:
             build_command = f"command time -v ninja {targets}"
 
+        if thinlto_cache_s3:
+            results.append(
+                Result.from_commands_run(
+                    name="Restore ThinLTO cache",
+                    command=restore_thinlto_cache,
+                    command_args=[thinlto_cache_s3],
+                )
+            )
+
         build_result_index = len(results)
         results.append(
             Result.from_commands_run(
@@ -462,6 +611,19 @@ def do_checkout():
             else:
                 results.append(retry_cmake)
 
+        if (
+            thinlto_cache_s3
+            and thinlto_cache_writable
+            and results[build_result_index].is_ok()
+        ):
+            results.append(
+                Result.from_commands_run(
+                    name="Upload ThinLTO cache",
+                    command=upload_thinlto_cache,
+                    command_args=[thinlto_cache_s3, thinlto_cache_flavour(cmake_cmd)],
+                )
+            )
+
         run_shell("sccache stats", "sccache --show-stats")
         if build_type in (BuildTypes.AMD_TIDY, BuildTypes.ARM_TIDY):
             run_shell("clang-tidy-cache stats", "clang-tidy-cache.py --show-stats")

ci/praktika/cache.py

@@ -39,6 +39,11 @@ def from_fs(cls, path):
         def from_dict(cls, obj):
             return Cache.CacheRecord(**obj)
 
+    @staticmethod
+    def submodule_archive_s3_path(cache_hash):
+        base = Settings.SUBMODULE_CACHE_S3_PATH or Settings.CACHE_S3_PATH
+        return f"{base}/submodules/{cache_hash}.tar.zst"
+
     def __init__(self):
         self.digest = Digest()
         self.success = {}  # type Dict[str, Any]

ci/praktika/native_jobs.py

@@ -10,6 +10,7 @@
 
 from . import Job, Workflow
 from ._environment import _Environment
+from .cache import Cache
 from .cidb import CIDB
 from .digest import Digest
 from .docker import Docker
@@ -252,13 +253,22 @@ def _prepare_submodule_cache(workflow, workflow_config: RunConfig) -> Result:
             )
 
         cache_hash = hashlib.sha256(submodule_shas.encode()).hexdigest()[:16]
-        s3_path = f"{Settings.CACHE_S3_PATH}/submodules/{cache_hash}.tar.zst"
+        s3_path = Cache.submodule_archive_s3_path(cache_hash)
         print(f"Submodule cache hash: {cache_hash}")
 
         # Check if cache already exists in S3
         if S3.head_object(s3_path):
             print(f"Submodule cache hit: {s3_path}")
             info = f"cache hit: {cache_hash}"
+        elif Info().is_community_pr:
+            # Like sccache, community PRs only read; jobs clone without a hash.
+            print(f"Submodule cache miss, community PR will not create it: {s3_path}")
+            return Result.create_from(
+                name="Submodule Cache",
+                status=Result.Status.OK,
+                stopwatch=stop_watch,
+                info=f"cache miss, not creating (read-only): {cache_hash}",
+            )
         else:
             print(f"Submodule cache miss, creating: {s3_path}")
             Shell.check("git submodule sync", verbose=True, strict=True)
@@ -270,6 +280,12 @@ def _prepare_submodule_cache(workflow, workflow_config: RunConfig) -> Result:
                 retries=3,
                 env=_submodule_auth_env(workflow),
             )
+            # The archive is public: never upload it with the clone token inside.
+            leaked = Shell.get_output(
+                "grep -rIlF x-access-token .git/config .git/modules --exclude-dir=objects"
+            )
+            if leaked:
+                raise RuntimeError(f"submodule cache would contain a GitHub token: {leaked}")
             archive_path = f"{Settings.TEMP_DIR}/submodules_{cache_hash}.tar.zst"
             Shell.check(
                 f"tar -cf - .git/modules | zstd -c -T0 > {archive_path}",
@@ -892,7 +908,11 @@ def check_affected_jobs():
             )
         )
 
-    if results[-1].is_ok() and workflow.enable_cache and Settings.ENABLE_SUBMODULE_CACHE:
+    if (
+        results[-1].is_ok()
+        and (workflow.enable_cache or Info().is_community_pr)
+        and Settings.ENABLE_SUBMODULE_CACHE
+    ):
         result = _prepare_submodule_cache(workflow, workflow_config)
         results.append(result)
 

ci/praktika/runner.py

@@ -11,6 +11,7 @@
 from pathlib import Path
 
 from ._environment import _Environment
+from .cache import Cache
 from .artifact import Artifact
 from .cidb import CIDB
 from .digest import Digest
@@ -362,7 +363,7 @@ def _restore_submodule_cache():
             if not cache_hash:
                 print("NOTE: No submodule cache hash in workflow config, skipping restore")
                 return
-            s3_path = f"{Settings.CACHE_S3_PATH}/submodules/{cache_hash}.tar.zst"
+            s3_path = Cache.submodule_archive_s3_path(cache_hash)
             local_archive = f"{Settings.TEMP_DIR}/submodules_{cache_hash}.tar.zst"
             print(f"Restoring submodule cache: {s3_path}")
             S3.copy_file_from_s3(s3_path=s3_path, local_path=local_archive, no_strict=True)