Antalya 26.8 - Store the submodule cache in Hetzner Object Storage

.github/workflows/fast_builds.yml

@@ -45,6 +45,8 @@ env:
   AZURE_CONTAINER_NAME: ${{ secrets.AZURE_CONTAINER_NAME }}
   AZURE_STORAGE_ACCOUNT_URL: "https://${{ secrets.AZURE_ACCOUNT_NAME }}.blob.core.windows.net/"
   ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }}
+  HETZNER_S3_ACCESS_KEY: ${{ secrets.HETZNER_S3_ACCESS_KEY }}
+  HETZNER_S3_SECRET_KEY: ${{ secrets.HETZNER_S3_SECRET_KEY }}
 
 
 

.github/workflows/master.yml

@@ -60,6 +60,8 @@ env:
   AZURE_CONTAINER_NAME: ${{ secrets.AZURE_CONTAINER_NAME }}
   AZURE_STORAGE_ACCOUNT_URL: "https://${{ secrets.AZURE_ACCOUNT_NAME }}.blob.core.windows.net/"
   ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }}
+  HETZNER_S3_ACCESS_KEY: ${{ secrets.HETZNER_S3_ACCESS_KEY }}
+  HETZNER_S3_SECRET_KEY: ${{ secrets.HETZNER_S3_SECRET_KEY }}
 
 
 

.github/workflows/pull_request.yml

@@ -35,6 +35,8 @@ env:
   AZURE_CONTAINER_NAME: ${{ secrets.AZURE_CONTAINER_NAME }}
   AZURE_STORAGE_ACCOUNT_URL: "https://${{ secrets.AZURE_ACCOUNT_NAME }}.blob.core.windows.net/"
   ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }}
+  HETZNER_S3_ACCESS_KEY: ${{ secrets.HETZNER_S3_ACCESS_KEY }}
+  HETZNER_S3_SECRET_KEY: ${{ secrets.HETZNER_S3_SECRET_KEY }}
 
 
 

ci/praktika/cache.py

@@ -39,6 +39,11 @@ def from_fs(cls, path):
         def from_dict(cls, obj):
             return Cache.CacheRecord(**obj)
 
+    @staticmethod
+    def submodule_archive_s3_path(cache_hash):
+        base = Settings.SUBMODULE_CACHE_S3_PATH or Settings.CACHE_S3_PATH
+        return f"{base}/submodules/{cache_hash}.tar.zst"
+
     def __init__(self):
         self.digest = Digest()
         self.success = {}  # type Dict[str, Any]

ci/praktika/native_jobs.py

@@ -10,6 +10,7 @@
 
 from . import Job, Workflow
 from ._environment import _Environment
+from .cache import Cache
 from .cidb import CIDB
 from .digest import Digest
 from .docker import Docker
@@ -252,13 +253,22 @@ def _prepare_submodule_cache(workflow, workflow_config: RunConfig) -> Result:
             )
 
         cache_hash = hashlib.sha256(submodule_shas.encode()).hexdigest()[:16]
-        s3_path = f"{Settings.CACHE_S3_PATH}/submodules/{cache_hash}.tar.zst"
+        s3_path = Cache.submodule_archive_s3_path(cache_hash)
         print(f"Submodule cache hash: {cache_hash}")
 
         # Check if cache already exists in S3
         if S3.head_object(s3_path):
             print(f"Submodule cache hit: {s3_path}")
             info = f"cache hit: {cache_hash}"
+        elif Info().is_community_pr:
+            # Like sccache, community PRs only read; jobs clone without a hash.
+            print(f"Submodule cache miss, community PR will not create it: {s3_path}")
+            return Result.create_from(
+                name="Submodule Cache",
+                status=Result.Status.OK,
+                stopwatch=stop_watch,
+                info=f"cache miss, not creating (read-only): {cache_hash}",
+            )
         else:
             print(f"Submodule cache miss, creating: {s3_path}")
             Shell.check("git submodule sync", verbose=True, strict=True)
@@ -270,6 +280,12 @@ def _prepare_submodule_cache(workflow, workflow_config: RunConfig) -> Result:
                 retries=3,
                 env=_submodule_auth_env(workflow),
             )
+            # The archive is public: never upload it with the clone token inside.
+            leaked = Shell.get_output(
+                "grep -rIlF x-access-token .git/config .git/modules --exclude-dir=objects"
+            )
+            if leaked:
+                raise RuntimeError(f"submodule cache would contain a GitHub token: {leaked}")
             archive_path = f"{Settings.TEMP_DIR}/submodules_{cache_hash}.tar.zst"
             Shell.check(
                 f"tar -cf - .git/modules | zstd -c -T0 > {archive_path}",
@@ -892,7 +908,11 @@ def check_affected_jobs():
             )
         )
 
-    if results[-1].is_ok() and workflow.enable_cache and Settings.ENABLE_SUBMODULE_CACHE:
+    if (
+        results[-1].is_ok()
+        and (workflow.enable_cache or Info().is_community_pr)
+        and Settings.ENABLE_SUBMODULE_CACHE
+    ):
         result = _prepare_submodule_cache(workflow, workflow_config)
         results.append(result)
 

ci/praktika/runner.py

@@ -11,6 +11,7 @@
 from pathlib import Path
 
 from ._environment import _Environment
+from .cache import Cache
 from .artifact import Artifact
 from .cidb import CIDB
 from .digest import Digest
@@ -362,7 +363,7 @@ def _restore_submodule_cache():
             if not cache_hash:
                 print("NOTE: No submodule cache hash in workflow config, skipping restore")
                 return
-            s3_path = f"{Settings.CACHE_S3_PATH}/submodules/{cache_hash}.tar.zst"
+            s3_path = Cache.submodule_archive_s3_path(cache_hash)
             local_archive = f"{Settings.TEMP_DIR}/submodules_{cache_hash}.tar.zst"
             print(f"Restoring submodule cache: {s3_path}")
             S3.copy_file_from_s3(s3_path=s3_path, local_path=local_archive, no_strict=True)

ci/praktika/s3.py

@@ -15,7 +15,10 @@
 
 try:
     import boto3
+    from botocore import UNSIGNED
+    from botocore.config import Config
     from botocore.exceptions import ClientError, NoCredentialsError
+    from boto3.s3.transfer import TransferConfig
 
     BOTO3_AVAILABLE = True
 except ImportError:
@@ -58,9 +61,64 @@ def clean_line(line):
 
 class S3:
     _boto3_client = None
+    _external_clients = {}
 
     @classmethod
-    def _get_boto3_client(cls):
+    def _external_bucket_config(cls, bucket):
+        return (Settings.S3_EXTERNAL_BUCKETS or {}).get(bucket)
+
+    @classmethod
+    def _reads_anonymously(cls, bucket):
+        """Reads of a public_read bucket are unsigned unless this job has the bucket's keys."""
+        config = cls._external_bucket_config(bucket)
+        return bool(
+            config
+            and config.get("public_read")
+            and not (
+                os.environ.get(config["access_key_env"])
+                and os.environ.get(config["secret_key_env"])
+            )
+        )
+
+    @classmethod
+    def _get_external_client(cls, bucket, config, anonymous):
+        assert BOTO3_AVAILABLE, f"boto3 is required for S3 bucket [{bucket}]"
+        client = cls._external_clients.get((bucket, anonymous))
+        if client is None:
+            client_config = {
+                "max_pool_connections": config.get("download_concurrency", 10)
+            }
+            if anonymous:
+                credentials = {
+                    "config": Config(signature_version=UNSIGNED, **client_config)
+                }
+            else:
+                access_key = os.environ.get(config["access_key_env"], "")
+                secret_key = os.environ.get(config["secret_key_env"], "")
+                # boto3 would otherwise send the ambient AWS credentials to this endpoint
+                assert (
+                    access_key and secret_key
+                ), f"S3 bucket [{bucket}] needs [{config['access_key_env']}] and [{config['secret_key_env']}] set"
+                credentials = {
+                    "aws_access_key_id": access_key,
+                    "aws_secret_access_key": secret_key,
+                    "config": Config(**client_config),
+                }
+            client = boto3.client(
+                "s3",
+                endpoint_url=config["endpoint_url"],
+                region_name=config["region"],
+                **credentials,
+            )
+            cls._external_clients[(bucket, anonymous)] = client
+        return client
+
+    @classmethod
+    def _get_boto3_client(cls, bucket=None, read=False):
+        config = cls._external_bucket_config(bucket)
+        if config:
+            anonymous = read and cls._reads_anonymously(bucket)
+            return cls._get_external_client(bucket, config, anonymous)
         if not BOTO3_AVAILABLE:
             return None
         if cls._boto3_client is None:
@@ -188,7 +246,7 @@ def copy_file_to_s3(
                 extra_args["Tagging"] = urlencode(tags)
 
             def _upload():
-                client = cls._get_boto3_client()
+                client = cls._get_boto3_client(bucket)
                 if extra_args:
                     client.upload_file(
                         str(local_path), bucket, key, ExtraArgs=extra_args
@@ -260,24 +318,69 @@ def put(
         s3_full_path = str(s3_full_path).removeprefix("s3://")
         bucket, key = s3_full_path.split("/", maxsplit=1)
 
-        command = (
-            f"aws s3api put-object --bucket {bucket} --key {key} --body {local_path}"
-        )
-        if if_none_matched:
-            command += ' --if-none-match "*"'
-        if if_match:
-            command += f' --if-match "{if_match}"'
-        if metadata:
-            for k, v in metadata.items():
-                command += f" --metadata {k}={v}"
-
-        if text:
-            command += ' --content-type "text/plain; charset=utf-8"'
-        res = cls.run_command_with_retries(command, no_strict=no_strict)
+        if cls._external_bucket_config(bucket):
+            res = cls._put_object_boto3(
+                bucket,
+                key,
+                local_path,
+                text,
+                metadata,
+                if_none_matched,
+                if_match,
+                no_strict,
+            )
+        else:
+            command = f"aws s3api put-object --bucket {bucket} --key {key} --body {local_path}"
+            if if_none_matched:
+                command += ' --if-none-match "*"'
+            if if_match:
+                command += f' --if-match "{if_match}"'
+            if metadata:
+                for k, v in metadata.items():
+                    command += f" --metadata {k}={v}"
+
+            if text:
+                command += ' --content-type "text/plain; charset=utf-8"'
+            res = cls.run_command_with_retries(command, no_strict=no_strict)
         if res:
             StorageUsage.add_uploaded(local_path)
         return res
 
+    @classmethod
+    def _put_object_boto3(
+        cls,
+        bucket,
+        key,
+        local_path,
+        text,
+        metadata,
+        if_none_matched,
+        if_match,
+        no_strict,
+    ):
+        args = {"Bucket": bucket, "Key": key}
+        if metadata:
+            args["Metadata"] = metadata
+        if text:
+            args["ContentType"] = "text/plain; charset=utf-8"
+        if if_none_matched:
+            args["IfNoneMatch"] = "*"
+        if if_match:
+            args["IfMatch"] = if_match
+        try:
+            with open(local_path, "rb") as f:
+                cls._get_boto3_client(bucket).put_object(Body=f, **args)
+        except ClientError as e:
+            error_code = e.response.get("Error", {}).get("Code", "")
+            if error_code in ("PreconditionFailed", "ConditionalRequestConflict"):
+                print(f"S3 conditional request failed (concurrent write detected): {error_code}")
+                return False
+            print(f"ERROR: Failed to put S3 object [{bucket}/{key}]: {error_code}")
+            if not no_strict:
+                raise
+            return False
+        return True
+
     @classmethod
     def run_command_with_retries(
         cls,
@@ -354,8 +457,18 @@ def copy_file_from_s3(
                     local_file.parent.mkdir(parents=True, exist_ok=True)
 
                     def _download():
-                        client = cls._get_boto3_client()
-                        client.download_file(bucket, key, str(local_file))
+                        client = cls._get_boto3_client(bucket, read=True)
+                        concurrency = (cls._external_bucket_config(bucket) or {}).get(
+                            "download_concurrency"
+                        )
+                        client.download_file(
+                            bucket,
+                            key,
+                            str(local_file),
+                            Config=TransferConfig(max_concurrency=concurrency)
+                            if concurrency
+                            else None,
+                        )
 
                     # Retry on transient credential failures
                     cls._retry_on_no_credentials(_download)
@@ -426,18 +539,35 @@ def copy_file_from_s3_matching_pattern(
     def head_object(cls, s3_path):
         s3_path = str(s3_path).removeprefix("s3://")
         bucket, key = s3_path.split("/", maxsplit=1)
-        output = Shell.get_output(
-            f"aws s3api head-object --bucket {bucket} --key {key}", verbose=True
-        )
-        if not output:
-            return None
-        data = json.loads(output)
+        if cls._external_bucket_config(bucket):
+            try:
+                data = cls._get_boto3_client(bucket, read=True).head_object(
+                    Bucket=bucket, Key=key
+                )
+            except ClientError as e:
+                if e.response.get("Error", {}).get("Code", "") in (
+                    "404",
+                    "NoSuchKey",
+                    "NotFound",
+                ):
+                    return None
+                # A public bucket answers a missing key with 403 to callers who may not list it
+                if e.response.get("Error", {}).get("Code", "") == "403" and cls._reads_anonymously(bucket):
+                    return None
+                raise
+        else:
+            output = Shell.get_output(
+                f"aws s3api head-object --bucket {bucket} --key {key}", verbose=True
+            )
+            if not output:
+                return None
+            data = json.loads(output)
         # HeadObject omits optional fields (Expiration) unless a lifecycle Expire
         # rule applies, and emits extra keys Object does not declare.
         return cls.Object(
             AcceptRanges=data.get("AcceptRanges", ""),
             Expiration=data.get("Expiration", ""),
-            LastModified=data.get("LastModified", ""),
+            LastModified=str(data.get("LastModified", "")),
             ContentLength=data.get("ContentLength", 0),
             ETag=data.get("ETag", ""),
             ContentType=data.get("ContentType", ""),
@@ -456,7 +586,7 @@ def assert_read_access(cls, s3_path):
         """
         s3_path = str(s3_path).removeprefix("s3://")
         bucket, key = s3_path.split("/", maxsplit=1)
-        client = cls._get_boto3_client()
+        client = cls._get_boto3_client(bucket, read=True)
         assert client, "boto3 client is required for S3 read-access probe"
 
         def _probe():

ci/praktika/settings.py

@@ -1,7 +1,7 @@
 import dataclasses
 import importlib.util
 from pathlib import Path
-from typing import Dict, Iterable, List, Optional
+from typing import Any, Dict, Iterable, List, Optional
 
 
 @dataclasses.dataclass
@@ -111,6 +111,8 @@ class _Settings:
     # If enabled, submodule clones authenticate with the App installation token
     # (required for private submodules); otherwise they run anonymously.
     ENABLE_SUBMODULE_CLONE_AUTH: bool = False
+    # bucket/prefix of the submodule archive, defaults to CACHE_S3_PATH
+    SUBMODULE_CACHE_S3_PATH: str = ""
 
     # v2: records carry the producing workflow event, used as the reuse trust
     # signal instead of the branch (see CacheRunnerHooks.configure).
@@ -127,6 +129,10 @@ class _Settings:
     S3_UPSTREAM_REPORT_BUCKET: str = ""
     HTML_PAGE_FILE: str = "./ci/praktika/json.html"
     S3_BUCKET_TO_HTTP_ENDPOINT: Optional[Dict[str, str]] = None
+    # Non-AWS buckets: {bucket: {endpoint_url, region, access_key_env, secret_key_env, public_read, download_concurrency}}
+    # The keys are read from the named env vars; with public_read, jobs without them read unsigned.
+    # download_concurrency (optional): parallel ranged GETs, for links with a low rate per connection.
+    S3_EXTERNAL_BUCKETS: Optional[Dict[str, Dict[str, Any]]] = None
     TEXT_CONTENT_EXTENSIONS: Iterable[str] = frozenset([".txt", ".log"])
     # Compress if text file size exceeds this threshold (in MB, 0 - disable compression)
     COMPRESS_THRESHOLD_MB: int = 0
@@ -177,6 +183,8 @@ class _Settings:
     "EVENT_FEED_S3_PATH",
     "AWS_REGION",
     "S3_BUCKET_TO_HTTP_ENDPOINT",
+    "S3_EXTERNAL_BUCKETS",
+    "SUBMODULE_CACHE_S3_PATH",
     "TEXT_CONTENT_EXTENSIONS",
     "TEMP_DIR",
     "OUTPUT_DIR",