diff --git a/.github/workflows/fast_builds.yml b/.github/workflows/fast_builds.yml index eaabcc3a63286..26a61911e0462 100644 --- a/.github/workflows/fast_builds.yml +++ b/.github/workflows/fast_builds.yml @@ -45,6 +45,8 @@ env: AZURE_CONTAINER_NAME: ${{ secrets.AZURE_CONTAINER_NAME }} AZURE_STORAGE_ACCOUNT_URL: "https://${{ secrets.AZURE_ACCOUNT_NAME }}.blob.core.windows.net/" ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }} + HETZNER_S3_ACCESS_KEY: ${{ secrets.HETZNER_S3_ACCESS_KEY }} + HETZNER_S3_SECRET_KEY: ${{ secrets.HETZNER_S3_SECRET_KEY }} diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 62f6d64627657..90d8b9e15d717 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -60,6 +60,8 @@ env: AZURE_CONTAINER_NAME: ${{ secrets.AZURE_CONTAINER_NAME }} AZURE_STORAGE_ACCOUNT_URL: "https://${{ secrets.AZURE_ACCOUNT_NAME }}.blob.core.windows.net/" ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }} + HETZNER_S3_ACCESS_KEY: ${{ secrets.HETZNER_S3_ACCESS_KEY }} + HETZNER_S3_SECRET_KEY: ${{ secrets.HETZNER_S3_SECRET_KEY }} diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index c638bc99d1e54..e16505d822493 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -35,6 +35,8 @@ env: AZURE_CONTAINER_NAME: ${{ secrets.AZURE_CONTAINER_NAME }} AZURE_STORAGE_ACCOUNT_URL: "https://${{ secrets.AZURE_ACCOUNT_NAME }}.blob.core.windows.net/" ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }} + HETZNER_S3_ACCESS_KEY: ${{ secrets.HETZNER_S3_ACCESS_KEY }} + HETZNER_S3_SECRET_KEY: ${{ secrets.HETZNER_S3_SECRET_KEY }} diff --git a/ci/praktika/cache.py b/ci/praktika/cache.py index 148f4edfbc521..6c1545eef742e 100644 --- a/ci/praktika/cache.py +++ b/ci/praktika/cache.py @@ -39,6 +39,11 @@ def from_fs(cls, path): def from_dict(cls, obj): return Cache.CacheRecord(**obj) + @staticmethod + def submodule_archive_s3_path(cache_hash): + base = Settings.SUBMODULE_CACHE_S3_PATH or Settings.CACHE_S3_PATH + return f"{base}/submodules/{cache_hash}.tar.zst" + def __init__(self): self.digest = Digest() self.success = {} # type Dict[str, Any] diff --git a/ci/praktika/native_jobs.py b/ci/praktika/native_jobs.py index e6c9a673f43cd..d907dc574795b 100644 --- a/ci/praktika/native_jobs.py +++ b/ci/praktika/native_jobs.py @@ -10,6 +10,7 @@ from . import Job, Workflow from ._environment import _Environment +from .cache import Cache from .cidb import CIDB from .digest import Digest from .docker import Docker @@ -252,13 +253,22 @@ def _prepare_submodule_cache(workflow, workflow_config: RunConfig) -> Result: ) cache_hash = hashlib.sha256(submodule_shas.encode()).hexdigest()[:16] - s3_path = f"{Settings.CACHE_S3_PATH}/submodules/{cache_hash}.tar.zst" + s3_path = Cache.submodule_archive_s3_path(cache_hash) print(f"Submodule cache hash: {cache_hash}") # Check if cache already exists in S3 if S3.head_object(s3_path): print(f"Submodule cache hit: {s3_path}") info = f"cache hit: {cache_hash}" + elif Info().is_community_pr: + # Like sccache, community PRs only read; jobs clone without a hash. + print(f"Submodule cache miss, community PR will not create it: {s3_path}") + return Result.create_from( + name="Submodule Cache", + status=Result.Status.OK, + stopwatch=stop_watch, + info=f"cache miss, not creating (read-only): {cache_hash}", + ) else: print(f"Submodule cache miss, creating: {s3_path}") Shell.check("git submodule sync", verbose=True, strict=True) @@ -270,6 +280,12 @@ def _prepare_submodule_cache(workflow, workflow_config: RunConfig) -> Result: retries=3, env=_submodule_auth_env(workflow), ) + # The archive is public: never upload it with the clone token inside. + leaked = Shell.get_output( + "grep -rIlF x-access-token .git/config .git/modules --exclude-dir=objects" + ) + if leaked: + raise RuntimeError(f"submodule cache would contain a GitHub token: {leaked}") archive_path = f"{Settings.TEMP_DIR}/submodules_{cache_hash}.tar.zst" Shell.check( f"tar -cf - .git/modules | zstd -c -T0 > {archive_path}", @@ -892,7 +908,11 @@ def check_affected_jobs(): ) ) - if results[-1].is_ok() and workflow.enable_cache and Settings.ENABLE_SUBMODULE_CACHE: + if ( + results[-1].is_ok() + and (workflow.enable_cache or Info().is_community_pr) + and Settings.ENABLE_SUBMODULE_CACHE + ): result = _prepare_submodule_cache(workflow, workflow_config) results.append(result) diff --git a/ci/praktika/runner.py b/ci/praktika/runner.py index 6d105b87468ac..a09b09c3471e3 100644 --- a/ci/praktika/runner.py +++ b/ci/praktika/runner.py @@ -11,6 +11,7 @@ from pathlib import Path from ._environment import _Environment +from .cache import Cache from .artifact import Artifact from .cidb import CIDB from .digest import Digest @@ -362,7 +363,7 @@ def _restore_submodule_cache(): if not cache_hash: print("NOTE: No submodule cache hash in workflow config, skipping restore") return - s3_path = f"{Settings.CACHE_S3_PATH}/submodules/{cache_hash}.tar.zst" + s3_path = Cache.submodule_archive_s3_path(cache_hash) local_archive = f"{Settings.TEMP_DIR}/submodules_{cache_hash}.tar.zst" print(f"Restoring submodule cache: {s3_path}") S3.copy_file_from_s3(s3_path=s3_path, local_path=local_archive, no_strict=True) diff --git a/ci/praktika/s3.py b/ci/praktika/s3.py index c064f513e635b..863718e8b915d 100644 --- a/ci/praktika/s3.py +++ b/ci/praktika/s3.py @@ -15,7 +15,10 @@ try: import boto3 + from botocore import UNSIGNED + from botocore.config import Config from botocore.exceptions import ClientError, NoCredentialsError + from boto3.s3.transfer import TransferConfig BOTO3_AVAILABLE = True except ImportError: @@ -58,9 +61,64 @@ def clean_line(line): class S3: _boto3_client = None + _external_clients = {} @classmethod - def _get_boto3_client(cls): + def _external_bucket_config(cls, bucket): + return (Settings.S3_EXTERNAL_BUCKETS or {}).get(bucket) + + @classmethod + def _reads_anonymously(cls, bucket): + """Reads of a public_read bucket are unsigned unless this job has the bucket's keys.""" + config = cls._external_bucket_config(bucket) + return bool( + config + and config.get("public_read") + and not ( + os.environ.get(config["access_key_env"]) + and os.environ.get(config["secret_key_env"]) + ) + ) + + @classmethod + def _get_external_client(cls, bucket, config, anonymous): + assert BOTO3_AVAILABLE, f"boto3 is required for S3 bucket [{bucket}]" + client = cls._external_clients.get((bucket, anonymous)) + if client is None: + client_config = { + "max_pool_connections": config.get("download_concurrency", 10) + } + if anonymous: + credentials = { + "config": Config(signature_version=UNSIGNED, **client_config) + } + else: + access_key = os.environ.get(config["access_key_env"], "") + secret_key = os.environ.get(config["secret_key_env"], "") + # boto3 would otherwise send the ambient AWS credentials to this endpoint + assert ( + access_key and secret_key + ), f"S3 bucket [{bucket}] needs [{config['access_key_env']}] and [{config['secret_key_env']}] set" + credentials = { + "aws_access_key_id": access_key, + "aws_secret_access_key": secret_key, + "config": Config(**client_config), + } + client = boto3.client( + "s3", + endpoint_url=config["endpoint_url"], + region_name=config["region"], + **credentials, + ) + cls._external_clients[(bucket, anonymous)] = client + return client + + @classmethod + def _get_boto3_client(cls, bucket=None, read=False): + config = cls._external_bucket_config(bucket) + if config: + anonymous = read and cls._reads_anonymously(bucket) + return cls._get_external_client(bucket, config, anonymous) if not BOTO3_AVAILABLE: return None if cls._boto3_client is None: @@ -188,7 +246,7 @@ def copy_file_to_s3( extra_args["Tagging"] = urlencode(tags) def _upload(): - client = cls._get_boto3_client() + client = cls._get_boto3_client(bucket) if extra_args: client.upload_file( str(local_path), bucket, key, ExtraArgs=extra_args @@ -260,24 +318,69 @@ def put( s3_full_path = str(s3_full_path).removeprefix("s3://") bucket, key = s3_full_path.split("/", maxsplit=1) - command = ( - f"aws s3api put-object --bucket {bucket} --key {key} --body {local_path}" - ) - if if_none_matched: - command += ' --if-none-match "*"' - if if_match: - command += f' --if-match "{if_match}"' - if metadata: - for k, v in metadata.items(): - command += f" --metadata {k}={v}" - - if text: - command += ' --content-type "text/plain; charset=utf-8"' - res = cls.run_command_with_retries(command, no_strict=no_strict) + if cls._external_bucket_config(bucket): + res = cls._put_object_boto3( + bucket, + key, + local_path, + text, + metadata, + if_none_matched, + if_match, + no_strict, + ) + else: + command = f"aws s3api put-object --bucket {bucket} --key {key} --body {local_path}" + if if_none_matched: + command += ' --if-none-match "*"' + if if_match: + command += f' --if-match "{if_match}"' + if metadata: + for k, v in metadata.items(): + command += f" --metadata {k}={v}" + + if text: + command += ' --content-type "text/plain; charset=utf-8"' + res = cls.run_command_with_retries(command, no_strict=no_strict) if res: StorageUsage.add_uploaded(local_path) return res + @classmethod + def _put_object_boto3( + cls, + bucket, + key, + local_path, + text, + metadata, + if_none_matched, + if_match, + no_strict, + ): + args = {"Bucket": bucket, "Key": key} + if metadata: + args["Metadata"] = metadata + if text: + args["ContentType"] = "text/plain; charset=utf-8" + if if_none_matched: + args["IfNoneMatch"] = "*" + if if_match: + args["IfMatch"] = if_match + try: + with open(local_path, "rb") as f: + cls._get_boto3_client(bucket).put_object(Body=f, **args) + except ClientError as e: + error_code = e.response.get("Error", {}).get("Code", "") + if error_code in ("PreconditionFailed", "ConditionalRequestConflict"): + print(f"S3 conditional request failed (concurrent write detected): {error_code}") + return False + print(f"ERROR: Failed to put S3 object [{bucket}/{key}]: {error_code}") + if not no_strict: + raise + return False + return True + @classmethod def run_command_with_retries( cls, @@ -354,8 +457,18 @@ def copy_file_from_s3( local_file.parent.mkdir(parents=True, exist_ok=True) def _download(): - client = cls._get_boto3_client() - client.download_file(bucket, key, str(local_file)) + client = cls._get_boto3_client(bucket, read=True) + concurrency = (cls._external_bucket_config(bucket) or {}).get( + "download_concurrency" + ) + client.download_file( + bucket, + key, + str(local_file), + Config=TransferConfig(max_concurrency=concurrency) + if concurrency + else None, + ) # Retry on transient credential failures cls._retry_on_no_credentials(_download) @@ -426,18 +539,35 @@ def copy_file_from_s3_matching_pattern( def head_object(cls, s3_path): s3_path = str(s3_path).removeprefix("s3://") bucket, key = s3_path.split("/", maxsplit=1) - output = Shell.get_output( - f"aws s3api head-object --bucket {bucket} --key {key}", verbose=True - ) - if not output: - return None - data = json.loads(output) + if cls._external_bucket_config(bucket): + try: + data = cls._get_boto3_client(bucket, read=True).head_object( + Bucket=bucket, Key=key + ) + except ClientError as e: + if e.response.get("Error", {}).get("Code", "") in ( + "404", + "NoSuchKey", + "NotFound", + ): + return None + # A public bucket answers a missing key with 403 to callers who may not list it + if e.response.get("Error", {}).get("Code", "") == "403" and cls._reads_anonymously(bucket): + return None + raise + else: + output = Shell.get_output( + f"aws s3api head-object --bucket {bucket} --key {key}", verbose=True + ) + if not output: + return None + data = json.loads(output) # HeadObject omits optional fields (Expiration) unless a lifecycle Expire # rule applies, and emits extra keys Object does not declare. return cls.Object( AcceptRanges=data.get("AcceptRanges", ""), Expiration=data.get("Expiration", ""), - LastModified=data.get("LastModified", ""), + LastModified=str(data.get("LastModified", "")), ContentLength=data.get("ContentLength", 0), ETag=data.get("ETag", ""), ContentType=data.get("ContentType", ""), @@ -456,7 +586,7 @@ def assert_read_access(cls, s3_path): """ s3_path = str(s3_path).removeprefix("s3://") bucket, key = s3_path.split("/", maxsplit=1) - client = cls._get_boto3_client() + client = cls._get_boto3_client(bucket, read=True) assert client, "boto3 client is required for S3 read-access probe" def _probe(): diff --git a/ci/praktika/settings.py b/ci/praktika/settings.py index 760a0caa90748..3385cd6b57368 100644 --- a/ci/praktika/settings.py +++ b/ci/praktika/settings.py @@ -1,7 +1,7 @@ import dataclasses import importlib.util from pathlib import Path -from typing import Dict, Iterable, List, Optional +from typing import Any, Dict, Iterable, List, Optional @dataclasses.dataclass @@ -111,6 +111,8 @@ class _Settings: # If enabled, submodule clones authenticate with the App installation token # (required for private submodules); otherwise they run anonymously. ENABLE_SUBMODULE_CLONE_AUTH: bool = False + # bucket/prefix of the submodule archive, defaults to CACHE_S3_PATH + SUBMODULE_CACHE_S3_PATH: str = "" # v2: records carry the producing workflow event, used as the reuse trust # signal instead of the branch (see CacheRunnerHooks.configure). @@ -127,6 +129,10 @@ class _Settings: S3_UPSTREAM_REPORT_BUCKET: str = "" HTML_PAGE_FILE: str = "./ci/praktika/json.html" S3_BUCKET_TO_HTTP_ENDPOINT: Optional[Dict[str, str]] = None + # Non-AWS buckets: {bucket: {endpoint_url, region, access_key_env, secret_key_env, public_read, download_concurrency}} + # The keys are read from the named env vars; with public_read, jobs without them read unsigned. + # download_concurrency (optional): parallel ranged GETs, for links with a low rate per connection. + S3_EXTERNAL_BUCKETS: Optional[Dict[str, Dict[str, Any]]] = None TEXT_CONTENT_EXTENSIONS: Iterable[str] = frozenset([".txt", ".log"]) # Compress if text file size exceeds this threshold (in MB, 0 - disable compression) COMPRESS_THRESHOLD_MB: int = 0 @@ -177,6 +183,8 @@ class _Settings: "EVENT_FEED_S3_PATH", "AWS_REGION", "S3_BUCKET_TO_HTTP_ENDPOINT", + "S3_EXTERNAL_BUCKETS", + "SUBMODULE_CACHE_S3_PATH", "TEXT_CONTENT_EXTENSIONS", "TEMP_DIR", "OUTPUT_DIR", diff --git a/ci/praktika/yaml_additional_templates.py b/ci/praktika/yaml_additional_templates.py index 8731d5e5d5c17..2869279da9962 100644 --- a/ci/praktika/yaml_additional_templates.py +++ b/ci/praktika/yaml_additional_templates.py @@ -8,6 +8,8 @@ class AltinityWorkflowTemplates: AZURE_CONTAINER_NAME: ${{{{ secrets.AZURE_CONTAINER_NAME }}}} AZURE_STORAGE_ACCOUNT_URL: "https://${{{{ secrets.AZURE_ACCOUNT_NAME }}}}.blob.core.windows.net/" ROBOT_TOKEN: ${{{{ secrets.ROBOT_TOKEN }}}} + HETZNER_S3_ACCESS_KEY: ${{{{ secrets.HETZNER_S3_ACCESS_KEY }}}} + HETZNER_S3_SECRET_KEY: ${{{{ secrets.HETZNER_S3_SECRET_KEY }}}} """ # Additional pre steps for all jobs JOB_SETUP_STEPS = """ diff --git a/ci/settings/altinity_overrides.py b/ci/settings/altinity_overrides.py index 9e965980b8ef5..101c6f492a047 100644 --- a/ci/settings/altinity_overrides.py +++ b/ci/settings/altinity_overrides.py @@ -63,6 +63,21 @@ class RunnerLabels: # after the burst of ~150 clones; send the ambient gh token instead. ENABLE_SUBMODULE_CLONE_AUTH = True +# Most runners are in Hetzner (hel1): cache there to avoid AWS egress. +CI_CACHE_BUCKET = "altinity-builds-cache" +SUBMODULE_CACHE_S3_PATH = f"{CI_CACHE_BUCKET}/ci_ch_cache" +S3_EXTERNAL_BUCKETS = { + CI_CACHE_BUCKET: { + "endpoint_url": "https://hel1.your-objectstorage.com", + "region": "hel1", + "access_key_env": "HETZNER_S3_ACCESS_KEY", + "secret_key_env": "HETZNER_S3_SECRET_KEY", + "public_read": True, + # netcup runners get ~2.5 MB/s per connection from hel1 + "download_concurrency": 32, + } +} + DISABLED_WORKFLOWS = [ "backport_branches.py", "custom_build_praktika.py", diff --git a/ci/tests/test_s3_external_bucket.py b/ci/tests/test_s3_external_bucket.py new file mode 100644 index 0000000000000..9746cbeb7a115 --- /dev/null +++ b/ci/tests/test_s3_external_bucket.py @@ -0,0 +1,197 @@ +""" +Buckets listed in Settings.S3_EXTERNAL_BUCKETS live on an S3-compatible store other than AWS. +S3.put and S3.head_object must reach them through a boto3 client with their own endpoint and credentials, +and must keep using the AWS CLI for every other bucket. +""" + +import datetime +import os +import sys + +import pytest + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../..")) + +pytest.importorskip("boto3") +from botocore.exceptions import ClientError + +from ci.praktika import s3 as s3_module +from ci.praktika.cache import Cache +from ci.praktika.s3 import S3 +from ci.praktika.settings import Settings + +BUCKET = "external-bucket" +EXTERNAL = { + BUCKET: { + "endpoint_url": "https://s3.example.test", + "region": "example-1", + "access_key_env": "EXTERNAL_S3_ACCESS_KEY", + "secret_key_env": "EXTERNAL_S3_SECRET_KEY", + "public_read": True, + } +} + + +class _FakeClient: + def __init__(self, error_code=None): + self.error_code = error_code + self.put_args = None + + def _maybe_fail(self, operation): + if self.error_code: + raise ClientError({"Error": {"Code": self.error_code}}, operation) + + def put_object(self, Body, **kwargs): + self._maybe_fail("PutObject") + self.put_args = kwargs + + def head_object(self, Bucket, Key): + self._maybe_fail("HeadObject") + return { + "ETag": '"etag"', + "ContentLength": 3, + "LastModified": datetime.datetime(2026, 1, 2, 3, 4, 5), + } + + +@pytest.fixture(autouse=True) +def external_bucket(monkeypatch): + monkeypatch.setattr(Settings, "S3_EXTERNAL_BUCKETS", EXTERNAL, raising=False) + monkeypatch.setattr(S3, "_external_clients", {}) + + def no_cli(*args, **kwargs): + raise AssertionError("the AWS CLI must not be used for an external bucket") + + monkeypatch.setattr(s3_module.Shell, "get_output", no_cli) + monkeypatch.setattr(s3_module.Shell, "get_res_stdout_stderr", no_cli) + + +def _use_client(monkeypatch, client): + """Serve the same fake for signed and anonymous clients.""" + monkeypatch.setattr( + S3, "_external_clients", {(BUCKET, False): client, (BUCKET, True): client} + ) + + +def _put(tmp_path, **kwargs): + local_file = tmp_path / "archive" + local_file.write_text("abc") + return S3.put(f"{BUCKET}/prefix/archive", str(local_file), **kwargs) + + +def test_put_sends_if_none_match(tmp_path, monkeypatch): + client = _FakeClient() + _use_client(monkeypatch, client) + assert _put(tmp_path, if_none_matched=True) is True + assert client.put_args == { + "Bucket": BUCKET, + "Key": "prefix/archive", + "IfNoneMatch": "*", + } + + +@pytest.mark.parametrize("code", ["PreconditionFailed", "ConditionalRequestConflict"]) +def test_put_lost_race_returns_false(tmp_path, monkeypatch, code): + _use_client(monkeypatch, _FakeClient(code)) + assert _put(tmp_path, if_none_matched=True) is False + + +def test_put_unexpected_error(tmp_path, monkeypatch): + _use_client(monkeypatch, _FakeClient("AccessDenied")) + with pytest.raises(ClientError): + _put(tmp_path) + assert _put(tmp_path, no_strict=True) is False + + +def test_head_object(monkeypatch): + _use_client(monkeypatch, _FakeClient()) + obj = S3.head_object(f"{BUCKET}/prefix/archive") + assert obj.ContentLength == 3 + assert obj.LastModified == "2026-01-02 03:04:05" + + +@pytest.mark.parametrize("code", ["404", "NoSuchKey", "NotFound"]) +def test_head_object_missing_is_none(monkeypatch, code): + _use_client(monkeypatch, _FakeClient(code)) + assert S3.head_object(f"{BUCKET}/prefix/archive") is None + + +def test_head_object_other_error_raises(monkeypatch): + _use_client(monkeypatch, _FakeClient("AccessDenied")) + with pytest.raises(ClientError): + S3.head_object(f"{BUCKET}/prefix/archive") + + +@pytest.fixture +def no_credentials(monkeypatch): + monkeypatch.delenv("EXTERNAL_S3_ACCESS_KEY", raising=False) + monkeypatch.delenv("EXTERNAL_S3_SECRET_KEY", raising=False) + + +def test_writes_need_credentials(tmp_path, no_credentials): + with pytest.raises(AssertionError, match="EXTERNAL_S3_ACCESS_KEY"): + _put(tmp_path) + + +def test_reads_of_a_public_bucket_need_no_credentials(no_credentials): + client = S3._get_boto3_client(BUCKET, read=True) + assert client.meta.config.signature_version is s3_module.UNSIGNED + assert client.meta.endpoint_url == "https://s3.example.test" + + +def test_reads_of_a_private_bucket_need_credentials(monkeypatch, no_credentials): + private = {BUCKET: {**EXTERNAL[BUCKET], "public_read": False}} + monkeypatch.setattr(Settings, "S3_EXTERNAL_BUCKETS", private, raising=False) + with pytest.raises(AssertionError, match="EXTERNAL_S3_ACCESS_KEY"): + S3.head_object(f"{BUCKET}/prefix/archive") + + +def test_signed_client_uses_the_configured_endpoint(monkeypatch): + monkeypatch.setenv("EXTERNAL_S3_ACCESS_KEY", "key") + monkeypatch.setenv("EXTERNAL_S3_SECRET_KEY", "secret") + client = S3._get_boto3_client(BUCKET) + assert client.meta.endpoint_url == "https://s3.example.test" + assert client.meta.region_name == "example-1" + assert S3._get_boto3_client(BUCKET) is client + assert S3._get_boto3_client(BUCKET, read=True) is client + + +def test_forbidden_is_a_miss_only_for_anonymous_reads(monkeypatch, no_credentials): + _use_client(monkeypatch, _FakeClient("403")) + assert S3.head_object(f"{BUCKET}/prefix/archive") is None + monkeypatch.setenv("EXTERNAL_S3_ACCESS_KEY", "key") + monkeypatch.setenv("EXTERNAL_S3_SECRET_KEY", "secret") + with pytest.raises(ClientError): + S3.head_object(f"{BUCKET}/prefix/archive") + + +def test_other_buckets_still_use_the_aws_cli(monkeypatch): + calls = [] + monkeypatch.setattr( + s3_module.Shell, "get_output", lambda cmd, **kw: calls.append(cmd) or "" + ) + assert S3.head_object("aws-bucket/key") is None + assert calls == ["aws s3api head-object --bucket aws-bucket --key key"] + + +def test_submodule_archive_path(monkeypatch): + monkeypatch.setattr(Settings, "CACHE_S3_PATH", "bucket/cache") + monkeypatch.setattr(Settings, "SUBMODULE_CACHE_S3_PATH", "") + assert Cache.submodule_archive_s3_path("abc") == "bucket/cache/submodules/abc.tar.zst" + monkeypatch.setattr(Settings, "SUBMODULE_CACHE_S3_PATH", f"{BUCKET}/cache") + assert Cache.submodule_archive_s3_path("abc") == f"{BUCKET}/cache/submodules/abc.tar.zst" + + +def test_download_concurrency(monkeypatch, tmp_path, no_credentials): + seen = {} + + class Client: + def download_file(self, bucket, key, filename, Config=None): + seen["concurrency"] = Config.max_concurrency + + monkeypatch.setattr(S3, "_external_clients", {(BUCKET, True): Client()}) + monkeypatch.setitem(EXTERNAL[BUCKET], "download_concurrency", 32) + assert S3.copy_file_from_s3(f"{BUCKET}/prefix/archive", str(tmp_path / "out")) + assert seen["concurrency"] == 32 + monkeypatch.setattr(S3, "_external_clients", {}) + assert S3._get_boto3_client(BUCKET, read=True).meta.config.max_pool_connections == 32 diff --git a/ci/tests/test_selected_tests_run.py b/ci/tests/test_selected_tests_run.py index 28c4d2d809da0..489823d625327 100644 --- a/ci/tests/test_selected_tests_run.py +++ b/ci/tests/test_selected_tests_run.py @@ -342,10 +342,10 @@ def test_pr_workflow_keeps_full_suite_msan_wasmedge_functional_tests(): if "amd_msan, WasmEdge" in name: full_suite_msan_wasmedge_jobs.append(name) continue - # The flaky, targeted and azure jobs run their own selection or config. + # The flaky, targeted, azure and cas jobs run their own selection or config. assert any( marker in name - for marker in ("selected tests", "flaky check", "targeted", "azure") + for marker in ("selected tests", "flaky check", "targeted", "azure", "cas ") ), f"full-suite sanitizer functional test job in the PR workflow: {name}" assert len(full_suite_msan_wasmedge_jobs) == 10