diff --git a/.github/actions/create_workflow_report/create_workflow_report.py b/.github/actions/create_workflow_report/create_workflow_report.py index 98d9bd70e3104..705c9ad6ff101 100755 --- a/.github/actions/create_workflow_report/create_workflow_report.py +++ b/.github/actions/create_workflow_report/create_workflow_report.py @@ -64,13 +64,28 @@ def query_dataframe_with_retry( wait = backoff_seconds * attempt time.sleep(wait) -def get_commit_statuses(sha: str) -> pd.DataFrame: +def _results_link_has_ref(target_url: str | None, ref: str) -> bool: + """ + Whether a CI status results link has `ref` in it:`json.html?REF=&...` for + praktika jobs, `/REFs//...` for regression jobs. + """ + target_url = target_url or "" + return f"REF={ref}&" in target_url or f"/REFs/{ref}/" in target_url + + +def get_commit_statuses(sha: str, pr_number: int, ref: str) -> pd.DataFrame: """ Fetch commit statuses for a given SHA and return as a pandas DataFrame. Handles pagination to get all statuses. + Statuses belong to the commit, not the ref, so a release tag and its branch + can write to the same job names. For release runs (`pr_number` 0), only + statuses of `ref` are kept. + Args: sha (str): Commit SHA to fetch statuses for. + pr_number (int): Pull request number, 0 for release runs. + ref (str): Ref of the run being reported. Returns: pd.DataFrame: DataFrame containing all statuses. @@ -119,6 +134,7 @@ def get_commit_statuses(sha: str) -> pd.DataFrame: "results_link": item["target_url"], } for item in all_data + if pr_number != 0 or _results_link_has_ref(item["target_url"], ref) ] # Create DataFrame @@ -362,9 +378,9 @@ def _find_rebase_baseline(commit_sha: str, cwd: str | None) -> str | None: return lines[0] -def _git_commit_date(sha: str, cwd: str | None) -> str | None: +def _git_commit_time(sha: str, cwd: str | None) -> str | None: p = subprocess.run( - ["git", "show", "-s", "--format=%cs", sha], + ["git", "show", "-s", "--format=%ct", sha], cwd=cwd, capture_output=True, text=True, @@ -372,8 +388,12 @@ def _git_commit_date(sha: str, cwd: str | None) -> str | None: ) if p.returncode != 0: return None - date = p.stdout.strip() - return date or None + timestamp = p.stdout.strip() + if not timestamp: + return None + return datetime.fromtimestamp(int(timestamp), timezone.utc).strftime( + "%Y-%m-%dT%H:%M:%SZ" + ) def _git_is_shallow(cwd: str | None) -> bool: @@ -492,7 +512,7 @@ def get_prs_in_release_dataframe( commit_sha is the workflow commit from the GitHub Actions API. - Returns (dataframe, baseline_date YYYY-MM-DD or None). + Returns (dataframe, baseline commit time as UTC YYYY-MM-DDTHH:MM:SSZ or None). """ _fetch_commit(commit_sha, cwd) resolved = _git_rev_parse(commit_sha, cwd) @@ -505,13 +525,11 @@ def get_prs_in_release_dataframe( ) _fetch_release_tags(releases, cwd) - _, baseline_sha, baseline_date = _find_release_baseline( + _, baseline_sha, _ = _find_release_baseline( branch_ref, releases, cwd, commit_sha=commit_sha ) if not baseline_sha: baseline_sha = _find_rebase_baseline(commit_sha, cwd) - if baseline_sha: - baseline_date = _git_commit_date(baseline_sha, cwd) if not baseline_sha: raise Exception( @@ -519,7 +537,10 @@ def get_prs_in_release_dataframe( ) df = _git_log_merge_prs(baseline_sha, commit_sha, cwd, repo) - return _enrich_prs_in_release_merge_prs(df, repo), baseline_date + return ( + _enrich_prs_in_release_merge_prs(df, repo), + _git_commit_time(baseline_sha, cwd), + ) def _parse_release_ref(ref_name: str) -> tuple[str, str, str | None] | None: @@ -543,8 +564,8 @@ def _parse_release_ref(ref_name: str) -> tuple[str, str, str | None] | None: return None -def _prs_in_release_search_url(branch_name: str, baseline_date: str | None) -> str | None: - if not baseline_date: +def _prs_in_release_search_url(branch_name: str, baseline_time: str | None) -> str | None: + if not baseline_time: return None # Tag-triggered runs expose the tag as head_branch; search needs the PR base # branch (e.g. v25.8.28.10001.altinitystable → stable-25.8). @@ -552,7 +573,7 @@ def _prs_in_release_search_url(branch_name: str, baseline_date: str | None) -> s if parsed: flavour, minor_version, _ = parsed branch_name = f"{flavour}-{minor_version}" - query = f"is:pr base:{branch_name} merged:>{baseline_date}" + query = f"is:pr base:{branch_name} merged:>{baseline_time}" return ( f"https://github.com/{GITHUB_REPO}/pulls?" + urllib.parse.urlencode({"q": query}) @@ -1370,7 +1391,7 @@ def create_workflow_report( results_dfs = { "prs_in_release": [], - "job_statuses": get_commit_statuses(commit_sha), + "job_statuses": get_commit_statuses(commit_sha, pr_number, branch_name), "checks_fails": add_labels_to_checks_fails( get_checks_fails(db_client, commit_sha, branch_name), workflow_result, @@ -1386,12 +1407,12 @@ def create_workflow_report( if pr_number == 0 and not mark_preview: try: - prs_df, baseline_date = get_prs_in_release_dataframe( + prs_df, baseline_time = get_prs_in_release_dataframe( branch_name, cwd=os.getcwd(), commit_sha=commit_sha ) results_dfs["prs_in_release"] = prs_df prs_in_release_search_url = _prs_in_release_search_url( - branch_name, baseline_date + branch_name, baseline_time ) except Exception as e: print(f"Error in get_prs_in_release_dataframe: {e}") diff --git a/.github/actions/create_workflow_report/test_report_queries.py b/.github/actions/create_workflow_report/test_report_queries.py index a3b2bb651f6b2..dd824c1b7bda4 100755 --- a/.github/actions/create_workflow_report/test_report_queries.py +++ b/.github/actions/create_workflow_report/test_report_queries.py @@ -36,7 +36,11 @@ from clickhouse_driver import Client from testflows.core import * -from create_workflow_report import _git_log_merge_prs, get_checks_fails +from create_workflow_report import ( + _git_log_merge_prs, + get_checks_fails, + get_commit_statuses, +) def check_result_matches_expect(df: pd.DataFrame, expect: list[str]) -> None: @@ -172,11 +176,55 @@ def test_git_log_merge_prs_includes_fork_prs(self): assert set(df["pr_number"]) == {2226, 2252} +@TestScenario +def test_commit_statuses_only_keep_requested_ref(self): + """Test statuses written by another ref on the same commit are ignored.""" + base = "https://altinity-build-artifacts.s3.amazonaws.com" + tag = "v26.6.8.20002.altinityantalya" + # GitHub returns statuses newest first. + statuses = [ + ("Stress test", "failure", f"{base}/json.html?REF=antalya-26.6&sha=abc"), + ("Regression release x", "failure", f"{base}/REFs/antalya-26.6/abc/r.html"), + ("Regression release y", "success", f"{base}/REFs/{tag}/abc/r.html"), + ("Stress test", "success", f"{base}/json.html?REF={tag}&sha=abc"), + ("PR job", "success", f"{base}/json.html?PR=1&sha=abc"), + ] + response = type( + "Response", + (), + { + "status_code": 200, + "headers": {}, + "json": lambda _: [ + {"context": c, "state": st, "description": "", "target_url": u} + for c, st, u in statuses + ], + }, + )() + with When("I fetch statuses for the release tag"): + with patch("create_workflow_report.requests.get", return_value=response): + df = get_commit_statuses("abc", 0, tag) + + with Then("only statuses of the tag remain"): + assert dict(zip(df["job_name"], df["job_status"])) == { + "Stress test": "success", + "Regression release y": "success", + } + + with When("I fetch statuses for a pull request run"): + with patch("create_workflow_report.requests.get", return_value=response): + df = get_commit_statuses("abc", 1, "some-branch") + + with Then("statuses are not filtered by ref"): + assert set(df["job_name"]) == {"Stress test", "Regression release x", "Regression release y", "PR job"} + + @Name("test report queries") @TestModule def test_report_queries(self): Scenario(run=test_checks_fails_query, flags=TE) Scenario(run=test_git_log_merge_prs_includes_fork_prs, flags=TE) + Scenario(run=test_commit_statuses_only_keep_requested_ref, flags=TE) if main(): diff --git a/.github/workflows/fast_builds.yml b/.github/workflows/fast_builds.yml index 1930eac75ca72..57c16c9e95c00 100644 --- a/.github/workflows/fast_builds.yml +++ b/.github/workflows/fast_builds.yml @@ -45,6 +45,8 @@ env: AZURE_CONTAINER_NAME: ${{ secrets.AZURE_CONTAINER_NAME }} AZURE_STORAGE_ACCOUNT_URL: "https://${{ secrets.AZURE_ACCOUNT_NAME }}.blob.core.windows.net/" ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }} + HETZNER_S3_ACCESS_KEY: ${{ secrets.HETZNER_S3_ACCESS_KEY }} + HETZNER_S3_SECRET_KEY: ${{ secrets.HETZNER_S3_SECRET_KEY }} diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 6dbee52fbde76..1f34c3f3ac984 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -60,6 +60,8 @@ env: AZURE_CONTAINER_NAME: ${{ secrets.AZURE_CONTAINER_NAME }} AZURE_STORAGE_ACCOUNT_URL: "https://${{ secrets.AZURE_ACCOUNT_NAME }}.blob.core.windows.net/" ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }} + HETZNER_S3_ACCESS_KEY: ${{ secrets.HETZNER_S3_ACCESS_KEY }} + HETZNER_S3_SECRET_KEY: ${{ secrets.HETZNER_S3_SECRET_KEY }} @@ -2322,11 +2324,11 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_tsan, sequential, 2/2)' --workflow "MasterCI" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_1_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_1_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_msan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAxLzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 1/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAxLzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 1/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -2341,7 +2343,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 1/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 1/6)" - name: Prepare env script run: | @@ -2371,13 +2373,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 1/8)' --workflow "MasterCI" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 1/6)' --workflow "MasterCI" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_2_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_2_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_msan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAyLzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 2/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAyLzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 2/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -2392,7 +2394,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 2/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 2/6)" - name: Prepare env script run: | @@ -2422,13 +2424,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 2/8)' --workflow "MasterCI" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 2/6)' --workflow "MasterCI" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_3_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_3_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_msan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAzLzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 3/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAzLzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 3/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -2443,7 +2445,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 3/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 3/6)" - name: Prepare env script run: | @@ -2473,13 +2475,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 3/8)' --workflow "MasterCI" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 3/6)' --workflow "MasterCI" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_4_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_4_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_msan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA0Lzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 4/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA0LzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 4/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -2494,7 +2496,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 4/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 4/6)" - name: Prepare env script run: | @@ -2524,13 +2526,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 4/8)' --workflow "MasterCI" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 4/6)' --workflow "MasterCI" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_5_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_5_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_msan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA1Lzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 5/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA1LzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 5/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -2545,7 +2547,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 5/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 5/6)" - name: Prepare env script run: | @@ -2575,13 +2577,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 5/8)' --workflow "MasterCI" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 5/6)' --workflow "MasterCI" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_6_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_6_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_msan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA2Lzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 6/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA2LzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 6/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -2596,7 +2598,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 6/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 6/6)" - name: Prepare env script run: | @@ -2626,13 +2628,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 6/8)' --workflow "MasterCI" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 6/6)' --workflow "MasterCI" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_7_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_sequential_1_4: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 32g] needs: [build_amd_msan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA3Lzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 7/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDEvNCk=') }} + name: "Stateless tests (amd_msan, WasmEdge, sequential, 1/4)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -2647,7 +2649,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 7/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 1/4)" - name: Prepare env script run: | @@ -2677,13 +2679,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 7/8)' --workflow "MasterCI" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 1/4)' --workflow "MasterCI" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_8_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_sequential_2_4: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 32g] needs: [build_amd_msan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA4Lzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 8/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDIvNCk=') }} + name: "Stateless tests (amd_msan, WasmEdge, sequential, 2/4)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -2698,7 +2700,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 8/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 2/4)" - name: Prepare env script run: | @@ -2728,13 +2730,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 8/8)' --workflow "MasterCI" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 2/4)' --workflow "MasterCI" --ci --timestamp - stateless_tests_amd_msan_wasmedge_sequential_1_2: + stateless_tests_amd_msan_wasmedge_sequential_3_4: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 32g] needs: [build_amd_msan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDEvMik=') }} - name: "Stateless tests (amd_msan, WasmEdge, sequential, 1/2)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDMvNCk=') }} + name: "Stateless tests (amd_msan, WasmEdge, sequential, 3/4)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -2749,7 +2751,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 1/2)" + test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 3/4)" - name: Prepare env script run: | @@ -2779,13 +2781,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 1/2)' --workflow "MasterCI" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 3/4)' --workflow "MasterCI" --ci --timestamp - stateless_tests_amd_msan_wasmedge_sequential_2_2: + stateless_tests_amd_msan_wasmedge_sequential_4_4: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 32g] needs: [build_amd_msan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDIvMik=') }} - name: "Stateless tests (amd_msan, WasmEdge, sequential, 2/2)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDQvNCk=') }} + name: "Stateless tests (amd_msan, WasmEdge, sequential, 4/4)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -2800,7 +2802,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 2/2)" + test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 4/4)" - name: Prepare env script run: | @@ -2830,7 +2832,7 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 2/2)' --workflow "MasterCI" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 4/4)' --workflow "MasterCI" --ci --timestamp stateless_tests_amd_debug_distributed_plan_s3_storage_parallel: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c] @@ -6753,7 +6755,7 @@ jobs: finish_workflow: runs-on: [self-hosted, altinity-on-demand, altinity-style-checker] - needs: [ast_fuzzer_amd_debug, ast_fuzzer_amd_msan, ast_fuzzer_amd_tsan, ast_fuzzer_arm_asan_ubsan, build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_llvm_coverage_per_test, build_amd_msan, build_amd_release, build_amd_release_pr_cache_warmup, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_release, build_arm_release_pr_cache_warmup, build_arm_tsan, buzzhouse_amd_debug, buzzhouse_amd_msan, buzzhouse_amd_tsan, buzzhouse_arm_asan_ubsan, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, integration_tests_amd_msan_10_10, integration_tests_amd_msan_1_10, integration_tests_amd_msan_2_10, integration_tests_amd_msan_3_10, integration_tests_amd_msan_4_10, integration_tests_amd_msan_5_10, integration_tests_amd_msan_6_10, integration_tests_amd_msan_7_10, integration_tests_amd_msan_8_10, integration_tests_amd_msan_9_10, integration_tests_amd_tsan_1_8, integration_tests_amd_tsan_2_8, integration_tests_amd_tsan_3_8, integration_tests_amd_tsan_4_8, integration_tests_amd_tsan_5_8, integration_tests_amd_tsan_6_8, integration_tests_amd_tsan_7_8, integration_tests_amd_tsan_8_8, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, sign_release_amd_release, sign_release_arm_release, source_upload, sqllogic_test, sqlstorm_test, sqltest, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_1_3, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_2_3, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_3_3, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_2_2, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_1_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_2_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_3_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_4_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_5_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_6_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_7_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_8_8, stateless_tests_amd_msan_wasmedge_parallel_1_8, stateless_tests_amd_msan_wasmedge_parallel_2_8, stateless_tests_amd_msan_wasmedge_parallel_3_8, stateless_tests_amd_msan_wasmedge_parallel_4_8, stateless_tests_amd_msan_wasmedge_parallel_5_8, stateless_tests_amd_msan_wasmedge_parallel_6_8, stateless_tests_amd_msan_wasmedge_parallel_7_8, stateless_tests_amd_msan_wasmedge_parallel_8_8, stateless_tests_amd_msan_wasmedge_sequential_1_2, stateless_tests_amd_msan_wasmedge_sequential_2_2, stateless_tests_amd_tsan_parallel_1_4, stateless_tests_amd_tsan_parallel_2_4, stateless_tests_amd_tsan_parallel_3_4, stateless_tests_amd_tsan_parallel_4_4, stateless_tests_amd_tsan_s3_storage_parallel_1_3, stateless_tests_amd_tsan_s3_storage_parallel_2_3, stateless_tests_amd_tsan_s3_storage_parallel_3_3, stateless_tests_amd_tsan_s3_storage_sequential_1_2, stateless_tests_amd_tsan_s3_storage_sequential_2_2, stateless_tests_amd_tsan_sequential_1_2, stateless_tests_amd_tsan_sequential_2_2, stateless_tests_arm_asan_ubsan_azure_parallel_1_8, stateless_tests_arm_asan_ubsan_azure_parallel_2_8, stateless_tests_arm_asan_ubsan_azure_parallel_3_8, stateless_tests_arm_asan_ubsan_azure_parallel_4_8, stateless_tests_arm_asan_ubsan_azure_parallel_5_8, stateless_tests_arm_asan_ubsan_azure_parallel_6_8, stateless_tests_arm_asan_ubsan_azure_parallel_7_8, stateless_tests_arm_asan_ubsan_azure_parallel_8_8, stateless_tests_arm_asan_ubsan_azure_sequential_1_2, stateless_tests_arm_asan_ubsan_azure_sequential_2_2, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, stress_test_amd_asan_ubsan, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_tsan, stress_test_arm_asan_ubsan, stress_test_arm_asan_ubsan_s3, stress_test_arm_debug, stress_test_arm_msan, stress_test_arm_release, stress_test_arm_tsan, stress_test_azure_amd_msan, stress_test_azure_amd_tsan, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] + needs: [ast_fuzzer_amd_debug, ast_fuzzer_amd_msan, ast_fuzzer_amd_tsan, ast_fuzzer_arm_asan_ubsan, build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_llvm_coverage_per_test, build_amd_msan, build_amd_release, build_amd_release_pr_cache_warmup, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_release, build_arm_release_pr_cache_warmup, build_arm_tsan, buzzhouse_amd_debug, buzzhouse_amd_msan, buzzhouse_amd_tsan, buzzhouse_arm_asan_ubsan, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, integration_tests_amd_msan_10_10, integration_tests_amd_msan_1_10, integration_tests_amd_msan_2_10, integration_tests_amd_msan_3_10, integration_tests_amd_msan_4_10, integration_tests_amd_msan_5_10, integration_tests_amd_msan_6_10, integration_tests_amd_msan_7_10, integration_tests_amd_msan_8_10, integration_tests_amd_msan_9_10, integration_tests_amd_tsan_1_8, integration_tests_amd_tsan_2_8, integration_tests_amd_tsan_3_8, integration_tests_amd_tsan_4_8, integration_tests_amd_tsan_5_8, integration_tests_amd_tsan_6_8, integration_tests_amd_tsan_7_8, integration_tests_amd_tsan_8_8, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, sign_release_amd_release, sign_release_arm_release, source_upload, sqllogic_test, sqlstorm_test, sqltest, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_1_3, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_2_3, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_3_3, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_2_2, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_1_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_2_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_3_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_4_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_5_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_6_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_7_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_8_8, stateless_tests_amd_msan_wasmedge_parallel_1_6, stateless_tests_amd_msan_wasmedge_parallel_2_6, stateless_tests_amd_msan_wasmedge_parallel_3_6, stateless_tests_amd_msan_wasmedge_parallel_4_6, stateless_tests_amd_msan_wasmedge_parallel_5_6, stateless_tests_amd_msan_wasmedge_parallel_6_6, stateless_tests_amd_msan_wasmedge_sequential_1_4, stateless_tests_amd_msan_wasmedge_sequential_2_4, stateless_tests_amd_msan_wasmedge_sequential_3_4, stateless_tests_amd_msan_wasmedge_sequential_4_4, stateless_tests_amd_tsan_parallel_1_4, stateless_tests_amd_tsan_parallel_2_4, stateless_tests_amd_tsan_parallel_3_4, stateless_tests_amd_tsan_parallel_4_4, stateless_tests_amd_tsan_s3_storage_parallel_1_3, stateless_tests_amd_tsan_s3_storage_parallel_2_3, stateless_tests_amd_tsan_s3_storage_parallel_3_3, stateless_tests_amd_tsan_s3_storage_sequential_1_2, stateless_tests_amd_tsan_s3_storage_sequential_2_2, stateless_tests_amd_tsan_sequential_1_2, stateless_tests_amd_tsan_sequential_2_2, stateless_tests_arm_asan_ubsan_azure_parallel_1_8, stateless_tests_arm_asan_ubsan_azure_parallel_2_8, stateless_tests_arm_asan_ubsan_azure_parallel_3_8, stateless_tests_arm_asan_ubsan_azure_parallel_4_8, stateless_tests_arm_asan_ubsan_azure_parallel_5_8, stateless_tests_arm_asan_ubsan_azure_parallel_6_8, stateless_tests_arm_asan_ubsan_azure_parallel_7_8, stateless_tests_arm_asan_ubsan_azure_parallel_8_8, stateless_tests_arm_asan_ubsan_azure_sequential_1_2, stateless_tests_arm_asan_ubsan_azure_sequential_2_2, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, stress_test_amd_asan_ubsan, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_tsan, stress_test_arm_asan_ubsan, stress_test_arm_asan_ubsan_s3, stress_test_arm_debug, stress_test_arm_msan, stress_test_arm_release, stress_test_arm_tsan, stress_test_azure_amd_msan, stress_test_azure_amd_tsan, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] if: ${{ !cancelled() && needs.config_workflow.outputs.pipeline_status != '' }} name: "Finish Workflow" outputs: @@ -6896,16 +6898,16 @@ jobs: - stateless_tests_amd_tsan_parallel_4_4 - stateless_tests_amd_tsan_sequential_1_2 - stateless_tests_amd_tsan_sequential_2_2 - - stateless_tests_amd_msan_wasmedge_parallel_1_8 - - stateless_tests_amd_msan_wasmedge_parallel_2_8 - - stateless_tests_amd_msan_wasmedge_parallel_3_8 - - stateless_tests_amd_msan_wasmedge_parallel_4_8 - - stateless_tests_amd_msan_wasmedge_parallel_5_8 - - stateless_tests_amd_msan_wasmedge_parallel_6_8 - - stateless_tests_amd_msan_wasmedge_parallel_7_8 - - stateless_tests_amd_msan_wasmedge_parallel_8_8 - - stateless_tests_amd_msan_wasmedge_sequential_1_2 - - stateless_tests_amd_msan_wasmedge_sequential_2_2 + - stateless_tests_amd_msan_wasmedge_parallel_1_6 + - stateless_tests_amd_msan_wasmedge_parallel_2_6 + - stateless_tests_amd_msan_wasmedge_parallel_3_6 + - stateless_tests_amd_msan_wasmedge_parallel_4_6 + - stateless_tests_amd_msan_wasmedge_parallel_5_6 + - stateless_tests_amd_msan_wasmedge_parallel_6_6 + - stateless_tests_amd_msan_wasmedge_sequential_1_4 + - stateless_tests_amd_msan_wasmedge_sequential_2_4 + - stateless_tests_amd_msan_wasmedge_sequential_3_4 + - stateless_tests_amd_msan_wasmedge_sequential_4_4 - stateless_tests_amd_debug_distributed_plan_s3_storage_parallel - stateless_tests_amd_debug_distributed_plan_s3_storage_sequential - stateless_tests_amd_tsan_s3_storage_parallel_1_3 diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 734e8779091b6..0782b84112eaa 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -35,6 +35,8 @@ env: AZURE_CONTAINER_NAME: ${{ secrets.AZURE_CONTAINER_NAME }} AZURE_STORAGE_ACCOUNT_URL: "https://${{ secrets.AZURE_ACCOUNT_NAME }}.blob.core.windows.net/" ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }} + HETZNER_S3_ACCESS_KEY: ${{ secrets.HETZNER_S3_ACCESS_KEY }} + HETZNER_S3_SECRET_KEY: ${{ secrets.HETZNER_S3_SECRET_KEY }} @@ -1357,11 +1359,11 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_debug, sequential)' --workflow "PR" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_1_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_1_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAxLzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 1/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAxLzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 1/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1376,7 +1378,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 1/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 1/6)" - name: Prepare env script run: | @@ -1404,13 +1406,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 1/8)' --workflow "PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 1/6)' --workflow "PR" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_2_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_2_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAyLzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 2/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAyLzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 2/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1425,7 +1427,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 2/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 2/6)" - name: Prepare env script run: | @@ -1453,13 +1455,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 2/8)' --workflow "PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 2/6)' --workflow "PR" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_3_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_3_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAzLzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 3/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAzLzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 3/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1474,7 +1476,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 3/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 3/6)" - name: Prepare env script run: | @@ -1502,13 +1504,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 3/8)' --workflow "PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 3/6)' --workflow "PR" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_4_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_4_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA0Lzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 4/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA0LzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 4/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1523,7 +1525,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 4/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 4/6)" - name: Prepare env script run: | @@ -1551,13 +1553,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 4/8)' --workflow "PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 4/6)' --workflow "PR" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_5_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_5_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA1Lzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 5/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA1LzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 5/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1572,7 +1574,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 5/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 5/6)" - name: Prepare env script run: | @@ -1600,13 +1602,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 5/8)' --workflow "PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 5/6)' --workflow "PR" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_6_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_6_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA2Lzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 6/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA2LzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 6/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1621,7 +1623,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 6/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 6/6)" - name: Prepare env script run: | @@ -1649,13 +1651,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 6/8)' --workflow "PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 6/6)' --workflow "PR" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_7_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_sequential_1_4: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 32g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA3Lzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 7/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDEvNCk=') }} + name: "Stateless tests (amd_msan, WasmEdge, sequential, 1/4)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1670,7 +1672,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 7/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 1/4)" - name: Prepare env script run: | @@ -1698,13 +1700,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 7/8)' --workflow "PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 1/4)' --workflow "PR" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_8_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_sequential_2_4: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 32g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA4Lzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 8/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDIvNCk=') }} + name: "Stateless tests (amd_msan, WasmEdge, sequential, 2/4)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1719,7 +1721,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 8/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 2/4)" - name: Prepare env script run: | @@ -1747,13 +1749,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 8/8)' --workflow "PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 2/4)' --workflow "PR" --ci --timestamp - stateless_tests_amd_msan_wasmedge_sequential_1_2: + stateless_tests_amd_msan_wasmedge_sequential_3_4: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 32g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDEvMik=') }} - name: "Stateless tests (amd_msan, WasmEdge, sequential, 1/2)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDMvNCk=') }} + name: "Stateless tests (amd_msan, WasmEdge, sequential, 3/4)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1768,7 +1770,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 1/2)" + test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 3/4)" - name: Prepare env script run: | @@ -1796,13 +1798,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 1/2)' --workflow "PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 3/4)' --workflow "PR" --ci --timestamp - stateless_tests_amd_msan_wasmedge_sequential_2_2: + stateless_tests_amd_msan_wasmedge_sequential_4_4: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 32g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDIvMik=') }} - name: "Stateless tests (amd_msan, WasmEdge, sequential, 2/2)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDQvNCk=') }} + name: "Stateless tests (amd_msan, WasmEdge, sequential, 4/4)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1817,7 +1819,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 2/2)" + test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 4/4)" - name: Prepare env script run: | @@ -1845,7 +1847,7 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 2/2)' --workflow "PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 4/4)' --workflow "PR" --ci --timestamp stateless_tests_amd_debug_distributed_plan_s3_storage_parallel: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c] @@ -5571,48 +5573,6 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'BuzzHouse (amd_msan)' --workflow "PR" --ci --timestamp - parser_memory_check: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64] - needs: [build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_release, build_arm_tsan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'UGFyc2VyIG1lbW9yeSBjaGVjaw==') }} - name: "Parser memory check" - outputs: - data: ${{ steps.run.outputs.DATA }} - pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} - steps: - - name: Checkout code - uses: actions/checkout@v6 - with: - ref: ${{ env.CHECKOUT_REF }} - - - name: Setup - uses: ./.github/actions/runner_setup - - name: Docker setup - uses: ./.github/actions/docker_setup - with: - test_name: "Parser memory check" - - - name: Prepare env script - run: | - rm -rf ./ci/tmp - mkdir -p ./ci/tmp - cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' - export PYTHONPATH=./ci:.: - - cat > ./ci/tmp/workflow_job.json << 'EOF' - ${{ toJson(job) }} - EOF - cat > ./ci/tmp/workflow_status.json << 'EOF' - ${{ toJson(needs) }} - EOF - ENV_SETUP_SCRIPT_EOF - - - name: Run - id: run - run: | - . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Parser memory check' --workflow "PR" --ci --timestamp - promql_compliance: runs-on: [self-hosted, altinity-on-demand, altinity-style-checker-aarch64] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, integration_tests_amd_msan_10_10, integration_tests_amd_msan_1_10, integration_tests_amd_msan_2_10, integration_tests_amd_msan_3_10, integration_tests_amd_msan_4_10, integration_tests_amd_msan_5_10, integration_tests_amd_msan_6_10, integration_tests_amd_msan_7_10, integration_tests_amd_msan_8_10, integration_tests_amd_msan_9_10, integration_tests_amd_tsan_1_8, integration_tests_amd_tsan_2_8, integration_tests_amd_tsan_3_8, integration_tests_amd_tsan_4_8, integration_tests_amd_tsan_5_8, integration_tests_amd_tsan_6_8, integration_tests_amd_tsan_7_8, integration_tests_amd_tsan_8_8, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] @@ -5918,7 +5878,7 @@ jobs: finish_workflow: runs-on: [self-hosted, altinity-on-demand, altinity-style-checker] - needs: [ast_fuzzer_amd_debug, ast_fuzzer_amd_debug_targeted, ast_fuzzer_amd_debug_targeted_old_compatibility, ast_fuzzer_amd_msan, ast_fuzzer_amd_tsan, ast_fuzzer_arm_asan_ubsan, bugfix_validation_unit_tests, build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_release, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_release, build_arm_tsan, build_toolchain_pgo_bolt_aarch64, build_toolchain_pgo_bolt_amd64, build_wasm_parser, buzzhouse_amd_debug, buzzhouse_amd_msan, buzzhouse_amd_tsan, buzzhouse_arm_asan_ubsan, ci_tests, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, integration_tests_amd_asan_ubsan_targeted, integration_tests_amd_msan_10_10, integration_tests_amd_msan_1_10, integration_tests_amd_msan_2_10, integration_tests_amd_msan_3_10, integration_tests_amd_msan_4_10, integration_tests_amd_msan_5_10, integration_tests_amd_msan_6_10, integration_tests_amd_msan_7_10, integration_tests_amd_msan_8_10, integration_tests_amd_msan_9_10, integration_tests_amd_tsan_1_8, integration_tests_amd_tsan_2_8, integration_tests_amd_tsan_3_8, integration_tests_amd_tsan_4_8, integration_tests_amd_tsan_5_8, integration_tests_amd_tsan_6_8, integration_tests_amd_tsan_7_8, integration_tests_amd_tsan_8_8, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, keeper_stress_tests_pr, parser_memory_check, promql_compliance, quick_functional_tests, source_upload, sqllogic_test, sqlstorm_test, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_msan_wasmedge_parallel_1_8, stateless_tests_amd_msan_wasmedge_parallel_2_8, stateless_tests_amd_msan_wasmedge_parallel_3_8, stateless_tests_amd_msan_wasmedge_parallel_4_8, stateless_tests_amd_msan_wasmedge_parallel_5_8, stateless_tests_amd_msan_wasmedge_parallel_6_8, stateless_tests_amd_msan_wasmedge_parallel_7_8, stateless_tests_amd_msan_wasmedge_parallel_8_8, stateless_tests_amd_msan_wasmedge_sequential_1_2, stateless_tests_amd_msan_wasmedge_sequential_2_2, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_amd_tsan_s3_storage_parallel_selected_tests, stateless_tests_amd_tsan_s3_storage_sequential_selected_tests, stateless_tests_amd_tsan_sequential_selected_tests, stateless_tests_arm_asan_ubsan_azure_parallel_1_8, stateless_tests_arm_asan_ubsan_azure_parallel_2_8, stateless_tests_arm_asan_ubsan_azure_parallel_3_8, stateless_tests_arm_asan_ubsan_azure_parallel_4_8, stateless_tests_arm_asan_ubsan_azure_parallel_5_8, stateless_tests_arm_asan_ubsan_azure_parallel_6_8, stateless_tests_arm_asan_ubsan_azure_parallel_7_8, stateless_tests_arm_asan_ubsan_azure_parallel_8_8, stateless_tests_arm_asan_ubsan_azure_sequential_1_2, stateless_tests_arm_asan_ubsan_azure_sequential_2_2, stateless_tests_arm_asan_ubsan_targeted, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, stress_test_amd_asan_ubsan, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_tsan, stress_test_arm_asan_ubsan, stress_test_arm_asan_ubsan_s3, stress_test_arm_debug, stress_test_arm_msan, stress_test_arm_release, stress_test_arm_tsan, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] + needs: [ast_fuzzer_amd_debug, ast_fuzzer_amd_debug_targeted, ast_fuzzer_amd_debug_targeted_old_compatibility, ast_fuzzer_amd_msan, ast_fuzzer_amd_tsan, ast_fuzzer_arm_asan_ubsan, bugfix_validation_unit_tests, build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_release, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_release, build_arm_tsan, build_toolchain_pgo_bolt_aarch64, build_toolchain_pgo_bolt_amd64, build_wasm_parser, buzzhouse_amd_debug, buzzhouse_amd_msan, buzzhouse_amd_tsan, buzzhouse_arm_asan_ubsan, ci_tests, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, integration_tests_amd_asan_ubsan_targeted, integration_tests_amd_msan_10_10, integration_tests_amd_msan_1_10, integration_tests_amd_msan_2_10, integration_tests_amd_msan_3_10, integration_tests_amd_msan_4_10, integration_tests_amd_msan_5_10, integration_tests_amd_msan_6_10, integration_tests_amd_msan_7_10, integration_tests_amd_msan_8_10, integration_tests_amd_msan_9_10, integration_tests_amd_tsan_1_8, integration_tests_amd_tsan_2_8, integration_tests_amd_tsan_3_8, integration_tests_amd_tsan_4_8, integration_tests_amd_tsan_5_8, integration_tests_amd_tsan_6_8, integration_tests_amd_tsan_7_8, integration_tests_amd_tsan_8_8, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, keeper_stress_tests_pr, promql_compliance, quick_functional_tests, source_upload, sqllogic_test, sqlstorm_test, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_msan_wasmedge_parallel_1_6, stateless_tests_amd_msan_wasmedge_parallel_2_6, stateless_tests_amd_msan_wasmedge_parallel_3_6, stateless_tests_amd_msan_wasmedge_parallel_4_6, stateless_tests_amd_msan_wasmedge_parallel_5_6, stateless_tests_amd_msan_wasmedge_parallel_6_6, stateless_tests_amd_msan_wasmedge_sequential_1_4, stateless_tests_amd_msan_wasmedge_sequential_2_4, stateless_tests_amd_msan_wasmedge_sequential_3_4, stateless_tests_amd_msan_wasmedge_sequential_4_4, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_amd_tsan_s3_storage_parallel_selected_tests, stateless_tests_amd_tsan_s3_storage_sequential_selected_tests, stateless_tests_amd_tsan_sequential_selected_tests, stateless_tests_arm_asan_ubsan_azure_parallel_1_8, stateless_tests_arm_asan_ubsan_azure_parallel_2_8, stateless_tests_arm_asan_ubsan_azure_parallel_3_8, stateless_tests_arm_asan_ubsan_azure_parallel_4_8, stateless_tests_arm_asan_ubsan_azure_parallel_5_8, stateless_tests_arm_asan_ubsan_azure_parallel_6_8, stateless_tests_arm_asan_ubsan_azure_parallel_7_8, stateless_tests_arm_asan_ubsan_azure_parallel_8_8, stateless_tests_arm_asan_ubsan_azure_sequential_1_2, stateless_tests_arm_asan_ubsan_azure_sequential_2_2, stateless_tests_arm_asan_ubsan_targeted, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, stress_test_amd_asan_ubsan, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_tsan, stress_test_arm_asan_ubsan, stress_test_arm_asan_ubsan_s3, stress_test_arm_debug, stress_test_arm_msan, stress_test_arm_release, stress_test_arm_tsan, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] if: ${{ !cancelled() && needs.config_workflow.outputs.pipeline_status != '' }} name: "Finish Workflow" outputs: @@ -6039,16 +5999,16 @@ jobs: - bugfix_validation_unit_tests - stateless_tests_amd_debug_parallel - stateless_tests_amd_debug_sequential - - stateless_tests_amd_msan_wasmedge_parallel_1_8 - - stateless_tests_amd_msan_wasmedge_parallel_2_8 - - stateless_tests_amd_msan_wasmedge_parallel_3_8 - - stateless_tests_amd_msan_wasmedge_parallel_4_8 - - stateless_tests_amd_msan_wasmedge_parallel_5_8 - - stateless_tests_amd_msan_wasmedge_parallel_6_8 - - stateless_tests_amd_msan_wasmedge_parallel_7_8 - - stateless_tests_amd_msan_wasmedge_parallel_8_8 - - stateless_tests_amd_msan_wasmedge_sequential_1_2 - - stateless_tests_amd_msan_wasmedge_sequential_2_2 + - stateless_tests_amd_msan_wasmedge_parallel_1_6 + - stateless_tests_amd_msan_wasmedge_parallel_2_6 + - stateless_tests_amd_msan_wasmedge_parallel_3_6 + - stateless_tests_amd_msan_wasmedge_parallel_4_6 + - stateless_tests_amd_msan_wasmedge_parallel_5_6 + - stateless_tests_amd_msan_wasmedge_parallel_6_6 + - stateless_tests_amd_msan_wasmedge_sequential_1_4 + - stateless_tests_amd_msan_wasmedge_sequential_2_4 + - stateless_tests_amd_msan_wasmedge_sequential_3_4 + - stateless_tests_amd_msan_wasmedge_sequential_4_4 - stateless_tests_amd_debug_distributed_plan_s3_storage_parallel - stateless_tests_amd_debug_distributed_plan_s3_storage_sequential - stateless_tests_arm_binary_parallel @@ -6129,7 +6089,6 @@ jobs: - buzzhouse_arm_asan_ubsan - buzzhouse_amd_tsan - buzzhouse_amd_msan - - parser_memory_check - promql_compliance - sqllogic_test - sqlstorm_test diff --git a/.github/workflows/pull_request_community.yml b/.github/workflows/pull_request_community.yml index 9968fd9cc635a..47411981ce299 100644 --- a/.github/workflows/pull_request_community.yml +++ b/.github/workflows/pull_request_community.yml @@ -1192,11 +1192,11 @@ jobs: if-no-files-found: ignore retention-days: 14 - stateless_tests_amd_msan_wasmedge_parallel_1_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_1_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAxLzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 1/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAxLzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 1/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1211,7 +1211,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 1/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 1/6)" - name: Prepare env script run: | @@ -1238,14 +1238,14 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 1/8)' --workflow "Community PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 1/6)' --workflow "Community PR" --ci --timestamp - name: Upload failure report artifact if: failure() uses: actions/upload-artifact@v7 continue-on-error: true with: - name: failure-stateless_tests_amd_msan_wasmedge_parallel_1_8 + name: failure-stateless_tests_amd_msan_wasmedge_parallel_1_6 path: | ci/tmp/result_*.json ci/tmp/test_result.txt @@ -1256,11 +1256,11 @@ jobs: if-no-files-found: ignore retention-days: 14 - stateless_tests_amd_msan_wasmedge_parallel_2_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_2_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAyLzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 2/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAyLzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 2/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1275,7 +1275,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 2/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 2/6)" - name: Prepare env script run: | @@ -1302,14 +1302,14 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 2/8)' --workflow "Community PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 2/6)' --workflow "Community PR" --ci --timestamp - name: Upload failure report artifact if: failure() uses: actions/upload-artifact@v7 continue-on-error: true with: - name: failure-stateless_tests_amd_msan_wasmedge_parallel_2_8 + name: failure-stateless_tests_amd_msan_wasmedge_parallel_2_6 path: | ci/tmp/result_*.json ci/tmp/test_result.txt @@ -1320,11 +1320,11 @@ jobs: if-no-files-found: ignore retention-days: 14 - stateless_tests_amd_msan_wasmedge_parallel_3_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_3_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAzLzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 3/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAzLzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 3/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1339,7 +1339,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 3/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 3/6)" - name: Prepare env script run: | @@ -1366,14 +1366,14 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 3/8)' --workflow "Community PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 3/6)' --workflow "Community PR" --ci --timestamp - name: Upload failure report artifact if: failure() uses: actions/upload-artifact@v7 continue-on-error: true with: - name: failure-stateless_tests_amd_msan_wasmedge_parallel_3_8 + name: failure-stateless_tests_amd_msan_wasmedge_parallel_3_6 path: | ci/tmp/result_*.json ci/tmp/test_result.txt @@ -1384,11 +1384,11 @@ jobs: if-no-files-found: ignore retention-days: 14 - stateless_tests_amd_msan_wasmedge_parallel_4_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_4_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA0Lzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 4/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA0LzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 4/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1403,7 +1403,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 4/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 4/6)" - name: Prepare env script run: | @@ -1430,14 +1430,14 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 4/8)' --workflow "Community PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 4/6)' --workflow "Community PR" --ci --timestamp - name: Upload failure report artifact if: failure() uses: actions/upload-artifact@v7 continue-on-error: true with: - name: failure-stateless_tests_amd_msan_wasmedge_parallel_4_8 + name: failure-stateless_tests_amd_msan_wasmedge_parallel_4_6 path: | ci/tmp/result_*.json ci/tmp/test_result.txt @@ -1448,11 +1448,11 @@ jobs: if-no-files-found: ignore retention-days: 14 - stateless_tests_amd_msan_wasmedge_parallel_5_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_5_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA1Lzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 5/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA1LzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 5/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1467,7 +1467,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 5/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 5/6)" - name: Prepare env script run: | @@ -1494,14 +1494,14 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 5/8)' --workflow "Community PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 5/6)' --workflow "Community PR" --ci --timestamp - name: Upload failure report artifact if: failure() uses: actions/upload-artifact@v7 continue-on-error: true with: - name: failure-stateless_tests_amd_msan_wasmedge_parallel_5_8 + name: failure-stateless_tests_amd_msan_wasmedge_parallel_5_6 path: | ci/tmp/result_*.json ci/tmp/test_result.txt @@ -1512,11 +1512,11 @@ jobs: if-no-files-found: ignore retention-days: 14 - stateless_tests_amd_msan_wasmedge_parallel_6_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_6_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA2Lzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 6/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA2LzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 6/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1531,7 +1531,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 6/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 6/6)" - name: Prepare env script run: | @@ -1558,14 +1558,14 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 6/8)' --workflow "Community PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 6/6)' --workflow "Community PR" --ci --timestamp - name: Upload failure report artifact if: failure() uses: actions/upload-artifact@v7 continue-on-error: true with: - name: failure-stateless_tests_amd_msan_wasmedge_parallel_6_8 + name: failure-stateless_tests_amd_msan_wasmedge_parallel_6_6 path: | ci/tmp/result_*.json ci/tmp/test_result.txt @@ -1576,11 +1576,11 @@ jobs: if-no-files-found: ignore retention-days: 14 - stateless_tests_amd_msan_wasmedge_parallel_7_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_sequential_1_4: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 32g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA3Lzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 7/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDEvNCk=') }} + name: "Stateless tests (amd_msan, WasmEdge, sequential, 1/4)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1595,7 +1595,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 7/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 1/4)" - name: Prepare env script run: | @@ -1622,14 +1622,14 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 7/8)' --workflow "Community PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 1/4)' --workflow "Community PR" --ci --timestamp - name: Upload failure report artifact if: failure() uses: actions/upload-artifact@v7 continue-on-error: true with: - name: failure-stateless_tests_amd_msan_wasmedge_parallel_7_8 + name: failure-stateless_tests_amd_msan_wasmedge_sequential_1_4 path: | ci/tmp/result_*.json ci/tmp/test_result.txt @@ -1640,11 +1640,11 @@ jobs: if-no-files-found: ignore retention-days: 14 - stateless_tests_amd_msan_wasmedge_parallel_8_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_sequential_2_4: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 32g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA4Lzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 8/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDIvNCk=') }} + name: "Stateless tests (amd_msan, WasmEdge, sequential, 2/4)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1659,7 +1659,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 8/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 2/4)" - name: Prepare env script run: | @@ -1686,14 +1686,14 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 8/8)' --workflow "Community PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 2/4)' --workflow "Community PR" --ci --timestamp - name: Upload failure report artifact if: failure() uses: actions/upload-artifact@v7 continue-on-error: true with: - name: failure-stateless_tests_amd_msan_wasmedge_parallel_8_8 + name: failure-stateless_tests_amd_msan_wasmedge_sequential_2_4 path: | ci/tmp/result_*.json ci/tmp/test_result.txt @@ -1704,11 +1704,11 @@ jobs: if-no-files-found: ignore retention-days: 14 - stateless_tests_amd_msan_wasmedge_sequential_1_2: + stateless_tests_amd_msan_wasmedge_sequential_3_4: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 32g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDEvMik=') }} - name: "Stateless tests (amd_msan, WasmEdge, sequential, 1/2)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDMvNCk=') }} + name: "Stateless tests (amd_msan, WasmEdge, sequential, 3/4)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1723,7 +1723,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 1/2)" + test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 3/4)" - name: Prepare env script run: | @@ -1750,14 +1750,14 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 1/2)' --workflow "Community PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 3/4)' --workflow "Community PR" --ci --timestamp - name: Upload failure report artifact if: failure() uses: actions/upload-artifact@v7 continue-on-error: true with: - name: failure-stateless_tests_amd_msan_wasmedge_sequential_1_2 + name: failure-stateless_tests_amd_msan_wasmedge_sequential_3_4 path: | ci/tmp/result_*.json ci/tmp/test_result.txt @@ -1768,11 +1768,11 @@ jobs: if-no-files-found: ignore retention-days: 14 - stateless_tests_amd_msan_wasmedge_sequential_2_2: + stateless_tests_amd_msan_wasmedge_sequential_4_4: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 32g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDIvMik=') }} - name: "Stateless tests (amd_msan, WasmEdge, sequential, 2/2)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDQvNCk=') }} + name: "Stateless tests (amd_msan, WasmEdge, sequential, 4/4)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1787,7 +1787,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 2/2)" + test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 4/4)" - name: Prepare env script run: | @@ -1814,14 +1814,14 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 2/2)' --workflow "Community PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 4/4)' --workflow "Community PR" --ci --timestamp - name: Upload failure report artifact if: failure() uses: actions/upload-artifact@v7 continue-on-error: true with: - name: failure-stateless_tests_amd_msan_wasmedge_sequential_2_2 + name: failure-stateless_tests_amd_msan_wasmedge_sequential_4_4 path: | ci/tmp/result_*.json ci/tmp/test_result.txt diff --git a/ci/defs/job_configs.py b/ci/defs/job_configs.py index 0d96a727b19ff..eaec79a77d782 100644 --- a/ci/defs/job_configs.py +++ b/ci/defs/job_configs.py @@ -975,25 +975,25 @@ class JobConfigs: for total_batches in (2,) for batch in range(1, total_batches + 1) ], - # One shard can lose worker concurrency mid-suite under MSan and - # overrun the shared 3.5h functional-test budget. - *[ + # NOTE (strtgbb): 64g keeps MSan at 6 workers, Lower mem can cause workers to be killed + *[ Job.ParamSet( parameter=f"amd_msan, WasmEdge, parallel, {batch}/{total_batches}", - runs_on=RunnerLabels.FUNC_TESTER_AMD, + runs_on=RunnerLabels.AMD_LARGE, requires=[ArtifactNames.CH_AMD_MSAN_GH], timeout=3600 * 4, ) - for total_batches in (8,) + for total_batches in (6,) for batch in range(1, total_batches + 1) ], + # NOTE (strtgbb): 2/2 was ~3.3h of sequential work and overran 3.5h in teardown. *[ Job.ParamSet( parameter=f"amd_msan, WasmEdge, sequential, {batch}/{total_batches}", runs_on=RunnerLabels.AMD_SMALL_MEM, requires=[ArtifactNames.CH_AMD_MSAN_GH], ) - for total_batches in (2,) + for total_batches in (4,) for batch in range(1, total_batches + 1) ], Job.ParamSet( diff --git a/ci/praktika/cache.py b/ci/praktika/cache.py index 148f4edfbc521..6c1545eef742e 100644 --- a/ci/praktika/cache.py +++ b/ci/praktika/cache.py @@ -39,6 +39,11 @@ def from_fs(cls, path): def from_dict(cls, obj): return Cache.CacheRecord(**obj) + @staticmethod + def submodule_archive_s3_path(cache_hash): + base = Settings.SUBMODULE_CACHE_S3_PATH or Settings.CACHE_S3_PATH + return f"{base}/submodules/{cache_hash}.tar.zst" + def __init__(self): self.digest = Digest() self.success = {} # type Dict[str, Any] diff --git a/ci/praktika/native_jobs.py b/ci/praktika/native_jobs.py index e6c9a673f43cd..d907dc574795b 100644 --- a/ci/praktika/native_jobs.py +++ b/ci/praktika/native_jobs.py @@ -10,6 +10,7 @@ from . import Job, Workflow from ._environment import _Environment +from .cache import Cache from .cidb import CIDB from .digest import Digest from .docker import Docker @@ -252,13 +253,22 @@ def _prepare_submodule_cache(workflow, workflow_config: RunConfig) -> Result: ) cache_hash = hashlib.sha256(submodule_shas.encode()).hexdigest()[:16] - s3_path = f"{Settings.CACHE_S3_PATH}/submodules/{cache_hash}.tar.zst" + s3_path = Cache.submodule_archive_s3_path(cache_hash) print(f"Submodule cache hash: {cache_hash}") # Check if cache already exists in S3 if S3.head_object(s3_path): print(f"Submodule cache hit: {s3_path}") info = f"cache hit: {cache_hash}" + elif Info().is_community_pr: + # Like sccache, community PRs only read; jobs clone without a hash. + print(f"Submodule cache miss, community PR will not create it: {s3_path}") + return Result.create_from( + name="Submodule Cache", + status=Result.Status.OK, + stopwatch=stop_watch, + info=f"cache miss, not creating (read-only): {cache_hash}", + ) else: print(f"Submodule cache miss, creating: {s3_path}") Shell.check("git submodule sync", verbose=True, strict=True) @@ -270,6 +280,12 @@ def _prepare_submodule_cache(workflow, workflow_config: RunConfig) -> Result: retries=3, env=_submodule_auth_env(workflow), ) + # The archive is public: never upload it with the clone token inside. + leaked = Shell.get_output( + "grep -rIlF x-access-token .git/config .git/modules --exclude-dir=objects" + ) + if leaked: + raise RuntimeError(f"submodule cache would contain a GitHub token: {leaked}") archive_path = f"{Settings.TEMP_DIR}/submodules_{cache_hash}.tar.zst" Shell.check( f"tar -cf - .git/modules | zstd -c -T0 > {archive_path}", @@ -892,7 +908,11 @@ def check_affected_jobs(): ) ) - if results[-1].is_ok() and workflow.enable_cache and Settings.ENABLE_SUBMODULE_CACHE: + if ( + results[-1].is_ok() + and (workflow.enable_cache or Info().is_community_pr) + and Settings.ENABLE_SUBMODULE_CACHE + ): result = _prepare_submodule_cache(workflow, workflow_config) results.append(result) diff --git a/ci/praktika/runner.py b/ci/praktika/runner.py index 6d105b87468ac..a09b09c3471e3 100644 --- a/ci/praktika/runner.py +++ b/ci/praktika/runner.py @@ -11,6 +11,7 @@ from pathlib import Path from ._environment import _Environment +from .cache import Cache from .artifact import Artifact from .cidb import CIDB from .digest import Digest @@ -362,7 +363,7 @@ def _restore_submodule_cache(): if not cache_hash: print("NOTE: No submodule cache hash in workflow config, skipping restore") return - s3_path = f"{Settings.CACHE_S3_PATH}/submodules/{cache_hash}.tar.zst" + s3_path = Cache.submodule_archive_s3_path(cache_hash) local_archive = f"{Settings.TEMP_DIR}/submodules_{cache_hash}.tar.zst" print(f"Restoring submodule cache: {s3_path}") S3.copy_file_from_s3(s3_path=s3_path, local_path=local_archive, no_strict=True) diff --git a/ci/praktika/s3.py b/ci/praktika/s3.py index c064f513e635b..863718e8b915d 100644 --- a/ci/praktika/s3.py +++ b/ci/praktika/s3.py @@ -15,7 +15,10 @@ try: import boto3 + from botocore import UNSIGNED + from botocore.config import Config from botocore.exceptions import ClientError, NoCredentialsError + from boto3.s3.transfer import TransferConfig BOTO3_AVAILABLE = True except ImportError: @@ -58,9 +61,64 @@ def clean_line(line): class S3: _boto3_client = None + _external_clients = {} @classmethod - def _get_boto3_client(cls): + def _external_bucket_config(cls, bucket): + return (Settings.S3_EXTERNAL_BUCKETS or {}).get(bucket) + + @classmethod + def _reads_anonymously(cls, bucket): + """Reads of a public_read bucket are unsigned unless this job has the bucket's keys.""" + config = cls._external_bucket_config(bucket) + return bool( + config + and config.get("public_read") + and not ( + os.environ.get(config["access_key_env"]) + and os.environ.get(config["secret_key_env"]) + ) + ) + + @classmethod + def _get_external_client(cls, bucket, config, anonymous): + assert BOTO3_AVAILABLE, f"boto3 is required for S3 bucket [{bucket}]" + client = cls._external_clients.get((bucket, anonymous)) + if client is None: + client_config = { + "max_pool_connections": config.get("download_concurrency", 10) + } + if anonymous: + credentials = { + "config": Config(signature_version=UNSIGNED, **client_config) + } + else: + access_key = os.environ.get(config["access_key_env"], "") + secret_key = os.environ.get(config["secret_key_env"], "") + # boto3 would otherwise send the ambient AWS credentials to this endpoint + assert ( + access_key and secret_key + ), f"S3 bucket [{bucket}] needs [{config['access_key_env']}] and [{config['secret_key_env']}] set" + credentials = { + "aws_access_key_id": access_key, + "aws_secret_access_key": secret_key, + "config": Config(**client_config), + } + client = boto3.client( + "s3", + endpoint_url=config["endpoint_url"], + region_name=config["region"], + **credentials, + ) + cls._external_clients[(bucket, anonymous)] = client + return client + + @classmethod + def _get_boto3_client(cls, bucket=None, read=False): + config = cls._external_bucket_config(bucket) + if config: + anonymous = read and cls._reads_anonymously(bucket) + return cls._get_external_client(bucket, config, anonymous) if not BOTO3_AVAILABLE: return None if cls._boto3_client is None: @@ -188,7 +246,7 @@ def copy_file_to_s3( extra_args["Tagging"] = urlencode(tags) def _upload(): - client = cls._get_boto3_client() + client = cls._get_boto3_client(bucket) if extra_args: client.upload_file( str(local_path), bucket, key, ExtraArgs=extra_args @@ -260,24 +318,69 @@ def put( s3_full_path = str(s3_full_path).removeprefix("s3://") bucket, key = s3_full_path.split("/", maxsplit=1) - command = ( - f"aws s3api put-object --bucket {bucket} --key {key} --body {local_path}" - ) - if if_none_matched: - command += ' --if-none-match "*"' - if if_match: - command += f' --if-match "{if_match}"' - if metadata: - for k, v in metadata.items(): - command += f" --metadata {k}={v}" - - if text: - command += ' --content-type "text/plain; charset=utf-8"' - res = cls.run_command_with_retries(command, no_strict=no_strict) + if cls._external_bucket_config(bucket): + res = cls._put_object_boto3( + bucket, + key, + local_path, + text, + metadata, + if_none_matched, + if_match, + no_strict, + ) + else: + command = f"aws s3api put-object --bucket {bucket} --key {key} --body {local_path}" + if if_none_matched: + command += ' --if-none-match "*"' + if if_match: + command += f' --if-match "{if_match}"' + if metadata: + for k, v in metadata.items(): + command += f" --metadata {k}={v}" + + if text: + command += ' --content-type "text/plain; charset=utf-8"' + res = cls.run_command_with_retries(command, no_strict=no_strict) if res: StorageUsage.add_uploaded(local_path) return res + @classmethod + def _put_object_boto3( + cls, + bucket, + key, + local_path, + text, + metadata, + if_none_matched, + if_match, + no_strict, + ): + args = {"Bucket": bucket, "Key": key} + if metadata: + args["Metadata"] = metadata + if text: + args["ContentType"] = "text/plain; charset=utf-8" + if if_none_matched: + args["IfNoneMatch"] = "*" + if if_match: + args["IfMatch"] = if_match + try: + with open(local_path, "rb") as f: + cls._get_boto3_client(bucket).put_object(Body=f, **args) + except ClientError as e: + error_code = e.response.get("Error", {}).get("Code", "") + if error_code in ("PreconditionFailed", "ConditionalRequestConflict"): + print(f"S3 conditional request failed (concurrent write detected): {error_code}") + return False + print(f"ERROR: Failed to put S3 object [{bucket}/{key}]: {error_code}") + if not no_strict: + raise + return False + return True + @classmethod def run_command_with_retries( cls, @@ -354,8 +457,18 @@ def copy_file_from_s3( local_file.parent.mkdir(parents=True, exist_ok=True) def _download(): - client = cls._get_boto3_client() - client.download_file(bucket, key, str(local_file)) + client = cls._get_boto3_client(bucket, read=True) + concurrency = (cls._external_bucket_config(bucket) or {}).get( + "download_concurrency" + ) + client.download_file( + bucket, + key, + str(local_file), + Config=TransferConfig(max_concurrency=concurrency) + if concurrency + else None, + ) # Retry on transient credential failures cls._retry_on_no_credentials(_download) @@ -426,18 +539,35 @@ def copy_file_from_s3_matching_pattern( def head_object(cls, s3_path): s3_path = str(s3_path).removeprefix("s3://") bucket, key = s3_path.split("/", maxsplit=1) - output = Shell.get_output( - f"aws s3api head-object --bucket {bucket} --key {key}", verbose=True - ) - if not output: - return None - data = json.loads(output) + if cls._external_bucket_config(bucket): + try: + data = cls._get_boto3_client(bucket, read=True).head_object( + Bucket=bucket, Key=key + ) + except ClientError as e: + if e.response.get("Error", {}).get("Code", "") in ( + "404", + "NoSuchKey", + "NotFound", + ): + return None + # A public bucket answers a missing key with 403 to callers who may not list it + if e.response.get("Error", {}).get("Code", "") == "403" and cls._reads_anonymously(bucket): + return None + raise + else: + output = Shell.get_output( + f"aws s3api head-object --bucket {bucket} --key {key}", verbose=True + ) + if not output: + return None + data = json.loads(output) # HeadObject omits optional fields (Expiration) unless a lifecycle Expire # rule applies, and emits extra keys Object does not declare. return cls.Object( AcceptRanges=data.get("AcceptRanges", ""), Expiration=data.get("Expiration", ""), - LastModified=data.get("LastModified", ""), + LastModified=str(data.get("LastModified", "")), ContentLength=data.get("ContentLength", 0), ETag=data.get("ETag", ""), ContentType=data.get("ContentType", ""), @@ -456,7 +586,7 @@ def assert_read_access(cls, s3_path): """ s3_path = str(s3_path).removeprefix("s3://") bucket, key = s3_path.split("/", maxsplit=1) - client = cls._get_boto3_client() + client = cls._get_boto3_client(bucket, read=True) assert client, "boto3 client is required for S3 read-access probe" def _probe(): diff --git a/ci/praktika/settings.py b/ci/praktika/settings.py index 760a0caa90748..3385cd6b57368 100644 --- a/ci/praktika/settings.py +++ b/ci/praktika/settings.py @@ -1,7 +1,7 @@ import dataclasses import importlib.util from pathlib import Path -from typing import Dict, Iterable, List, Optional +from typing import Any, Dict, Iterable, List, Optional @dataclasses.dataclass @@ -111,6 +111,8 @@ class _Settings: # If enabled, submodule clones authenticate with the App installation token # (required for private submodules); otherwise they run anonymously. ENABLE_SUBMODULE_CLONE_AUTH: bool = False + # bucket/prefix of the submodule archive, defaults to CACHE_S3_PATH + SUBMODULE_CACHE_S3_PATH: str = "" # v2: records carry the producing workflow event, used as the reuse trust # signal instead of the branch (see CacheRunnerHooks.configure). @@ -127,6 +129,10 @@ class _Settings: S3_UPSTREAM_REPORT_BUCKET: str = "" HTML_PAGE_FILE: str = "./ci/praktika/json.html" S3_BUCKET_TO_HTTP_ENDPOINT: Optional[Dict[str, str]] = None + # Non-AWS buckets: {bucket: {endpoint_url, region, access_key_env, secret_key_env, public_read, download_concurrency}} + # The keys are read from the named env vars; with public_read, jobs without them read unsigned. + # download_concurrency (optional): parallel ranged GETs, for links with a low rate per connection. + S3_EXTERNAL_BUCKETS: Optional[Dict[str, Dict[str, Any]]] = None TEXT_CONTENT_EXTENSIONS: Iterable[str] = frozenset([".txt", ".log"]) # Compress if text file size exceeds this threshold (in MB, 0 - disable compression) COMPRESS_THRESHOLD_MB: int = 0 @@ -177,6 +183,8 @@ class _Settings: "EVENT_FEED_S3_PATH", "AWS_REGION", "S3_BUCKET_TO_HTTP_ENDPOINT", + "S3_EXTERNAL_BUCKETS", + "SUBMODULE_CACHE_S3_PATH", "TEXT_CONTENT_EXTENSIONS", "TEMP_DIR", "OUTPUT_DIR", diff --git a/ci/praktika/yaml_additional_templates.py b/ci/praktika/yaml_additional_templates.py index 8731d5e5d5c17..2869279da9962 100644 --- a/ci/praktika/yaml_additional_templates.py +++ b/ci/praktika/yaml_additional_templates.py @@ -8,6 +8,8 @@ class AltinityWorkflowTemplates: AZURE_CONTAINER_NAME: ${{{{ secrets.AZURE_CONTAINER_NAME }}}} AZURE_STORAGE_ACCOUNT_URL: "https://${{{{ secrets.AZURE_ACCOUNT_NAME }}}}.blob.core.windows.net/" ROBOT_TOKEN: ${{{{ secrets.ROBOT_TOKEN }}}} + HETZNER_S3_ACCESS_KEY: ${{{{ secrets.HETZNER_S3_ACCESS_KEY }}}} + HETZNER_S3_SECRET_KEY: ${{{{ secrets.HETZNER_S3_SECRET_KEY }}}} """ # Additional pre steps for all jobs JOB_SETUP_STEPS = """ diff --git a/ci/settings/altinity_overrides.py b/ci/settings/altinity_overrides.py index 9e965980b8ef5..101c6f492a047 100644 --- a/ci/settings/altinity_overrides.py +++ b/ci/settings/altinity_overrides.py @@ -63,6 +63,21 @@ class RunnerLabels: # after the burst of ~150 clones; send the ambient gh token instead. ENABLE_SUBMODULE_CLONE_AUTH = True +# Most runners are in Hetzner (hel1): cache there to avoid AWS egress. +CI_CACHE_BUCKET = "altinity-builds-cache" +SUBMODULE_CACHE_S3_PATH = f"{CI_CACHE_BUCKET}/ci_ch_cache" +S3_EXTERNAL_BUCKETS = { + CI_CACHE_BUCKET: { + "endpoint_url": "https://hel1.your-objectstorage.com", + "region": "hel1", + "access_key_env": "HETZNER_S3_ACCESS_KEY", + "secret_key_env": "HETZNER_S3_SECRET_KEY", + "public_read": True, + # netcup runners get ~2.5 MB/s per connection from hel1 + "download_concurrency": 32, + } +} + DISABLED_WORKFLOWS = [ "backport_branches.py", "custom_build_praktika.py", diff --git a/ci/tests/test_s3_external_bucket.py b/ci/tests/test_s3_external_bucket.py new file mode 100644 index 0000000000000..9746cbeb7a115 --- /dev/null +++ b/ci/tests/test_s3_external_bucket.py @@ -0,0 +1,197 @@ +""" +Buckets listed in Settings.S3_EXTERNAL_BUCKETS live on an S3-compatible store other than AWS. +S3.put and S3.head_object must reach them through a boto3 client with their own endpoint and credentials, +and must keep using the AWS CLI for every other bucket. +""" + +import datetime +import os +import sys + +import pytest + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../..")) + +pytest.importorskip("boto3") +from botocore.exceptions import ClientError + +from ci.praktika import s3 as s3_module +from ci.praktika.cache import Cache +from ci.praktika.s3 import S3 +from ci.praktika.settings import Settings + +BUCKET = "external-bucket" +EXTERNAL = { + BUCKET: { + "endpoint_url": "https://s3.example.test", + "region": "example-1", + "access_key_env": "EXTERNAL_S3_ACCESS_KEY", + "secret_key_env": "EXTERNAL_S3_SECRET_KEY", + "public_read": True, + } +} + + +class _FakeClient: + def __init__(self, error_code=None): + self.error_code = error_code + self.put_args = None + + def _maybe_fail(self, operation): + if self.error_code: + raise ClientError({"Error": {"Code": self.error_code}}, operation) + + def put_object(self, Body, **kwargs): + self._maybe_fail("PutObject") + self.put_args = kwargs + + def head_object(self, Bucket, Key): + self._maybe_fail("HeadObject") + return { + "ETag": '"etag"', + "ContentLength": 3, + "LastModified": datetime.datetime(2026, 1, 2, 3, 4, 5), + } + + +@pytest.fixture(autouse=True) +def external_bucket(monkeypatch): + monkeypatch.setattr(Settings, "S3_EXTERNAL_BUCKETS", EXTERNAL, raising=False) + monkeypatch.setattr(S3, "_external_clients", {}) + + def no_cli(*args, **kwargs): + raise AssertionError("the AWS CLI must not be used for an external bucket") + + monkeypatch.setattr(s3_module.Shell, "get_output", no_cli) + monkeypatch.setattr(s3_module.Shell, "get_res_stdout_stderr", no_cli) + + +def _use_client(monkeypatch, client): + """Serve the same fake for signed and anonymous clients.""" + monkeypatch.setattr( + S3, "_external_clients", {(BUCKET, False): client, (BUCKET, True): client} + ) + + +def _put(tmp_path, **kwargs): + local_file = tmp_path / "archive" + local_file.write_text("abc") + return S3.put(f"{BUCKET}/prefix/archive", str(local_file), **kwargs) + + +def test_put_sends_if_none_match(tmp_path, monkeypatch): + client = _FakeClient() + _use_client(monkeypatch, client) + assert _put(tmp_path, if_none_matched=True) is True + assert client.put_args == { + "Bucket": BUCKET, + "Key": "prefix/archive", + "IfNoneMatch": "*", + } + + +@pytest.mark.parametrize("code", ["PreconditionFailed", "ConditionalRequestConflict"]) +def test_put_lost_race_returns_false(tmp_path, monkeypatch, code): + _use_client(monkeypatch, _FakeClient(code)) + assert _put(tmp_path, if_none_matched=True) is False + + +def test_put_unexpected_error(tmp_path, monkeypatch): + _use_client(monkeypatch, _FakeClient("AccessDenied")) + with pytest.raises(ClientError): + _put(tmp_path) + assert _put(tmp_path, no_strict=True) is False + + +def test_head_object(monkeypatch): + _use_client(monkeypatch, _FakeClient()) + obj = S3.head_object(f"{BUCKET}/prefix/archive") + assert obj.ContentLength == 3 + assert obj.LastModified == "2026-01-02 03:04:05" + + +@pytest.mark.parametrize("code", ["404", "NoSuchKey", "NotFound"]) +def test_head_object_missing_is_none(monkeypatch, code): + _use_client(monkeypatch, _FakeClient(code)) + assert S3.head_object(f"{BUCKET}/prefix/archive") is None + + +def test_head_object_other_error_raises(monkeypatch): + _use_client(monkeypatch, _FakeClient("AccessDenied")) + with pytest.raises(ClientError): + S3.head_object(f"{BUCKET}/prefix/archive") + + +@pytest.fixture +def no_credentials(monkeypatch): + monkeypatch.delenv("EXTERNAL_S3_ACCESS_KEY", raising=False) + monkeypatch.delenv("EXTERNAL_S3_SECRET_KEY", raising=False) + + +def test_writes_need_credentials(tmp_path, no_credentials): + with pytest.raises(AssertionError, match="EXTERNAL_S3_ACCESS_KEY"): + _put(tmp_path) + + +def test_reads_of_a_public_bucket_need_no_credentials(no_credentials): + client = S3._get_boto3_client(BUCKET, read=True) + assert client.meta.config.signature_version is s3_module.UNSIGNED + assert client.meta.endpoint_url == "https://s3.example.test" + + +def test_reads_of_a_private_bucket_need_credentials(monkeypatch, no_credentials): + private = {BUCKET: {**EXTERNAL[BUCKET], "public_read": False}} + monkeypatch.setattr(Settings, "S3_EXTERNAL_BUCKETS", private, raising=False) + with pytest.raises(AssertionError, match="EXTERNAL_S3_ACCESS_KEY"): + S3.head_object(f"{BUCKET}/prefix/archive") + + +def test_signed_client_uses_the_configured_endpoint(monkeypatch): + monkeypatch.setenv("EXTERNAL_S3_ACCESS_KEY", "key") + monkeypatch.setenv("EXTERNAL_S3_SECRET_KEY", "secret") + client = S3._get_boto3_client(BUCKET) + assert client.meta.endpoint_url == "https://s3.example.test" + assert client.meta.region_name == "example-1" + assert S3._get_boto3_client(BUCKET) is client + assert S3._get_boto3_client(BUCKET, read=True) is client + + +def test_forbidden_is_a_miss_only_for_anonymous_reads(monkeypatch, no_credentials): + _use_client(monkeypatch, _FakeClient("403")) + assert S3.head_object(f"{BUCKET}/prefix/archive") is None + monkeypatch.setenv("EXTERNAL_S3_ACCESS_KEY", "key") + monkeypatch.setenv("EXTERNAL_S3_SECRET_KEY", "secret") + with pytest.raises(ClientError): + S3.head_object(f"{BUCKET}/prefix/archive") + + +def test_other_buckets_still_use_the_aws_cli(monkeypatch): + calls = [] + monkeypatch.setattr( + s3_module.Shell, "get_output", lambda cmd, **kw: calls.append(cmd) or "" + ) + assert S3.head_object("aws-bucket/key") is None + assert calls == ["aws s3api head-object --bucket aws-bucket --key key"] + + +def test_submodule_archive_path(monkeypatch): + monkeypatch.setattr(Settings, "CACHE_S3_PATH", "bucket/cache") + monkeypatch.setattr(Settings, "SUBMODULE_CACHE_S3_PATH", "") + assert Cache.submodule_archive_s3_path("abc") == "bucket/cache/submodules/abc.tar.zst" + monkeypatch.setattr(Settings, "SUBMODULE_CACHE_S3_PATH", f"{BUCKET}/cache") + assert Cache.submodule_archive_s3_path("abc") == f"{BUCKET}/cache/submodules/abc.tar.zst" + + +def test_download_concurrency(monkeypatch, tmp_path, no_credentials): + seen = {} + + class Client: + def download_file(self, bucket, key, filename, Config=None): + seen["concurrency"] = Config.max_concurrency + + monkeypatch.setattr(S3, "_external_clients", {(BUCKET, True): Client()}) + monkeypatch.setitem(EXTERNAL[BUCKET], "download_concurrency", 32) + assert S3.copy_file_from_s3(f"{BUCKET}/prefix/archive", str(tmp_path / "out")) + assert seen["concurrency"] == 32 + monkeypatch.setattr(S3, "_external_clients", {}) + assert S3._get_boto3_client(BUCKET, read=True).meta.config.max_pool_connections == 32 diff --git a/ci/workflows/pull_request.py b/ci/workflows/pull_request.py index ab1c92dc9763e..fd402def0eaf9 100644 --- a/ci/workflows/pull_request.py +++ b/ci/workflows/pull_request.py @@ -215,7 +215,8 @@ # job.set_run_after(CORE_BLOCKING_JOB_NAMES) # for job in JobConfigs.performance_comparison_with_master_head_jobs # ], # NOTE (strtgbb): failed previously due to GH secrets not being handled properly, try again later - JobConfigs.parser_memory_check_job, + # NOTE (strtgbb): ooks up master clickhouse-examples on upstream S3 + # JobConfigs.parser_memory_check_job, # ClickBench runs on PRs only when files in its digest change # (see `clickbench_jobs.digest_config`), so the cost is bounded. # *[ diff --git a/docker/keeper/Dockerfile b/docker/keeper/Dockerfile index b8c0961ea8bbd..85b7712ef4acd 100644 --- a/docker/keeper/Dockerfile +++ b/docker/keeper/Dockerfile @@ -16,7 +16,7 @@ RUN arch=${TARGETARCH:-amd64} \ esac -FROM alpine:3.21 +FROM alpine ENV LANG=en_US.UTF-8 \ LANGUAGE=en_US:en \ @@ -121,6 +121,8 @@ RUN clickhouse-keeper --version \ && chown root:clickhouse "${DEFAULT_LOG_DIR}" \ && chmod ugo+Xrw -R "${DEFAULT_DATA_DIR}" "${DEFAULT_LOG_DIR}" "${DEFAULT_CONFIG_DIR}" +RUN apk update && apk upgrade --no-cache + # /var/lib/clickhouse is necessary due to the current default configuration for Keeper VOLUME "${DEFAULT_DATA_DIR}" /var/lib/clickhouse EXPOSE 2181 10181 44444 9181 diff --git a/docker/server/Dockerfile.alpine b/docker/server/Dockerfile.alpine index 11082d291d566..3aaf63a034b48 100644 --- a/docker/server/Dockerfile.alpine +++ b/docker/server/Dockerfile.alpine @@ -123,8 +123,7 @@ RUN mkdir -p \ && chown root:clickhouse "${DEFAULT_LOG_DIR}" \ && chmod ugo+Xrw -R "${DEFAULT_DATA_DIR}" "${DEFAULT_LOG_DIR}" "${DEFAULT_CLIENT_CONFIG_DIR}" "${DEFAULT_SERVER_CONFIG_DIR}" -RUN apk update -RUN apk add --upgrade libssl3 libcrypto3 +RUN apk update && apk upgrade --no-cache VOLUME "${DEFAULT_DATA_DIR}" EXPOSE 9000 8123 9009 diff --git a/tests/integration/test_s3_cluster/test.py b/tests/integration/test_s3_cluster/test.py index 30d5a6190217c..76b8f0df2881b 100644 --- a/tests/integration/test_s3_cluster/test.py +++ b/tests/integration/test_s3_cluster/test.py @@ -2,23 +2,21 @@ import logging import os import shutil -import uuid +import time +from email.errors import HeaderParseError import pytest from helpers.cluster import ClickHouseCluster -from helpers.config_cluster import minio_access_key, minio_secret_key +from helpers.config_cluster import minio_secret_key from helpers.mock_servers import start_mock_servers from helpers.test_tools import TSV -from helpers.utility import random_string logging.getLogger().setLevel(logging.INFO) logging.getLogger().addHandler(logging.StreamHandler()) SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) - -# Base CSV files committed to git. Immutable on disk, safe to read concurrently. -S3_BASE_FILES = [ +S3_DATA = [ "data/clickhouse/part1.csv", "data/clickhouse/part123.csv", "data/database/part2.csv", @@ -26,48 +24,14 @@ ] -def _generated_host_dir(): - """Per-xdist-worker host directory for the dynamically generated CSV files. - - Concurrent xdist workers running this module (especially under - `--dist=each`, used by the targeted/flaky integration jobs) all execute - this fixture in parallel. Their MinIO containers are isolated by - `ClickHouseCluster` (it appends `PYTEST_XDIST_WORKER` to `project_name`), - but the host filesystem path is not. With a shared host directory, one - worker's teardown (`shutil.rmtree(...)`) deletes files while another - worker is still uploading them, producing setup-phase - `FileNotFoundError` on `data/generated/file_*.csv` and failing every - test in the module via fixture error. - - The MinIO bucket key for tests stays `data/generated/file_N.csv` (each - worker uploads to its own MinIO instance); only the host write/read path - is scoped per worker. - """ - worker_id = os.environ.get("PYTEST_XDIST_WORKER", "") - suffix = f"_{worker_id}" if worker_id else "" - return os.path.join(SCRIPT_DIR, f"data/generated{suffix}") - - def create_buckets_s3(cluster): minio = cluster.minio_client - host_dir = _generated_host_dir() - os.makedirs(host_dir, exist_ok=True) - - files_to_upload = [] # list of (bucket_object_key, host_file_path) - - # Base files: committed in git, read from the shared SCRIPT_DIR location. - for relative_path in S3_BASE_FILES: - files_to_upload.append( - (relative_path, os.path.join(SCRIPT_DIR, relative_path)) - ) - - # Generated files: written into a per-worker host directory but uploaded - # to MinIO under the canonical `data/generated/` prefix that tests query. for file_number in range(100): - bucket_object_key = f"data/generated/file_{file_number}.csv" - host_path = os.path.join(host_dir, f"file_{file_number}.csv") - with open(host_path, "w+", encoding="utf-8") as f: + file_name = f"data/generated/file_{file_number}.csv" + os.makedirs(os.path.join(SCRIPT_DIR, "data/generated/"), exist_ok=True) + S3_DATA.append(file_name) + with open(os.path.join(SCRIPT_DIR, file_name), "w+", encoding="utf-8") as f: # a String, b UInt64 data = [] @@ -79,13 +43,12 @@ def create_buckets_s3(cluster): writer = csv.writer(f) writer.writerows(data) - files_to_upload.append((bucket_object_key, host_path)) - for bucket_object_key, host_path in files_to_upload: + for file in S3_DATA: minio.fput_object( bucket_name=cluster.minio_bucket, - object_name=bucket_object_key, - file_path=host_path, + object_name=file, + file_path=os.path.join(SCRIPT_DIR, file), ) for obj in minio.list_objects(cluster.minio_bucket, recursive=True): print(obj.object_name) @@ -104,7 +67,6 @@ def run_s3_mocks(started_cluster): @pytest.fixture(scope="module") def started_cluster(): - cluster = None try: cluster = ClickHouseCluster(__file__) cluster.add_instance( @@ -140,10 +102,8 @@ def started_cluster(): yield cluster finally: - # Only this worker's directory; never touches other workers' data. - shutil.rmtree(_generated_host_dir(), ignore_errors=True) - if cluster is not None: - cluster.shutdown() + shutil.rmtree(os.path.join(SCRIPT_DIR, "data/generated/")) + cluster.shutdown() def test_select_all(started_cluster): @@ -549,218 +509,3 @@ def test_cluster_default_expression(started_cluster): ) assert result == expected_result - - -@pytest.mark.parametrize("allow_experimental_analyzer", [0, 1]) -@pytest.mark.parametrize("use_partition_strategy", [False, True]) -def test_hive_partitioning(started_cluster, allow_experimental_analyzer, use_partition_strategy): - node = started_cluster.instances["s0_0_0"] - - data_path = f"root/data/hive_{allow_experimental_analyzer}/{random_string(6)}" - - if use_partition_strategy: - node.query( - f""" - INSERT - INTO FUNCTION s3( - 'http://minio1:9001/{data_path}', - 'minio', - '{minio_secret_key}', - format='Parquet', - partition_strategy='hive', - structure='key Int32, value Int32') - PARTITION BY key - SELECT number+1 as key, number+1 as value - FROM system.numbers - LIMIT 4 - """ - ) - else: - for i in range(1, 5): - node.query( - f""" - INSERT - INTO FUNCTION s3('http://minio1:9001/{data_path}/key={i}/data.parquet', 'minio', '{minio_secret_key}', 'Parquet', 'key Int32, value Int32') - SELECT {i}, {i} - """ - ) - - # With use_partition_strategy=1 and use_hive_partitioning=0 we have no 'key' column - if not use_partition_strategy: - query_id_full = str(uuid.uuid4()) - result = node.query( - f""" - SELECT count() - FROM s3('http://minio1:9001/{data_path}/key=**.parquet', 'minio', '{minio_secret_key}', 'Parquet', 'key Int32, value Int32') - WHERE key <= 2 - FORMAT TSV - SETTINGS enable_filesystem_cache = 0, use_query_cache = 0, use_cache_for_count_from_files = 0, use_hive_partitioning = 0, allow_experimental_analyzer={allow_experimental_analyzer} - """, - query_id=query_id_full, - ) - result = int(result) - assert result == 2 - - query_id_optimized = str(uuid.uuid4()) - result = node.query( - f""" - SELECT count() - FROM s3('http://minio1:9001/{data_path}/key=**.parquet', 'minio', '{minio_secret_key}', 'Parquet', 'key Int32, value Int32') - WHERE key <= 2 - FORMAT TSV - SETTINGS enable_filesystem_cache = 0, use_query_cache = 0, use_cache_for_count_from_files = 0, use_hive_partitioning = 1, allow_experimental_analyzer={allow_experimental_analyzer} - """, - query_id=query_id_optimized, - ) - result = int(result) - assert result == 2 - - if not use_partition_strategy: - query_id_cluster_full = str(uuid.uuid4()) - result = node.query( - f""" - SELECT count() - FROM s3Cluster(cluster_simple, 'http://minio1:9001/{data_path}/key=**.parquet', 'minio', '{minio_secret_key}', 'Parquet', 'key Int32, value Int32') - WHERE key <= 2 - FORMAT TSV - SETTINGS enable_filesystem_cache = 0, use_query_cache = 0, use_cache_for_count_from_files = 0, use_hive_partitioning = 0, allow_experimental_analyzer={allow_experimental_analyzer} - """, - query_id=query_id_cluster_full, - ) - result = int(result) - assert result == 2 - - query_id_cluster_optimized = str(uuid.uuid4()) - result = node.query( - f""" - SELECT count() - FROM s3Cluster(cluster_simple, 'http://minio1:9001/{data_path}/key=**.parquet', 'minio', '{minio_secret_key}', 'Parquet', 'key Int32, value Int32') - WHERE key <= 2 - FORMAT TSV - SETTINGS enable_filesystem_cache = 0, use_query_cache = 0, use_cache_for_count_from_files = 0, use_hive_partitioning = 1, allow_experimental_analyzer={allow_experimental_analyzer} - """, - query_id=query_id_cluster_optimized, - ) - result = int(result) - assert result == 2 - - node.query("SYSTEM FLUSH LOGS ON CLUSTER 'cluster_simple'") - - event = "EngineFileLikeReadFiles" - - if not use_partition_strategy: - full_traffic = node.query( - f""" - SELECT sum(ProfileEvents['{event}']) - FROM clusterAllReplicas(cluster_simple, system.query_log) - WHERE type='QueryFinish' AND initial_query_id='{query_id_full}' - FORMAT TSV - """ - ) - full_traffic = int(full_traffic) - assert full_traffic == 4 - - optimized_traffic = node.query( - f""" - SELECT sum(ProfileEvents['{event}']) - FROM clusterAllReplicas(cluster_simple, system.query_log) - WHERE type='QueryFinish' AND initial_query_id='{query_id_optimized}' - FORMAT TSV - """ - ) - optimized_traffic = int(optimized_traffic) - assert optimized_traffic == 2 - - if not use_partition_strategy: - cluster_full_traffic = node.query( - f""" - SELECT sum(ProfileEvents['{event}']) - FROM clusterAllReplicas(cluster_simple, system.query_log) - WHERE type='QueryFinish' AND initial_query_id='{query_id_cluster_full}' - FORMAT TSV - """ - ) - cluster_full_traffic = int(cluster_full_traffic) - assert cluster_full_traffic == full_traffic - - cluster_optimized_traffic = node.query( - f""" - SELECT sum(ProfileEvents['{event}']) - FROM clusterAllReplicas(cluster_simple, system.query_log) - WHERE type='QueryFinish' AND initial_query_id='{query_id_cluster_optimized}' - FORMAT TSV - """ - ) - cluster_optimized_traffic = int(cluster_optimized_traffic) - assert cluster_optimized_traffic == optimized_traffic - -def test_iceberg_s3_cluster_read_task_failpoint(started_cluster): - """INSERT INTO SELECT FROM icebergS3Cluster fails and does not hang. - Reproduces the scenario from https://github.com/ClickHouse/ClickHouse/issues/98165 - """ - node = started_cluster.instances["s0_0_0"] - run_id = uuid.uuid4().hex[:8] - iceberg_table = f"iceberg_src_{run_id}" - dst_table = f"local_dst_{run_id}" - iceberg_url = f"http://minio1:9001/root/{iceberg_table}/" - - # Create and populate the source Iceberg table before enabling the failpoint. - node.query( - f""" - CREATE TABLE {iceberg_table} (id UInt64, data String) - ENGINE = IcebergS3('{iceberg_url}', '{minio_access_key}', '{minio_secret_key}') - """ - ) - node.query( - f"INSERT INTO {iceberg_table} SELECT number AS id, randomString(10) AS data FROM numbers(50)", - settings={"allow_insert_into_iceberg": 1}, - ) - - # Local destination table — we only care that the query fails, not the data. - node.query( - f""" - CREATE TABLE {dst_table} (id UInt64, data String) - ENGINE = MergeTree() ORDER BY id - """ - ) - - all_node_names = ["s0_0_0", "s0_0_1", "s0_1_0"] - try: - for name in all_node_names: - started_cluster.instances[name].query( - "SYSTEM ENABLE FAILPOINT storage_cluster_read_sleep" - ) - - # The failpoint sleeps 10s uninterruptibly before max_execution_time can - # fire, so the server's legitimate abort lands well after 5s. The client - # timeout must outlast that abort (plus sanitizer/runner slowdown), not - # the 5s execution limit, otherwise a slow runner raises a false - # "Client timed out!". - _, error = node.query_and_get_answer_with_error( - f""" - INSERT INTO {dst_table} - SELECT * FROM icebergS3Cluster( - 'cluster_simple', - '{iceberg_url}', - '{minio_access_key}', '{minio_secret_key}') - SETTINGS max_execution_time = 5 - """, - timeout=120, - ) - - assert error, ( - "Expected a timeout error but the query succeeded. " - "The `storage_cluster_read_sleep` failpoint should have caused the query " - "to hang inside ReadTaskIterator::next() on every worker, " - "and `max_execution_time` should have aborted it." - ) - assert "Timeout" in error or "timeout" in error.lower(), ( - f"Expected a timeout-related error, got: {error}" - ) - finally: - for name in all_node_names: - started_cluster.instances[name].query( - "SYSTEM DISABLE FAILPOINT storage_cluster_read_sleep" - ) - node.query(f"DROP TABLE IF EXISTS {dst_table}") - node.query(f"DROP TABLE IF EXISTS {iceberg_table}") diff --git a/tests/queries/0_stateless/02354_vector_search_rescoring_distance_in_select_list.sql b/tests/queries/0_stateless/02354_vector_search_rescoring_distance_in_select_list.sql index e0e49b0609047..25efb3237844b 100644 --- a/tests/queries/0_stateless/02354_vector_search_rescoring_distance_in_select_list.sql +++ b/tests/queries/0_stateless/02354_vector_search_rescoring_distance_in_select_list.sql @@ -62,6 +62,8 @@ INSERT INTO tab_bf16 VALUES (0, [1.0, 0.0]), (9, [0.0, 2.4]); -- The nearest neighbours to [0.0, 0.2] are 5,6,7,8 +-- Round distances: ARM sanitizer builds disable SimSIMD and scalar USearch +-- differs in the 3rd decimal (0.203125 vs 0.1875). Neighbor order is unchanged. SELECT 'Column: Array(Float32)';