FIPS 25.3: Several CI fixes

.github/actions/create_workflow_report/ci_run_report.html.jinja

@@ -5,6 +5,9 @@
   <meta http-equiv="content-type" content="text/html; charset=UTF-8">
   <meta charset="UTF-8">
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  {%- if is_preview %}
+  <meta http-equiv="Cache-Control" content="max-age=300">
+  {%- endif %}
   <style>
     /* Base colors for Altinity */
     :root {
@@ -223,6 +226,8 @@
           element.className = element.className.replace(regex_dir, '') + dir
         }
         function getValue(element) {
+          var childWithSort = element.querySelector('[data-sort]')
+          if (childWithSort) return childWithSort.getAttribute('data-sort')
           return (
             (alt_sort && element.getAttribute('data-sort-alt')) ||
             element.getAttribute('data-sort') || element.innerText

.github/grype/run_grype_scan.sh

@@ -3,7 +3,7 @@ set -e
 
 IMAGE=$1
 
-GRYPE_VERSION=${GRYPE_VERSION:-"v0.92.2"}
+GRYPE_VERSION=${GRYPE_VERSION:-"v0.115.0"}
 
 docker pull $IMAGE
 docker pull anchore/grype:${GRYPE_VERSION}

.github/workflows/fast_builds.yml

@@ -0,0 +1,301 @@
+# yamllint disable rule:comments-indentation
+name: Fast Builds
+run-name: "${{ github.event.inputs.workflow_name || ' ' }}"
+
+env:
+  # Force the stdout and stderr streams to be unbuffered
+  PYTHONUNBUFFERED: 1
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  CLICKHOUSE_TEST_STAT_LOGIN: ${{ secrets.CLICKHOUSE_TEST_STAT_LOGIN }}
+  CLICKHOUSE_TEST_STAT_PASSWORD: ${{ secrets.CLICKHOUSE_TEST_STAT_PASSWORD }}
+  CLICKHOUSE_TEST_STAT_URL: ${{ secrets.CLICKHOUSE_TEST_STAT_URL }}
+  DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+  ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }}
+
+on: # yamllint disable-line rule:truthy
+  workflow_dispatch:
+    inputs:
+      workflow_name:
+        description: 'Name of the workflow'
+        required: false
+        type: string
+
+jobs:
+  RunConfig:
+    runs-on: [self-hosted, altinity-on-demand, altinity-style-checker-aarch64]
+    outputs:
+      data: ${{ steps.runconfig.outputs.CI_DATA }}
+    steps:
+      - name: Check out repository code
+        uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6
+        with:
+          clear-repository: true # to ensure correct digests
+          fetch-depth: 0 # to get version
+          filter: tree:0
+      - name: Debug Info
+        uses: ./.github/actions/debug
+      - name: PrepareRunConfig
+        id: runconfig
+        run: |
+            echo "::group::configure CI run"
+            python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --configure --workflow FastBuilds --outfile ${{ runner.temp }}/ci_run_data.json
+            echo "::endgroup::"
+            echo "::group::CI run configure results"
+            python3 -m json.tool ${{ runner.temp }}/ci_run_data.json
+            echo "::endgroup::"
+            {
+              echo 'CI_DATA<<EOF'
+              cat  ${{ runner.temp }}/ci_run_data.json
+              echo 'EOF'
+            } >> "$GITHUB_OUTPUT"
+      - name: Re-create GH statuses for skipped jobs if any
+        run: |
+            python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ runner.temp }}/ci_run_data.json --update-gh-statuses
+      - name: Note report location to summary
+        env:
+          PR_NUMBER: ${{ github.event.pull_request.number || 0 }}
+          COMMIT_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+        run: |
+          REPORT_LINK=https://s3.amazonaws.com/altinity-build-artifacts/$PR_NUMBER/$COMMIT_SHA/ci_run_report.html
+          echo "Workflow Run Report: [View Report]($REPORT_LINK)" >> $GITHUB_STEP_SUMMARY
+
+  BuildDockers:
+    needs: [RunConfig]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/docker_test_images.yml
+    secrets: inherit
+    with:
+      data: ${{ needs.RunConfig.outputs.data }}
+
+  BuilderDebRelease:
+    needs: [RunConfig, BuildDockers]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/reusable_build.yml
+    secrets: inherit
+    with:
+      build_name: package_release
+      checkout_depth: 0
+      data: ${{ needs.RunConfig.outputs.data }}
+      # always rebuild on release branches to be able to publish from any commit
+      force: true
+
+  BuilderDebAarch64:
+    needs: [RunConfig, BuildDockers]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/reusable_build.yml
+    secrets: inherit
+    with:
+      build_name: package_aarch64
+      checkout_depth: 0
+      data: ${{ needs.RunConfig.outputs.data }}
+      # always rebuild on release branches to be able to publish from any commit
+      force: true
+      runner_type: builder-aarch64
+
+  BuilderDebAsan:
+    needs: [RunConfig, BuildDockers]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/reusable_build.yml
+    secrets: inherit
+    with:
+      build_name: package_asan
+      data: ${{ needs.RunConfig.outputs.data }}
+
+  BuilderDebUBsan:
+    needs: [RunConfig, BuildDockers]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/reusable_build.yml
+    secrets: inherit
+    with:
+      build_name: package_ubsan
+      data: ${{ needs.RunConfig.outputs.data }}
+
+  BuilderDebTsan:
+    needs: [RunConfig, BuildDockers]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/reusable_build.yml
+    secrets: inherit
+    with:
+      build_name: package_tsan
+      data: ${{ needs.RunConfig.outputs.data }}
+
+  BuilderDebDebug:
+    needs: [RunConfig, BuildDockers]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/reusable_build.yml
+    secrets: inherit
+    with:
+      build_name: package_debug
+      data: ${{ needs.RunConfig.outputs.data }}
+      force: true
+
+  DockerServerImage:
+    needs: [RunConfig, BuilderDebRelease, BuilderDebAarch64]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/reusable_test.yml
+    secrets: inherit
+    with:
+      test_name: Docker server image
+      runner_type: altinity-func-tester
+      data: ${{ needs.RunConfig.outputs.data }}
+
+  DockerKeeperImage:
+    needs: [RunConfig, BuilderDebRelease, BuilderDebAarch64]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/reusable_test.yml
+    secrets: inherit
+    with:
+      test_name: Docker keeper image
+      runner_type: altinity-func-tester
+      data: ${{ needs.RunConfig.outputs.data }}
+
+  GrypeScan:
+    needs: [RunConfig, DockerServerImage, DockerKeeperImage]
+    if: ${{ !failure() && !cancelled() }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - image: server
+            suffix: ''
+          - image: server
+            suffix: '-alpine'
+          - image: keeper
+            suffix: ''
+    uses: ./.github/workflows/grype_scan.yml
+    secrets: inherit
+    with:
+      docker_image: altinityinfra/clickhouse-${{ matrix.image }}
+      version: ${{ fromJson(needs.RunConfig.outputs.data).version }}
+      tag-suffix: ${{ matrix.suffix }}
+
+  Builds_Report:
+    # run report check for failed builds to indicate the CI error
+    if: ${{ !cancelled() && needs.RunConfig.result == 'success' && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'Builds') }}
+    needs: [RunConfig, BuilderDebRelease, BuilderDebAarch64, BuilderDebAsan, BuilderDebUBsan, BuilderDebTsan, BuilderDebDebug]
+    runs-on: [self-hosted, altinity-on-demand, altinity-style-checker-aarch64]
+    steps:
+      - name: Check out repository code
+        uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6
+      - name: Download reports
+        run: |
+          python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(needs.RunConfig.outputs.data) }} --pre --job-name Builds
+      - name: Builds report
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 ./build_report_check.py --reports package_release package_aarch64 package_asan package_ubsan package_tsan package_debug
+      - name: Set status
+        # NOTE(vnemkov): generate and upload the report even if previous step failed
+        if: success() || failure()
+        run: |
+          python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(needs.RunConfig.outputs.data) }} --post --job-name Builds
+
+  InstallPackagesTestRelease:
+    needs: [RunConfig, BuilderDebRelease]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/reusable_test.yml
+    secrets: inherit
+    with:
+      test_name: Install packages (release)
+      runner_type: altinity-func-tester
+      data: ${{ needs.RunConfig.outputs.data }}
+      run_command: |
+        python3 install_check.py "$CHECK_NAME"
+
+  InstallPackagesTestAarch64:
+    needs: [RunConfig, BuilderDebAarch64]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/reusable_test.yml
+    secrets: inherit
+    with:
+      test_name: Install packages (aarch64)
+      runner_type: altinity-func-tester-aarch64
+      data: ${{ needs.RunConfig.outputs.data }}
+      run_command: |
+        python3 install_check.py "$CHECK_NAME"
+
+  FunctionalStatelessTestRelease:
+    needs: [RunConfig, BuilderDebRelease]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/reusable_test.yml
+    secrets: inherit
+    with:
+      test_name: Stateless tests (release)
+      runner_type: altinity-func-tester, 16c
+      data: ${{ needs.RunConfig.outputs.data }}
+
+  FunctionalStatelessTestAarch64:
+    needs: [RunConfig, BuilderDebAarch64]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/reusable_test.yml
+    secrets: inherit
+    with:
+      test_name: Stateless tests (aarch64)
+      runner_type: altinity-func-tester-aarch64, 16c
+      data: ${{ needs.RunConfig.outputs.data }}
+
+  SignRelease:
+    needs: [RunConfig, BuilderDebRelease]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/reusable_sign.yml
+    secrets: inherit
+    with:
+      test_name: Sign release
+      runner_type: altinity-style-checker
+      data: ${{ needs.RunConfig.outputs.data }}
+
+  SignAarch64:
+    needs: [RunConfig, BuilderDebAarch64]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/reusable_sign.yml
+    secrets: inherit
+    with:
+      test_name: Sign aarch64
+      runner_type: altinity-style-checker-aarch64
+      data: ${{ needs.RunConfig.outputs.data }}
+
+  FinishCheck:
+    if: ${{ !cancelled() }}
+    needs:
+      - RunConfig
+      - DockerServerImage
+      - DockerKeeperImage
+      - Builds_Report
+      - FunctionalStatelessTestRelease
+      - FunctionalStatelessTestAarch64
+      - InstallPackagesTestRelease
+      - InstallPackagesTestAarch64
+      - GrypeScan
+      - SignRelease
+      - SignAarch64
+    runs-on: [self-hosted, altinity-on-demand, altinity-style-checker-aarch64]
+    steps:
+      - name: Check out repository code
+        uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6
+        with:
+          clear-repository: true
+      - name: Finish label
+        if: ${{ !failure() }}
+        run: |
+          # update overall ci report
+          python3 ./tests/ci/finish_check.py --wf-status ${{ contains(needs.*.result, 'failure') && 'failure' || 'success' }}
+      - name: Check Workflow results
+        if: ${{ !cancelled() }}
+        run: |
+          export WORKFLOW_RESULT_FILE="/tmp/workflow_results.json"
+          cat > "$WORKFLOW_RESULT_FILE" << 'EOF'
+          ${{ toJson(needs) }}
+          EOF
+          python3 ./tests/ci/ci_buddy.py --check-wf-status
+      - name: Finalize workflow report
+        if: ${{ !cancelled() }}
+        uses: ./.github/actions/create_workflow_report
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          CHECKS_DATABASE_HOST: ${{ secrets.CHECKS_DATABASE_HOST }}
+          CHECKS_DATABASE_USER: ${{ secrets.CLICKHOUSE_TEST_STAT_LOGIN }}
+          CHECKS_DATABASE_PASSWORD: ${{ secrets.CLICKHOUSE_TEST_STAT_PASSWORD }}
+        with:
+          final: true

.github/workflows/grype_scan.yml

@@ -30,7 +30,7 @@ env:
   AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
   AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
   AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-  GRYPE_VERSION: "v0.92.2-arm64v8"
+  GRYPE_VERSION: "v0.115.0-arm64v8"
 
 jobs:
     grype_scan:
@@ -50,7 +50,7 @@ jobs:
             sudo apt-get install -y python3-pip python3-venv
             python3 -m venv venv
             source venv/bin/activate
-            pip install --upgrade requests chardet urllib3 unidiff boto3 PyGithub
+            pip install --upgrade requests chardet urllib3 unidiff 'boto3==1.43.33' PyGithub
             pip install testflows==$TESTFLOWS_VERSION awscli==1.33.28
             echo PATH=$PATH >>$GITHUB_ENV
 

.github/workflows/regression-reusable-suite.yml

@@ -67,7 +67,7 @@ jobs:
       AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
       # Docker credentials
       DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
-      DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+      DOCKER_PASSWORD: ${{ secrets.DOCKER_TOKEN }}
       # Database credentials
       CHECKS_DATABASE_HOST: ${{ secrets.CHECKS_DATABASE_HOST }}
       CHECKS_DATABASE_USER: ${{ secrets.CLICKHOUSE_TEST_STAT_LOGIN }}