diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml new file mode 100644 index 00000000000..2fac7f5826a --- /dev/null +++ b/.github/workflows/coverity.yml @@ -0,0 +1,242 @@ +name: Coverity Scan + +# Public static analysis via the free Coverity Scan service (scan.coverity.com, +# operated by Black Duck). Coverity is a compiled-language analyzer: it must wrap +# the real C/C++ build with `cov-build`, so this workflow installs oneAPI, builds +# the SYCL extensions under capture, and uploads the result for analysis. +# +# One-time setup required before the first run: +# 1. Register IntelPython/dpnp at https://scan.coverity.com/github +# (sign in with GitHub; the project name must match COVERITY_PROJECT below). +# 2. Add two repository secrets (Settings -> Secrets and variables -> Actions): +# COVERITY_SCAN_TOKEN - the project token from the Project Settings tab +# COVERITY_SCAN_EMAIL - a maintainer email for build notifications +# +# dpnp is ~140K lines of C++/Cython, which puts it in the 100K-500K LOC free-tier +# bucket: 21 builds/week, max 3/day. Running weekly plus on demand stays well +# under that, and matters more here than for the smaller IntelPython projects +# because a full dpnp build under cov-build takes hours. + +on: + schedule: + - cron: "0 1 * * 1" # Mondays 01:00 UTC; well under the free build quota + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: coverity-${{ github.ref }} + cancel-in-progress: true + +env: + COVERITY_PROJECT: IntelPython/dpnp + ONEAPI_ROOT: /opt/intel/oneapi + +jobs: + coverity-scan: + # Forks lack the COVERITY_SCAN_* secrets; only run on the canonical repo. + if: github.repository == 'IntelPython/dpnp' + runs-on: ubuntu-latest + # Backstop for the whole job. dpnp's own conda-package build budgets 90 min + # and Coverity's emit roughly doubles compile time, so this is deliberately + # close to the 6 h hard cap GitHub imposes on a job. + timeout-minutes: 350 + + steps: + - name: Checkout repo + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # versioneer needs the tags to compute the package version + fetch-depth: 0 + persist-credentials: false + + - name: Add Intel repository + run: | + wget -qO- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \ + | gpg --dearmor | sudo tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null + echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" \ + | sudo tee /etc/apt/sources.list.d/oneAPI.list + sudo apt-get update + + - name: Install latest Intel OneAPI + # Same package set the coverage workflow uses: dpnp's CMakeLists requires + # IntelSYCL, MKL, TBB and oneDPL (libdpstd) to be found at configure time. + run: | + sudo apt-get install -y hwloc \ + intel-oneapi-mkl \ + intel-oneapi-umf \ + intel-oneapi-mkl-devel \ + intel-oneapi-tbb-devel \ + intel-oneapi-libdpstd-devel \ + intel-oneapi-compiler-dpcpp-cpp + sudo apt-get install -y ninja-build + + - name: Setup Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.12' + architecture: x64 + + - name: Install dpnp build dependencies + # Mirrors the build-time requirements in pyproject.toml. dpnp still uses + # classic scikit-build, so setup.py is the build entry point. + # Cython is pinned here only (not in pyproject.toml) to keep the generated + # code stable between scans, so Coverity CIDs and their triage survive; + # bump it deliberately, as the mkl_fft and mkl-service scans do. + run: | + pip install cmake "cython==3.3.0" ninja numpy scikit-build setuptools wheel \ + "versioneer[toml]==0.29" + + - name: Install dpctl + # dpctl must come from pip, not conda: the conda package has a runtime + # dependency on DPC++ RT, which would be resolved independently of the + # apt-installed compiler above and can end up newer than it. DPC++ only + # guarantees backwards compatibility, so that skew breaks the build. + run: pip install -r environments/dpctl_pkg.txt + + - name: Report compiler version + run: | + source "${ONEAPI_ROOT}/setvars.sh" + icpx --version + + - name: Download Coverity Build Tool + timeout-minutes: 15 + env: + COVERITY_SCAN_TOKEN: ${{ secrets.COVERITY_SCAN_TOKEN }} + run: | + curl --location --no-progress-meter --fail-with-body \ + --retry 5 --retry-connrefused --retry-delay 5 \ + --data-urlencode "token=${COVERITY_SCAN_TOKEN}" \ + --data-urlencode "project=${COVERITY_PROJECT}" \ + --output cov-analysis.tar.gz \ + https://scan.coverity.com/download/linux64 + # An invalid token/project is answered with a small HTML error page and + # HTTP 200, not the multi-hundred-MB tarball, so --fail-with-body does + # not catch it. Fail loudly with a clear hint instead. + if [ "$(stat -c '%s' cov-analysis.tar.gz)" -lt 1000000 ]; then + echo "::error::Coverity build tool download failed. Verify the COVERITY_SCAN_TOKEN secret and that the registered project name matches '${COVERITY_PROJECT}'." + head -c 512 cov-analysis.tar.gz || true + exit 1 + fi + mkdir -p cov-analysis + tar -xzf cov-analysis.tar.gz --strip 1 -C cov-analysis + echo "${PWD}/cov-analysis/bin" >> "$GITHUB_PATH" + + - name: Configure Coverity for the DPC++ compiler + run: | + source "${ONEAPI_ROOT}/setvars.sh" + # Cython-generated sources and the pybind11 extensions go through + # icpx, the sysroot/driver probing bits still use gcc + cov-configure --gcc + # Register icx/icpx with Coverity's native Intel oneAPI compiler type. + # The generic clangcc/clangcxx templates parse without + # SYCL mode and fail on device-only builtins (__ocl_sampler_t, + # __ocl_image2d_ro_t, ...): dpctl's scan with that setup captures only + # 9 of its 30 sources, and nearly all of dpnp's sources include SYCL. + # The native type emits each SYCL source twice, as a host pass and a + # spir64 device pass; see the capture check below for the latter. + # intel_oneapi_icx is current; intel_icx:linux is the older spelling + # the Scan build tool still ships (what mkl_umath's scan resolves to). + comptypes="$(cov-configure --list-compiler-types | cut -d, -f1)" + comptype="" + for candidate in intel_oneapi_icx intel_oneapi_icx:linux intel_icx:linux; do + if grep -qxF "${candidate}" <<< "${comptypes}"; then + comptype="${candidate}" + break + fi + done + if [ -z "${comptype}" ]; then + echo "::error::No Intel oneAPI compiler type in this cov-analysis; refusing to fall back to the generic Clang configuration." + exit 1 + fi + echo "Using Coverity compiler type: ${comptype}" + # With --template, --compiler takes a bare executable name; capture + # matches on the name of the spawned executable, so this also covers + # the absolute icx/icpx paths CMake resolves. + cov-configure --template --comptype "${comptype}" --compiler icx + cov-configure --template --comptype "${comptype}" --compiler icpx + + - name: Build under cov-build + # dpnp has ~250 translation units, many of them template-heavy SYCL + # kernels, and cov-build compiles each one a second time with its own + # frontend. Cap the step so a wedged capture fails before the job + # timeout eats the whole 6 h budget. + timeout-minutes: 300 + run: | + set -o pipefail + source "${ONEAPI_ROOT}/setvars.sh" + # --skip-editable: the in-place build_ext already compiled everything + # Coverity needs to see, a pip install would only repeat it + cov-build --dir cov-int \ + python scripts/build_locally.py --oneapi --skip-editable --verbose \ + 2>&1 | tee cov-build.log + { + echo "### Coverity capture" + echo '```' + grep -E "Emitted .*compilation unit|are ready for analysis" cov-build.log || true + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + # A partial capture still uploads happily and uses up a scan, so + # require every project source to be emitted. This is checked per + # source rather than via the "Emitted N (P%)" total because that + # total also counts the spir64 device passes, which the Scan build + # tool's emitter cannot parse (it trips over and then crashes in the + # SPIR-V builtin headers), capping P near 50% for dpnp. The host + # pass of each source is what gets analyzed. CMake's own try-compile + # probes under CMakeFiles/ are not project code and often fail. + # `|| true` because a no-match grep would otherwise trip pipefail and + # end the step before the errors below are printed. + log=cov-int/build-log.txt + compiled="$(grep -oP '\[STATUS\] Compiling \K\S+' "${log}" | grep -F "${GITHUB_WORKSPACE}/" | grep -vF /CMakeFiles/ | sort -u || true)" + # Unanchored: parallel compiles can splice another unit's output onto + # the same line of the log. + emitted="$(grep -oP "Emit for file '\K[^']+(?=' \(TU [0-9]+\) complete\.)" "${log}" | sort -u || true)" + missing="$(comm -23 <(echo "${compiled}") <(echo "${emitted}"))" + total="$(grep -c . <<< "${compiled}" || true)" + nmissing="$(grep -c . <<< "${missing}" || true)" + echo "Coverity emitted $(( total - nmissing )) of ${total} project sources" | tee -a "$GITHUB_STEP_SUMMARY" + if [ "${total}" -eq 0 ]; then + echo "::error::No project sources were compiled under cov-build." + exit 1 + fi + if [ "${nmissing}" -gt 0 ]; then + echo "${missing}" + echo "::error::Coverity did not emit ${nmissing} of ${total} project sources (listed above)." + exit 1 + fi + + - name: Upload Coverity build log + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: coverity-build-log + path: | + cov-build.log + cov-int/build-log.txt + retention-days: 7 + if-no-files-found: warn + + - name: Submit results to Coverity Scan + # Scan keeps only the latest build, so a run dispatched on a feature + # branch would replace master's results; such runs stop after capture. + if: github.ref == 'refs/heads/master' + timeout-minutes: 30 + env: + COVERITY_SCAN_TOKEN: ${{ secrets.COVERITY_SCAN_TOKEN }} + COVERITY_SCAN_EMAIL: ${{ secrets.COVERITY_SCAN_EMAIL }} + run: | + tar -czf cov-int.tgz cov-int + # dpnp's emit database is far larger than the other IntelPython + # projects'; log the size because Scan rejects oversized submissions + # and the resulting error is otherwise opaque. + echo "Submitting $(du -h cov-int.tgz | cut -f1) of intermediate results" + curl --no-progress-meter --fail-with-body \ + --retry 5 --retry-connrefused --retry-delay 5 \ + --form token="${COVERITY_SCAN_TOKEN}" \ + --form email="${COVERITY_SCAN_EMAIL}" \ + --form file=@cov-int.tgz \ + --form version="${GITHUB_SHA}" \ + --form description="GitHub Actions ${GITHUB_REF_NAME} (run ${GITHUB_RUN_ID})" \ + --form project="${COVERITY_PROJECT}" \ + https://scan.coverity.com/builds diff --git a/README.md b/README.md index 5241ddc96b2..28cdf3fdd4c 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ [![Conda package](https://github.com/IntelPython/dpnp/actions/workflows/conda-package.yml/badge.svg?branch=master&event=push)](https://github.com/IntelPython/dpnp/actions/workflows/conda-package.yml) [![Coverage Status](https://coveralls.io/repos/github/IntelPython/dpnp/badge.svg?branch=master)](https://coveralls.io/github/IntelPython/dpnp?branch=master) [![Build Sphinx](https://github.com/IntelPython/dpnp/workflows/Build%20Sphinx/badge.svg)](https://intelpython.github.io/dpnp) +[![Coverity Scan Build Status](https://scan.coverity.com/projects/intelpython-dpnp/badge.svg)](https://scan.coverity.com/projects/intelpython-dpnp) [![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/IntelPython/dpnp/badge)](https://securityscorecards.dev/viewer/?uri=github.com/IntelPython/dpnp) oneAPI logo