diff --git a/.github/workflows/.java-version b/.github/workflows/.java-version index b4de394767..98d9bcb75a 100644 --- a/.github/workflows/.java-version +++ b/.github/workflows/.java-version @@ -1 +1 @@ -11 +17 diff --git a/.github/workflows/check-dependencies.yml b/.github/workflows/check-dependencies.yml index fc6de5b560..ef9ff04fc3 100644 --- a/.github/workflows/check-dependencies.yml +++ b/.github/workflows/check-dependencies.yml @@ -22,6 +22,11 @@ on: permissions: contents: read +# Keep the newest PR run; preserve independent push, release and manual runs. +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: dependency-check: runs-on: ubuntu-latest diff --git a/.github/workflows/cluster-test-ci.yml b/.github/workflows/cluster-test-ci.yml index 3edcfcb874..c545ff8c80 100644 --- a/.github/workflows/cluster-test-ci.yml +++ b/.github/workflows/cluster-test-ci.yml @@ -27,6 +27,11 @@ on: - '!**/AGENTS.md' - '!**/CLAUDE.md' +# Keep the newest PR run; preserve independent push, release and manual runs. +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: cluster-test: runs-on: ubuntu-latest @@ -59,8 +64,27 @@ jobs: - name: Run simple cluster test run: | - mvn test -pl hugegraph-cluster-test/hugegraph-clustertest-test -am -P simple-cluster-test + timeout 45m mvn test -pl hugegraph-cluster-test/hugegraph-clustertest-test \ + -am -P simple-cluster-test - name: Run multi cluster test run: | - mvn test -pl hugegraph-cluster-test/hugegraph-clustertest-test -am -P multi-cluster-test + timeout 45m mvn test -pl hugegraph-cluster-test/hugegraph-clustertest-test \ + -am -P multi-cluster-test + + - name: Show cluster diagnostics on failure + if: failure() + run: | + echo "[ci] java processes:" + ps -ef | grep -E "HugeGraph|hg-|java" | grep -v grep || true + echo "[ci] listening tcp ports:" + (ss -ltnp || netstat -ltnp || true) 2>&1 + find hugegraph-cluster-test -path '*/logs/*' -type f | sort | while read -r log; do + echo "--- tail -n 200 $log ---" + tail -n 200 "$log" || true + done + find . -path '*/surefire-reports/*' -type f \ + \( -name '*.txt' -o -name '*.xml' \) | sort | while read -r report; do + echo "--- tail -n 120 $report ---" + tail -n 120 "$report" || true + done diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 23f1807dc2..5b017919cf 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -9,6 +9,11 @@ on: schedule: - cron: '33 0 * * 5' +# Keep the newest PR run; preserve independent push, release and manual runs. +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: analyze: env: diff --git a/.github/workflows/commons-ci.yml b/.github/workflows/commons-ci.yml index 0e7d4d8d95..cdfea36d8b 100644 --- a/.github/workflows/commons-ci.yml +++ b/.github/workflows/commons-ci.yml @@ -9,6 +9,11 @@ on: - 'test-*' pull_request: +# Keep the newest PR run; preserve independent push, release and manual runs. +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: java-runtime: uses: ./.github/workflows/java-runtime.yml @@ -53,11 +58,13 @@ jobs: - name: Run common test run: | - mvn test -pl hugegraph-commons/hugegraph-common -Dtest=UnitTestSuite -DskipCommonsTests=false + mvn test -pl hugegraph-commons/hugegraph-common -Dtest=UnitTestSuite \ + -DskipCommonsTests=false -Dsurefire.failIfNoSpecifiedTests=true - name: Run rpc test run: | - mvn test -pl hugegraph-commons/hugegraph-rpc -Dtest=UnitTestSuite -DskipCommonsTests=false + mvn test -pl hugegraph-commons/hugegraph-rpc -Dtest=UnitTestSuite \ + -DskipCommonsTests=false -Dsurefire.failIfNoSpecifiedTests=true - name: Upload coverage to Codecov uses: codecov/codecov-action@v7 diff --git a/.github/workflows/docker-build-ci.yml b/.github/workflows/docker-build-ci.yml index 368791dbea..3c87855b0c 100644 --- a/.github/workflows/docker-build-ci.yml +++ b/.github/workflows/docker-build-ci.yml @@ -44,6 +44,11 @@ on: - '!**/AGENTS.md' - '!**/CLAUDE.md' +# Keep the newest PR run; preserve independent push, release and manual runs. +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: docker-bake-check: runs-on: ubuntu-latest @@ -137,6 +142,15 @@ jobs: echo "Healthcheck: $HC" [[ "$HC" != "null" ]] || { echo "ERROR: HEALTHCHECK missing in ${{ matrix.dockerfile }}"; exit 1; } + - name: Verify Java 17 runtime in ${{ matrix.dockerfile }} + run: | + JAVA_VERSION=$(docker run --rm --entrypoint java "$IMAGE_ID" -version 2>&1) + echo "$JAVA_VERSION" + grep -Eq 'version "17\.' <<< "$JAVA_VERSION" || { + echo "ERROR: expected a Java 17 runtime in ${{ matrix.dockerfile }}" + exit 1 + } + - name: Test server entrypoint property mapping if: matrix.dockerfile == 'hugegraph-server/Dockerfile' run: bash hugegraph-server/hugegraph-dist/docker/test/test-docker-entrypoint.sh diff --git a/.github/workflows/helm-chart-ci.yml b/.github/workflows/helm-chart-ci.yml index 1ca124731f..e78dd03655 100644 --- a/.github/workflows/helm-chart-ci.yml +++ b/.github/workflows/helm-chart-ci.yml @@ -27,6 +27,11 @@ on: permissions: contents: read +# Keep the newest PR run; preserve independent push, release and manual runs. +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: lint-and-render: runs-on: ubuntu-latest diff --git a/.github/workflows/licence-checker.yml b/.github/workflows/licence-checker.yml index 3d36ad4597..491ccf57b5 100644 --- a/.github/workflows/licence-checker.yml +++ b/.github/workflows/licence-checker.yml @@ -7,6 +7,11 @@ on: - 'release-*' pull_request: +# Keep the newest PR run; preserve independent push, release and manual runs. +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: check-license: runs-on: ubuntu-latest diff --git a/.github/workflows/pd-store-ci.yml b/.github/workflows/pd-store-ci.yml index 108e83d3a5..a61c5673ad 100644 --- a/.github/workflows/pd-store-ci.yml +++ b/.github/workflows/pd-store-ci.yml @@ -28,6 +28,11 @@ on: - '!**/CLAUDE.md' # TODO: consider merge to one ci.yml file +# Keep the newest PR run; preserve independent push, release and manual runs. +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: rocksdb-compatibility: permissions: @@ -184,6 +189,31 @@ jobs: mvn test -pl hugegraph-pd/hg-pd-test -am \ -P pd-core-test -Djacoco.sessionId=pd-core-test + - name: Verify Surefire single-class routing + run: | + SUREFIRE_DIR=hugegraph-pd/hg-pd-test/target/surefire-reports + REPORT="$SUREFIRE_DIR/TEST-org.apache.hugegraph.pd.core.PDConfigTest.xml" + rm -f "$REPORT" + mvn test -pl hugegraph-pd/hg-pd-test -am \ + -P pd-core-test -Dtest=PDConfigTest -ntp + test -s "$REPORT" || { + echo "::error::PD single-class test produced no report" + exit 1 + } + NEGATIVE_LOG=$(mktemp) + trap 'rm -f "$NEGATIVE_LOG"' EXIT + if mvn test -pl hugegraph-pd/hg-pd-test -am \ + -P pd-core-test -Dtest=NoSuchJava17ContractTest -ntp \ + >"$NEGATIVE_LOG" 2>&1; then + cat "$NEGATIVE_LOG" + echo "::error::Surefire accepted a missing PD test class" + exit 1 + fi + cat "$NEGATIVE_LOG" + grep -Fq 'No tests matching pattern "NoSuchJava17ContractTest"' \ + "$NEGATIVE_LOG" + grep -Fq '(pd-core-test) @ hg-pd-test' "$NEGATIVE_LOG" + - name: Check startup test prerequisites (PD) id: pd-preflight run: | @@ -258,6 +288,23 @@ jobs: "$REPORT_FILE" \ hg-pd-grpc hg-pd-common hg-pd-client hg-pd-core hg-pd-service hg-pd-dist + - name: Show PD diagnostics on failure + if: failure() + run: | + VERSION=$(grep -E '^VersionInBash=' \ + hugegraph-commons/hugegraph-common/src/main/resources/version.properties | cut -d'=' -f2-) + [[ "$VERSION" =~ ^[0-9A-Za-z._-]+$ ]] || { + echo "Invalid VersionInBash: $VERSION" + exit 1 + } + PD_DIR=hugegraph-pd/apache-hugegraph-pd-$VERSION + bash $TRAVIS_DIR/ci-service-utils.sh dump "$PD_DIR" HugeGraphPD || true + find . -path '*/surefire-reports/*' -type f \ + \( -name '*.txt' -o -name '*.xml' \) | sort | while read -r report; do + echo "--- tail -n 120 $report ---" + tail -n 120 "$report" || true + done + - name: Upload coverage to Codecov uses: codecov/codecov-action@v7 with: @@ -301,6 +348,31 @@ jobs: run: | mvn clean package -U -Dmaven.javadoc.skip=true -Dmaven.test.skip=true -ntp --fail-at-end + - name: Verify Surefire single-class routing + run: | + SUREFIRE_DIR=hugegraph-store/hg-store-test/target/surefire-reports + REPORT="$SUREFIRE_DIR/TEST-org.apache.hugegraph.store.raftcore.ZeroByteStringHelperTest.xml" + rm -f "$REPORT" + mvn test -pl hugegraph-store/hg-store-test -am \ + -P store-raftcore-test -Dtest=ZeroByteStringHelperTest -ntp + test -s "$REPORT" || { + echo "::error::Store single-class test produced no report" + exit 1 + } + NEGATIVE_LOG=$(mktemp) + trap 'rm -f "$NEGATIVE_LOG"' EXIT + if mvn test -pl hugegraph-store/hg-store-test -am \ + -P store-raftcore-test -Dtest=NoSuchJava17ContractTest -ntp \ + >"$NEGATIVE_LOG" 2>&1; then + cat "$NEGATIVE_LOG" + echo "::error::Surefire accepted a missing Store test class" + exit 1 + fi + cat "$NEGATIVE_LOG" + grep -Fq 'No tests matching pattern "NoSuchJava17ContractTest"' \ + "$NEGATIVE_LOG" + grep -Fq '(store-raftcore-test) @ hg-store-test' "$NEGATIVE_LOG" + - name: Check source formatting run: | mvn editorconfig:check -pl hugegraph-store/hg-store-test -am -ntp @@ -397,6 +469,25 @@ jobs: "$REPORT_FILE" \ hg-store-grpc hg-store-common hg-store-client hg-store-rocksdb hg-store-core + - name: Show Store diagnostics on failure + if: failure() + run: | + VERSION=$(grep -E '^VersionInBash=' \ + hugegraph-commons/hugegraph-common/src/main/resources/version.properties | cut -d'=' -f2-) + [[ "$VERSION" =~ ^[0-9A-Za-z._-]+$ ]] || { + echo "Invalid VersionInBash: $VERSION" + exit 1 + } + PD_DIR=hugegraph-pd/apache-hugegraph-pd-$VERSION + STORE_DIR=hugegraph-store/apache-hugegraph-store-$VERSION + bash $TRAVIS_DIR/ci-service-utils.sh dump "$PD_DIR" HugeGraphPD || true + bash $TRAVIS_DIR/ci-service-utils.sh dump "$STORE_DIR" HugeGraphStore || true + find . -path '*/surefire-reports/*' -type f \ + \( -name '*.txt' -o -name '*.xml' \) | sort | while read -r report; do + echo "--- tail -n 120 $report ---" + tail -n 120 "$report" || true + done + - name: Upload coverage to Codecov uses: codecov/codecov-action@v7 with: @@ -442,6 +533,39 @@ jobs: - name: Prepare env and service run: | + VERSION=$(grep -E '^VersionInBash=' \ + hugegraph-commons/hugegraph-common/src/main/resources/version.properties | cut -d'=' -f2-) + [[ "$VERSION" =~ ^[0-9A-Za-z._-]+$ ]] || { + echo "Invalid VersionInBash: $VERSION" + exit 1 + } + STORE_CONFIG=hugegraph-store/apache-hugegraph-store-$VERSION/conf/application-pd.yml + [[ -f "$STORE_CONFIG" ]] || { + echo "Store config not found: $STORE_CONFIG" + exit 1 + } + [[ $(grep -Ec '^[[:space:]]*min_write_buffer_number_to_merge:' \ + "$STORE_CONFIG") -eq 1 ]] || { + echo "Unable to locate RocksDB config insertion point: $STORE_CONFIG" + exit 1 + } + # Bound file-handle caching for this CI topology without changing + # the released Store default for deployments with different limits. + sed -i \ + -e '/^[[:space:]]*max_open_files:/d' \ + -e '/^[[:space:]]*min_write_buffer_number_to_merge:/a\ max_open_files: 4096' \ + "$STORE_CONFIG" + grep -n 'max_open_files' "$STORE_CONFIG" + + SOFT_OPEN_FILES=$(ulimit -Sn) + echo "[ci] open-file limit before HStore startup:" \ + "soft=$SOFT_OPEN_FILES, hard=$(ulimit -Hn)" + if [[ "$SOFT_OPEN_FILES" != "unlimited" ]] && + (( SOFT_OPEN_FILES < 65535 )); then + ulimit -Sn 65535 + fi + echo "[ci] open-file limit for HStore:" \ + "soft=$(ulimit -Sn), hard=$(ulimit -Hn)" $TRAVIS_DIR/install-backend.sh $BACKEND - name: Run unit test @@ -449,6 +573,9 @@ jobs: $TRAVIS_DIR/run-unit-test.sh $BACKEND - name: Run core test + # Bound stuck HStore scans so the diagnostic steps below can run + # before the GitHub-hosted job reaches its global execution limit. + timeout-minutes: 45 env: HUGEGRAPH_TEST_TASK_LOG_LEVEL: WARN run: | @@ -468,6 +595,146 @@ jobs: run: | $TRAVIS_DIR/run-tinkerpop-test.sh $BACKEND tinkerpop + + - name: Collect HStore diagnostics on failure or timeout + if: ${{ failure() || cancelled() }} + timeout-minutes: 10 + run: | + set +e + DIAGNOSTICS_DIR=hstore-diagnostics + mkdir -p "$DIAGNOSTICS_DIR/threads" \ + "$DIAGNOSTICS_DIR/surefire" \ + "$DIAGNOSTICS_DIR/services" + exec > >(tee -a "$DIAGNOSTICS_DIR/summary.log") 2>&1 + + VERSION=$(grep -E '^VersionInBash=' \ + hugegraph-commons/hugegraph-common/src/main/resources/version.properties | cut -d'=' -f2-) + [[ "$VERSION" =~ ^[0-9A-Za-z._-]+$ ]] || { + echo "Invalid VersionInBash: $VERSION" + exit 0 + } + PD_DIR=hugegraph-pd/apache-hugegraph-pd-$VERSION + STORE_DIR=hugegraph-store/apache-hugegraph-store-$VERSION + SERVER_DIR=hugegraph-server/apache-hugegraph-server-$VERSION + + echo "::group::HugeGraphStore file descriptors" + STORE_PID_FILE="$STORE_DIR/bin/pid" + if [[ -f "$STORE_PID_FILE" ]]; then + STORE_PID=$(tr -d '[:space:]' < "$STORE_PID_FILE") + if [[ "$STORE_PID" =~ ^[0-9]+$ ]] && + [[ -r "/proc/$STORE_PID/limits" ]] && + [[ -d "/proc/$STORE_PID/fd" ]]; then + cat "/proc/$STORE_PID/limits" | + tee "$DIAGNOSTICS_DIR/services/store-process-limits.txt" || true + + FD_TARGETS="$DIAGNOSTICS_DIR/services/store-fd-targets.txt" + find "/proc/$STORE_PID/fd" -mindepth 1 -maxdepth 1 -type l \ + -printf '%l\n' >"$FD_TARGETS" \ + 2>"$DIAGNOSTICS_DIR/services/store-fd-errors.txt" || true + ls -l "/proc/$STORE_PID/fd" \ + >"$DIAGNOSTICS_DIR/services/store-fds.txt" 2>&1 || true + + STORE_FD_COUNT=$(wc -l < "$FD_TARGETS" | tr -d '[:space:]') + echo "[ci] Store PID $STORE_PID open fd count: $STORE_FD_COUNT" | + tee "$DIAGNOSTICS_DIR/services/store-fd-count.txt" || true + + awk ' + /\.sst/ { types["rocksdb-sst"]++; next } + /^socket:/ { types["socket"]++; next } + /^pipe:/ { types["pipe"]++; next } + /^anon_inode:/ { types["anon-inode"]++; next } + /\/storage\// { types["rocksdb-other"]++; next } + { types["other"]++ } + END { + for (type in types) { + print types[type], type + } + } + ' "$FD_TARGETS" | sort -nr | + tee "$DIAGNOSTICS_DIR/services/store-fd-types.txt" || true + + sort "$FD_TARGETS" | uniq -c | sort -nr \ + >"$DIAGNOSTICS_DIR/services/store-fd-target-counts.txt" || true + echo "[ci] Store top fd targets:" + head -n 50 \ + "$DIAGNOSTICS_DIR/services/store-fd-target-counts.txt" || true + else + echo "[ci] Store PID is unavailable for fd diagnostics: $STORE_PID" + fi + else + echo "[ci] Store pid file not found: $STORE_PID_FILE" + fi + echo "::endgroup::" + + echo "::group::Runner process snapshot" + ps -eo pid,ppid,stat,etime,%cpu,%mem,args --sort=pid | + tee "$DIAGNOSTICS_DIR/processes.txt" || true + (ss -ltnp || netstat -ltnp || true) 2>&1 | + tee "$DIAGNOSTICS_DIR/listening-ports.txt" || true + echo "::endgroup::" + + echo "::group::Java thread dumps" + if command -v jcmd >/dev/null 2>&1; then + jcmd -l | tee "$DIAGNOSTICS_DIR/jcmd-list.txt" || true + while read -r pid _; do + [[ "$pid" =~ ^[0-9]+$ ]] || continue + kill -0 "$pid" >/dev/null 2>&1 || continue + thread_dump="$DIAGNOSTICS_DIR/threads/java-$pid.txt" + timeout 30s jcmd "$pid" Thread.print -l >"$thread_dump" 2>&1 + if [[ $? -ne 0 ]] && command -v jstack >/dev/null 2>&1; then + timeout 30s jstack -l "$pid" >"$thread_dump" 2>&1 || true + fi + echo "[ci] captured Java thread dump: $thread_dump" + done < "$DIAGNOSTICS_DIR/jcmd-list.txt" + else + echo "[ci] jcmd is unavailable" + fi + echo "::endgroup::" + + bash $TRAVIS_DIR/ci-service-utils.sh dump "$PD_DIR" HugeGraphPD || true + bash $TRAVIS_DIR/ci-service-utils.sh dump "$STORE_DIR" HugeGraphStore || true + bash $TRAVIS_DIR/ci-service-utils.sh dump \ + "$SERVER_DIR" HugeGraphServer || true + + for service in \ + "pd:$PD_DIR/logs" \ + "store:$STORE_DIR/logs" \ + "server:$SERVER_DIR/logs"; do + service_name="${service%%:*}" + log_dir="${service#*:}" + if [[ -d "$log_dir" ]]; then + mkdir -p "$DIAGNOSTICS_DIR/services/$service_name" + cp -a "$log_dir/." \ + "$DIAGNOSTICS_DIR/services/$service_name/" || true + fi + done + + echo "::group::Surefire reports" + while read -r report; do + destination="$DIAGNOSTICS_DIR/surefire/${report#./}" + mkdir -p "$(dirname "$destination")" + cp "$report" "$destination" || true + echo "--- tail -n 200 $report ---" + tail -n 200 "$report" || true + done < <( + find . -path "./$DIAGNOSTICS_DIR" -prune -o \ + -path '*/surefire-reports/*' -type f \ + \( -name '*.txt' -o -name '*.xml' -o \ + -name '*.dump' -o -name '*.dumpstream' \) \ + -print | sort + ) + echo "::endgroup::" + exit 0 + + - name: Upload HStore diagnostics + if: ${{ failure() || cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: hstore-diagnostics-${{ github.run_id }}-${{ github.run_attempt }} + path: hstore-diagnostics + if-no-files-found: warn + retention-days: 14 + - name: Upload coverage to Codecov uses: codecov/codecov-action@v7 with: diff --git a/.github/workflows/riscv64-ci.yml b/.github/workflows/riscv64-ci.yml index 51d161a978..9812ecfa99 100644 --- a/.github/workflows/riscv64-ci.yml +++ b/.github/workflows/riscv64-ci.yml @@ -28,20 +28,23 @@ env: tonistiigi/binfmt@sha256:400a4873b838d1b89194d982c45e5fb3cda4593fbfd7e08a02e76b03b21166f0 RISCV64_BASE_IMAGE: >- ubuntu@sha256:4edded5722eb644868b7b976033d241d2ab3fff0a170924df69b200a59a2b994 - DRAGONWELL_RISCV64_ARCHIVE: >- - Alibaba_Dragonwell_Extended_11.0.31.28.11_riscv64_linux.tar.gz - DRAGONWELL_RISCV64_SHA256: >- - 7df2d308f0dca7a779d2854e6da19214f99cd77aa6d4982c3bf981a77266a79b - # This verified vendor artifact stays separate from setup-java's project runtime. - # A RISC-V runtime upgrade must update the archive, URL and checksum together. - DRAGONWELL_RISCV64_URL: >- - https://github.com/dragonwell-project/dragonwell11/releases/download/dragonwell-extended-11.0.31.28_jdk-11.0.31-ga/Alibaba_Dragonwell_Extended_11.0.31.28.11_riscv64_linux.tar.gz + TEMURIN_RISCV64_ARCHIVE: >- + OpenJDK17U-jdk_riscv64_linux_hotspot_17.0.20_8.tar.gz + TEMURIN_RISCV64_SHA256: >- + bc36e8044c88df9f4ec2967c5277d1c8fae572378c12c0bac44dcae15d3af2f7 + TEMURIN_RISCV64_BASE_URL: >- + https://github.com/adoptium/temurin17-binaries/releases/download + TEMURIN_RISCV64_RELEASE: jdk-17.0.20%2B8 + EXPECTED_JAVA_MAJOR: '17' + EXPECTED_RISCV64_JAVA_VERSION: '17.0.20' + EXPECTED_RISCV64_JAVA_VENDOR: Eclipse Adoptium RISCV64_CONTAINER: >- hugegraph-riscv64-ci-${{ github.run_id }}-${{ github.run_attempt }} jobs: build-server-riscv64: runs-on: ubuntu-24.04 + continue-on-error: true timeout-minutes: 90 steps: @@ -57,24 +60,28 @@ jobs: --name "$RISCV64_CONTAINER" "$RISCV64_BASE_IMAGE" sleep infinity test "$(docker exec "$RISCV64_CONTAINER" uname -m)" = riscv64 - - name: Prepare source and Java 11 + - name: Prepare source and Java 17 run: | - ARCHIVE_PATH="$RUNNER_TEMP/$DRAGONWELL_RISCV64_ARCHIVE" + ARCHIVE_PATH="$RUNNER_TEMP/$TEMURIN_RISCV64_ARCHIVE" curl --fail --location --retry 3 --show-error \ - "$DRAGONWELL_RISCV64_URL" --output "$ARCHIVE_PATH" - echo "$DRAGONWELL_RISCV64_SHA256 $ARCHIVE_PATH" | sha256sum -c - + "$TEMURIN_RISCV64_BASE_URL/$TEMURIN_RISCV64_RELEASE/$TEMURIN_RISCV64_ARCHIVE" \ + --output "$ARCHIVE_PATH" + echo "$TEMURIN_RISCV64_SHA256 $ARCHIVE_PATH" | sha256sum -c - - docker exec "$RISCV64_CONTAINER" mkdir -p /workspace /opt/dragonwell + docker exec "$RISCV64_CONTAINER" mkdir -p /workspace /opt/temurin git ls-files -z | tar --null --files-from=- -cf - | \ docker exec -i "$RISCV64_CONTAINER" tar -xf - -C /workspace docker cp "$ARCHIVE_PATH" \ - "$RISCV64_CONTAINER:/tmp/$DRAGONWELL_RISCV64_ARCHIVE" + "$RISCV64_CONTAINER:/tmp/$TEMURIN_RISCV64_ARCHIVE" - name: Build and smoke test on RISC-V run: | docker exec \ - --env DRAGONWELL_RISCV64_ARCHIVE="$DRAGONWELL_RISCV64_ARCHIVE" \ - --env DRAGONWELL_RISCV64_SHA256="$DRAGONWELL_RISCV64_SHA256" \ + --env TEMURIN_RISCV64_ARCHIVE="$TEMURIN_RISCV64_ARCHIVE" \ + --env TEMURIN_RISCV64_SHA256="$TEMURIN_RISCV64_SHA256" \ + --env EXPECTED_JAVA_MAJOR="$EXPECTED_JAVA_MAJOR" \ + --env EXPECTED_RISCV64_JAVA_VERSION="$EXPECTED_RISCV64_JAVA_VERSION" \ + --env EXPECTED_RISCV64_JAVA_VENDOR="$EXPECTED_RISCV64_JAVA_VENDOR" \ "$RISCV64_CONTAINER" bash -euo pipefail -c ' test "$(uname -m)" = riscv64 apt-get -q update @@ -83,11 +90,11 @@ jobs: lsof maven procps \ protobuf-compiler protobuf-compiler-grpc-java-plugin - ARCHIVE_PATH="/tmp/$DRAGONWELL_RISCV64_ARCHIVE" - echo "$DRAGONWELL_RISCV64_SHA256 $ARCHIVE_PATH" | sha256sum -c - - tar -xzf "$ARCHIVE_PATH" --strip-components=1 -C /opt/dragonwell + ARCHIVE_PATH="/tmp/$TEMURIN_RISCV64_ARCHIVE" + echo "$TEMURIN_RISCV64_SHA256 $ARCHIVE_PATH" | sha256sum -c - + tar -xzf "$ARCHIVE_PATH" --strip-components=1 -C /opt/temurin rm "$ARCHIVE_PATH" - export JAVA_HOME=/opt/dragonwell + export JAVA_HOME=/opt/temurin export PATH="$JAVA_HOME/bin:$PATH" java -XshowSettings:vm -version diff --git a/.github/workflows/server-ci.yml b/.github/workflows/server-ci.yml index ef47e8f294..3c0e3f8492 100644 --- a/.github/workflows/server-ci.yml +++ b/.github/workflows/server-ci.yml @@ -8,6 +8,11 @@ on: - 'test-*' pull_request: +# Keep the newest PR run; preserve independent push, release and manual runs. +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: java-runtime: uses: ./.github/workflows/java-runtime.yml @@ -27,6 +32,9 @@ jobs: - name: Run wait-storage.sh peer failover tests run: hugegraph-server/hugegraph-dist/src/assembly/travis/test-wait-storage.sh + - name: Run RISC-V Java runtime contract tests + run: hugegraph-server/hugegraph-dist/src/assembly/travis/test-riscv64-java-runtime.sh + # Required coverage: memory on the project runtime and any additional test JDKs. # The shared runtime comes from .github/workflows/.java-version, independent of bytecode targets. # Keep the job ID for the Codecov configuration validator. @@ -45,7 +53,7 @@ jobs: BASE_BRANCH_NAME: ${{ github.base_ref }} TARGET_BRANCH_NAME: ${{ github.base_ref != '' && github.base_ref || github.ref_name }} RELEASE_BRANCH: ${{ startsWith(github.ref_name, 'release-') || startsWith(github.ref_name, 'test-') }} - RAFT_MODE: ${{ startsWith(github.head_ref, 'test') || startsWith(github.head_ref, 'raft') }} + RAFT_MODE: ${{ startsWith(github.ref_name, 'raft-') || startsWith(github.head_ref, 'raft-') }} strategy: fail-fast: false @@ -114,7 +122,34 @@ jobs: mvn package -Dmaven.test.skip=true -pl hugegraph-server/hugegraph-dist -am -ntp VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) SERVER_DIR=hugegraph-server/apache-hugegraph-server-$VERSION/ - $TRAVIS_DIR/test-java-security-properties.sh $SERVER_DIR + $TRAVIS_DIR/test-java-security-properties.sh \ + "$SERVER_DIR" "$GITHUB_WORKSPACE" + + - name: Verify Surefire single-class routing + if: ${{ env.BACKEND == 'rocksdb' }} + run: | + SUREFIRE_DIR=hugegraph-server/hugegraph-test/target/surefire-reports + REPORT="$SUREFIRE_DIR/TEST-org.apache.hugegraph.unit.auth.HugeGraphAuthProxyTest.xml" + rm -f "$REPORT" + mvn test -pl hugegraph-server/hugegraph-test -am \ + -P unit-test -Dtest=HugeGraphAuthProxyTest -ntp + test -s "$REPORT" || { + echo "::error::Server single-class test produced no report" + exit 1 + } + NEGATIVE_LOG=$(mktemp) + trap 'rm -f "$NEGATIVE_LOG"' EXIT + if mvn test -pl hugegraph-server/hugegraph-test -am \ + -P unit-test -Dtest=NoSuchJava17ContractTest -ntp \ + >"$NEGATIVE_LOG" 2>&1; then + cat "$NEGATIVE_LOG" + echo "::error::Surefire accepted a missing Server test class" + exit 1 + fi + cat "$NEGATIVE_LOG" + grep -Fq 'No tests matching pattern "NoSuchJava17ContractTest"' \ + "$NEGATIVE_LOG" + grep -Fq '(unit-test) @ hugegraph-test' "$NEGATIVE_LOG" - name: Check startup test prerequisites id: server-preflight @@ -142,6 +177,8 @@ jobs: run: | VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) SERVER_DIR=hugegraph-server/apache-hugegraph-server-$VERSION/ + bash $TRAVIS_DIR/test-ci-service-utils.sh "$TRAVIS_DIR/ci-service-utils.sh" + bash $TRAVIS_DIR/test-start-hugegraph-signal.sh "$SERVER_DIR/bin/start-hugegraph.sh" $TRAVIS_DIR/test-start-hugegraph.sh $SERVER_DIR - name: Startup tests skipped (missing prerequisites) @@ -168,6 +205,7 @@ jobs: # TODO: disable raft test in normal PR due to the always timeout problem - name: Run raft test if: ${{ env.RAFT_MODE == 'true' && env.BACKEND == 'rocksdb' }} + timeout-minutes: 45 run: | $TRAVIS_DIR/run-api-test-for-raft.sh $BACKEND $REPORT_DIR @@ -304,6 +342,8 @@ jobs: fi build-server-riscv64: + # Keep the emulated Java 17 RISC-V native-runtime signal visible while the + # platform lane stabilizes, but do not use it as a gate yet. uses: ./.github/workflows/riscv64-ci.yml permissions: contents: read diff --git a/.serena/memories/code_style_and_conventions.md b/.serena/memories/code_style_and_conventions.md index 701757a1b6..ce3809cdf4 100644 --- a/.serena/memories/code_style_and_conventions.md +++ b/.serena/memories/code_style_and_conventions.md @@ -24,5 +24,5 @@ - Validate: `mvn apache-rat:check -ntp` + `mvn editorconfig:check` ## Build -- Java 11 target except Commons (Java 8 target); compiler/dependency versions come from module POMs. +- Java 17 release target, `-Xlint:unchecked`, Lombok 1.18.30 (provided/optional) - Swagger: `io.swagger.core.v3:swagger-jaxrs2-jakarta` for REST API docs diff --git a/.serena/memories/implementation_patterns_and_guidelines.md b/.serena/memories/implementation_patterns_and_guidelines.md index 26d19ac26f..c5518b8cca 100644 --- a/.serena/memories/implementation_patterns_and_guidelines.md +++ b/.serena/memories/implementation_patterns_and_guidelines.md @@ -35,7 +35,7 @@ - **Profiles**: `unit-test`, `core-test`, `api-test`, `tinkerpop-structure-test`, `tinkerpop-process-test` - **Backends in CI**: memory, rocksdb, hbase (matrix) - **Single test class**: `mvn test -pl hugegraph-server/hugegraph-test -am -P core-test,memory -Dtest=ClassName` -- TinkerPop tests: only on `release-*`/`test-*` branches +- TinkerPop tests in this Java 17 foundation branch: only `release-*` and `test-*` are selected by `server-ci.yml`; the TinkerPop 3.8 upgrade branch adds its own CI condition in the third PR. - Server Raft API tests are branch-gated in `server-ci.yml`; Store raft-core tests run in normal `pd-store-ci.yml` CI. ## Docker @@ -44,6 +44,6 @@ - Container logs: stdout-based ## CI Pipelines -- `server-ci.yml`: compile + unit/core/API tests (memory/rocksdb/hbase × Java 11) +- `server-ci.yml`: compile + unit/core/API tests (memory/rocksdb/hbase × Java 17) - `rerun-ci.yml`: auto-rerun flaky failures (max 2 reruns, 180s delay) - `auto-pr-review.yml`: auto-comment on new PRs diff --git a/.serena/memories/key_file_locations.md b/.serena/memories/key_file_locations.md index ac10c668e8..bde5926a3c 100644 --- a/.serena/memories/key_file_locations.md +++ b/.serena/memories/key_file_locations.md @@ -29,7 +29,7 @@ - Dist: `hugegraph-store/hg-store-dist/src/assembly/static/` ## CI Workflows (.github/workflows/) -- `server-ci.yml` — Server tests (matrix: memory/rocksdb/hbase × Java 11) +- `server-ci.yml` — Server tests (matrix: memory/rocksdb/hbase × Java 17) - `pd-store-ci.yml` — PD, Store & HStore tests - `commons-ci.yml` — Commons tests - `cluster-test-ci.yml` — Cluster integration diff --git a/.serena/memories/project_overview.md b/.serena/memories/project_overview.md index a08aea9ec6..d3870668c6 100644 --- a/.serena/memories/project_overview.md +++ b/.serena/memories/project_overview.md @@ -14,8 +14,8 @@ Apache HugeGraph is a fast-speed, highly-scalable graph database supporting 10+ - Integration with Flink/Spark/HDFS ## Technology Stack -- **Language**: Java 11+ (required) -- **Build**: Maven 3.5+ +- **Language**: Java 17 (required; currently supported release) +- **Build**: Maven 3.6.3+ - **Graph Framework**: Apache TinkerPop 3.5.1 - **RPC**: gRPC + Protocol Buffers - **API Docs**: Swagger (io.swagger.core.v3) diff --git a/.serena/memories/task_completion_checklist.md b/.serena/memories/task_completion_checklist.md index 22a12b4db7..58a32fb0e4 100644 --- a/.serena/memories/task_completion_checklist.md +++ b/.serena/memories/task_completion_checklist.md @@ -22,7 +22,7 @@ mvn clean compile -Dmaven.javadoc.skip=true # Compile warnings 4. Run `./install-dist/scripts/dependency/regenerate_known_dependencies.sh` ## 4. CI Awareness -- `server-ci.yml`: memory/rocksdb/hbase × Java 11 +- `server-ci.yml`: memory/rocksdb/hbase × Java 17 - `rerun-ci.yml`: auto-retries flaky failures - `licence-checker.yml`: header validation - Server Raft API tests are branch-gated; Store raft-core and core tests run in normal PD/Store CI. diff --git a/AGENTS.md b/AGENTS.md index a38801710f..e7768ad5eb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -40,7 +40,7 @@ Commons: shared utilities; independent compiler and test settings ## Build and validation -Commands run from the repository root. Java 11+ and Maven 3.5+; +Commands run from the repository root. Java 17 and Maven 3.6.3+; versions come from [pom.xml](pom.xml), including `${revision}`. ```bash diff --git a/README.md b/README.md index bd9895f2ef..36c8783700 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ HugeGraph supports both **standalone** and **distributed** deployments: │ HugeGraph Server (:8080) │ │ ┌──────────┐ ┌──────────┐ ┌──────────────────┐ │ │ │ REST API │ │ Gremlin │ │ Cypher Engine │ │ - │ │(Jersey 3)│ │ (TP 3.5) │ │ (OpenCypher) │ │ + │ │(Jersey 3)│ │(TP 3.5.1)│ │ (OpenCypher) │ │ │ └────┬─────┘ └────┬─────┘ └────────┬─────────┘ │ │ └─────────────┼─────────────────┘ │ │ ┌────────▼────────┐ │ @@ -125,7 +125,7 @@ flowchart TB subgraph Server["HugeGraph Server :8080"] API[REST API
Jersey 3] - GS[Gremlin Server
TinkerPop 3.5] + GS[Gremlin Server
TinkerPop 3.5.1] CS[Cypher Engine
OpenCypher] CORE[Graph Engine
hugegraph-core] @@ -176,8 +176,12 @@ curl -X POST http://localhost:8080/gremlin \ ### Prerequisites -- **Java 11+** (required) -- **Maven 3.5+** (for building from source) +- **Java 17** (required and currently the only supported Java release) +- **Maven 3.6.3+** (for building from source) + +The launch scripts reject Java versions older than 17. That minimum-version +check does not qualify later Java releases; use Java 17 unless another release +is explicitly listed as supported. ### Option 1: Docker (Fastest) @@ -336,7 +340,7 @@ For repository constraints and module guidance, see [AGENTS.md](AGENTS.md). - Review the [Architecture Diagram](#architecture) above 2. **Set Up Your Environment** - - Install Java 11+ and Maven 3.5+ + - Install Java 17 and Maven 3.6.3+ - Follow [BUILDING.md](docs/BUILDING.md) for build instructions - Configure your IDE to use `.editorconfig` for code style and `style/checkstyle.xml` for Checkstyle rules diff --git a/docs/BUILDING.md b/docs/BUILDING.md index d4c807c748..7725868752 100644 --- a/docs/BUILDING.md +++ b/docs/BUILDING.md @@ -3,8 +3,17 @@ Building HugeGraph Required: -* Java 11 -* Maven 3.5+ +* Java 17 (currently the only supported Java release) +* Maven 3.6.3+ + +The launch scripts enforce Java 17 as the minimum runtime. This check does not +qualify later Java releases; build HugeGraph with Java 17 unless another +release is explicitly listed as supported. + +The published `hugegraph-common` and `hugegraph-rpc` libraries use Java 11 +bytecode for downstream compatibility. Their build and tests still run on +Java 17; this library bytecode target does not lower HugeGraph's runtime +requirement. To build without executing tests: `mvn clean package -Dmaven.test.skip=true` @@ -39,3 +48,6 @@ To find the Java binary in your environment, run the appropriate command for you * Linux/macOS: `which java` * Windows: `for %i in (java.exe) do @echo. %~$PATH:i` +The Java 17 baseline retains TinkerPop 3.5.1 and uses Groovy 2.5.23 as an +intermediate compatibility update. Groovy 2.5.14 cannot read Java 17 class files. +The subsequent TinkerPop upgrade replaces this compatibility pin with Groovy 4. diff --git a/hugegraph-cluster-test/hugegraph-clustertest-dist/src/assembly/static/conf/rest-server.properties.template b/hugegraph-cluster-test/hugegraph-clustertest-dist/src/assembly/static/conf/rest-server.properties.template index 43979ec120..a16f69fe36 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-dist/src/assembly/static/conf/rest-server.properties.template +++ b/hugegraph-cluster-test/hugegraph-clustertest-dist/src/assembly/static/conf/rest-server.properties.template @@ -19,7 +19,7 @@ # could use '0.0.0.0' or specified (real)IP to expose external network access restserver.url=http://$REST_SERVER_ADDRESS$ # gremlin server url, need to be consistent with host and port in gremlin-server.yaml -#gremlinserver.url=http://$REST_SERVER_ADDRESS$ +gremlinserver.url=http://$GREMLIN_SERVER_ADDRESS$ graphs=./conf/graphs @@ -39,6 +39,7 @@ arthas.disabled_commands=jad auth.authenticator=org.apache.hugegraph.auth.StandardAuthenticator # true if using hstore mode usePD=true +pd.peers=$PD_PEERS_LIST$ # default password auth.admin_pa=pa diff --git a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/pom.xml b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/pom.xml index b59648304f..e6b364651a 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/pom.xml +++ b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/pom.xml @@ -30,8 +30,6 @@ - 11 - 11 UTF-8 diff --git a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/base/ClusterConstant.java b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/base/ClusterConstant.java index 730bbc53ed..e1bad5bf36 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/base/ClusterConstant.java +++ b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/base/ClusterConstant.java @@ -49,6 +49,7 @@ public class ClusterConstant { public static final String GRAPH_TEMPLATE_FILE = "hugegraph.properties.template"; public static final String GREMLIN_DRIVER_SETTING_FILE = "gremlin-driver-settings.yaml"; public static final String GREMLIN_SERVER_FILE = "gremlin-server.yaml"; + public static final String JVM_MODULE_OPTIONS_FILE = "jvm-module.options"; public static final String REMOTE_SETTING_FILE = "remote.yaml"; public static final String REMOTE_OBJECTS_SETTING_FILE = "remote-objects.yaml"; public static final String EMPTY_SAMPLE_GROOVY_FILE = "scripts/empty-sample.groovy"; @@ -106,7 +107,7 @@ public static String getFileInDir(String path, String fileName) { return ""; } - public static boolean isJava11OrHigher() { + public static boolean isJava17OrHigher() { String version = System.getProperty("java.version"); if (version.startsWith("1.")) { version = version.substring(2, 3); @@ -117,7 +118,7 @@ public static boolean isJava11OrHigher() { } } int versionNumber = Integer.parseInt(version); - return versionNumber >= 11; + return versionNumber >= 17; } public static String getProjectDir() { diff --git a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/base/EnvUtil.java b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/base/EnvUtil.java index 4d4bab3831..1c5f4c1357 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/base/EnvUtil.java +++ b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/base/EnvUtil.java @@ -18,7 +18,9 @@ package org.apache.hugegraph.ct.base; import java.io.IOException; +import java.net.InetSocketAddress; import java.net.ServerSocket; +import java.net.Socket; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; @@ -30,6 +32,7 @@ public class EnvUtil { private static final Logger LOG = HGTestLogger.UTIL_LOG; + private static final int PORT_CONNECT_TIMEOUT_MILLIS = 500; private static final Set ports = new HashSet<>(); public static int getAvailablePort() { @@ -48,6 +51,16 @@ public static int getAvailablePort() { } } + public static boolean isPortOpen(String host, int port) { + try (Socket socket = new Socket()) { + socket.connect(new InetSocketAddress(host, port), + PORT_CONNECT_TIMEOUT_MILLIS); + return true; + } catch (IOException ignored) { + return false; + } + } + public static void copyFileToDestination(Path source, Path destination) { try { ensureParentDirectoryExists(destination); diff --git a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/config/ClusterConfig.java b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/config/ClusterConfig.java index c71e4b07e1..6e1e9881f0 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/config/ClusterConfig.java +++ b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/config/ClusterConfig.java @@ -59,6 +59,7 @@ public ClusterConfig(int pdCnt, int storeCnt, int serverCnt) { for (int i = 0; i < serverCnt; i++) { ServerConfig serverConfig = new ServerConfig(); + serverConfig.setPDPeersList(pdGrpcList); serverConfigs.add(serverConfig); GraphConfig graphConfig = new GraphConfig(); graphConfig.setPDPeersList(pdGrpcList); diff --git a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/config/GremlinServerConfig.java b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/config/GremlinServerConfig.java new file mode 100644 index 0000000000..eeba926cab --- /dev/null +++ b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/config/GremlinServerConfig.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.ct.config; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.regex.Pattern; + +public final class GremlinServerConfig { + + private static final Pattern HOST_SETTING = + Pattern.compile("^#?host\\s*:.*$"); + private static final Pattern PORT_SETTING = + Pattern.compile("^#?port\\s*:.*$"); + + private GremlinServerConfig() { + throw new IllegalStateException("Utility class"); + } + + public static void update(Path configPath, String host, int port) { + if (port < 0 || port > 65535) { + throw new IllegalArgumentException("Invalid Gremlin port: " + port); + } + + try { + List lines = Files.readAllLines(configPath, + StandardCharsets.UTF_8); + boolean hostUpdated = false; + boolean portUpdated = false; + for (int i = 0; i < lines.size(); i++) { + String line = lines.get(i); + if (HOST_SETTING.matcher(line).matches()) { + lines.set(i, "host: " + host); + hostUpdated = true; + } else if (PORT_SETTING.matcher(line).matches()) { + lines.set(i, "port: " + port); + portUpdated = true; + } + } + + if (!hostUpdated || !portUpdated) { + throw new IllegalStateException( + "Missing host or port setting in " + configPath); + } + Files.write(configPath, lines, StandardCharsets.UTF_8); + } catch (IOException e) { + throw new IllegalStateException( + "Failed to update Gremlin server config " + configPath, e); + } + } +} diff --git a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/config/ServerConfig.java b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/config/ServerConfig.java index 569a11dddf..2840c946b9 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/config/ServerConfig.java +++ b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/config/ServerConfig.java @@ -24,6 +24,7 @@ import static org.apache.hugegraph.ct.base.EnvUtil.getAvailablePort; import java.nio.file.Paths; +import java.util.List; import lombok.Getter; @@ -32,16 +33,24 @@ public class ServerConfig extends AbstractConfig { private final int rpcPort; private final int restPort; + private final int gremlinPort; public ServerConfig() { readTemplate(Paths.get(CONFIG_FILE_PATH + SERVER_TEMPLATE_FILE)); this.fileName = SERVER_PROPERTIES; this.rpcPort = getAvailablePort(); this.restPort = getAvailablePort(); + this.gremlinPort = getAvailablePort(); properties.put("REST_SERVER_ADDRESS", LOCALHOST + ":" + this.restPort); + properties.put("GREMLIN_SERVER_ADDRESS", + LOCALHOST + ":" + this.gremlinPort); properties.put("RPC_PORT", String.valueOf(this.rpcPort)); } + public void setPDPeersList(List pdPeersList) { + setProperty("PD_PEERS_LIST", String.join(",", pdPeersList)); + } + public void setServerID(String serverID) { setProperty("SERVER_ID", serverID); } @@ -51,4 +60,3 @@ public void setRole(String role) { } } - diff --git a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/env/AbstractEnv.java b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/env/AbstractEnv.java index 0c24860929..ccf0582a00 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/env/AbstractEnv.java +++ b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/env/AbstractEnv.java @@ -21,6 +21,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.concurrent.TimeUnit; import org.apache.hugegraph.ct.base.HGTestLogger; import org.apache.hugegraph.ct.config.ClusterConfig; @@ -28,6 +29,7 @@ import org.apache.hugegraph.ct.config.PDConfig; import org.apache.hugegraph.ct.config.ServerConfig; import org.apache.hugegraph.ct.config.StoreConfig; +import org.apache.hugegraph.ct.node.AbstractNodeWrapper; import org.apache.hugegraph.ct.node.PDNodeWrapper; import org.apache.hugegraph.ct.node.ServerNodeWrapper; import org.apache.hugegraph.ct.node.StoreNodeWrapper; @@ -40,6 +42,8 @@ public abstract class AbstractEnv implements BaseEnv { private static final Logger LOG = HGTestLogger.ENV_LOG; + private static final int NODE_START_TIMEOUT_SECONDS = 120; + private static final int NODE_START_POLL_MILLIS = 1000; protected ClusterConfig clusterConfig; protected List pdNodeWrappers; @@ -71,9 +75,12 @@ protected void init(int pdCnt, int storeCnt, int serverCnt) { } for (int i = 0; i < serverCnt; i++) { - ServerNodeWrapper serverNodeWrapper = new ServerNodeWrapper(cluster_id, i); - serverNodeWrappers.add(serverNodeWrapper); ServerConfig serverConfig = clusterConfig.getServerConfig(i); + ServerNodeWrapper serverNodeWrapper = + new ServerNodeWrapper(cluster_id, i, + serverConfig.getRestPort(), + serverConfig.getGremlinPort()); + serverNodeWrappers.add(serverNodeWrapper); serverConfig.setServerID(serverNodeWrapper.getID()); GraphConfig graphConfig = clusterConfig.getGraphConfig(i); if (i == 0) { @@ -88,34 +95,54 @@ protected void init(int pdCnt, int storeCnt, int serverCnt) { public void startCluster() { for (PDNodeWrapper pdNodeWrapper : pdNodeWrappers) { - pdNodeWrapper.start(); - while (!pdNodeWrapper.isStarted()) { - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - } + startNode(pdNodeWrapper); } for (StoreNodeWrapper storeNodeWrapper : storeNodeWrappers) { - storeNodeWrapper.start(); - while (!storeNodeWrapper.isStarted()) { - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - } + startNode(storeNodeWrapper); } for (ServerNodeWrapper serverNodeWrapper : serverNodeWrappers) { - serverNodeWrapper.start(); - while (!serverNodeWrapper.isStarted()) { - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } + startNode(serverNodeWrapper); + } + } + + private void startNode(AbstractNodeWrapper nodeWrapper) { + System.out.printf("[cluster-test] starting %s in %s%n", + nodeWrapper.getID(), nodeWrapper.getNodePath()); + nodeWrapper.start(); + waitUntilStarted(nodeWrapper); + } + + private static void waitUntilStarted(AbstractNodeWrapper nodeWrapper) { + long deadline = System.nanoTime() + + TimeUnit.SECONDS.toNanos(NODE_START_TIMEOUT_SECONDS); + while (System.nanoTime() < deadline) { + if (!nodeWrapper.isAlive()) { + nodeWrapper.dumpLog(); + throw new AssertionError(String.format( + "%s failed to start, process status: %s", + nodeWrapper.getID(), nodeWrapper.processStatus())); + } + if (nodeWrapper.isStarted()) { + System.out.printf("[cluster-test] %s started%n", + nodeWrapper.getID()); + return; } + sleepBeforeRetry(); + } + + nodeWrapper.dumpLog(); + throw new AssertionError(String.format( + "%s did not start within %s seconds, process status: %s", + nodeWrapper.getID(), NODE_START_TIMEOUT_SECONDS, + nodeWrapper.processStatus())); + } + + private static void sleepBeforeRetry() { + try { + TimeUnit.MILLISECONDS.sleep(NODE_START_POLL_MILLIS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError("Interrupted while waiting node start", e); } } @@ -131,6 +158,22 @@ public void stopCluster() { } } + public void dumpClusterStatus() { + System.out.println("===== cluster node diagnostics ====="); + dumpNodeStatus(pdNodeWrappers); + dumpNodeStatus(storeNodeWrappers); + dumpNodeStatus(serverNodeWrappers); + } + + private static void dumpNodeStatus( + List nodeWrappers) { + for (AbstractNodeWrapper nodeWrapper : nodeWrappers) { + System.out.printf("[cluster-test] %s process status: %s%n", + nodeWrapper.getID(), nodeWrapper.processStatus()); + nodeWrapper.dumpLog(); + } + } + public ClusterConfig getConf() { return this.clusterConfig; } diff --git a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/env/BaseEnv.java b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/env/BaseEnv.java index f6c4ba5fb6..422901959b 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/env/BaseEnv.java +++ b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/env/BaseEnv.java @@ -29,6 +29,8 @@ public interface BaseEnv { /* clear the cluster env and all config*/ void stopCluster(); + void dumpClusterStatus(); + ClusterConfig getConf(); void init(); diff --git a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/AbstractNodeWrapper.java b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/AbstractNodeWrapper.java index 8236bb1392..81d3a73923 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/AbstractNodeWrapper.java +++ b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/AbstractNodeWrapper.java @@ -20,8 +20,6 @@ import static org.apache.hugegraph.ct.base.ClusterConstant.CT_PACKAGE_PATH; import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileReader; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -147,12 +145,17 @@ public void updateConfigPath(String ConfigPath) { @Override public boolean isStarted() { - try (Scanner sc = new Scanner(new FileReader(getLogPath()))) { + if (!isAlive()) { + return false; + } + + try (Scanner sc = new Scanner(Paths.get(getLogPath()), + StandardCharsets.UTF_8.name())) { while (sc.hasNextLine()) { String line = sc.nextLine(); if (line.contains(startLine)) return true; } - } catch (FileNotFoundException ignored) { + } catch (IOException ignored) { } return false; } @@ -161,6 +164,13 @@ public void stop() { if (this.instance == null) { return; } + if (!this.instance.isAlive()) { + System.out.printf("[cluster-test] %s stopped unexpectedly: %s%n", + getID(), processStatus()); + dumpLog(); + deleteDir(); + return; + } this.instance.destroy(); try { if (!this.instance.waitFor(20, TimeUnit.SECONDS)) { @@ -174,7 +184,36 @@ public void stop() { } public boolean isAlive() { - return this.instance.isAlive(); + return this.instance != null && this.instance.isAlive(); + } + + public String processStatus() { + if (this.instance == null) { + return "not started"; + } + if (this.instance.isAlive()) { + return "alive"; + } + return "exited with code " + this.instance.exitValue(); + } + + public void dumpLog() { + Path logPath = Paths.get(getLogPath()); + System.out.println("===== " + getID() + " log: " + logPath + " ====="); + if (!Files.exists(logPath)) { + System.out.println("Log file does not exist"); + return; + } + + try { + List lines = Files.readAllLines(logPath, StandardCharsets.UTF_8); + int start = Math.max(0, lines.size() - 200); + for (int i = start; i < lines.size(); i++) { + System.out.println(lines.get(i)); + } + } catch (IOException e) { + System.out.println("Failed to read log file: " + e.getMessage()); + } } protected ProcessBuilder runCmd(List startCmd, File stdoutFile) throws IOException { diff --git a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/PDNodeWrapper.java b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/PDNodeWrapper.java index a89c614c4c..e7a9e0f24f 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/PDNodeWrapper.java +++ b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/PDNodeWrapper.java @@ -24,7 +24,7 @@ import static org.apache.hugegraph.ct.base.ClusterConstant.PD_LIB_PATH; import static org.apache.hugegraph.ct.base.ClusterConstant.PD_TEMPLATE_PATH; import static org.apache.hugegraph.ct.base.ClusterConstant.getFileInDir; -import static org.apache.hugegraph.ct.base.ClusterConstant.isJava11OrHigher; +import static org.apache.hugegraph.ct.base.ClusterConstant.isJava17OrHigher; import java.io.File; import java.io.IOException; @@ -62,8 +62,8 @@ public void start() { File stdoutFile = new File(getLogPath()); List startCmd = new ArrayList<>(); startCmd.add(JAVA_CMD); - if (!isJava11OrHigher()) { - LOG.error("Please make sure that the JDK is installed and the version >= 11"); + if (!isJava17OrHigher()) { + LOG.error("Please make sure that the JDK is installed and the version >= 17"); return; } diff --git a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/ServerNodeWrapper.java b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/ServerNodeWrapper.java index e16b96781e..332615b8e2 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/ServerNodeWrapper.java +++ b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/ServerNodeWrapper.java @@ -17,6 +17,7 @@ package org.apache.hugegraph.ct.node; +import static org.apache.hugegraph.ct.base.ClusterConstant.BIN_DIR; import static org.apache.hugegraph.ct.base.ClusterConstant.CONF_DIR; import static org.apache.hugegraph.ct.base.ClusterConstant.EMPTY_SAMPLE_GROOVY_FILE; import static org.apache.hugegraph.ct.base.ClusterConstant.EXAMPLE_GROOVY_FILE; @@ -24,15 +25,18 @@ import static org.apache.hugegraph.ct.base.ClusterConstant.GREMLIN_DRIVER_SETTING_FILE; import static org.apache.hugegraph.ct.base.ClusterConstant.GREMLIN_SERVER_FILE; import static org.apache.hugegraph.ct.base.ClusterConstant.JAVA_CMD; +import static org.apache.hugegraph.ct.base.ClusterConstant.JVM_MODULE_OPTIONS_FILE; import static org.apache.hugegraph.ct.base.ClusterConstant.LIB_DIR; import static org.apache.hugegraph.ct.base.ClusterConstant.LOG4J_FILE; +import static org.apache.hugegraph.ct.base.ClusterConstant.LOCALHOST; import static org.apache.hugegraph.ct.base.ClusterConstant.PLUGINS_DIR; import static org.apache.hugegraph.ct.base.ClusterConstant.REMOTE_OBJECTS_SETTING_FILE; import static org.apache.hugegraph.ct.base.ClusterConstant.REMOTE_SETTING_FILE; import static org.apache.hugegraph.ct.base.ClusterConstant.SERVER_LIB_PATH; import static org.apache.hugegraph.ct.base.ClusterConstant.SERVER_PACKAGE_PATH; import static org.apache.hugegraph.ct.base.ClusterConstant.SERVER_TEMPLATE_PATH; -import static org.apache.hugegraph.ct.base.ClusterConstant.isJava11OrHigher; +import static org.apache.hugegraph.ct.base.ClusterConstant.isJava17OrHigher; +import static org.apache.hugegraph.ct.base.EnvUtil.isPortOpen; import java.io.BufferedReader; import java.io.File; @@ -46,16 +50,37 @@ import java.util.Collections; import java.util.List; +import org.apache.hugegraph.ct.config.GremlinServerConfig; + public class ServerNodeWrapper extends AbstractNodeWrapper { private static List hgJars = loadHgJarsOnce(); + private final int restPort; + private final int gremlinPort; + public ServerNodeWrapper(int clusterIndex, int index) { + this(clusterIndex, index, -1, -1); + } + + public ServerNodeWrapper(int clusterIndex, int index, int restPort) { + this(clusterIndex, index, restPort, -1); + } + + public ServerNodeWrapper(int clusterIndex, int index, int restPort, + int gremlinPort) { super(clusterIndex, index); + this.restPort = restPort; + this.gremlinPort = gremlinPort; this.fileNames = new ArrayList<>( List.of(LOG4J_FILE, GREMLIN_SERVER_FILE, GREMLIN_DRIVER_SETTING_FILE, REMOTE_SETTING_FILE, REMOTE_OBJECTS_SETTING_FILE)); this.workPath = SERVER_LIB_PATH; createNodeDir(Paths.get(SERVER_TEMPLATE_PATH), getNodePath() + CONF_DIR + File.separator); + if (this.gremlinPort >= 0) { + GremlinServerConfig.update( + Paths.get(getNodePath(), CONF_DIR, GREMLIN_SERVER_FILE), + LOCALHOST, this.gremlinPort); + } this.fileNames = new ArrayList<>(List.of(EMPTY_SAMPLE_GROOVY_FILE, EXAMPLE_GROOVY_FILE)); this.startLine = "INFO: [HttpServer] Started."; createNodeDir(Paths.get(SERVER_PACKAGE_PATH), getNodePath()); @@ -111,8 +136,8 @@ public void start() { File stdoutFile = new File(getLogPath()); List startCmd = new ArrayList<>(); startCmd.add(JAVA_CMD); - if (!isJava11OrHigher()) { - LOG.error("Please make sure that the JDK is installed and the version >= 11"); + if (!isJava17OrHigher()) { + LOG.error("Please make sure that the JDK is installed and the version >= 17"); return; } @@ -124,9 +149,8 @@ public void start() { startCmd.addAll(Arrays.asList( "-Dname=HugeGraphServer" + this.index, - "--add-exports=java.base/jdk.internal.reflect=ALL-UNNAMED", - "--add-modules=jdk.unsupported", - "--add-exports=java.base/sun.nio.ch=ALL-UNNAMED", + "@" + Paths.get(SERVER_PACKAGE_PATH, BIN_DIR, + JVM_MODULE_OPTIONS_FILE), "-cp", storeClassPath, "org.apache.hugegraph.dist.HugeGraphServer", "./conf/gremlin-server.yaml", @@ -138,6 +162,14 @@ public void start() { } } + @Override + public boolean isStarted() { + return super.isStarted() && + (this.restPort < 0 || isPortOpen(LOCALHOST, this.restPort)) && + (this.gremlinPort < 0 || + isPortOpen(LOCALHOST, this.gremlinPort)); + } + @Override public String getID() { return "Server" + this.index; diff --git a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/StoreNodeWrapper.java b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/StoreNodeWrapper.java index 1cb0f67eae..d76ceaa0b8 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/StoreNodeWrapper.java +++ b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/StoreNodeWrapper.java @@ -24,7 +24,7 @@ import static org.apache.hugegraph.ct.base.ClusterConstant.STORE_LIB_PATH; import static org.apache.hugegraph.ct.base.ClusterConstant.STORE_TEMPLATE_PATH; import static org.apache.hugegraph.ct.base.ClusterConstant.getFileInDir; -import static org.apache.hugegraph.ct.base.ClusterConstant.isJava11OrHigher; +import static org.apache.hugegraph.ct.base.ClusterConstant.isJava17OrHigher; import java.io.File; import java.io.IOException; @@ -59,8 +59,8 @@ public void start() { File stdoutFile = new File(getLogPath()); List startCmd = new ArrayList<>(); startCmd.add(JAVA_CMD); - if (!isJava11OrHigher()) { - LOG.error("Please make sure that the JDK is installed and the version >= 11"); + if (!isJava17OrHigher()) { + LOG.error("Please make sure that the JDK is installed and the version >= 17"); return; } diff --git a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/test/java/org/apache/hugegraph/ct/base/EnvUtilTest.java b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/test/java/org/apache/hugegraph/ct/base/EnvUtilTest.java new file mode 100644 index 0000000000..00fc93bf8d --- /dev/null +++ b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/test/java/org/apache/hugegraph/ct/base/EnvUtilTest.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.ct.base; + +import java.net.ServerSocket; + +import org.junit.Assert; +import org.junit.Test; + +public class EnvUtilTest { + + @Test + public void testPortOpen() throws Exception { + int port; + try (ServerSocket socket = new ServerSocket(0)) { + port = socket.getLocalPort(); + Assert.assertTrue(EnvUtil.isPortOpen("127.0.0.1", port)); + } + + Assert.assertFalse(EnvUtil.isPortOpen("127.0.0.1", port)); + } +} diff --git a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/test/java/org/apache/hugegraph/ct/config/GremlinServerConfigTest.java b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/test/java/org/apache/hugegraph/ct/config/GremlinServerConfigTest.java new file mode 100644 index 0000000000..ae051b4d0a --- /dev/null +++ b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/test/java/org/apache/hugegraph/ct/config/GremlinServerConfigTest.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.ct.config; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.List; + +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class GremlinServerConfigTest { + + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void testUpdateCommentedHostAndPort() throws Exception { + File config = temporaryFolder.newFile("gremlin-server.yaml"); + String content = "#host: 127.0.0.1\n" + + "#port: 8182\n" + + "evaluationTimeout: 30000\n"; + Files.write(config.toPath(), content.getBytes(StandardCharsets.UTF_8)); + + GremlinServerConfig.update(config.toPath(), "127.0.0.1", 12345); + + List updated = Files.readAllLines(config.toPath(), + StandardCharsets.UTF_8); + Assert.assertTrue(updated.toString(), + updated.contains("host: 127.0.0.1")); + Assert.assertTrue(updated.toString(), updated.contains("port: 12345")); + Assert.assertFalse(updated.toString(), updated.contains("port: 8182")); + } +} diff --git a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/test/java/org/apache/hugegraph/ct/node/AbstractNodeWrapperTest.java b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/test/java/org/apache/hugegraph/ct/node/AbstractNodeWrapperTest.java new file mode 100644 index 0000000000..2d452f04d3 --- /dev/null +++ b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/test/java/org/apache/hugegraph/ct/node/AbstractNodeWrapperTest.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.ct.node; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.PrintStream; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.concurrent.TimeUnit; + +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class AbstractNodeWrapperTest { + + private static final String START_LINE = "node is ready"; + + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void testStartedRequiresLiveProcess() throws Exception { + TestNodeWrapper wrapper = new TestNodeWrapper(newNodeDirectory()); + Files.write(Paths.get(wrapper.getLogPath()), + START_LINE.getBytes(StandardCharsets.UTF_8)); + + Assert.assertFalse(wrapper.isStarted()); + + Process process = startSleeperProcess(); + wrapper.attach(process); + try { + Assert.assertTrue(process.isAlive()); + Assert.assertTrue(wrapper.isStarted()); + } finally { + process.destroyForcibly(); + process.waitFor(10, TimeUnit.SECONDS); + } + } + + @Test + public void testStopDumpsLogOfUnexpectedlyExitedProcess() throws Exception { + TestNodeWrapper wrapper = new TestNodeWrapper(newNodeDirectory()); + String failure = "fatal startup failure"; + Files.write(Paths.get(wrapper.getLogPath()), + failure.getBytes(StandardCharsets.UTF_8)); + Process process = startExitedProcess(); + wrapper.attach(process); + + PrintStream originalOut = System.out; + ByteArrayOutputStream output = new ByteArrayOutputStream(); + try { + System.setOut(new PrintStream(output, true, + StandardCharsets.UTF_8.name())); + wrapper.stop(); + } finally { + System.setOut(originalOut); + } + + String diagnostics = output.toString(StandardCharsets.UTF_8.name()); + Assert.assertTrue(diagnostics, diagnostics.contains(failure)); + } + + private File newNodeDirectory() throws Exception { + File nodeDirectory = temporaryFolder.newFolder(); + File logDirectory = new File(nodeDirectory, "logs"); + Assert.assertTrue(logDirectory.mkdir()); + return nodeDirectory; + } + + private static Process startSleeperProcess() throws Exception { + String java = javaExecutable(); + String classpath = testClassesPath(); + return new ProcessBuilder(java, "-cp", classpath, + Sleeper.class.getName()).start(); + } + + private static Process startExitedProcess() throws Exception { + Process process = new ProcessBuilder(javaExecutable(), "-version").start(); + Assert.assertTrue(process.waitFor(10, TimeUnit.SECONDS)); + return process; + } + + private static String javaExecutable() { + String executable = System.getProperty("os.name").startsWith("Windows") ? + "java.exe" : "java"; + return Paths.get(System.getProperty("java.home"), "bin", executable) + .toString(); + } + + private static String testClassesPath() throws URISyntaxException { + return Paths.get(AbstractNodeWrapperTest.class.getProtectionDomain() + .getCodeSource() + .getLocation() + .toURI()) + .toString(); + } + + public static class Sleeper { + + public static void main(String[] args) throws Exception { + TimeUnit.SECONDS.sleep(30); + } + } + + private static class TestNodeWrapper extends AbstractNodeWrapper { + + private File nodeDirectory; + + private TestNodeWrapper(File nodeDirectory) { + this.nodeDirectory = nodeDirectory; + this.startLine = START_LINE; + } + + private void attach(Process process) { + this.instance = process; + } + + @Override + public void start() { + throw new UnsupportedOperationException(); + } + + @Override + public String getID() { + return "TestNode"; + } + + @Override + public String getNodePath() { + if (this.nodeDirectory == null) { + return System.getProperty("java.io.tmpdir") + File.separator; + } + return this.nodeDirectory.getAbsolutePath() + File.separator; + } + } +} diff --git a/hugegraph-cluster-test/hugegraph-clustertest-test/pom.xml b/hugegraph-cluster-test/hugegraph-clustertest-test/pom.xml index 735ea66b43..5c5acbfa57 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-test/pom.xml +++ b/hugegraph-cluster-test/hugegraph-clustertest-test/pom.xml @@ -29,8 +29,6 @@ hugegraph-clustertest-test - 11 - 11 UTF-8 @@ -81,7 +79,6 @@ org.apache.maven.plugins maven-surefire-plugin - 2.20 simple-cluster-test diff --git a/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/MultiClusterTest/BaseMultiClusterTest.java b/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/MultiClusterTest/BaseMultiClusterTest.java index 9e90933026..fa5d4501eb 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/MultiClusterTest/BaseMultiClusterTest.java +++ b/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/MultiClusterTest/BaseMultiClusterTest.java @@ -115,8 +115,21 @@ protected static String assertResponseStatus(int status, public static Response createAndAssert(RestClient client, String path, String body, int status) { - Response r = client.post(path, body); - assertResponseStatus(status, r); - return r; + try { + Response r = client.post(path, body); + assertResponseStatus(status, r); + return r; + } catch (RuntimeException | AssertionError e) { + System.out.printf("[cluster-test] POST %s%s failed: %s%n", + client.target().getUri(), path, e.getMessage()); + try { + env.dumpClusterStatus(); + } catch (Throwable diagnosticError) { + if (diagnosticError != e) { + e.addSuppressed(diagnosticError); + } + } + throw e; + } } } diff --git a/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/SimpleClusterTest/BaseSimpleTest.java b/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/SimpleClusterTest/BaseSimpleTest.java index f0f0c33461..d6df82bcfa 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/SimpleClusterTest/BaseSimpleTest.java +++ b/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/SimpleClusterTest/BaseSimpleTest.java @@ -74,7 +74,10 @@ public static void initEnv() { public static void clearEnv() throws InterruptedException { env.stopCluster(); Thread.sleep(2000); - client.close(); + if (client != null) { + client.close(); + client = null; + } } protected String execCmd(String[] cmds) throws IOException { diff --git a/hugegraph-cluster-test/pom.xml b/hugegraph-cluster-test/pom.xml index cd54ac0ffe..fc176e21f8 100644 --- a/hugegraph-cluster-test/pom.xml +++ b/hugegraph-cluster-test/pom.xml @@ -39,12 +39,22 @@ - 11 - 11 UTF-8 apache-${release.name}-ct-${project.version} + + + + + org.apache.commons + commons-text + 1.10.0 + + + + com.google.code.findbugs @@ -96,7 +106,6 @@ org.apache.maven.plugins maven-surefire-plugin - 2.20 simple-cluster-test @@ -120,7 +129,6 @@ org.apache.maven.plugins maven-surefire-plugin - 2.20 multi-cluster-test diff --git a/hugegraph-commons/AGENTS.md b/hugegraph-commons/AGENTS.md index 179e01300c..58f4512c35 100644 --- a/hugegraph-commons/AGENTS.md +++ b/hugegraph-commons/AGENTS.md @@ -6,8 +6,8 @@ Shared rules: [root AGENTS.md](../AGENTS.md). Paths here are relative to this mo - Config, locks, events, iterators and REST: `hugegraph-common/src/main/java/org/apache/hugegraph/`. - RPC framework: `hugegraph-rpc/`, which depends on `hugegraph-common`. -- Commons targets Java 8 bytecode in [pom.xml](pom.xml), although CI uses JDK 11. - Do not infer its language/API baseline from Server's JDK requirement. +- Commons and RPC use Java 11 bytecode while building and testing on JDK 17; + see [pom.xml](pom.xml) and the [root POM](../pom.xml). ## Tests diff --git a/hugegraph-commons/hugegraph-common/build.sh b/hugegraph-commons/hugegraph-common/build.sh index b2cb6211b9..2e4b0bbc20 100644 --- a/hugegraph-commons/hugegraph-common/build.sh +++ b/hugegraph-commons/hugegraph-common/build.sh @@ -16,8 +16,5 @@ # limitations under the License. # -export MAVEN_HOME=/home/scmtools/buildkit/maven/apache-maven-3.3.9/ -export JAVA_HOME=/home/scmtools/buildkit/java/jdk1.8.0_25/ -export PATH=$JAVA_HOME/bin:$MAVEN_HOME/bin:$PATH - -mvn clean test -Dtest=UnitTestSuite +mvn clean test -Dtest=UnitTestSuite \ + -Dsurefire.failIfNoSpecifiedTests=true -DskipCommonsTests=false diff --git a/hugegraph-commons/hugegraph-common/pom.xml b/hugegraph-commons/hugegraph-common/pom.xml index 14f7cc217c..d703a2407d 100644 --- a/hugegraph-commons/hugegraph-common/pom.xml +++ b/hugegraph-commons/hugegraph-common/pom.xml @@ -257,7 +257,6 @@ org.jacoco jacoco-maven-plugin - 0.8.2 pre-unit-test diff --git a/hugegraph-commons/hugegraph-common/src/test/java/org/apache/hugegraph/testutil/AssertTest.java b/hugegraph-commons/hugegraph-common/src/test/java/org/apache/hugegraph/testutil/AssertTest.java index 53f60247e9..21f935aa0a 100644 --- a/hugegraph-commons/hugegraph-common/src/test/java/org/apache/hugegraph/testutil/AssertTest.java +++ b/hugegraph-commons/hugegraph-common/src/test/java/org/apache/hugegraph/testutil/AssertTest.java @@ -422,8 +422,6 @@ public void testAssertContains() { Assert.assertThrows(NullPointerException.class, () -> { Assert.assertContains(null, "null"); - }, e -> { - Assert.assertNull(e.getMessage()); }); } diff --git a/hugegraph-commons/hugegraph-common/src/test/java/org/apache/hugegraph/unit/util/DateUtilTest.java b/hugegraph-commons/hugegraph-common/src/test/java/org/apache/hugegraph/unit/util/DateUtilTest.java index 7c51406b36..6c8214e3fe 100644 --- a/hugegraph-commons/hugegraph-common/src/test/java/org/apache/hugegraph/unit/util/DateUtilTest.java +++ b/hugegraph-commons/hugegraph-common/src/test/java/org/apache/hugegraph/unit/util/DateUtilTest.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import org.junit.Test; @@ -81,9 +82,11 @@ public void testNow() { @Test public void testParseCornerDateValue() throws InterruptedException { final CountDownLatch latch = new CountDownLatch(1); + final Date expected = DateUtil.parse("0", "yyyy"); int threadCount = 10; List threads = new ArrayList<>(threadCount); AtomicInteger errorCount = new AtomicInteger(0); + AtomicReference firstError = new AtomicReference<>(); for (int t = 0; t < threadCount; t++) { Thread thread = new Thread(() -> { try { @@ -92,9 +95,9 @@ public void testParseCornerDateValue() throws InterruptedException { throw new RuntimeException(e); } try { - Assert.assertEquals(new Date(-62167248343000L), - DateUtil.parse("0", "yyyy")); - } catch (Exception e) { + Assert.assertEquals(expected, DateUtil.parse("0", "yyyy")); + } catch (Throwable e) { + firstError.compareAndSet(null, e); errorCount.incrementAndGet(); } }); @@ -109,6 +112,15 @@ public void testParseCornerDateValue() throws InterruptedException { thread.join(); } + Throwable error = firstError.get(); + if (error != null) { + AssertionError assertion = new AssertionError(String.format( + "Expected concurrent parses to match " + + "baseline result, but got %s failures", + errorCount.get())); + assertion.initCause(error); + throw assertion; + } Assert.assertEquals(0, errorCount.get()); } diff --git a/hugegraph-commons/hugegraph-rpc/pom.xml b/hugegraph-commons/hugegraph-rpc/pom.xml index 23f0a32077..822c983cd7 100644 --- a/hugegraph-commons/hugegraph-rpc/pom.xml +++ b/hugegraph-commons/hugegraph-rpc/pom.xml @@ -124,7 +124,6 @@ org.jacoco jacoco-maven-plugin - 0.8.2 pre-unit-test diff --git a/hugegraph-commons/hugegraph-rpc/src/test/java/org/apache/hugegraph/unit/BaseUnitTest.java b/hugegraph-commons/hugegraph-rpc/src/test/java/org/apache/hugegraph/unit/BaseUnitTest.java index 919ba151f0..34912bc76b 100644 --- a/hugegraph-commons/hugegraph-rpc/src/test/java/org/apache/hugegraph/unit/BaseUnitTest.java +++ b/hugegraph-commons/hugegraph-rpc/src/test/java/org/apache/hugegraph/unit/BaseUnitTest.java @@ -17,7 +17,9 @@ package org.apache.hugegraph.unit; +import java.net.URI; import java.net.URL; +import java.nio.file.Paths; import org.junit.BeforeClass; @@ -39,7 +41,7 @@ protected static HugeConfig config(boolean server) { protected static HugeConfig config(String type) { String name = String.format("rpc-%s.properties", type); URL conf = BaseUnitTest.class.getClassLoader().getResource(name); - return new HugeConfig(conf.getPath()); + return new HugeConfig(Paths.get(URI.create(conf.toExternalForm())).toString()); } protected static void startServer(RpcServer rpcServer) { diff --git a/hugegraph-commons/hugegraph-rpc/src/test/java/org/apache/hugegraph/unit/ServerClientTest.java b/hugegraph-commons/hugegraph-rpc/src/test/java/org/apache/hugegraph/unit/ServerClientTest.java index a1c64c0585..591732749a 100644 --- a/hugegraph-commons/hugegraph-rpc/src/test/java/org/apache/hugegraph/unit/ServerClientTest.java +++ b/hugegraph-commons/hugegraph-rpc/src/test/java/org/apache/hugegraph/unit/ServerClientTest.java @@ -558,7 +558,7 @@ public void testServiceProxy() { @Test public void testAddServiceMultiTimesOfSameService() { - RpcServer rpcServerExport = new RpcServer(config(true)); + RpcServer rpcServerExport = new RpcServer(config("server-random")); rpcServerExport.config().addService(HelloService.class, new HelloServiceImpl()); @@ -578,7 +578,7 @@ public void testAddServiceMultiTimesOfSameService() { @Test public void testExportMultiTimesOfSameServer() { - RpcServer rpcServerExport = new RpcServer(config(true)); + RpcServer rpcServerExport = new RpcServer(config("server-random")); rpcServerExport.config().addService(HelloService.class, new HelloServiceImpl()); rpcServerExport.exportAll(); @@ -589,7 +589,7 @@ public void testExportMultiTimesOfSameServer() { @Test public void testExportMultiTimesOfSameService() { - RpcServer rpcServerExport = new RpcServer(config(true)); + RpcServer rpcServerExport = new RpcServer(config("server-random")); rpcServerExport.config().addService(HelloService.class, new HelloServiceImpl()); rpcServerExport.exportAll(); @@ -603,7 +603,7 @@ public void testExportMultiTimesOfSameService() { @Test public void testExportNoneService() { - RpcServer rpcServerNoneService = new RpcServer(config(true)); + RpcServer rpcServerNoneService = new RpcServer(config("server-random")); // Will be ignored if none service added rpcServerNoneService.exportAll(); @@ -611,57 +611,70 @@ public void testExportNoneService() { stopServer(rpcServerNoneService); } + private static RpcClientProvider clientFor(RpcServer server) { + HugeConfig config = config(false); + String remoteUrlKey = org.apache.hugegraph.config.RpcOptions.RPC_REMOTE_URL.name(); + config.setProperty(remoteUrlKey, server.host() + ":" + server.port()); + return new RpcClientProvider(config); + } + @Test - public void testUnexportService() { - RpcServer rpcServerUnexport = new RpcServer(config(true)); + public void testTemporaryServerDoesNotShareMainListener() { + rpcServer.config().addService(HelloService.class, new HelloServiceImpl()); + startServer(rpcServer); + RpcServer temporary = new RpcServer(config("server-random")); + try { + temporary.config().addService(HelloService.class, new HelloServiceImpl()); + startServer(temporary); + Assert.assertNotEquals(rpcServer.port(), temporary.port()); + } finally { + stopServer(temporary); + } + HelloService client = rpcClient.config().serviceProxy(HelloService.class); + Assert.assertEquals("hello tom!", client.hello("tom")); + } + @Test + public void testUnexportService() { + RpcServer rpcServerUnexport = new RpcServer(config("server-random")); RpcProviderConfig serverConfig = rpcServerUnexport.config(); String service = serverConfig.addService(HelloService.class, new HelloServiceImpl()); rpcServerUnexport.exportAll(); - - RpcConsumerConfig clientConfig = rpcClient.config(); - HelloService client = clientConfig.serviceProxy(HelloService.class); - - Assert.assertEquals("hello tom!", client.hello("tom")); - - rpcServerUnexport.unexport(service); - - Assert.assertThrows(SofaRpcException.class, () -> { - client.hello("tom"); - }); - - stopServer(rpcServerUnexport); + RpcClientProvider isolatedClient = clientFor(rpcServerUnexport); + try { + HelloService client = isolatedClient.config().serviceProxy(HelloService.class); + Assert.assertEquals("hello tom!", client.hello("tom")); + rpcServerUnexport.unexport(service); + Assert.assertThrows(SofaRpcException.class, () -> client.hello("tom")); + } finally { + isolatedClient.destroy(); + stopServer(rpcServerUnexport); + } } @Test public void testUnexportAllService() { - RpcServer rpcServerUnexport = new RpcServer(config(true)); - + RpcServer rpcServerUnexport = new RpcServer(config("server-random")); RpcProviderConfig serverConfig = rpcServerUnexport.config(); serverConfig.addService(HelloService.class, new HelloServiceImpl()); serverConfig.addService("graph", HelloService.class, new GraphHelloServiceImpl("graph")); rpcServerUnexport.exportAll(); - - RpcConsumerConfig clientConfig = rpcClient.config(); - HelloService client = clientConfig.serviceProxy(HelloService.class); - HelloService clientG = clientConfig.serviceProxy("graph", - HelloService.class); - - Assert.assertEquals("hello tom!", client.hello("tom")); - Assert.assertEquals("graph: hello tom!", clientG.hello("tom")); - - rpcServerUnexport.unexportAll(); - - Assert.assertThrows(SofaRpcException.class, () -> { - client.hello("tom"); - }); - Assert.assertThrows(SofaRpcException.class, () -> { - clientG.hello("tom"); - }); - - stopServer(rpcServerUnexport); + RpcClientProvider isolatedClient = clientFor(rpcServerUnexport); + try { + RpcConsumerConfig clientConfig = isolatedClient.config(); + HelloService client = clientConfig.serviceProxy(HelloService.class); + HelloService clientG = clientConfig.serviceProxy("graph", HelloService.class); + Assert.assertEquals("hello tom!", client.hello("tom")); + Assert.assertEquals("graph: hello tom!", clientG.hello("tom")); + rpcServerUnexport.unexportAll(); + Assert.assertThrows(SofaRpcException.class, () -> client.hello("tom")); + Assert.assertThrows(SofaRpcException.class, () -> clientG.hello("tom")); + } finally { + isolatedClient.destroy(); + stopServer(rpcServerUnexport); + } } @Test diff --git a/hugegraph-commons/pom.xml b/hugegraph-commons/pom.xml index b9e780bd32..7d0e9cfc98 100644 --- a/hugegraph-commons/pom.xml +++ b/hugegraph-commons/pom.xml @@ -94,14 +94,13 @@ 1.7.0 UTF-8 ${project.basedir}/.. - 1.8 - 1.8 + 11 2.18.0 1.10 - 2.8.0 + 2.10.1 1.9.4 3.2.2 - 3.12.0 + 3.18.0 2.7 1.13 30.0-jre @@ -132,8 +131,6 @@ maven-compiler-plugin - ${compiler.source} - ${compiler.target} 500 @@ -193,7 +190,6 @@ org.jacoco jacoco-maven-plugin - 0.8.7 pre-unit-test @@ -288,9 +284,12 @@ org.apache.maven.plugins maven-surefire-plugin - 2.20 ${skipCommonsTests} + + @{argLine} + "@${project.parent.basedir}/../hugegraph-server/hugegraph-test/conf/jvm-test-module.options" + diff --git a/hugegraph-pd/Dockerfile b/hugegraph-pd/Dockerfile index 655860d407..c7050826ac 100644 --- a/hugegraph-pd/Dockerfile +++ b/hugegraph-pd/Dockerfile @@ -18,7 +18,7 @@ # Dockerfile for HugeGraph PD # 1st stage: build source code -FROM --platform=$BUILDPLATFORM maven:3.9.0-eclipse-temurin-11 AS build +FROM --platform=$BUILDPLATFORM maven:3.9.16-eclipse-temurin-17 AS build WORKDIR /pkg @@ -35,7 +35,7 @@ RUN --mount=type=cache,id=hugegraph-maven-${SOURCE_REVISION},target=/root/.m2,sh # 2nd stage: runtime env # Note: ZGC (The Z Garbage Collector) is only supported on ARM-Mac with java > 13 -FROM eclipse-temurin:11-jre-jammy +FROM eclipse-temurin:17-jre-jammy LABEL maintainer="HugeGraph Docker Maintainers " diff --git a/hugegraph-pd/README.md b/hugegraph-pd/README.md index 5050025565..8f890efd98 100644 --- a/hugegraph-pd/README.md +++ b/hugegraph-pd/README.md @@ -36,8 +36,8 @@ For detailed architecture and design, see [Architecture Documentation](docs/arch ### Prerequisites -- **Java**: 11 or higher -- **Maven**: 3.5 or higher +- **Java**: 17 (currently the only supported release) +- **Maven**: 3.6.3 or higher - **Disk Space**: At least 1GB for PD data directory ### Build diff --git a/hugegraph-pd/docs/architecture.md b/hugegraph-pd/docs/architecture.md index 080189be95..6fd56dd212 100644 --- a/hugegraph-pd/docs/architecture.md +++ b/hugegraph-pd/docs/architecture.md @@ -54,7 +54,7 @@ HugeGraph PD (Placement Driver) is the control plane for HugeGraph distributed d - **Storage**: RocksDB for persistent metadata - **Communication**: gRPC with Protocol Buffers - **Framework**: Spring Boot for REST APIs and dependency injection -- **Language**: Java 11+ +- **Language**: Java 17 (currently supported release) ## Module Architecture diff --git a/hugegraph-pd/docs/development.md b/hugegraph-pd/docs/development.md index 514bd989a1..61ed03f3b5 100644 --- a/hugegraph-pd/docs/development.md +++ b/hugegraph-pd/docs/development.md @@ -18,10 +18,10 @@ This document provides comprehensive guidance for developing, testing, and contr Ensure you have the following tools installed: -| Tool | Minimum Version | Recommended | Purpose | +| Tool | Version Requirement | Recommended | Purpose | |------|----------------|-------------|---------| -| **JDK** | 11 | 11 or 17 | Java runtime and compilation | -| **Maven** | 3.5.0 | 3.8+ | Build tool and dependency management | +| **JDK** | 17 (only supported release) | 17 LTS | Java runtime and compilation | +| **Maven** | 3.6.3+ | 3.8+ | Build tool and dependency management | | **Git** | 2.0+ | Latest | Version control | | **IDE** | N/A | IntelliJ IDEA | Development environment | @@ -30,11 +30,11 @@ Ensure you have the following tools installed: ```bash # Check Java version java -version -# Expected: openjdk version "11.0.x" or later +# Expected: openjdk version "17.0.x" # Check Maven version mvn -version -# Expected: Apache Maven 3.5.0 or later +# Expected: Apache Maven 3.6.3 or later # Check Git version git --version @@ -77,8 +77,8 @@ Required for Lombok support: #### Configure JDK 1. **File → Project Structure → Project** -2. **Project SDK**: Select JDK 11 or 17 -3. **Project language level**: 11 +2. **Project SDK**: Select JDK 17 +3. **Project language level**: 17 4. **Apply** and **OK** ## Building from Source @@ -196,10 +196,10 @@ mvn test -pl hugegraph-pd/hg-pd-test -am -P pd-rest-test ```bash # Run specific test class -mvn -pl hugegraph-pd/hg-pd-test test -Dtest=PartitionServiceTest -DfailIfNoTests=false +mvn -pl hugegraph-pd/hg-pd-test test -Dtest=PartitionServiceTest # Run specific test method -mvn -pl hugegraph-pd/hg-pd-test test -Dtest=PartitionServiceTest#testSplitPartition -DfailIfNoTests=false +mvn -pl hugegraph-pd/hg-pd-test test -Dtest=PartitionServiceTest#testSplitPartition ``` #### Test from IDE @@ -382,7 +382,7 @@ if (store == null) { - **Main class**: `org.apache.hugegraph.pd.HgPdApplication` (in `hg-pd-service`) - **Program arguments**: `--spring.config.location=file:./conf/application.yml` - **Working directory**: `hugegraph-pd/hg-pd-dist/target/hugegraph-pd-/` - - **JRE**: 11 or 17 + - **JRE**: 17 2. Set breakpoints in code diff --git a/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/start-hugegraph-pd.sh b/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/start-hugegraph-pd.sh index 1329df2271..82a92dc9f9 100755 --- a/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/start-hugegraph-pd.sh +++ b/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/start-hugegraph-pd.sh @@ -69,7 +69,7 @@ ensure_path_writable "$PLUGINS" # The maximum and minimum heap memory that service can use MAX_MEM=$((32 * 1024)) MIN_MEM=$((1 * 512)) -EXPECT_JDK_VERSION=11 +EXPECT_JDK_VERSION=17 # Change to $BIN's parent cd "${TOP}" || exit @@ -82,8 +82,11 @@ else fi # check jdk version -JAVA_VERSION=$($JAVA -version 2>&1 | awk 'NR==1{gsub(/"/,""); print $3}' | awk -F'_' '{print $1}') -if [[ $? -ne 0 || $JAVA_VERSION < $EXPECT_JDK_VERSION ]]; then +JAVA_VERSION=$($JAVA -version 2>&1 | + awk -F'"' '/^(java|openjdk) version "/ {print $2; exit}' | + sed 's/^1\.//' | cut -d'.' -f1) +JAVA_VERSION="${JAVA_VERSION%%[!0-9]*}" +if [[ -z $JAVA_VERSION || $JAVA_VERSION -lt $EXPECT_JDK_VERSION ]]; then echo "Please make sure that the JDK is installed and the version >= $EXPECT_JDK_VERSION" >> ${OUTPUT} exit 1 fi @@ -111,7 +114,7 @@ case "$GC_OPTION" in -XX:InitiatingHeapOccupancyPercent=50 -XX:G1RSetUpdatingPauseTimePercent=5" ;; zgc|ZGC) - echo "Using ZGC as the default garbage collector (Only support Java 11+)" + echo "Using ZGC as the default garbage collector (requires Java 17 or later)" JAVA_OPTIONS="${JAVA_OPTIONS} -XX:+UseZGC -XX:+UnlockExperimentalVMOptions \ -XX:ConcGCThreads=2 -XX:ParallelGCThreads=6 \ -XX:ZCollectionInterval=120 -XX:ZAllocationSpikeTolerance=5 \ diff --git a/hugegraph-pd/hg-pd-test/pom.xml b/hugegraph-pd/hg-pd-test/pom.xml index 72cdef81aa..b473457b5d 100644 --- a/hugegraph-pd/hg-pd-test/pom.xml +++ b/hugegraph-pd/hg-pd-test/pom.xml @@ -46,7 +46,6 @@ org.jacoco jacoco-maven-plugin - 0.8.8 coverage-report @@ -191,12 +190,6 @@ 2.0.0-RC.3 compile - - org.apache.tinkerpop - gremlin-shaded - 3.5.1 - compile - @@ -204,11 +197,18 @@ org.apache.maven.plugins maven-surefire-plugin - 2.20 + + + org.apache.maven.surefire + surefire-junit4 + ${maven.surefire.plugin.version} + + pd-client-test + true ${basedir}/src/main/java/ ${basedir}/target/classes/ @@ -221,6 +221,7 @@ pd-core-test + true ${basedir}/src/main/java/ ${basedir}/target/classes/ @@ -233,6 +234,7 @@ pd-common-test + true ${basedir}/src/main/java/ ${basedir}/target/classes/ @@ -245,6 +247,7 @@ pd-rest-test + true ${basedir}/src/main/java/ ${basedir}/target/classes/ @@ -259,7 +262,6 @@ org.jacoco jacoco-maven-plugin - 0.8.8 org/apache/hugegraph/pd/rest/*.class diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/PDClientTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/PDClientTest.java index 3676122612..3e7bbda39a 100644 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/PDClientTest.java +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/PDClientTest.java @@ -21,7 +21,7 @@ import java.util.ArrayList; import java.util.List; -import org.apache.tinkerpop.shaded.minlog.Log; +import org.junit.Assert; import org.junit.Test; import org.mockito.Mockito; @@ -323,14 +323,12 @@ public void testChangePeerList() { @Test public void testSplitData() { try { - Metapb.PDConfig config = pdClient.getPDConfig(); - pdClient.setPDConfig(config.toBuilder() - .setMaxShardsPerStore(12) - .build()); - System.out.println(pdClient.getPDConfig()); pdClient.splitData(); + Assert.fail("Expected splitData() to reject an unready cluster"); } catch (PDException e) { - Log.error("testSplitData", e); + Assert.assertEquals( + Pdpb.ErrorType.Cluster_State_Forbid_Splitting_VALUE, + e.getErrorCode()); } } diff --git a/hugegraph-pd/pom.xml b/hugegraph-pd/pom.xml index 83b39c8377..12c9500e5c 100644 --- a/hugegraph-pd/pom.xml +++ b/hugegraph-pd/pom.xml @@ -74,7 +74,6 @@ org.jacoco jacoco-maven-plugin - 0.8.8 true @@ -151,7 +150,6 @@ org.apache.maven.plugins maven-surefire-plugin - 2.20 pd-common-test @@ -175,7 +173,6 @@ org.apache.maven.plugins maven-surefire-plugin - 2.20 pd-core-test @@ -199,7 +196,6 @@ org.apache.maven.plugins maven-surefire-plugin - 2.20 pd-client-test @@ -223,7 +219,6 @@ org.apache.maven.plugins maven-surefire-plugin - 2.20 pd-rest-test diff --git a/hugegraph-server/Dockerfile b/hugegraph-server/Dockerfile index 67866974cf..1ec3ce8bdb 100644 --- a/hugegraph-server/Dockerfile +++ b/hugegraph-server/Dockerfile @@ -18,7 +18,7 @@ # Dockerfile for HugeGraph Server # 1st stage: build source code -FROM --platform=$BUILDPLATFORM maven:3.9.0-eclipse-temurin-11 AS build +FROM --platform=$BUILDPLATFORM maven:3.9.16-eclipse-temurin-17 AS build WORKDIR /pkg @@ -35,13 +35,12 @@ RUN --mount=type=cache,id=hugegraph-maven-${SOURCE_REVISION},target=/root/.m2,sh # 2nd stage: runtime env # Note: ZGC (The Z Garbage Collector) is only supported on ARM-Mac with java > 13 -FROM eclipse-temurin:11-jre-jammy +FROM eclipse-temurin:17-jre-jammy LABEL maintainer="HugeGraph Docker Maintainers " # TODO: use g1gc or zgc as default -# Note: --add-exports is required for Java 11+ to access jdk.internal.reflect for auth proxy -ENV JAVA_OPTS="-XX:+UnlockExperimentalVMOptions -XX:+UseContainerSupport -XX:MaxRAMPercentage=50 -XshowSettings:vm --add-exports=java.base/jdk.internal.reflect=ALL-UNNAMED" \ +ENV JAVA_OPTS="-XX:+UnlockExperimentalVMOptions -XX:+UseContainerSupport -XX:MaxRAMPercentage=50 -XshowSettings:vm" \ HUGEGRAPH_HOME="hugegraph-server" \ STDOUT_MODE="true" diff --git a/hugegraph-server/Dockerfile-hstore b/hugegraph-server/Dockerfile-hstore index e90979563b..424db9c3e9 100644 --- a/hugegraph-server/Dockerfile-hstore +++ b/hugegraph-server/Dockerfile-hstore @@ -18,7 +18,7 @@ # Dockerfile for HugeGraph Server (hstore backend) # 1st stage: build source code -FROM --platform=$BUILDPLATFORM maven:3.9.0-eclipse-temurin-11 AS build +FROM --platform=$BUILDPLATFORM maven:3.9.16-eclipse-temurin-17 AS build WORKDIR /pkg @@ -35,7 +35,7 @@ RUN --mount=type=cache,id=hugegraph-maven-${SOURCE_REVISION},target=/root/.m2,sh # 2nd stage: runtime env # Note: ZGC (The Z Garbage Collector) is only supported on ARM-Mac with java > 13 -FROM eclipse-temurin:11-jre-jammy +FROM eclipse-temurin:17-jre-jammy LABEL maintainer="HugeGraph Docker Maintainers " diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/SerializerFactory.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/SerializerFactory.java index db5fe0a8cd..213cf2b264 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/SerializerFactory.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/SerializerFactory.java @@ -72,13 +72,11 @@ public static void register(String name, String classPath) { "AbstractSerializer: '%s'", classPath); } - // Check exists - if (serializers.containsKey(name)) { + // Register atomically: identical re-registration is a no-op + Class registered = serializers.putIfAbsent(name, (Class) clazz); + if (registered != null && !registered.equals(clazz)) { throw new BackendException("Exists serializer: %s(Class '%s')", - name, serializers.get(name).getName()); + name, registered.getName()); } - - // Register class - serializers.put(name, (Class) clazz); } } diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/BackendProviderFactory.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/BackendProviderFactory.java index ac1e0018ce..2612ea3a07 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/BackendProviderFactory.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/BackendProviderFactory.java @@ -114,12 +114,10 @@ public static void register(String name, String classPath) { BackendException.check(subclass, "Class '%s' is not a subclass of " + "class BackendStoreProvider", classPath); - // Check exists - BackendException.check(!providers.containsKey(name), + // Register atomically: identical re-registration is a no-op + Class registered = providers.putIfAbsent(name, (Class) clazz); + BackendException.check(registered == null || registered.equals(clazz), "Exists BackendStoreProvider: %s (%s)", - name, providers.get(name)); - - // Register class - providers.put(name, (Class) clazz); + name, registered); } } diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/util/Reflection.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/util/Reflection.java index 6e5fd07527..14e7b77120 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/util/Reflection.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/util/Reflection.java @@ -20,6 +20,8 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.Arrays; +import java.util.LinkedHashSet; +import java.util.Set; import org.apache.hugegraph.HugeException; import org.apache.hugegraph.exception.NotSupportException; @@ -42,14 +44,14 @@ public class Reflection { registerFieldsToFilterMethodTemp = reflectionClazzTemp.getMethod("registerFieldsToFilter", - Class.class, String[].class); + Class.class, Set.class); registerMethodsToFilterMethodTemp = reflectionClazzTemp.getMethod("registerMethodsToFilter", - Class.class, String[].class); + Class.class, Set.class); } catch (ClassNotFoundException e) { LOG.error("Can't find jdk.internal.reflect.Reflection class, " + - "please ensure you are using Java 11", e); + "please ensure you are using Java 17", e); } catch (NoSuchMethodException e) { LOG.error("Can't find reflection filter methods", e); } @@ -62,34 +64,59 @@ public class Reflection { public static void registerFieldsToFilter(Class containingClass, String... fieldNames) { if (REGISTER_FILEDS_TO_FILTER_METHOD == null) { throw new NotSupportException("Reflection.registerFieldsToFilter() - " + - "requires Java 11 or higher"); + "requires Java 17 or higher"); } try { REGISTER_FILEDS_TO_FILTER_METHOD.setAccessible(true); - REGISTER_FILEDS_TO_FILTER_METHOD.invoke(REFLECTION_CLAZZ, containingClass, fieldNames); - } catch (IllegalAccessException | InvocationTargetException e) { - throw new HugeException("Failed to register class '%s' fields to filter: %s", - containingClass, Arrays.toString(fieldNames)); + REGISTER_FILEDS_TO_FILTER_METHOD.invoke(REFLECTION_CLAZZ, containingClass, + toFilterSet(fieldNames)); + } catch (IllegalAccessException e) { + throw new HugeException( + "Failed to register class '%s' fields to filter: %s", + e, containingClass, Arrays.toString(fieldNames)); + } catch (InvocationTargetException e) { + throwInvocationTargetException(e, containingClass, fieldNames, "fields"); } } public static void registerMethodsToFilter(Class containingClass, String... methodNames) { if (REGISTER_METHODS_TO_FILTER_METHOD == null) { throw new NotSupportException("Reflection.registerMethodsToFilter() - " + - "requires Java 11 or higher"); + "requires Java 17 or higher"); } try { REGISTER_METHODS_TO_FILTER_METHOD.setAccessible(true); REGISTER_METHODS_TO_FILTER_METHOD.invoke(REFLECTION_CLAZZ, containingClass, - methodNames); - } catch (IllegalAccessException | InvocationTargetException e) { - throw new HugeException("Failed to register class '%s' methods to filter: %s", - containingClass, Arrays.toString(methodNames)); + toFilterSet(methodNames)); + } catch (IllegalAccessException e) { + throw new HugeException( + "Failed to register class '%s' methods to filter: %s", + e, containingClass, Arrays.toString(methodNames)); + } catch (InvocationTargetException e) { + throwInvocationTargetException(e, containingClass, methodNames, "methods"); } } + private static Set toFilterSet(String... members) { + return new LinkedHashSet<>(Arrays.asList(members)); + } + + private static void throwInvocationTargetException(InvocationTargetException exception, + Class containingClass, + String[] members, + String type) { + Throwable cause = exception.getCause(); + if (cause instanceof IllegalArgumentException) { + throw (IllegalArgumentException) cause; + } + Throwable failure = cause != null ? cause : exception; + throw new HugeException( + "Failed to register class '%s' %s to filter: %s", + failure, containingClass, type, Arrays.toString(members)); + } + public static Class loadClass(String clazz) { try { return Class.forName(clazz); diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh index caffedc482..2c93aa085c 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh @@ -63,11 +63,12 @@ ensure_path_writable "$PLUGINS" # The maximum and minimum heap memory that service can use MAX_MEM=$((32 * 1024)) MIN_MEM=$((1 * 512)) -MIN_JAVA_VERSION=11 +MIN_JAVA_VERSION=17 # JDK 24 removed the Security Manager (JEP 486): "-Djava.security.manager=allow" # is a fatal VM initialization error there and System.setSecurityManager() always # throws, so HugeSecurityManager cannot be installed on newer runtimes. MAX_SECURITY_JAVA_VERSION=23 +JVM_MODULE_OPTIONS="${BIN}/jvm-module.options" # Add the slf4j-log4j12 binding CP=$(find -L $LIB -name 'log4j-slf4j-impl*.jar' | sort | tr '\n' ':') @@ -114,6 +115,11 @@ if [[ -z $JAVA_VERSION || $JAVA_VERSION -lt $MIN_JAVA_VERSION ]]; then exit 1 fi +if [[ ! -r ${JVM_MODULE_OPTIONS} ]]; then + echo "Missing or unreadable JVM module options file: ${JVM_MODULE_OPTIONS}" >> "${OUTPUT}" + exit 1 +fi + # Set Java options if [ "$JAVA_OPTIONS" = "" ]; then XMX=$(calc_xmx $MIN_MEM $MAX_MEM) @@ -128,12 +134,6 @@ if [ "$JAVA_OPTIONS" = "" ]; then # -Xloggc:./logs/gc.log -XX:+PrintHeapAtGC -XX:+PrintGCDetails -XX:+PrintGCDateStamps" fi -if [[ $JAVA_VERSION -gt 9 ]]; then - JAVA_OPTIONS="${JAVA_OPTIONS} --add-exports=java.base/jdk.internal.reflect=ALL-UNNAMED \ - --add-modules=jdk.unsupported \ - --add-exports=java.base/sun.nio.ch=ALL-UNNAMED " -fi - # Using G1GC as the default garbage collector (Recommended for large memory machines) # mention: zgc is only available on ARM-Mac with java > 13 case "$GC_OPTION" in @@ -144,7 +144,7 @@ case "$GC_OPTION" in -XX:G1RSetUpdatingPauseTimePercent=5" ;; zgc|ZGC) - echo "Using ZGC as the default garbage collector (Only support Java 11+)" + echo "Using ZGC as the default garbage collector (requires Java 17 or later)" JAVA_OPTIONS="${JAVA_OPTIONS} -XX:+UseZGC -XX:+UnlockExperimentalVMOptions \ -XX:ConcGCThreads=2 -XX:ParallelGCThreads=6 \ -XX:ZCollectionInterval=120 -XX:ZAllocationSpikeTolerance=5 \ @@ -258,12 +258,12 @@ fi # Turn on security check if [[ "${STDOUT_MODE:-false}" == "true" ]]; then - exec ${JAVA} -Dname="HugeGraphServer" ${JVM_OPTIONS} ${JAVA_OPTIONS} \ + exec ${JAVA} @"${JVM_MODULE_OPTIONS}" -Dname="HugeGraphServer" ${JVM_OPTIONS} ${JAVA_OPTIONS} \ ${SECURITY_MANAGER_OPTION} -cp ${CLASSPATH}: \ org.apache.hugegraph.bootstrap.HugeGraphServerBootstrap \ ${OPEN_SECURITY_CHECK} ${GREMLIN_SERVER_CONF} ${REST_SERVER_CONF} else - exec ${JAVA} -Dname="HugeGraphServer" ${JVM_OPTIONS} ${JAVA_OPTIONS} \ + exec ${JAVA} @"${JVM_MODULE_OPTIONS}" -Dname="HugeGraphServer" ${JVM_OPTIONS} ${JAVA_OPTIONS} \ ${SECURITY_MANAGER_OPTION} -cp ${CLASSPATH}: \ org.apache.hugegraph.bootstrap.HugeGraphServerBootstrap \ ${OPEN_SECURITY_CHECK} ${GREMLIN_SERVER_CONF} ${REST_SERVER_CONF} \ diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/init-store.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/init-store.sh index 74ec0bb731..d934192fad 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/init-store.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/init-store.sh @@ -30,6 +30,7 @@ TOP="$(cd "${BIN}"/../ && pwd)" CONF="$TOP/conf" LIB="$TOP/lib" PLUGINS="$TOP/plugins" +JVM_MODULE_OPTIONS="${BIN}/jvm-module.options" . "${BIN}"/util.sh @@ -38,15 +39,16 @@ ensure_path_writable "${PLUGINS}" if [[ -n "$JAVA_HOME" ]]; then JAVA="$JAVA_HOME"/bin/java - EXT="$JAVA_HOME/jre/lib/ext:$LIB:$PLUGINS" else JAVA=java - EXT="$LIB:$PLUGINS" fi cd "${TOP}" || exit -DEFAULT_JAVA_OPTIONS="--add-exports=java.base/jdk.internal.reflect=ALL-UNNAMED" +if [[ ! -r ${JVM_MODULE_OPTIONS} ]]; then + echo "Missing or unreadable JVM module options file: ${JVM_MODULE_OPTIONS}" >&2 + exit 1 +fi echo "Initializing HugeGraph Store..." @@ -54,7 +56,7 @@ echo "Initializing HugeGraph Store..." CP=$(find -L "${LIB}" -name 'hugegraph*.jar' | sort | tr '\n' ':') CP="$CP":$(find -L "${LIB}" -name '*.jar' \! -name 'hugegraph*' | sort | tr '\n' ':') CP="$CP":$(find -L "${PLUGINS}" -name '*.jar' | sort | tr '\n' ':') -$JAVA -cp $CP ${DEFAULT_JAVA_OPTIONS} \ +"${JAVA}" @"${JVM_MODULE_OPTIONS}" -cp "$CP" \ org.apache.hugegraph.cmd.InitStore "${CONF}"/rest-server.properties INIT_STORE_STATUS=$? if [[ ${INIT_STORE_STATUS} -ne 0 ]]; then diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/jvm-module.options b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/jvm-module.options new file mode 100644 index 0000000000..eeb6114c5c --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/jvm-module.options @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +--add-exports=java.base/jdk.internal.reflect=ALL-UNNAMED +--add-modules=jdk.unsupported +--add-exports=java.base/sun.nio.ch=ALL-UNNAMED diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh index 2c99238327..46044e932b 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh @@ -99,6 +99,22 @@ if [[ $PRELOAD == "true" ]]; then sed -i -e '/registerBackends/d; /serverStarted/d' "${SCRIPTS}/${EXAMPLE_SCRIPT}" fi +function forward_signal_and_wait() { + local signal="$1" + local exit_code="$2" + + trap - SIGHUP SIGINT SIGQUIT SIGTERM + if kill -0 "$PID" 2>/dev/null; then + kill "-$signal" "$PID" 2>/dev/null || true + # The foreground wait is interrupted before the trap runs, so retry + # until the child has exited and been reaped. + while kill -0 "$PID" 2>/dev/null; do + wait "$PID" 2>/dev/null || true + done + fi + exit "$exit_code" +} + if [[ $DAEMON == "true" ]]; then echo "Starting HugeGraphServer in daemon mode..." "${BIN}"/hugegraph-server.sh "${CONF}/${GREMLIN_SERVER_CONF}" "${CONF}"/rest-server.properties \ @@ -133,7 +149,14 @@ else PID="$!" # Write pid to file echo "$PID" > "$PID_FILE" - trap 'kill $PID; wait $PID; exit $?' SIGHUP SIGINT SIGQUIT SIGTERM + trap 'forward_signal_and_wait HUP 129' SIGHUP + # The background JVM can inherit an ignored SIGINT disposition, so use + # SIGTERM to guarantee that Ctrl-C shuts it down while retaining exit 130. + trap 'forward_signal_and_wait TERM 130' SIGINT + # Forward TERM instead of QUIT: the JVM only dumps threads on SIGQUIT + # and keeps running, which would leave the wait loop below stuck. + trap 'forward_signal_and_wait TERM 131' SIGQUIT + trap 'forward_signal_and_wait TERM 143' SIGTERM wait $PID exit $? fi diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-monitor.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-monitor.sh index 570d07b545..ffe1a2243d 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-monitor.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-monitor.sh @@ -31,7 +31,7 @@ TOP="$(cd $BIN/../ && pwd)" . $BIN/util.sh if [ "$JAVA_HOME" == "" ]; then - echo "Must set JAVA_HOME environment variable and installed jdk >= 1.8" + echo "Must set JAVA_HOME environment variable and install JDK >= 17" exit 1 fi diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/ci-service-utils.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/ci-service-utils.sh new file mode 100644 index 0000000000..934db1f171 --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/ci-service-utils.sh @@ -0,0 +1,211 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +function dump_service_diagnostics() { + local service_dir="$1" + local service_name="$2" + local log_dir="${service_dir}/logs" + + echo "::group::${service_name} diagnostics" + echo "[ci] service dir: ${service_dir}" + echo "[ci] java processes:" + ps -ef | grep -E "HugeGraph|hg-|java" | grep -v grep || true + echo "[ci] listening tcp ports:" + (ss -ltnp || netstat -ltnp || true) 2>&1 + + if [ -d "${log_dir}" ]; then + find "${log_dir}" -maxdepth 2 -type f | sort | while read -r log_file; do + echo "--- tail -n 200 ${log_file} ---" + tail -n 200 "${log_file}" || true + done + else + echo "[ci] log dir not found: ${log_dir}" + fi + echo "::endgroup::" +} + +function wait_for_tcp_port() { + local service_name="$1" + local host="$2" + local port="$3" + local pid_file="$4" + local service_dir="$5" + local timeout_seconds="${6:-90}" + + echo "[ci] waiting for ${service_name} at ${host}:${port}" + for second in $(seq 1 "${timeout_seconds}"); do + if bash -c "echo > /dev/tcp/${host}/${port}" >/dev/null 2>&1; then + echo "[ci] ${service_name} is listening on ${host}:${port}" + return 0 + fi + + if [ -f "${pid_file}" ]; then + local pid + pid="$(cat "${pid_file}")" + if [ -n "${pid}" ] && ! kill -0 "${pid}" >/dev/null 2>&1; then + echo "[ci] ${service_name} process ${pid} exited before readiness" + dump_service_diagnostics "${service_dir}" "${service_name}" + return 1 + fi + fi + + if [ "$((second % 10))" -eq 0 ]; then + echo "[ci] still waiting for ${service_name} (${second}s)" + fi + sleep 1 + done + + echo "[ci] timeout waiting for ${service_name} at ${host}:${port}" + dump_service_diagnostics "${service_dir}" "${service_name}" + return 1 +} + +function http_status_is_accepted() { + local status="$1" + local accepted_statuses="$2" + + case ",${accepted_statuses}," in + *",${status},"*) + return 0 + ;; + *) + return 1 + ;; + esac +} + +function wait_for_http_status() { + local service_name="$1" + local url="$2" + local pid_file="$3" + local service_dir="$4" + local timeout_seconds="${5:-90}" + local accepted_statuses="${6:-200}" + local connect_timeout_seconds=2 + local max_request_seconds=5 + local started_at="${SECONDS}" + local deadline=$((started_at + timeout_seconds)) + local next_log_at=10 + + echo "[ci] waiting for ${service_name} HTTP readiness at ${url}" + echo "[ci] accepted HTTP statuses: ${accepted_statuses}" + while (( SECONDS < deadline )); do + local remaining=$((deadline - SECONDS)) + local request_timeout="${max_request_seconds}" + if (( remaining < request_timeout )); then + request_timeout="${remaining}" + fi + if (( request_timeout < 1 )); then + break + fi + + local status + status="$(curl -s -o /dev/null -w "%{http_code}" \ + --connect-timeout "${connect_timeout_seconds}" \ + --max-time "${request_timeout}" \ + "${url}" 2>/dev/null)" || status="000" + if http_status_is_accepted "${status}" "${accepted_statuses}"; then + echo "[ci] ${service_name} is HTTP ready at ${url}" \ + "(status ${status})" + return 0 + fi + + if [ -f "${pid_file}" ]; then + local pid + pid="$(cat "${pid_file}")" + if [ -n "${pid}" ] && ! kill -0 "${pid}" >/dev/null 2>&1; then + echo "[ci] ${service_name} process ${pid} exited before" \ + "HTTP readiness" + dump_service_diagnostics "${service_dir}" "${service_name}" + return 1 + fi + fi + + local elapsed=$((SECONDS - started_at)) + if (( elapsed >= next_log_at )); then + echo "[ci] still waiting for ${service_name} HTTP readiness" \ + "(${elapsed}s, last status ${status})" + next_log_at=$((next_log_at + 10)) + fi + if (( SECONDS >= deadline )); then + break + fi + sleep 1 + done + + echo "[ci] timeout waiting for ${service_name} HTTP readiness at ${url}" + dump_service_diagnostics "${service_dir}" "${service_name}" + return 1 +} + +function process_is_running() { + local pid="$1" + local state + + if [[ ! "${pid}" =~ ^[0-9]+$ ]]; then + return 1 + fi + + if ! kill -0 "${pid}" 2>/dev/null; then + return 1 + fi + + state="$(ps -o stat= -p "${pid}" 2>/dev/null | tr -d '[:space:]')" || + state="" + [[ "${state}" != Z* ]] +} + +function wait_for_process_exit() { + local pid="$1" + local timeout_seconds="${2:-10}" + local deadline=$((SECONDS + timeout_seconds)) + + while process_is_running "${pid}"; do + if (( SECONDS >= deadline )); then + return 1 + fi + sleep 1 + done + return 0 +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + command="$1" + shift || true + case "${command}" in + dump) + dump_service_diagnostics "$@" + exit $? + ;; + wait) + wait_for_tcp_port "$@" + exit $? + ;; + wait-http) + wait_for_http_status "$@" + exit $? + ;; + *) + echo "Usage: $0 dump SERVICE_DIR SERVICE_NAME" + echo " $0 wait SERVICE_NAME HOST PORT PID_FILE SERVICE_DIR [TIMEOUT_SECONDS]" + echo " $0 wait-http SERVICE_NAME URL PID_FILE SERVICE_DIR" \ + "[TIMEOUT_SECONDS] [ACCEPTED_STATUSES]" + exit 2 + ;; + esac +fi diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-native-runtime-smoke-test.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-native-runtime-smoke-test.sh index 7cf09e30ee..e8570eef01 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-native-runtime-smoke-test.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-native-runtime-smoke-test.sh @@ -26,7 +26,6 @@ TRAVIS_DIR=$(cd "$(dirname "$0")" && pwd) SERVER_DIR=$(cd "$1" && pwd) SERVER_URL=${SERVER_URL:-http://127.0.0.1:8080} EXPECTED_ARCH=${EXPECTED_ARCH:-riscv64} -EXPECTED_JAVA_MAJOR=${EXPECTED_JAVA_MAJOR:-11} SERVER_START_ATTEMPTED=false SERVER_STARTUP_TIMEOUT=${SERVER_STARTUP_TIMEOUT:-300} SERVER_START_COMMAND_TIMEOUT=$((SERVER_STARTUP_TIMEOUT + 30)) @@ -34,7 +33,7 @@ STORE_DUMP_TIMEOUT=${STORE_DUMP_TIMEOUT:-120} WORK_DIR=$(mktemp -d "${TMPDIR:-/tmp}/hugegraph-native-runtime-smoke.XXXXXX") RUN_ID="$(date +%s)_$$" -export EXPECTED_ARCH EXPECTED_JAVA_MAJOR +export EXPECTED_ARCH cleanup() { local status=$? diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-rocksdb-jni-smoke-test.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-rocksdb-jni-smoke-test.sh index 5603431d5d..81d88796d0 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-rocksdb-jni-smoke-test.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-rocksdb-jni-smoke-test.sh @@ -25,7 +25,9 @@ fi TRAVIS_DIR=$(cd "$(dirname "$0")" && pwd) SERVER_DIR=$(cd "$1" && pwd) EXPECTED_ARCH=${EXPECTED_ARCH:-} -EXPECTED_JAVA_MAJOR=${EXPECTED_JAVA_MAJOR:-11} +EXPECTED_JAVA_MAJOR=${EXPECTED_JAVA_MAJOR:-17} +EXPECTED_RISCV64_JAVA_VERSION=${EXPECTED_RISCV64_JAVA_VERSION:-} +EXPECTED_RISCV64_JAVA_VENDOR=${EXPECTED_RISCV64_JAVA_VENDOR:-} ACTUAL_ARCH=$(uname -m) if [[ -n "${JAVA_HOME:-}" ]]; then @@ -88,8 +90,8 @@ fi configure_riscv64_libatomic if [[ "$ACTUAL_ARCH" == "riscv64" ]]; then - EXPECTED_RISCV64_JAVA_VERSION=${EXPECTED_RISCV64_JAVA_VERSION:-11.0.31.28} - if [[ "$JAVA_VERSION" != "$EXPECTED_RISCV64_JAVA_VERSION" ]]; then + if [[ -n "$EXPECTED_RISCV64_JAVA_VERSION" && \ + "$JAVA_VERSION" != "$EXPECTED_RISCV64_JAVA_VERSION" ]]; then echo "Expected RISC-V Java $EXPECTED_RISCV64_JAVA_VERSION, got $JAVA_VERSION" >&2 exit 1 fi @@ -97,8 +99,10 @@ if [[ "$ACTUAL_ARCH" == "riscv64" ]]; then echo "Expected RISC-V Server VM, got $JAVA_VM_NAME" >&2 exit 1 fi - if [[ "$JAVA_VM_VENDOR" != "Alibaba" ]]; then - echo "Expected RISC-V Java vendor Alibaba, got $JAVA_VM_VENDOR" >&2 + if [[ -n "$EXPECTED_RISCV64_JAVA_VENDOR" && \ + "$JAVA_VM_VENDOR" != "$EXPECTED_RISCV64_JAVA_VENDOR" ]]; then + echo "Expected RISC-V Java vendor $EXPECTED_RISCV64_JAVA_VENDOR," \ + "got $JAVA_VM_VENDOR" >&2 exit 1 fi if [[ "$JAVA_VM_INFO" != *"mixed mode"* ]]; then diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/start-pd.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/start-pd.sh index 94d4f82fed..dc199321bd 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/start-pd.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/start-pd.sh @@ -30,6 +30,9 @@ else fi PD_DIR=$HOME_DIR/hugegraph-pd/apache-hugegraph-pd-$VersionInBash +TRAVIS_DIR=$(dirname "$0") + +source "$TRAVIS_DIR"/ci-service-utils.sh pushd $PD_DIR # conf/application.yml ships auth.secret-key empty on purpose, so PD would @@ -42,5 +45,6 @@ pushd $PD_DIR export SPRING_APPLICATION_JSON='{"auth":{"secret-key":"pd-ci-test-secret-not-for-production"}}' . bin/start-hugegraph-pd.sh ) -sleep 10 +wait_for_http_status HugeGraphPD http://127.0.0.1:8620/v1/ready \ + "$PD_DIR"/bin/pid "$PD_DIR" 90 200 popd diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/start-store.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/start-store.sh index 353034201a..0f45483cb5 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/start-store.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/start-store.sh @@ -30,8 +30,12 @@ else fi STORE_DIR=$HOME_DIR/hugegraph-store/apache-hugegraph-store-$VersionInBash +TRAVIS_DIR=$(dirname "$0") + +source "$TRAVIS_DIR"/ci-service-utils.sh pushd $STORE_DIR . bin/start-hugegraph-store.sh -sleep 10 +wait_for_http_status HugeGraphStore http://127.0.0.1:8520/v1/health \ + "$STORE_DIR"/bin/pid "$STORE_DIR" 90 200,401 popd diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-jacoco-report.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-jacoco-report.sh index 133026e594..59b4caf32c 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-jacoco-report.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-jacoco-report.sh @@ -370,12 +370,12 @@ def goals(plugin): def check_module(module, test_module): parent = ET.parse(ROOT / module / "pom.xml").getroot() parent_plugin = jacoco_plugin(parent.find(NS + "build")) - assert child_text(parent_plugin, "version") == "0.8.8" + assert child_text(parent_plugin, "version") == "", "inherit the managed JaCoCo version" assert child_text(parent_plugin.find(NS + "configuration"), "append") == "true" test = ET.parse(ROOT / module / test_module / "pom.xml").getroot() default_plugin = jacoco_plugin(test.find(NS + "build")) - assert child_text(default_plugin, "version") == "0.8.8" + assert child_text(default_plugin, "version") == "", "inherit the managed JaCoCo version" assert "report-aggregate" not in goals(default_plugin) profile = None @@ -385,7 +385,7 @@ def check_module(module, test_module): break assert profile is not None profile_plugin = jacoco_plugin(profile.find(NS + "build")) - assert child_text(profile_plugin, "version") == "0.8.8" + assert child_text(profile_plugin, "version") == "", "inherit the managed JaCoCo version" executions = profile_plugin.findall(".//" + NS + "execution") aggregates = [execution for execution in executions if "report-aggregate" in goals(execution)] @@ -393,6 +393,11 @@ def check_module(module, test_module): assert child_text(aggregates[0], "phase") == "verify" +root_pom = ET.parse(ROOT / "pom.xml").getroot() +managed_plugin = jacoco_plugin(root_pom.find(NS + "build/" + NS + "pluginManagement")) +assert child_text(managed_plugin, "version") == "${jacoco.maven.plugin.version}" +assert child_text(root_pom.find(NS + "properties"), "jacoco.maven.plugin.version") == "0.8.15" + check_module("hugegraph-pd", "hg-pd-test") check_module("hugegraph-store", "hg-store-test") diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-ci-service-utils.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-ci-service-utils.sh new file mode 100644 index 0000000000..a776591a1e --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-ci-service-utils.sh @@ -0,0 +1,69 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +UTILS="${1:-${SCRIPT_DIR}/ci-service-utils.sh}" +ACTIVE_PID="" + +cleanup() { + if [[ -n "${ACTIVE_PID}" ]]; then + kill "${ACTIVE_PID}" 2>/dev/null || true + wait "${ACTIVE_PID}" 2>/dev/null || true + fi +} +trap cleanup EXIT + +source "${UTILS}" + +if ! declare -F process_is_running >/dev/null || + ! declare -F wait_for_process_exit >/dev/null; then + echo "FAIL: process exit helpers are not available" + exit 1 +fi + +sleep 10 & +ACTIVE_PID=$! +if wait_for_process_exit "${ACTIVE_PID}" 1; then + echo "FAIL: a running process was reported as exited" + exit 1 +fi +kill "${ACTIVE_PID}" 2>/dev/null || true +wait "${ACTIVE_PID}" 2>/dev/null || true +ACTIVE_PID="" + +sleep 1 & +ACTIVE_PID=$! +if ! wait_for_process_exit "${ACTIVE_PID}" 5; then + echo "FAIL: a terminated process was reported as running" + exit 1 +fi +wait "${ACTIVE_PID}" 2>/dev/null || true +ACTIVE_PID="" + +ps() { + echo "Z" +} +if process_is_running "$$"; then + echo "FAIL: a zombie process was reported as running" + exit 1 +fi +unset -f ps + +echo "PASS: process exit helpers handle running, terminated, and zombie states" diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh index 796d69c83b..f009d91ece 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh @@ -17,11 +17,13 @@ set -euo pipefail -SERVER_ROOT_INPUT="${1:?Usage: $0 PATH_TO_SERVER_DIST}" +SERVER_ROOT_INPUT="${1:?Usage: $0 PATH_TO_SERVER_DIST [SOURCE_ROOT]}" +SOURCE_ROOT_INPUT="${2:-}" SERVER_ROOT=$(cd "$SERVER_ROOT_INPUT" && pwd) SERVER_SCRIPT="${SERVER_ROOT}/bin/hugegraph-server.sh" CONF="${SERVER_ROOT}/conf" SECURITY_PROPERTIES="${CONF}/java-security.properties" +JVM_MODULE_OPTIONS="${SERVER_ROOT}/bin/jvm-module.options" fail() { echo "FAIL: $1" >&2 @@ -43,12 +45,136 @@ assert_no_argument() { fi } +assert_source_consumer() { + local source_file="$1" + local expected="$2" + [[ -f "$source_file" ]] || fail "source consumer is missing: $source_file" + grep -Fq -- "$expected" "$source_file" || + fail "JVM module options consumer is not wired: $source_file" +} + +assert_surefire_arg_lines() { + local pom="$1" + local expected="$2" + local total + local wired + local jacoco_wired + read -r total wired jacoco_wired < <( + awk -v expected="$expected" ' + /maven-surefire-plugin<\/artifactId>/ { + in_surefire = 1 + } + in_surefire && /]*)?>/ { + in_arg_line = 1 + arg_line = "" + } + in_arg_line { + arg_line = arg_line $0 + } + in_arg_line && /<\/argLine>/ { + total++ + if (index(arg_line, expected) != 0) { + wired++ + } + if (index(arg_line, "@{argLine}") != 0) { + jacoco_wired++ + } + in_arg_line = 0 + } + in_surefire && /<\/plugin>/ { + in_surefire = 0 + } + END { + print total + 0, wired + 0, jacoco_wired + 0 + } + ' "$pom" + ) + if [[ "$total" -eq 0 || "$wired" -ne "$total" ]]; then + fail "all Surefire argLine values must use jvm-module.options: $pom" + fi + if [[ "$jacoco_wired" -ne "$total" ]]; then + fail "all Surefire argLine values must preserve @{argLine}: $pom" + fi +} + +assert_no_inline_module_options() { + local pattern + local source_file + pattern="--add-(exports|opens)([[:space:]]+|=)[\"']?java\\.base/|" + pattern="${pattern}--add-modules([[:space:]]+|=)[\"']?jdk\.unsupported" + for source_file in "$@"; do + [[ -f "$source_file" ]] || fail "source consumer is missing: $source_file" + done + if grep -En -- "$pattern" "$@"; then + fail "JVM module options must only be declared in jvm-module.options" + fi +} + if [[ ! -x "$SERVER_SCRIPT" ]]; then fail "server script is not executable: $SERVER_SCRIPT" fi if [[ ! -f "$SECURITY_PROPERTIES" ]]; then fail "security properties file is missing: $SECURITY_PROPERTIES" fi +if [[ ! -f "$JVM_MODULE_OPTIONS" ]]; then + fail "JVM module options file is missing: $JVM_MODULE_OPTIONS" +fi + +assert_argument "--add-exports=java.base/jdk.internal.reflect=ALL-UNNAMED" \ + "$JVM_MODULE_OPTIONS" +assert_argument "--add-modules=jdk.unsupported" "$JVM_MODULE_OPTIONS" +assert_argument "--add-exports=java.base/sun.nio.ch=ALL-UNNAMED" \ + "$JVM_MODULE_OPTIONS" + +if [[ -n "$SOURCE_ROOT_INPUT" ]]; then + if [[ ! -d "$SOURCE_ROOT_INPUT" ]]; then + fail "source root is not a directory: $SOURCE_ROOT_INPUT" + fi + SOURCE_ROOT=$(cd "$SOURCE_ROOT_INPUT" && pwd) + SERVER_DIST_SOURCE="${SOURCE_ROOT}/hugegraph-server/hugegraph-dist" + CLUSTER_SOURCE="${SOURCE_ROOT}/hugegraph-cluster-test/"\ +"hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct" + SERVER_LAUNCHER_SOURCE="${SERVER_DIST_SOURCE}/src/assembly/static/bin/"\ +"hugegraph-server.sh" + INIT_STORE_SOURCE="${SERVER_DIST_SOURCE}/src/assembly/static/bin/init-store.sh" + SUREFIRE_POM="${SOURCE_ROOT}/hugegraph-server/hugegraph-test/pom.xml" + TEST_JVM_MODULE_OPTIONS="${SOURCE_ROOT}/hugegraph-server/hugegraph-test/"\ +"conf/jvm-test-module.options" + COMMONS_POM="${SOURCE_ROOT}/hugegraph-commons/pom.xml" + CLUSTER_WRAPPER="${CLUSTER_SOURCE}/node/ServerNodeWrapper.java" + SERVER_DOCKERFILE="${SOURCE_ROOT}/hugegraph-server/Dockerfile" + HSTORE_DOCKERFILE="${SOURCE_ROOT}/hugegraph-server/Dockerfile-hstore" + SERVER_WORKFLOW="${SOURCE_ROOT}/.github/workflows/server-ci.yml" + DOCKER_WORKFLOW="${SOURCE_ROOT}/.github/workflows/docker-build-ci.yml" + UPGRADE_CONTRACT_SCRIPT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/"\ +"test-java17-upgrade-contracts.sh" + + [[ -x "$UPGRADE_CONTRACT_SCRIPT" ]] || \ + fail "Java 17 upgrade contract script is missing: $UPGRADE_CONTRACT_SCRIPT" + "$UPGRADE_CONTRACT_SCRIPT" "$SERVER_ROOT" "$SOURCE_ROOT" + + assert_source_consumer "$SERVER_LAUNCHER_SOURCE" '@"${JVM_MODULE_OPTIONS}"' + assert_source_consumer "$INIT_STORE_SOURCE" '@"${JVM_MODULE_OPTIONS}"' + assert_surefire_arg_lines "$SUREFIRE_POM" \ + '@${project.basedir}/../hugegraph-dist/src/assembly/static/bin/jvm-module.options' + [[ -f "$TEST_JVM_MODULE_OPTIONS" ]] || \ + fail "JVM test module options file is missing: $TEST_JVM_MODULE_OPTIONS" + assert_argument \ + "--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED" \ + "$TEST_JVM_MODULE_OPTIONS" + assert_argument "--add-opens=java.base/java.lang=ALL-UNNAMED" \ + "$TEST_JVM_MODULE_OPTIONS" + assert_surefire_arg_lines "$SUREFIRE_POM" \ + '@${project.basedir}/conf/jvm-test-module.options' + assert_surefire_arg_lines "$COMMONS_POM" \ + '@${project.parent.basedir}/../hugegraph-server/hugegraph-test/conf/jvm-test-module.options' + assert_source_consumer "$CLUSTER_WRAPPER" \ + '"@" + Paths.get(SERVER_PACKAGE_PATH, BIN_DIR,' + assert_no_inline_module_options \ + "$SERVER_LAUNCHER_SOURCE" "$INIT_STORE_SOURCE" "$SUREFIRE_POM" \ + "$COMMONS_POM" "$CLUSTER_WRAPPER" "$SERVER_DOCKERFILE" \ + "$HSTORE_DOCKERFILE" "$SERVER_WORKFLOW" "$DOCKER_WORKFLOW" +fi if [[ -n "${JAVA_HOME:-}" ]]; then JAVA_BIN="${JAVA_HOME}/bin/java" @@ -409,7 +535,7 @@ if [[ " $* " == *" -version "* ]]; then if [[ -n "${MOCK_JAVA_PREAMBLE:-}" ]]; then echo "${MOCK_JAVA_PREAMBLE}" >&2 fi - echo "openjdk version \"${MOCK_JAVA_VERSION:-11}.0.0\"" >&2 + echo "openjdk version \"${MOCK_JAVA_VERSION:-17}.0.0\"" >&2 exit 0 fi printf '%s\n' "$@" > "$CAPTURE_FILE" @@ -425,6 +551,7 @@ CAPTURE_FILE="$ENABLED_CAPTURE" JAVA_HOME="$MOCK_JAVA_HOME" \ assert_argument \ "-Djava.security.properties=${SECURITY_PROPERTIES}" "$ENABLED_CAPTURE" +assert_argument "@${JVM_MODULE_OPTIONS}" "$ENABLED_CAPTURE" assert_no_argument '^-Djava\.security\.manager=' "$ENABLED_CAPTURE" assert_argument \ "org.apache.hugegraph.bootstrap.HugeGraphServerBootstrap" "$ENABLED_CAPTURE" @@ -503,10 +630,10 @@ assert_argument "-Djava.security.manager=allow" "$AGENT_JDK21_CAPTURE" # ... and trip the JDK 24+ security guard when the agent version is high. HIGH_AGENT_PREAMBLE=$'Picked up JAVA_TOOL_OPTIONS: -javaagent:apm-agent.jar\nAPM agent version "24.0.1" is starting' -HIGH_AGENT_CAPTURE="${TEMP_DIR}/agent-preamble-jdk11.args" -HIGH_AGENT_ERROR="${TEMP_DIR}/agent-preamble-jdk11.err" +HIGH_AGENT_CAPTURE="${TEMP_DIR}/agent-preamble-jdk17.args" +HIGH_AGENT_ERROR="${TEMP_DIR}/agent-preamble-jdk17.err" CAPTURE_FILE="$HIGH_AGENT_CAPTURE" JAVA_HOME="$MOCK_JAVA_HOME" \ - MOCK_JAVA_VERSION=11 MOCK_JAVA_PREAMBLE="$HIGH_AGENT_PREAMBLE" \ + MOCK_JAVA_VERSION=17 MOCK_JAVA_PREAMBLE="$HIGH_AGENT_PREAMBLE" \ STDOUT_MODE=true "$SERVER_SCRIPT" \ "${CONF}/gremlin-server.yaml" "${CONF}/rest-server.properties" true \ >/dev/null 2>"$HIGH_AGENT_ERROR" @@ -518,6 +645,15 @@ assert_argument \ "org.apache.hugegraph.bootstrap.HugeGraphServerBootstrap" "$HIGH_AGENT_CAPTURE" assert_no_argument '^-Djava\.security\.manager=' "$HIGH_AGENT_CAPTURE" +JDK11_ERROR="${TEMP_DIR}/jdk11.err" +if JAVA_HOME="$MOCK_JAVA_HOME" MOCK_JAVA_VERSION=11 STDOUT_MODE=true \ + "$SERVER_SCRIPT" "${CONF}/gremlin-server.yaml" \ + "${CONF}/rest-server.properties" false >/dev/null 2>"$JDK11_ERROR"; then + fail "launcher accepted a Java 11 runtime" +fi +grep -Fq "version >= 17, current is 11" "${SERVER_ROOT}/logs/hugegraph-server.log" || + fail "launcher did not report the Java 17 minimum" + JDK24_DISABLED_CAPTURE="${TEMP_DIR}/jdk24-disabled.args" CAPTURE_FILE="$JDK24_DISABLED_CAPTURE" JAVA_HOME="$MOCK_JAVA_HOME" \ MOCK_JAVA_VERSION=24 STDOUT_MODE=true "$SERVER_SCRIPT" \ diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java17-upgrade-contracts.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java17-upgrade-contracts.sh new file mode 100755 index 0000000000..b371003713 --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java17-upgrade-contracts.sh @@ -0,0 +1,220 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +SERVER_ROOT_INPUT="${1:?Usage: $0 PATH_TO_SERVER_DIST PATH_TO_SOURCE_ROOT}" +SOURCE_ROOT_INPUT="${2:?Usage: $0 PATH_TO_SERVER_DIST PATH_TO_SOURCE_ROOT}" +SERVER_ROOT=$(cd "$SERVER_ROOT_INPUT" && pwd) +SOURCE_ROOT=$(cd "$SOURCE_ROOT_INPUT" && pwd) + +fail() { + echo "FAIL: $1" >&2 + exit 1 +} + +assert_argument() { + local argument="$1" + local capture="$2" + grep -Fxq -- "$argument" "$capture" || \ + fail "missing JVM argument: $argument" +} + +assert_default_test_is_tolerant() { + local pom="$1" + + python3 - "$pom" <<'PY' +import sys +import xml.etree.ElementTree as ET + +pom = sys.argv[1] +namespace = {"m": "http://maven.apache.org/POM/4.0.0"} +root = ET.parse(pom).getroot() +value = root.find( + "m:properties/m:surefire.failIfNoSpecifiedTests", namespace +) +if value is None or (value.text or "").strip() != "false": + raise SystemExit( + "{}: default-test must tolerate -Dtest misses in reactor modules".format(pom) + ) +PY +} + +assert_supported_java_contract() { + local pom="$1" + + python3 - "$pom" <<'PY' +import sys +import xml.etree.ElementTree as ET + +pom = sys.argv[1] +namespace = {"m": "http://maven.apache.org/POM/4.0.0"} +root = ET.parse(pom).getroot() +properties = root.find("m:properties", namespace) +if properties is None: + raise SystemExit("{}: Maven properties are missing".format(pom)) + +release = properties.find("m:maven.compiler.release", namespace) +if release is None or (release.text or "").strip() != "17": + raise SystemExit("{}: compiler release must remain 17".format(pom)) + +supported_range = properties.find("m:java.supported.version.range", namespace) +if supported_range is None or (supported_range.text or "").strip() != "[17,18)": + raise SystemExit("{}: supported JDK range must be [17,18)".format(pom)) + +expected_reference = "${java.supported.version.range}" +actual_references = [] +for plugin in root.findall("m:build/m:plugins/m:plugin", namespace): + artifact_id = plugin.find("m:artifactId", namespace) + if artifact_id is None or artifact_id.text != "maven-enforcer-plugin": + continue + for rule in plugin.findall( + "m:executions/m:execution/m:configuration/m:rules/m:requireJavaVersion", + namespace, + ): + version = rule.find("m:version", namespace) + if version is not None: + actual_references.append((version.text or "").strip()) + +if actual_references != [expected_reference]: + raise SystemExit( + "{}: requireJavaVersion must consume {} exactly once; found {}".format( + pom, expected_reference, actual_references + ) + ) +PY +} + +assert_surefire_execution_scope() { + local pom="$1" + shift + + python3 - "$pom" "$@" <<'PY' +import sys +import xml.etree.ElementTree as ET + +pom = sys.argv[1] +expected_ids = sys.argv[2:] +namespace = {"m": "http://maven.apache.org/POM/4.0.0"} +root = ET.parse(pom).getroot() + +properties = root.find("m:properties", namespace) +if properties is not None: + module_value = properties.find("m:surefire.failIfNoSpecifiedTests", namespace) + if module_value is not None and (module_value.text or "").strip() == "true": + raise SystemExit( + "{}: leaf-wide failIfNoSpecifiedTests=true breaks default-test".format(pom) + ) + +surefire = None +for plugin in root.findall("m:build/m:plugins/m:plugin", namespace): + artifact_id = plugin.find("m:artifactId", namespace) + if artifact_id is not None and artifact_id.text == "maven-surefire-plugin": + surefire = plugin + break + +if surefire is None: + raise SystemExit("{}: maven-surefire-plugin is missing".format(pom)) + +strict_executions = set() +for execution in surefire.findall("m:executions/m:execution", namespace): + execution_id = execution.find("m:id", namespace) + strict = execution.find("m:configuration/m:failIfNoSpecifiedTests", namespace) + if execution_id is None or strict is None: + continue + if (strict.text or "").strip() == "true": + strict_executions.add(execution_id.text) + +missing = sorted(set(expected_ids) - strict_executions) +if missing: + raise SystemExit( + "{}: named Surefire executions are not strict: {}".format( + pom, ", ".join(missing) + ) + ) +PY +} + +assert_supported_java_contract "${SOURCE_ROOT}/pom.xml" +assert_default_test_is_tolerant "${SOURCE_ROOT}/pom.xml" +assert_surefire_execution_scope \ + "${SOURCE_ROOT}/hugegraph-server/hugegraph-test/pom.xml" \ + core-test unit-test api-test tinkerpop-structure-test tinkerpop-process-test +assert_surefire_execution_scope \ + "${SOURCE_ROOT}/hugegraph-pd/hg-pd-test/pom.xml" \ + pd-client-test pd-core-test pd-common-test pd-rest-test +assert_surefire_execution_scope \ + "${SOURCE_ROOT}/hugegraph-store/hg-store-test/pom.xml" \ + store-client-test store-core-test store-common-test store-rocksdb-test \ + store-server-test store-raftcore-test + +MODULE_OPTIONS="${SERVER_ROOT}/bin/jvm-module.options" +SERVER_SCRIPT="${SERVER_ROOT}/bin/hugegraph-server.sh" +INIT_STORE_SCRIPT="${SERVER_ROOT}/bin/init-store.sh" +UTIL_SCRIPT="${SERVER_ROOT}/bin/util.sh" +CONF_SOURCE="${SERVER_ROOT}/conf" + +for source_file in "$MODULE_OPTIONS" "$SERVER_SCRIPT" \ + "$INIT_STORE_SCRIPT" "$UTIL_SCRIPT"; do + [[ -f "$source_file" ]] || fail "runtime asset is missing: $source_file" +done +[[ -d "$CONF_SOURCE" ]] || fail "server conf is missing: $CONF_SOURCE" + +TEMP_DIR=$(mktemp -d) +trap 'rm -rf "$TEMP_DIR"' EXIT + +DIST_ROOT="${TEMP_DIR}/server" +MOCK_JAVA_HOME="${TEMP_DIR}/mock-java-home" +mkdir -p "${DIST_ROOT}"/{bin,conf,ext,lib,logs,plugins} \ + "${MOCK_JAVA_HOME}/bin" +cp "$MODULE_OPTIONS" "$SERVER_SCRIPT" "$INIT_STORE_SCRIPT" "$UTIL_SCRIPT" \ + "${DIST_ROOT}/bin/" +cp -R "${CONF_SOURCE}/." "${DIST_ROOT}/conf/" + +# Model a full pre-Phase-2 conf/ directory: it has no module argfile. Both +# launchers must get the immutable runtime copy from bin/ instead. +if [[ -e "${DIST_ROOT}/conf/jvm-module.options" ]]; then + fail "legacy conf unexpectedly contains jvm-module.options" +fi + +cat > "${MOCK_JAVA_HOME}/bin/java" <<'MOCK' +#!/bin/bash +for argument in "$@"; do + if [[ "$argument" == "-version" ]]; then + echo 'openjdk version "17.0.0"' >&2 + exit 0 + fi +done +printf '%s\n' "$@" > "${CAPTURE_FILE:?}" +MOCK +chmod +x "${MOCK_JAVA_HOME}/bin/java" "${DIST_ROOT}/bin/hugegraph-server.sh" \ + "${DIST_ROOT}/bin/init-store.sh" + +SERVER_CAPTURE="${TEMP_DIR}/server.args" +CAPTURE_FILE="$SERVER_CAPTURE" JAVA_HOME="$MOCK_JAVA_HOME" \ + JAVA_OPTIONS="-Xms512m -Xmx512m" STDOUT_MODE=true \ + "${DIST_ROOT}/bin/hugegraph-server.sh" \ + "${DIST_ROOT}/conf/gremlin-server.yaml" \ + "${DIST_ROOT}/conf/rest-server.properties" false >/dev/null +assert_argument "@${DIST_ROOT}/bin/jvm-module.options" "$SERVER_CAPTURE" + +INIT_STORE_CAPTURE="${TEMP_DIR}/init-store.args" +CAPTURE_FILE="$INIT_STORE_CAPTURE" JAVA_HOME="$MOCK_JAVA_HOME" \ + "${DIST_ROOT}/bin/init-store.sh" >/dev/null +assert_argument "@${DIST_ROOT}/bin/jvm-module.options" "$INIT_STORE_CAPTURE" + +echo "PASS: Java 17 upgrade contracts" diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-riscv64-java-runtime.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-riscv64-java-runtime.sh new file mode 100755 index 0000000000..aa3131543b --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-riscv64-java-runtime.sh @@ -0,0 +1,155 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -euo pipefail + +TRAVIS_DIR=$(cd "$(dirname "$0")" && pwd) +SMOKE_SCRIPT="$TRAVIS_DIR/run-rocksdb-jni-smoke-test.sh" +NATIVE_SMOKE_SCRIPT="$TRAVIS_DIR/run-native-runtime-smoke-test.sh" +WORK_DIR=$(mktemp -d "${TMPDIR:-/tmp}/hugegraph-riscv64-java-test.XXXXXX") +MOCK_BIN="$WORK_DIR/bin" +MOCK_JAVA_HOME="$WORK_DIR/java-home" +SERVER_DIR="$WORK_DIR/server" + +cleanup() { + rm -rf "$WORK_DIR" +} +trap cleanup EXIT + +mkdir -p "$MOCK_BIN" "$MOCK_JAVA_HOME/bin" "$SERVER_DIR/bin" "$SERVER_DIR/lib" + +cat > "$MOCK_BIN/uname" <<'EOF' +#!/bin/bash +echo riscv64 +EOF + +cat > "$MOCK_JAVA_HOME/bin/java" <<'EOF' +#!/bin/bash +set -euo pipefail + +JAVA_VERSION=${MOCK_JAVA_VERSION:-17.0.20} +JAVA_VENDOR=${MOCK_JAVA_VENDOR:-Eclipse Adoptium} + +case "${1:-}" in + -version) + echo "openjdk version \"$JAVA_VERSION\"" >&2 + ;; + -XshowSettings:properties) + echo " java.vm.name = OpenJDK 64-Bit Server VM" >&2 + echo " java.vm.vendor = $JAVA_VENDOR" >&2 + echo " java.vm.version = $JAVA_VERSION+8" >&2 + echo " java.vm.info = mixed mode, sharing" >&2 + echo "openjdk version \"$JAVA_VERSION\"" >&2 + ;; + -cp) + echo "rocksdb-jni-smoke-ok" + ;; + *) + echo "Unexpected Java arguments: $*" >&2 + exit 1 + ;; +esac +EOF + +cat > "$SERVER_DIR/bin/util.sh" <<'EOF' +#!/bin/bash +configure_riscv64_libatomic() { + LD_PRELOAD=libatomic.so.1 +} +EOF + +cat > "$SERVER_DIR/bin/init-store.sh" <<'EOF' +#!/bin/bash +exit 42 +EOF + +chmod +x "$MOCK_BIN/uname" "$MOCK_JAVA_HOME/bin/java" \ + "$SERVER_DIR/bin/init-store.sh" + +run_smoke() { + env -u LD_PRELOAD \ + PATH="$MOCK_BIN:$PATH" \ + JAVA_HOME="$MOCK_JAVA_HOME" \ + "$@" "$SMOKE_SCRIPT" "$SERVER_DIR" +} + +run_native_smoke() { + env -u LD_PRELOAD \ + PATH="$MOCK_BIN:$PATH" \ + JAVA_HOME="$MOCK_JAVA_HOME" \ + "$@" "$NATIVE_SMOKE_SCRIPT" "$SERVER_DIR" +} + +if ! DEFAULT_OUTPUT=$(run_smoke 2>&1); then + echo "$DEFAULT_OUTPUT" >&2 + echo "RISC-V smoke rejected the Java 17 baseline" >&2 + exit 1 +fi +grep -q '^rocksdb-jni-smoke-ok$' <<< "$DEFAULT_OUTPUT" + +set +e +NATIVE_OUTPUT=$(run_native_smoke 2>&1) +NATIVE_STATUS=$? +set -e +if [[ $NATIVE_STATUS -ne 42 ]]; then + echo "$NATIVE_OUTPUT" >&2 + echo "Native smoke did not reach the controlled post-JNI boundary" >&2 + exit 1 +fi +grep -q '^rocksdb-jni-smoke-ok$' <<< "$NATIVE_OUTPUT" + +for JAVA_MAJOR_MISMATCH in 11.0.31 21.0.8; do + if MAJOR_OUTPUT=$(run_smoke \ + "MOCK_JAVA_VERSION=$JAVA_MAJOR_MISMATCH" 2>&1); then + echo "$MAJOR_OUTPUT" >&2 + echo "RISC-V smoke accepted Java $JAVA_MAJOR_MISMATCH" >&2 + exit 1 + fi + grep -Fq "Expected Java 17, got $JAVA_MAJOR_MISMATCH" <<< "$MAJOR_OUTPUT" +done + +EXPECTED_ARGS=( + EXPECTED_JAVA_MAJOR=17 + EXPECTED_RISCV64_JAVA_VERSION=17.0.20 + "EXPECTED_RISCV64_JAVA_VENDOR=Eclipse Adoptium" +) +if ! EXPECTED_OUTPUT=$(run_smoke "${EXPECTED_ARGS[@]}" 2>&1); then + echo "$EXPECTED_OUTPUT" >&2 + echo "RISC-V smoke rejected the configured Temurin 17 runtime" >&2 + exit 1 +fi +grep -q '^rocksdb-jni-smoke-ok$' <<< "$EXPECTED_OUTPUT" + +if VERSION_OUTPUT=$(run_smoke "${EXPECTED_ARGS[@]}" \ + MOCK_JAVA_VERSION=17.0.21 2>&1); then + echo "$VERSION_OUTPUT" >&2 + echo "RISC-V smoke accepted an unexpected Java version" >&2 + exit 1 +fi +grep -Fq 'Expected RISC-V Java 17.0.20, got 17.0.21' <<< "$VERSION_OUTPUT" + +if VENDOR_OUTPUT=$(run_smoke "${EXPECTED_ARGS[@]}" \ + "MOCK_JAVA_VENDOR=Unknown Vendor" 2>&1); then + echo "$VENDOR_OUTPUT" >&2 + echo "RISC-V smoke accepted an unexpected Java vendor" >&2 + exit 1 +fi +grep -Fq \ + 'Expected RISC-V Java vendor Eclipse Adoptium, got Unknown Vendor' \ + <<< "$VENDOR_OUTPUT" + +echo "PASS: RISC-V Java runtime contract" diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-signal.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-signal.sh new file mode 100644 index 0000000000..cd3df0814c --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-signal.sh @@ -0,0 +1,107 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Verifies that foreground start-hugegraph.sh exits on SIGINT when its +# background server child ignores SIGINT, as a JVM may do in this launch mode. + +set -uo pipefail + +START_SCRIPT="${1:-}" +if [[ -z "$START_SCRIPT" ]]; then + echo "Usage: $0 " + exit 2 +fi + +if [[ ! -f "$START_SCRIPT" ]]; then + echo "ERROR: start script not found: $START_SCRIPT" + exit 2 +fi + +if ! command -v timeout >/dev/null 2>&1; then + echo "SKIP: required tool 'timeout' not found" + exit 77 +fi + +TEST_ROOT=$(mktemp -d) +PID_FILE="$TEST_ROOT/bin/pid" + +cleanup() { + if [[ -s "$PID_FILE" ]]; then + kill -TERM "$(cat "$PID_FILE")" 2>/dev/null || true + fi + rm -rf "$TEST_ROOT" +} +trap cleanup EXIT + +mkdir -p "$TEST_ROOT/bin" "$TEST_ROOT/conf" "$TEST_ROOT/logs" "$TEST_ROOT/scripts" +cp "$START_SCRIPT" "$TEST_ROOT/bin/start-hugegraph.sh" + +cat > "$TEST_ROOT/bin/util.sh" <<'EOF' +read_property() { + local file="$1" + local property="$2" + grep "^${property}=" "$file" | head -n 1 | cut -d '=' -f 2- +} + +check_port() { + : +} +EOF + +cat > "$TEST_ROOT/bin/hugegraph-server.sh" <<'EOF' +#!/bin/bash +trap 'exit 0' TERM +trap '' INT +while true; do + sleep 1 +done +EOF + +cat > "$TEST_ROOT/conf/rest-server.properties" <<'EOF' +gremlinserver.url=http://127.0.0.1:8182 +restserver.url=http://127.0.0.1:8080 +EOF + +chmod +x "$TEST_ROOT/bin/start-hugegraph.sh" "$TEST_ROOT/bin/hugegraph-server.sh" + +export PID_FILE +export START_SCRIPT="$TEST_ROOT/bin/start-hugegraph.sh" + +timeout --signal=TERM --kill-after=5s 10s bash -c ' + target_pid=$$ + ( + while [[ ! -s "$PID_FILE" ]]; do + sleep 0.05 + done + sleep 0.1 + kill -INT "$target_pid" + ) & + exec "$START_SCRIPT" -d false +' +ACTUAL_EXIT=$? + +if [[ "$ACTUAL_EXIT" -ne 130 ]]; then + echo "FAIL: expected exit 130 after SIGINT, got $ACTUAL_EXIT" + exit 1 +fi + +if [[ -s "$PID_FILE" ]] && kill -0 "$(cat "$PID_FILE")" 2>/dev/null; then + echo "FAIL: server child is still running after SIGINT" + exit 1 +fi + +echo "PASS: SIGINT terminates the foreground wrapper and its server child" diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph.sh index 9f0bcfaa63..81a7ddd9fa 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph.sh @@ -30,6 +30,9 @@ set -uo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/ci-service-utils.sh" + HUGEGRAPH_ROOT="${1:-$(pwd)}" BIN="$HUGEGRAPH_ROOT/bin" START_SCRIPT="$BIN/start-hugegraph.sh" @@ -417,9 +420,9 @@ else wait_script_exit "$SCRIPT_PID" ACTUAL_EXIT=$? - # If the trap fired correctly, the wrapper's `wait $PID` already reaped Java. - # If wait_script_exit timed out (killer fired), Java may still be running — also a failure. - if ! ps -p "$FG_PID" >/dev/null 2>&1; then + # Allow a bounded shutdown window and treat a zombie as already terminated. + # If wait_script_exit timed out, a live Java process remains a failure. + if wait_for_process_exit "$FG_PID" "$SETTLE_WAIT"; then pass "Java process terminated after SIGTERM sent to wrapper" else fail "Java process still running after SIGTERM — signal not forwarded" diff --git a/hugegraph-server/hugegraph-test/conf/jvm-test-module.options b/hugegraph-server/hugegraph-test/conf/jvm-test-module.options new file mode 100644 index 0000000000..32b2b8b036 --- /dev/null +++ b/hugegraph-server/hugegraph-test/conf/jvm-test-module.options @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Test-only access required by TinkerPop and its Kryo dependencies on +# Java 17. Keep these permissions out of the Server runtime argument file. +--add-exports=java.base/sun.security.x509=ALL-UNNAMED +--add-exports=java.base/sun.security.action=ALL-UNNAMED +--add-opens=java.base/java.io=ALL-UNNAMED +--add-opens=java.base/java.nio=ALL-UNNAMED +--add-opens=java.base/sun.nio.cs=ALL-UNNAMED +--add-opens=java.base/java.lang=ALL-UNNAMED +--add-opens=java.base/java.lang.invoke=ALL-UNNAMED +--add-opens=java.base/java.lang.reflect=ALL-UNNAMED +--add-opens=java.base/java.util=ALL-UNNAMED +--add-opens=java.base/java.util.concurrent=ALL-UNNAMED +--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED +--add-opens=java.base/java.net=ALL-UNNAMED diff --git a/hugegraph-server/hugegraph-test/pom.xml b/hugegraph-server/hugegraph-test/pom.xml index 259d5a9b9a..768a83e88f 100644 --- a/hugegraph-server/hugegraph-test/pom.xml +++ b/hugegraph-server/hugegraph-test/pom.xml @@ -71,7 +71,6 @@ gremlin-groovy-test 3.2.11 - org.glassfish.jersey.test-framework.providers jersey-test-framework-provider-grizzly2 @@ -114,11 +113,18 @@ org.apache.maven.plugins maven-surefire-plugin - 2.20 + + + @{argLine} + "@${project.basedir}/../hugegraph-dist/src/assembly/static/bin/jvm-module.options" + "@${project.basedir}/conf/jvm-test-module.options" + + core-test + true ${basedir}/src/main/java/ ${basedir}/target/classes/ @@ -131,6 +137,7 @@ unit-test + true ${basedir}/src/main/java/ ${basedir}/target/classes/ @@ -143,6 +150,7 @@ api-test + true @@ -161,6 +169,7 @@ tinkerpop-structure-test + true ${basedir}/src/main/java/ ${basedir}/target/classes/ @@ -173,8 +182,14 @@ tinkerpop-process-test + true - -Dbuild.dir=${project.build.directory} + + @{argLine} + "@${project.basedir}/../hugegraph-dist/src/assembly/static/bin/jvm-module.options" + "@${project.basedir}/conf/jvm-test-module.options" + "-Dbuild.dir=${project.build.directory}" + ${basedir}/src/main/java/ ${basedir}/target/classes/ @@ -210,7 +225,6 @@ org.jacoco jacoco-maven-plugin - 0.8.8 org/apache/hugegraph/traversal/algorithm/*.class diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/store/client/OrderedScanSecurityTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/store/client/OrderedScanSecurityTest.java index 6d81e66487..c27e04b359 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/store/client/OrderedScanSecurityTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/store/client/OrderedScanSecurityTest.java @@ -81,6 +81,9 @@ private static void assertScanUnderSandbox(ExecutorService executor) throws Exce String factoryScript = "org.apache.hugegraph.store.client.util.ExecutorPool." + "newThreadFactory('untrusted').newThread({} as Runnable)"; engine.eval(factoryScript); + // Load Groovy's Thread metadata before restricting filesystem access. + // The assertion below must exercise thread-group access, not class loading. + engine.eval("new Thread()"); SecurityManager previous = System.getSecurityManager(); String name = Thread.currentThread().getName(); Thread.currentThread().setName("gremlin-server-exec-ordered-scan-test"); diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java index 4189c1692d..eddcda37d9 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java @@ -45,6 +45,7 @@ import org.apache.hugegraph.unit.cmd.InitStoreConfigTest; import org.apache.hugegraph.unit.core.AnalyzerTest; import org.apache.hugegraph.unit.core.BackendMutationTest; +import org.apache.hugegraph.unit.core.BackendProviderFactoryTest; import org.apache.hugegraph.unit.core.BackendStoreInfoTest; import org.apache.hugegraph.unit.core.ConditionQueryFlattenTest; import org.apache.hugegraph.unit.core.ConditionTest; @@ -156,6 +157,7 @@ RowLockTest.class, AnalyzerTest.class, BackendMutationTest.class, + BackendProviderFactoryTest.class, ConditionTest.class, StandardHugeGraphClearBackendTest.class, ConditionQueryFlattenTest.class, diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/auth/HugeGraphAuthProxyTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/auth/HugeGraphAuthProxyTest.java index 37d70af13f..60d2e097eb 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/auth/HugeGraphAuthProxyTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/auth/HugeGraphAuthProxyTest.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.concurrent.atomic.AtomicReference; +import org.apache.hugegraph.HugeException; import org.apache.hugegraph.HugeGraph; import org.apache.hugegraph.auth.AuthManager; import org.apache.hugegraph.auth.HugeAuthenticator; @@ -42,6 +43,7 @@ import org.apache.hugegraph.testutil.Assert; import org.apache.hugegraph.testutil.Whitebox; import org.apache.hugegraph.unit.BaseUnitTest; +import org.apache.hugegraph.util.Reflection; import org.apache.hugegraph.util.RateLimiter; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; @@ -59,6 +61,43 @@ public class HugeGraphAuthProxyTest extends BaseUnitTest { + @Test + public void testJdk17ReflectionFilters() { + Reflection.registerFieldsToFilter(ReflectionFilterTarget.class, "field"); + Reflection.registerMethodsToFilter(ReflectionFilterTarget.class, "method"); + + Assert.assertThrows(NoSuchFieldException.class, + () -> ReflectionFilterTarget.class.getDeclaredField("field")); + Assert.assertThrows(NoSuchMethodException.class, + () -> ReflectionFilterTarget.class.getDeclaredMethod("method")); + Assert.assertThrows(IllegalArgumentException.class, + () -> Reflection.registerFieldsToFilter( + ReflectionFilterTarget.class, "field")); + } + + @Test + public void testJdk17ReflectionFilterFailureCause() { + Throwable exception = Assert.assertThrows( + HugeException.class, + () -> Reflection.registerFieldsToFilter( + ReflectionFailureTarget.class, (String) null)); + + Assert.assertInstanceOf(NullPointerException.class, exception.getCause()); + } + + private static class ReflectionFailureTarget { + } + + private static class ReflectionFilterTarget { + + @SuppressWarnings("unused") + private String field; + + @SuppressWarnings("unused") + private void method() { + } + } + private static HugeGraphAuthProxy.Context setContext( HugeGraphAuthProxy.Context context) { try { diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/BackendProviderFactoryTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/BackendProviderFactoryTest.java new file mode 100644 index 0000000000..767febd620 --- /dev/null +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/BackendProviderFactoryTest.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.unit.core; + +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import org.apache.hugegraph.backend.BackendException; +import org.apache.hugegraph.backend.store.BackendProviderFactory; +import org.apache.hugegraph.backend.store.memory.InMemoryDBStoreProvider; +import org.apache.hugegraph.backend.store.raft.RaftBackendStoreProvider; +import org.apache.hugegraph.testutil.Assert; +import org.apache.hugegraph.unit.BaseUnitTest; +import org.junit.Test; + +public class BackendProviderFactoryTest extends BaseUnitTest { + + @Test + public void testRegister() { + String name = "fake-provider"; + String provider = InMemoryDBStoreProvider.class.getName(); + + BackendProviderFactory.register(name, provider); + BackendProviderFactory.register(name, provider); + + Assert.assertThrows(BackendException.class, () -> { + BackendProviderFactory.register( + name, RaftBackendStoreProvider.class.getName()); + }, e -> { + Assert.assertContains("Exists BackendStoreProvider:", + e.getMessage()); + }); + } + + @Test + public void testRegisterConcurrently() throws Exception { + String name = "concurrent-provider-" + System.nanoTime(); + CountDownLatch ready = new CountDownLatch(2); + CountDownLatch start = new CountDownLatch(1); + ExecutorService executor = Executors.newFixedThreadPool(2); + try { + List>> registrations = List.of( + executor.submit(() -> { + ready.countDown(); + start.await(); + BackendProviderFactory.register( + name, + InMemoryDBStoreProvider.class.getName()); + return InMemoryDBStoreProvider.class; + }), + executor.submit(() -> { + ready.countDown(); + start.await(); + BackendProviderFactory.register( + name, + RaftBackendStoreProvider.class.getName()); + return RaftBackendStoreProvider.class; + })); + + Assert.assertTrue(ready.await(5L, TimeUnit.SECONDS)); + start.countDown(); + + int successes = 0; + int conflicts = 0; + Class registered = null; + for (Future> registration : registrations) { + try { + registered = registration.get(5L, TimeUnit.SECONDS); + successes++; + } catch (ExecutionException e) { + Assert.assertInstanceOf(BackendException.class, + e.getCause()); + Assert.assertContains("Exists BackendStoreProvider:", + e.getCause().getMessage()); + conflicts++; + } + } + + Assert.assertEquals(1, successes); + Assert.assertEquals(1, conflicts); + Assert.assertNotNull(registered); + BackendProviderFactory.register(name, registered.getName()); + } finally { + start.countDown(); + executor.shutdownNow(); + } + } +} diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/serializer/SerializerFactoryTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/serializer/SerializerFactoryTest.java index 67afa69842..15294439db 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/serializer/SerializerFactoryTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/serializer/SerializerFactoryTest.java @@ -17,6 +17,14 @@ package org.apache.hugegraph.unit.serializer; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + import org.apache.hugegraph.backend.BackendException; import org.apache.hugegraph.backend.serializer.AbstractSerializer; import org.apache.hugegraph.backend.serializer.BinaryScatterSerializer; @@ -58,9 +66,14 @@ public void testRegister() { Assert.assertEquals(FakeSerializer.class, SerializerFactory.serializer(config, "fake").getClass()); + // Identical registration is idempotent + SerializerFactory.register("fake", FakeSerializer.class.getName()); + Assert.assertEquals(FakeSerializer.class, + SerializerFactory.serializer(config, "fake").getClass()); + Assert.assertThrows(BackendException.class, () -> { - // exist - SerializerFactory.register("fake", FakeSerializer.class.getName()); + // conflict + SerializerFactory.register("fake", TextSerializer.class.getName()); }, e -> { Assert.assertContains("Exists serializer:", e.getMessage()); }); @@ -81,6 +94,63 @@ public void testRegister() { }); } + @Test + public void testRegisterConcurrently() throws Exception { + String name = "concurrent-serializer-" + System.nanoTime(); + CountDownLatch ready = new CountDownLatch(2); + CountDownLatch start = new CountDownLatch(1); + ExecutorService executor = Executors.newFixedThreadPool(2); + try { + List>> registrations = List.of( + executor.submit(() -> { + ready.countDown(); + start.await(); + SerializerFactory.register( + name, FakeSerializer.class.getName()); + return FakeSerializer.class; + }), + executor.submit(() -> { + ready.countDown(); + start.await(); + SerializerFactory.register( + name, TextSerializer.class.getName()); + return TextSerializer.class; + })); + + Assert.assertTrue(ready.await(5L, TimeUnit.SECONDS)); + start.countDown(); + + int successes = 0; + int conflicts = 0; + Class registered = null; + for (Future> registration : registrations) { + try { + registered = registration.get(5L, TimeUnit.SECONDS); + successes++; + } catch (ExecutionException e) { + Assert.assertInstanceOf(BackendException.class, + e.getCause()); + Assert.assertContains("Exists serializer:", + e.getCause().getMessage()); + conflicts++; + } + } + + Assert.assertEquals(1, successes); + Assert.assertEquals(1, conflicts); + Assert.assertNotNull(registered); + + SerializerFactory.register(name, registered.getName()); + HugeConfig config = FakeObjects.newConfig(); + Assert.assertEquals( + registered, + SerializerFactory.serializer(config, name).getClass()); + } finally { + start.countDown(); + executor.shutdownNow(); + } + } + public static class FakeSerializer extends BinarySerializer { public FakeSerializer(HugeConfig config) { diff --git a/hugegraph-store/Dockerfile b/hugegraph-store/Dockerfile index 988b472b09..474fe59846 100644 --- a/hugegraph-store/Dockerfile +++ b/hugegraph-store/Dockerfile @@ -18,7 +18,7 @@ # Dockerfile for HugeGraph Store # 1st stage: build source code -FROM --platform=$BUILDPLATFORM maven:3.9.0-eclipse-temurin-11 AS build +FROM --platform=$BUILDPLATFORM maven:3.9.16-eclipse-temurin-17 AS build WORKDIR /pkg @@ -35,7 +35,7 @@ RUN --mount=type=cache,id=hugegraph-maven-${SOURCE_REVISION},target=/root/.m2,sh # 2nd stage: runtime env # Note: ZGC (The Z Garbage Collector) is only supported on ARM-Mac with java > 13 -FROM eclipse-temurin:11-jre-jammy +FROM eclipse-temurin:17-jre-jammy LABEL maintainer="HugeGraph Docker Maintainers " diff --git a/hugegraph-store/README.md b/hugegraph-store/README.md index 0c873f8fad..7d9575fb8c 100644 --- a/hugegraph-store/README.md +++ b/hugegraph-store/README.md @@ -23,7 +23,7 @@ HugeGraph Store is a distributed storage backend for HugeGraph that provides hig - **Storage Engine**: RocksDB 8.10.2 (see the [upgrade guidance](../docs/rocksdb-upgrade.md)) - **Consensus Protocol**: Apache JRaft (Ant Financial's Raft implementation) - **RPC Framework**: gRPC + Protocol Buffers -- **Deployment**: Java 11+, Docker/Kubernetes support +- **Deployment**: Java 17 (currently supported release), Docker/Kubernetes support ### When to Use HugeGraph Store @@ -87,8 +87,8 @@ For detailed architecture, Raft consensus mechanisms, and partition management, ### Prerequisites -- **Java**: 11 or higher -- **Maven**: 3.5 or higher +- **Java**: 17 (currently the only supported release) +- **Maven**: 3.6.3 or higher - **HugeGraph PD Cluster**: Store requires a running PD cluster for metadata coordination (see [PD README](../hugegraph-pd/README.md)) - **Disk Space**: At least 10GB per Store node for data and Raft logs - **Network**: Low-latency network (<5ms) between Store nodes for Raft consensus diff --git a/hugegraph-store/docs/deployment-guide.md b/hugegraph-store/docs/deployment-guide.md index ca39ae509c..a3339f6518 100644 --- a/hugegraph-store/docs/deployment-guide.md +++ b/hugegraph-store/docs/deployment-guide.md @@ -392,7 +392,7 @@ graph.name=hugegraph **On all nodes**: ```bash -# Check Java version (11+ required) +# Check Java version (Java 17 is the only currently supported release) java -version # Check Maven (for building from source) diff --git a/hugegraph-store/docs/development-guide.md b/hugegraph-store/docs/development-guide.md index 44136776e7..bdc36e07ef 100644 --- a/hugegraph-store/docs/development-guide.md +++ b/hugegraph-store/docs/development-guide.md @@ -18,8 +18,8 @@ Comprehensive guide for developing, testing, and contributing to HugeGraph Store ### Prerequisites **Required**: -- Java: 11 or higher (OpenJDK or Oracle JDK) -- Maven: 3.5 or higher +- Java: 17 (currently the only supported release; OpenJDK or Oracle JDK) +- Maven: 3.6.3 or higher - Git: Latest version - IDE: IntelliJ IDEA (recommended) or Eclipse @@ -303,12 +303,12 @@ mvn test -P store-core-test **Specific test class**: ```bash -mvn test -Dtest=HgStoreEngineTest +mvn test -pl hg-store-test -am -Dtest=HgStoreEngineTest ``` **Specific test method**: ```bash -mvn test -Dtest=HgStoreEngineTest#testPartitionCreation +mvn test -pl hg-store-test -am -Dtest=HgStoreEngineTest#testPartitionCreation ``` **From IntelliJ**: diff --git a/hugegraph-store/hg-store-core/pom.xml b/hugegraph-store/hg-store-core/pom.xml index 4eed6d8e47..f119543006 100644 --- a/hugegraph-store/hg-store-core/pom.xml +++ b/hugegraph-store/hg-store-core/pom.xml @@ -88,7 +88,7 @@ com.alibaba fastjson - 1.2.83 + 1.2.84 diff --git a/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/GraphIdManager.java b/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/GraphIdManager.java index 3c95919591..76eada307b 100644 --- a/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/GraphIdManager.java +++ b/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/GraphIdManager.java @@ -20,18 +20,25 @@ import static org.apache.hugegraph.store.constant.HugeServerTables.VERTEX_TABLE; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.Arrays; -import java.util.List; +import java.util.Collections; +import java.util.HashMap; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; +import org.apache.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.rocksdb.access.SessionOperator; import org.apache.hugegraph.store.meta.base.DBSessionBuilder; import org.apache.hugegraph.store.meta.base.PartitionMetaStore; import org.apache.hugegraph.store.term.Bits; +import org.apache.hugegraph.store.util.Asserts; import org.apache.hugegraph.store.util.HgStoreException; import com.google.protobuf.Int64Value; +import com.google.protobuf.InvalidProtocolBufferException; import lombok.extern.slf4j.Slf4j; @@ -43,7 +50,7 @@ public class GraphIdManager extends PartitionMetaStore { protected static final String GRAPH_ID_PREFIX = "@GRAPH_ID@"; - // FIXME: we need to ensure the right num & proper logic for it (IMPORTANT) + // Missing-graph sentinel; allocatable graph IDs are [0, maxGraphID) protected static int maxGraphID = 65535 - 1; static Object graphIdLock = new Object(); static Object cidLock = new Object(); @@ -68,13 +75,18 @@ public long getGraphId(String graphName) { byte[] key = MetadataKeyHelper.getGraphIDKey(graphName); Int64Value id = get(Int64Value.parser(), key); if (id == null) { - id = Int64Value.of(maxGraphID); + l = (long) maxGraphID; + } else { + l = checkGraphId(graphName, id.getValue(), + this.partitionId); } - l = id.getValue(); graphIdCache.put(graphName, l); } } } + if (l != maxGraphID) { + checkGraphId(graphName, l, this.partitionId); + } return l; } @@ -87,26 +99,152 @@ public long getGraphIdOrCreate(String graphName) { byte[] key = MetadataKeyHelper.getGraphIDKey(graphName); Int64Value id = get(Int64Value.parser(), key); if (id == null) { - id = Int64Value.of(getCId(GRAPH_ID_PREFIX, maxGraphID - 1)); + id = Int64Value.of(getCId(GRAPH_ID_PREFIX, maxGraphID)); if (id.getValue() == -1) { throw new HgStoreException(HgStoreException.EC_FAIL, "The number of graphs exceeds the maximum " + - "65535"); + maxGraphID); } log.info("partition: {}, Graph ID {} is allocated for graph {}, stack: {}", this.partitionId, id.getValue(), graphName, Arrays.toString(Thread.currentThread().getStackTrace())); put(key, id); flush(); + } else { + checkGraphId(graphName, id.getValue(), + this.partitionId); } l = id.getValue(); graphIdCache.put(graphName, l); } } } + checkGraphId(graphName, l, this.partitionId); return l; } + public static long checkGraphId(String graphName, long graphId, + int partitionId) { + if (graphId < 0L || graphId >= maxGraphID) { + throw new HgStoreException( + HgStoreException.EC_FAIL, + "Invalid graph ID %s for graph '%s' in partition %s, " + + "expected a value in [0, %s)", + graphId, graphName, partitionId, maxGraphID); + } + return graphId; + } + + public static void checkGraphIds(Map graphIds, + int partitionId) { + Map graphNames = new HashMap<>(); + graphIds.forEach((graphName, graphId) -> { + if (graphId == null) { + throw new HgStoreException(HgStoreException.EC_FAIL, + "Invalid null graph ID for graph '%s' " + + "in partition %s", + graphName, partitionId); + } + checkGraphId(graphName, graphId, partitionId); + String previous = graphNames.putIfAbsent(graphId, graphName); + if (previous != null && !previous.equals(graphName)) { + throw new HgStoreException( + HgStoreException.EC_FAIL, + "Graph ID %s is assigned to multiple graphs '%s' and '%s' " + + "in partition %s", + graphId, previous, graphName, partitionId); + } + }); + } + + public void updateGraphIds(Map graphIds) { + Map updates = new HashMap<>(graphIds); + checkGraphIds(updates, this.partitionId); + synchronized (graphIdLock) { + Map finalGraphIds = this.graphIds(); + Set previousGraphIds = updates.keySet().stream() + .map(finalGraphIds::get) + .filter(graphId -> graphId != null && + graphId >= 0L && + graphId < maxGraphID) + .collect(Collectors.toSet()); + finalGraphIds.putAll(updates); + this.checkUniqueGraphIds(finalGraphIds); + this.writeGraphIds(updates, previousGraphIds); + this.graphIdCache.putAll(updates); + } + } + + private Map graphIds() { + byte[] prefix = MetadataKeyHelper.getGraphIDKey(""); + Map graphIds = new HashMap<>(); + for (RocksDBSession.BackendColumn column : scan(prefix)) { + String graphName = new String(column.name, prefix.length, + column.name.length - prefix.length, + StandardCharsets.UTF_8); + try { + long graphId = Int64Value.parseFrom(column.value).getValue(); + graphIds.put(graphName, graphId); + } catch (InvalidProtocolBufferException e) { + throw new HgStoreException(HgStoreException.EC_FAIL, e); + } + } + return graphIds; + } + + private void checkUniqueGraphIds(Map graphIds) { + Map graphNames = new HashMap<>(); + graphIds.forEach((graphName, graphId) -> { + // Ignore legacy invalid IDs here so this repair path can replace them. + if (graphId < 0L || graphId >= maxGraphID) { + return; + } + String previous = graphNames.putIfAbsent(graphId, graphName); + if (previous != null && !previous.equals(graphName)) { + throw new HgStoreException( + HgStoreException.EC_FAIL, + "Graph ID %s is assigned to multiple graphs '%s' and '%s' " + + "in partition %s", + graphId, previous, graphName, this.partitionId); + } + }); + } + + private void writeGraphIds(Map graphIds, + Set previousGraphIds) { + try (RocksDBSession dbSession = getRocksDBSession()) { + SessionOperator operator = dbSession.sessionOp(); + try { + operator.prepare(); + // A repaired mapping does not prove that the old graph's data + // prefix is empty. Keep its ID reserved even if the slot was + // absent in legacy metadata. + for (Long graphId : previousGraphIds) { + byte[] slotKey = genCIDSlotKey(GRAPH_ID_PREFIX, graphId); + operator.put(getCFName(), slotKey, + Int64Value.of(graphId).toByteArray()); + } + // Publish mappings only after all IDs are reserved in the same batch. + for (Map.Entry entry : graphIds.entrySet()) { + Int64Value value = Int64Value.of(entry.getValue()); + byte[] slotKey = genCIDSlotKey(GRAPH_ID_PREFIX, + entry.getValue()); + operator.put(getCFName(), slotKey, value.toByteArray()); + } + for (Map.Entry entry : graphIds.entrySet()) { + Int64Value value = Int64Value.of(entry.getValue()); + byte[] graphIdKey = + MetadataKeyHelper.getGraphIDKey(entry.getKey()); + operator.put(getCFName(), graphIdKey, value.toByteArray()); + } + operator.commit(); + } catch (RuntimeException e) { + operator.rollback(); + throw e; + } + } + } + /** * Release a graph id */ @@ -132,10 +270,11 @@ private boolean checkCount(long l) { Bits.putShort(start, 0, (short) l); try (var session = sessionBuilder.getSession(partitionId)) { if (!session.tableIsExist(VERTEX_TABLE)) { + // Scanning a missing table creates it and requires a write lock return true; } - try (var itr = session.sessionOp().scan(VERTEX_TABLE, start)) { - return itr == null || !itr.hasNext(); + try (var iterator = session.sessionOp().scan(VERTEX_TABLE, start)) { + return iterator == null || !iterator.hasNext(); } } } @@ -149,41 +288,58 @@ private boolean checkCount(long l) { * @return id */ protected long getCId(String key, long max) { + Asserts.isTrue(max > 0L, "The maximum cyclic ID must be positive"); byte[] cidNextKey = MetadataKeyHelper.getCidKey(key); synchronized (cidLock) { Int64Value value = get(Int64Value.parser(), cidNextKey); - long current = value != null ? value.getValue() : 0L; - long last = current == 0 ? max - 1 : current - 1; - // Find an unused cid - List ids = - scan(Int64Value.parser(), genCIDSlotKey(key, current), genCIDSlotKey(key, max)); - var idSet = ids.stream().map(Int64Value::getValue).collect(Collectors.toSet()); - - while (idSet.contains(current) || !checkCount(current)) { - current++; + long start = Math.floorMod(value != null ? value.getValue() : 0L, max); + Set assignedGraphIds = this.assignedGraphIds(key); + long current = this.findAvailableCId(key, start, max, + assignedGraphIds); + if (current == -1L && start > 0L) { + current = this.findAvailableCId(key, 0L, start, + assignedGraphIds); } - - if (current == max - 1) { - current = 0; - ids = scan(Int64Value.parser(), genCIDSlotKey(key, current), - genCIDSlotKey(key, last)); - idSet = ids.stream().map(Int64Value::getValue).collect(Collectors.toSet()); - while (idSet.contains(current) || !checkCount(current)) { - current++; - } + if (current == -1L) { + return -1L; } - if (current == last) { - return -1; - } // Save current id, mark as used put(genCIDSlotKey(key, current), Int64Value.of(current)); - // Save the id for the next traversal - put(cidNextKey, Int64Value.of(current + 1)); + // Keep the next traversal position inside [0, max) + long next = current + 1L; + put(cidNextKey, Int64Value.of(next == max ? 0L : next)); return current; } } + private long findAvailableCId(String key, long start, long end, + Set assignedGraphIds) { + Set idSet = scan(Int64Value.parser(), genCIDSlotKey(key, start), + genCIDSlotKey(key, end)) + .stream() + .map(Int64Value::getValue) + .collect(Collectors.toSet()); + idSet.addAll(assignedGraphIds); + for (long current = start; current < end; current++) { + if (!idSet.contains(current) && checkCount(current)) { + return current; + } + } + return -1L; + } + + private Set assignedGraphIds(String key) { + if (!GRAPH_ID_PREFIX.equals(key)) { + return Collections.emptySet(); + } + // A persisted mapping stays authoritative even if its slot is missing. + return scan(Int64Value.parser(), MetadataKeyHelper.getGraphIDKey("")) + .stream() + .map(Int64Value::getValue) + .collect(Collectors.toSet()); + } + /** * Return key with used Cid */ diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh index 88165e47b6..15006bb777 100755 --- a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh +++ b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh @@ -122,7 +122,7 @@ ensure_path_writable "$PLUGINS" # The maximum and minimum heap memory that service can use (for production env set it 36GB) MAX_MEM=$((2 * 1024)) MIN_MEM=$((1 * 512)) -EXPECT_JDK_VERSION=11 +EXPECT_JDK_VERSION=17 # Change to $BIN's parent cd ${TOP} || exit @@ -135,8 +135,11 @@ else fi # check jdk version -JAVA_VERSION=$($JAVA -version 2>&1 | awk 'NR==1{gsub(/"/,""); print $3}' | awk -F'_' '{print $1}') -if [[ $? -ne 0 || $JAVA_VERSION < $EXPECT_JDK_VERSION ]]; then +JAVA_VERSION=$($JAVA -version 2>&1 | + awk -F'"' '/^(java|openjdk) version "/ {print $2; exit}' | + sed 's/^1\.//' | cut -d'.' -f1) +JAVA_VERSION="${JAVA_VERSION%%[!0-9]*}" +if [[ -z $JAVA_VERSION || $JAVA_VERSION -lt $EXPECT_JDK_VERSION ]]; then echo "Please make sure that the JDK is installed and the version >= $EXPECT_JDK_VERSION" >> ${OUTPUT} exit 1 fi @@ -164,7 +167,7 @@ case "$GC_OPTION" in -XX:InitiatingHeapOccupancyPercent=50 -XX:G1RSetUpdatingPauseTimePercent=5" ;; zgc|ZGC) - echo "Using ZGC as the default garbage collector (Only support Java 11+)" + echo "Using ZGC as the default garbage collector (requires Java 17 or later)" JAVA_OPTIONS="${JAVA_OPTIONS} -XX:+UseZGC -XX:+UnlockExperimentalVMOptions \ -XX:ConcGCThreads=2 -XX:ParallelGCThreads=6 \ -XX:ZCollectionInterval=120 -XX:ZAllocationSpikeTolerance=5 \ diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/controller/FixGraphIdController.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/controller/FixGraphIdController.java index 43a97e814c..e721d66469 100644 --- a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/controller/FixGraphIdController.java +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/controller/FixGraphIdController.java @@ -106,16 +106,14 @@ public String getNextId(@PathVariable(value = "partition_id") int pid) throws IO @PostMapping(value = "/update_graph_id/{partition_id}", produces = "application/json") public String updateGraphId(@PathVariable(value = "partition_id") int pid, @RequestBody Map idMap) throws IOException { + GraphIdManager.checkGraphIds(idMap, pid); + var handler = (BusinessHandlerImpl) nodeService.getStoreEngine().getBusinessHandler(); try (var manager = new GraphIdManager(handler, pid)) { - idMap.forEach((graphName, graphId) -> { - log.info("update graph id of {} to {}, partition, {}", graphName, graphId, pid); - var graphIdKey = MetadataKeyHelper.getGraphIDKey(graphName); - var slotKey = manager.genCIDSlotKey(GRAPH_ID_PREFIX, graphId); - var value = Int64Value.of(graphId); - manager.put(graphIdKey, value); - manager.put(slotKey, value); - }); + idMap.forEach((graphName, graphId) -> + log.info("update graph id of {} to {}, partition, {}", + graphName, graphId, pid)); + manager.updateGraphIds(idMap); manager.flush(); } handler.getKeyCreator().clearCache(pid); diff --git a/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/controller/FixGraphIdControllerTest.java b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/controller/FixGraphIdControllerTest.java new file mode 100644 index 0000000000..2a14fe31ac --- /dev/null +++ b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/controller/FixGraphIdControllerTest.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.node.controller; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +import org.apache.hugegraph.store.util.HgStoreException; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class FixGraphIdControllerTest { + + @Test + public void testUpdateGraphIdRejectsIdsOutsideAllocatableDomain() { + FixGraphIdController controller = new FixGraphIdController(); + long[] invalidGraphIds = {-1L, 65534L, 65536L}; + + for (long graphId : invalidGraphIds) { + String graphName = "invalid-" + graphId; + Map graphIds = + Collections.singletonMap(graphName, graphId); + + HgStoreException exception = Assertions.assertThrows( + HgStoreException.class, + () -> controller.updateGraphId(0, graphIds)); + Assertions.assertTrue(exception.getMessage().contains("Invalid graph ID")); + Assertions.assertTrue(exception.getMessage().contains(String.valueOf(graphId))); + Assertions.assertTrue(exception.getMessage().contains(graphName)); + } + } + + @Test + public void testUpdateGraphIdRejectsDuplicateIds() { + FixGraphIdController controller = new FixGraphIdController(); + Map graphIds = new LinkedHashMap<>(); + graphIds.put("graph-a", 5L); + graphIds.put("graph-b", 5L); + + HgStoreException exception = Assertions.assertThrows( + HgStoreException.class, + () -> controller.updateGraphId(0, graphIds)); + Assertions.assertTrue(exception.getMessage().contains("graph-a")); + Assertions.assertTrue(exception.getMessage().contains("graph-b")); + Assertions.assertTrue(exception.getMessage().contains("5")); + } +} diff --git a/hugegraph-store/hg-store-rocksdb/pom.xml b/hugegraph-store/hg-store-rocksdb/pom.xml index 232aecffa1..61e1ce1dfa 100644 --- a/hugegraph-store/hg-store-rocksdb/pom.xml +++ b/hugegraph-store/hg-store-rocksdb/pom.xml @@ -71,7 +71,7 @@ com.alibaba fastjson - 1.2.83 + 1.2.84 diff --git a/hugegraph-store/hg-store-test/pom.xml b/hugegraph-store/hg-store-test/pom.xml index 2eab2c3e87..edc045ba88 100644 --- a/hugegraph-store/hg-store-test/pom.xml +++ b/hugegraph-store/hg-store-test/pom.xml @@ -51,7 +51,6 @@ org.jacoco jacoco-maven-plugin - 0.8.8 @@ -221,11 +220,18 @@ org.apache.maven.plugins maven-surefire-plugin - 2.20 + + + org.apache.maven.surefire + surefire-junit4 + ${maven.surefire.plugin.version} + + store-client-test + true ${basedir}/src/main/java/ ${basedir}/target/classes/ @@ -238,6 +244,7 @@ store-core-test + true ${basedir}/src/main/java/ ${basedir}/target/classes/ @@ -253,12 +260,14 @@ **/PartitionStateMachineTest.java **/PartitionEngineErrorTest.java **/BatchGraphIsolationTest.java + **/GraphIDManagerTest.java store-common-test + true ${basedir}/src/main/java/ ${basedir}/target/classes/ @@ -271,6 +280,7 @@ store-rocksdb-test + true ${basedir}/src/main/java/ ${basedir}/target/classes/ @@ -283,6 +293,7 @@ store-server-test + true ${basedir}/src/main/java/ ${basedir}/target/classes/ @@ -298,6 +309,7 @@ store-raftcore-test + true ${basedir}/src/main/java/ ${basedir}/target/classes/ @@ -312,7 +324,6 @@ org.jacoco jacoco-maven-plugin - 0.8.8 **/grpc/**/* diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/OrderedKvIteratorTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/OrderedKvIteratorTest.java index 3073c1bfec..77802ca2c3 100644 --- a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/OrderedKvIteratorTest.java +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/OrderedKvIteratorTest.java @@ -298,7 +298,8 @@ public void testConcurrentInitializeFailsWithoutWaitingForSlowSource() TestIterator slow = new TestIterator(1); TestIterator failed = new TestIterator(2); slow.blockFirstHasNext(slowStarted, releaseSlow); - failed.failOnHasNextAfter(0); + // Start the slow source before failure can cancel its task. + failed.blockAndFailFirstHasNext(new CountDownLatch(0), slowStarted); Future result = caller.submit(() -> { OrderedKvIterator iterator = new OrderedKvIterator( diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/BatchGraphIsolationTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/BatchGraphIsolationTest.java index 3d78821a63..da159100ce 100644 --- a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/BatchGraphIsolationTest.java +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/BatchGraphIsolationTest.java @@ -63,6 +63,7 @@ public class BatchGraphIsolationTest { private static final int PARTITION_ID = 0; private static final int EMPTY_PARTITION_ID = 1; + private static final int ALLOCATION_PARTITION_ID = 2; private static final int KEY_CODE = 0; private static final int EMPTY_PARTITION_KEY_CODE = 32768; private static final byte[] SHARED_KEY = @@ -86,7 +87,7 @@ public static void setup() throws IOException { HgStoreEngineOptions.FakePdOptions fakePdOptions = new HgStoreEngineOptions.FakePdOptions(); - fakePdOptions.setPartitionCount(2); + fakePdOptions.setPartitionCount(3); fakePdOptions.setPeersList("127.0.0.1"); fakePdOptions.setStoreList("127.0.0.1"); options.setFakePdOptions(fakePdOptions); @@ -101,7 +102,8 @@ public String getDbDataPath(int partitionId, String dbName) { @Override public boolean hasPartition(String graphName, int partitionId) { - return partitionId == PARTITION_ID || partitionId == EMPTY_PARTITION_ID; + return partitionId == PARTITION_ID || partitionId == EMPTY_PARTITION_ID || + partitionId == ALLOCATION_PARTITION_ID; } @Override @@ -123,6 +125,18 @@ public static void teardown() { } } + @Test + public void testGraphIdAllocationDoesNotCreateVertexTable() { + String graph = "graph-id-allocation"; + + Assert.assertFalse(handler.existsTable(graph, ALLOCATION_PARTITION_ID, VERTEX_TABLE)); + + ((BusinessHandlerImpl) handler).getKeyCreator() + .getGraphIdOrCreate(ALLOCATION_PARTITION_ID, graph); + + Assert.assertFalse(handler.existsTable(graph, ALLOCATION_PARTITION_ID, VERTEX_TABLE)); + } + @Test(timeout = 5000L) public void testFirstBatchOnEmptyPartitionCompletes() { String graph = "first-batch-empty-partition"; diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/meta/GraphIDManagerTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/meta/GraphIDManagerTest.java index 60bb542a9b..ac0d5d0d1e 100644 --- a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/meta/GraphIDManagerTest.java +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/meta/GraphIDManagerTest.java @@ -17,69 +17,426 @@ package org.apache.hugegraph.store.meta; +import static org.apache.hugegraph.store.constant.HugeServerTables.VERTEX_TABLE; +import static org.apache.hugegraph.store.constant.HugeServerTables.OUT_EDGE_TABLE; +import static org.apache.hugegraph.store.constant.HugeServerTables.IN_EDGE_TABLE; +import static org.apache.hugegraph.store.constant.HugeServerTables.INDEX_TABLE; + import java.io.File; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; -import org.apache.hugegraph.pd.common.PDException; import org.apache.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.rocksdb.access.SessionOperator; import org.apache.hugegraph.store.UnitTestBase; +import org.apache.hugegraph.store.business.BusinessHandlerImpl; import org.apache.hugegraph.store.meta.base.DBSessionBuilder; +import org.apache.hugegraph.store.term.Bits; +import org.apache.hugegraph.store.util.HgStoreException; +import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import com.google.protobuf.Int64Value; + public class GraphIDManagerTest extends UnitTestBase { + + private static final int PARTITION_ID = 0; + private static final int GRAPH_ID_LIMIT = 8; + + private String dbName; + private File dbPath; + private RocksDBSession session; + private DBSessionBuilder sessionBuilder; + private int previousMaxGraphId; + @Before public void init() { - String dbPath = "/tmp/junit"; - UnitTestBase.deleteDir(new File(dbPath)); - super.initDB(dbPath); - } - - @Test - public void test() throws PDException { - GraphIdManager.maxGraphID = 64; - int max = GraphIdManager.maxGraphID; - try (RocksDBSession session = getDBSession("test")) { - GraphIdManager gid = new GraphIdManager(new DBSessionBuilder() { - @Override - public RocksDBSession getSession(int partId) { - return session.clone(); - } - }, 0); - for (int i = 0; i < max; i++) { - Assert.assertEquals(i, gid.getCId("Test", max)); - } + this.previousMaxGraphId = GraphIdManager.maxGraphID; + GraphIdManager.maxGraphID = GRAPH_ID_LIMIT; + this.dbName = "graph-id-manager-" + System.nanoTime(); + this.dbPath = new File("target/graph-id-manager-test", this.dbName); + Map config = new HashMap<>(); + config.put("rocksdb.write_buffer_size", "1048576"); + config.put("rocksdb.bloom_filter_bits_per_key", "10"); + BusinessHandlerImpl.initRocksdb(config, null); + this.session = factory.createGraphDB(this.dbPath.getAbsolutePath(), + this.dbName); + this.sessionBuilder = partId -> this.session.clone(); + } + + @After + public void clear() { + GraphIdManager.maxGraphID = this.previousMaxGraphId; + if (this.session != null) { + this.session.close(); + } + if (this.dbName != null) { + factory.releaseGraphDB(this.dbName); + } + if (this.dbPath != null) { + UnitTestBase.deleteDir(this.dbPath); + } + } - Assert.assertEquals(-1, gid.getCId("Test", max)); + @Test + public void testAllocateBoundaryIdBeforeWrap() { + GraphIdManager manager = this.newManager(); - gid.delCId("Test", 3); - Assert.assertEquals(3, gid.getCId("Test", max)); - Assert.assertEquals(-1, gid.getCId("Test", max)); + Assert.assertEquals(0L, manager.getCId("boundary", 4)); + Assert.assertEquals(1L, manager.getCId("boundary", 4)); + Assert.assertEquals(2L, manager.getCId("boundary", 4)); + Assert.assertEquals(3L, manager.getCId("boundary", 4)); + } - long start = System.currentTimeMillis(); - for (int i = 0; i < GraphIdManager.maxGraphID; i++) { - long id = gid.getGraphId("g" + i); - Assert.assertEquals(i, id); - } - System.out.println("time is " + (System.currentTimeMillis() - start)); - { - gid.releaseGraphId("g" + 10); - long id = gid.getGraphId("g" + 10); - Assert.assertEquals(10, id); - } - start = System.currentTimeMillis(); - for (int i = 0; i < GraphIdManager.maxGraphID; i++) { - long id = gid.releaseGraphId("g" + i); - Assert.assertEquals(i, id); + @Test + public void testReturnMinusOneWhenAllIdsAreUsed() { + GraphIdManager manager = this.newManager(); + + for (int i = 0; i < 4; i++) { + Assert.assertEquals(i, manager.getCId("full", 4)); + } + Assert.assertEquals(-1L, manager.getCId("full", 4)); + } + + @Test + public void testReuseReleasedId() { + GraphIdManager manager = this.newManager(); + + for (int i = 0; i < 4; i++) { + Assert.assertEquals(i, manager.getCId("release", 4)); + } + manager.delCId("release", 1L); + + Assert.assertEquals(1L, manager.getCId("release", 4)); + Assert.assertEquals(-1L, manager.getCId("release", 4)); + } + + @Test + public void testWrapToBeginningAfterBoundary() { + GraphIdManager manager = this.newManager(); + + Assert.assertEquals(0L, manager.getCId("wrap", 4)); + Assert.assertEquals(1L, manager.getCId("wrap", 4)); + manager.delCId("wrap", 0L); + Assert.assertEquals(2L, manager.getCId("wrap", 4)); + Assert.assertEquals(3L, manager.getCId("wrap", 4)); + + Assert.assertEquals(0L, manager.getCId("wrap", 4)); + } + + @Test + public void testPersistGraphIdAcrossManagerRestart() { + GraphIdManager firstManager = this.newManager(); + + Assert.assertEquals(0L, firstManager.getGraphIdOrCreate("first")); + + this.reopenDatabase(); + GraphIdManager restartedManager = this.newManager(); + Assert.assertEquals(0L, restartedManager.getGraphId("first")); + Assert.assertEquals(1L, restartedManager.getGraphIdOrCreate("second")); + } + + @Test + public void testSkipGraphIdMappingWithoutSlot() { + this.persistGraphId("existing", 0L); + + Assert.assertEquals(1L, + this.newManager().getGraphIdOrCreate("new")); + } + + @Test + public void testUpdateGraphIdsRejectsPersistedCollisionBeforeWriting() { + this.persistGraphId("existing", 2L); + Map graphIds = new LinkedHashMap<>(); + graphIds.put("graph-a", 1L); + graphIds.put("graph-b", 2L); + + HgStoreException exception = Assert.assertThrows( + HgStoreException.class, + () -> this.newManager().updateGraphIds(graphIds)); + Assert.assertTrue(exception.getMessage().contains("existing")); + Assert.assertTrue(exception.getMessage().contains("graph-b")); + Assert.assertTrue(exception.getMessage().contains("2")); + Assert.assertEquals(GRAPH_ID_LIMIT, + this.newManager().getGraphId("graph-a")); + Assert.assertEquals(GRAPH_ID_LIMIT, + this.newManager().getGraphId("graph-b")); + } + + @Test + public void testUpdateGraphIdsPersistsMappingAndSlot() { + GraphIdManager manager = this.newManager(); + + manager.updateGraphIds(Collections.singletonMap("graph-a", 3L)); + + Assert.assertEquals(3L, this.newManager().getGraphId("graph-a")); + Int64Value slot = manager.get( + Int64Value.parser(), + manager.genCIDSlotKey(GraphIdManager.GRAPH_ID_PREFIX, 3L)); + Assert.assertNotNull(slot); + Assert.assertEquals(3L, slot.getValue()); + } + + @Test + public void testUpdateGraphIdsAllowsIdempotentAssignment() { + GraphIdManager manager = this.newManager(); + Map graphIds = + Collections.singletonMap("graph-a", 3L); + + manager.updateGraphIds(graphIds); + manager.updateGraphIds(graphIds); + + Assert.assertEquals(3L, this.newManager().getGraphId("graph-a")); + } + + @Test + public void testKeepPreviousGraphIdReservedAfterRemap() { + GraphIdManager manager = this.newManager(); + + manager.updateGraphIds(Collections.singletonMap("graph-a", 3L)); + manager.updateGraphIds(Collections.singletonMap("graph-a", 4L)); + manager.put(MetadataKeyHelper.getCidKey(GraphIdManager.GRAPH_ID_PREFIX), + Int64Value.of(3L)); + manager.flush(); + + Assert.assertNotEquals(3L, + this.newManager().getGraphIdOrCreate("graph-b")); + } + + @Test + public void testReservePreviousGraphIdWhenLegacySlotIsMissing() { + this.persistGraphId("graph-a", 3L); + GraphIdManager manager = this.newManager(); + manager.updateGraphIds(Collections.singletonMap("graph-a", 4L)); + manager.put(MetadataKeyHelper.getCidKey(GraphIdManager.GRAPH_ID_PREFIX), + Int64Value.of(3L)); + manager.flush(); + + this.reopenDatabase(); + Assert.assertNotEquals(3L, + this.newManager().getGraphIdOrCreate("graph-b")); + } + + @Test + public void testKeepOutEdgeResidueWhenVertexTableMissing() { + this.assertRepairKeepsOldIdReserved(OUT_EDGE_TABLE, false, false); + } + + @Test + public void testKeepInEdgeResidueWhenVertexTableMissing() { + this.assertRepairKeepsOldIdReserved(IN_EDGE_TABLE, false, false); + } + + @Test + public void testKeepIndexResidueWhenVertexTableMissing() { + this.assertRepairKeepsOldIdReserved(INDEX_TABLE, false, false); + } + + @Test + public void testKeepOutEdgeResidueWhenVertexTableEmpty() { + this.assertRepairKeepsOldIdReserved(OUT_EDGE_TABLE, true, false); + } + + @Test + public void testKeepInEdgeResidueWhenVertexTableEmpty() { + this.assertRepairKeepsOldIdReserved(IN_EDGE_TABLE, true, false); + } + + @Test + public void testKeepIndexResidueWhenVertexTableEmptyAfterReopen() { + this.assertRepairKeepsOldIdReserved(INDEX_TABLE, true, true); + } + + @Test + public void testKeepPreviousGraphIdSlotWhenStillAssigned() { + GraphIdManager manager = this.newManager(); + manager.updateGraphIds(Collections.singletonMap("graph-a", 3L)); + this.persistGraphId("graph-b", 3L); + + manager.updateGraphIds(Collections.singletonMap("graph-a", 4L)); + + Int64Value slot = manager.get( + Int64Value.parser(), + manager.genCIDSlotKey(GraphIdManager.GRAPH_ID_PREFIX, 3L)); + Assert.assertNotNull(slot); + Assert.assertEquals(3L, slot.getValue()); + Assert.assertEquals(3L, this.newManager().getGraphId("graph-b")); + } + + @Test + public void testRejectPersistedGraphIdsOutsideAllocatableDomain() { + long[] invalidGraphIds = {-1L, GRAPH_ID_LIMIT, 65536L}; + + for (long graphId : invalidGraphIds) { + String readGraph = "invalid-read-" + graphId; + this.persistGraphId(readGraph, graphId); + this.assertInvalidPersistedGraphId( + readGraph, graphId, + () -> this.newManager().getGraphId(readGraph)); + + String writeGraph = "invalid-write-" + graphId; + this.persistGraphId(writeGraph, graphId); + this.assertInvalidPersistedGraphId( + writeGraph, graphId, + () -> this.newManager().getGraphIdOrCreate(writeGraph)); + } + } + + @Test + public void testSkipVertexDataWithoutGraphIdSlot() { + GraphIdManager manager = this.newManager(); + this.persistVertexWithGraphId(0); + + Assert.assertEquals(1L, manager.getCId("stale-forward", 4)); + } + + @Test + public void testSkipVertexDataWithoutGraphIdSlotAfterWrap() { + String key = "stale-wrap"; + GraphIdManager manager = this.newManager(); + this.persistVertexWithGraphId(0); + manager.put(MetadataKeyHelper.getCidKey(key), Int64Value.of(2L)); + manager.put(manager.genCIDSlotKey(key, 2L), Int64Value.of(2L)); + manager.put(manager.genCIDSlotKey(key, 3L), Int64Value.of(3L)); + + Assert.assertEquals(1L, manager.getCId(key, 4)); + } + + @Test + public void testGraphIdDomainExcludesMissingSentinel() { + GraphIdManager manager = this.newManager(); + + for (int i = 0; i < GRAPH_ID_LIMIT; i++) { + Assert.assertEquals(i, manager.getGraphIdOrCreate("graph-" + i)); + } + Assert.assertEquals(GRAPH_ID_LIMIT, manager.getGraphId("missing")); + Assert.assertThrows(HgStoreException.class, + () -> manager.getGraphIdOrCreate("overflow")); + } + + @Test + public void testConcurrentCreateSameGraphReturnsSameId() throws Exception { + int threadCount = 8; + ExecutorService executor = Executors.newFixedThreadPool(threadCount); + CountDownLatch start = new CountDownLatch(1); + List> futures = new ArrayList<>(threadCount); + + try { + for (int i = 0; i < threadCount; i++) { + futures.add(executor.submit(() -> { + GraphIdManager manager = this.newManager(); + start.await(); + return manager.getGraphIdOrCreate("same-graph"); + })); } - System.out.println("time is " + (System.currentTimeMillis() - start)); - start = System.currentTimeMillis(); - for (int i = 0; i < GraphIdManager.maxGraphID; i++) { - long id = gid.getCId(GraphIdManager.GRAPH_ID_PREFIX, GraphIdManager.maxGraphID); - // long id = gid.getGraphId("g" + i); - Assert.assertTrue(id >= 0); + start.countDown(); + + Set graphIds = new HashSet<>(); + for (Future future : futures) { + graphIds.add(future.get(10L, TimeUnit.SECONDS)); } - System.out.println("time is " + (System.currentTimeMillis() - start)); + Assert.assertEquals(1, graphIds.size()); + Assert.assertTrue(graphIds.contains(0L)); + } finally { + executor.shutdownNow(); + } + } + + private GraphIdManager newManager() { + return new GraphIdManager(this.sessionBuilder, PARTITION_ID); + } + + private void assertRepairKeepsOldIdReserved(String residualTable, + boolean vertexTableExists, + boolean reopen) { + GraphIdManager manager = this.newManager(); + manager.updateGraphIds(Collections.singletonMap("graph-a", 3L)); + if (vertexTableExists) { + this.session.createTables(VERTEX_TABLE); + } else { + Assert.assertFalse(this.session.tableIsExist(VERTEX_TABLE)); + } + this.session.createTables(residualTable); + byte[] key = new byte[5]; + Bits.putShort(key, 0, 3); + key[2] = 1; + Bits.putShort(key, 3, 0); + SessionOperator operator = this.session.sessionOp(); + try { + operator.prepare(); + operator.put(residualTable, key, new byte[]{42}); + operator.commit(); + } catch (RuntimeException e) { + operator.rollback(); + throw e; + } + + manager.updateGraphIds(Collections.singletonMap("graph-a", 4L)); + manager.put(MetadataKeyHelper.getCidKey(GraphIdManager.GRAPH_ID_PREFIX), + Int64Value.of(3L)); + manager.flush(); + if (reopen) { + this.reopenDatabase(); + } + + Assert.assertArrayEquals(new byte[]{42}, + this.session.sessionOp().get(residualTable, key)); + Assert.assertNotEquals("Residual data must keep the old graph ID reserved", + 3L, this.newManager().getGraphIdOrCreate("graph-b")); + } + + private void persistGraphId(String graphName, long graphId) { + GraphIdManager manager = this.newManager(); + manager.put(MetadataKeyHelper.getGraphIDKey(graphName), + Int64Value.of(graphId)); + manager.flush(); + } + + private void persistVertexWithGraphId(int graphId) { + this.session.createTables(VERTEX_TABLE); + byte[] key = new byte[3]; + Bits.putShort(key, 0, graphId); + key[2] = 1; + + SessionOperator operator = this.session.sessionOp(); + try { + operator.prepare(); + operator.put(VERTEX_TABLE, key, new byte[]{1}); + operator.commit(); + } catch (RuntimeException e) { + operator.rollback(); + throw e; } } + + private void assertInvalidPersistedGraphId(String graphName, long graphId, + Runnable action) { + HgStoreException exception = + Assert.assertThrows(HgStoreException.class, action::run); + Assert.assertTrue(exception.getMessage().contains("Invalid graph ID")); + Assert.assertTrue(exception.getMessage().contains(String.valueOf(graphId))); + Assert.assertTrue(exception.getMessage().contains(graphName)); + } + + private void reopenDatabase() { + this.session.close(); + this.session = null; + factory.releaseGraphDB(this.dbName); + this.session = factory.createGraphDB(this.dbPath.getAbsolutePath(), + this.dbName); + this.sessionBuilder = partId -> this.session.clone(); + } } diff --git a/hugegraph-store/pom.xml b/hugegraph-store/pom.xml index a8212b70db..b1228486f3 100644 --- a/hugegraph-store/pom.xml +++ b/hugegraph-store/pom.xml @@ -98,7 +98,6 @@ org.jacoco jacoco-maven-plugin - 0.8.8 true @@ -171,7 +170,6 @@ org.apache.maven.plugins maven-surefire-plugin - 2.20 store-client-test @@ -195,7 +193,6 @@ org.apache.maven.plugins maven-surefire-plugin - 2.20 store-core-test @@ -219,7 +216,6 @@ org.apache.maven.plugins maven-surefire-plugin - 2.20 store-common-test @@ -243,7 +239,6 @@ org.apache.maven.plugins maven-surefire-plugin - 2.20 store-rocksdb-test @@ -267,7 +262,6 @@ org.apache.maven.plugins maven-surefire-plugin - 2.20 store-server-test @@ -291,7 +285,6 @@ org.apache.maven.plugins maven-surefire-plugin - 2.20 store-raftcore-test diff --git a/hugegraph-struct/pom.xml b/hugegraph-struct/pom.xml index b88d0ae204..3a6bf21922 100644 --- a/hugegraph-struct/pom.xml +++ b/hugegraph-struct/pom.xml @@ -30,8 +30,6 @@ - 11 - 11 UTF-8 25.1-jre 3.5.1 diff --git a/install-dist/release-docs/LICENSE b/install-dist/release-docs/LICENSE index ec93fad372..94aa4757fa 100644 --- a/install-dist/release-docs/LICENSE +++ b/install-dist/release-docs/LICENSE @@ -259,7 +259,7 @@ The following components are provided under the Apache 2.0 License. See project Per-component LICENSE-[project].txt files are included only when additional non-Apache-2.0 license text is required for a bundled component. - https://central.sonatype.com/artifact/com.alibaba/fastjson/1.2.83 -> Apache 2.0 + https://central.sonatype.com/artifact/com.alibaba/fastjson/1.2.84 -> Apache 2.0 https://central.sonatype.com/artifact/com.alipay.sofa.common/sofa-common-tools/1.0.12 -> Apache 2.0 https://central.sonatype.com/artifact/com.alipay.sofa.lookout/lookout-api/1.4.1 -> Apache 2.0 https://central.sonatype.com/artifact/com.alipay.sofa/bolt/1.6.2 -> Apache 2.0 @@ -486,13 +486,14 @@ non-Apache-2.0 license text is required for a bundled component. https://central.sonatype.com/artifact/org.ansj/ansj_seg/5.1.6 -> Apache 2.0 https://central.sonatype.com/artifact/org.apache.commons/commons-collections4/4.4 -> Apache 2.0 https://central.sonatype.com/artifact/org.apache.commons/commons-compress/1.21 -> Apache 2.0 + https://central.sonatype.com/artifact/org.apache.commons/commons-configuration2/2.10.1 -> Apache 2.0 https://central.sonatype.com/artifact/org.apache.commons/commons-configuration2/2.8.0 -> Apache 2.0 https://central.sonatype.com/artifact/org.apache.commons/commons-lang3/3.12.0 -> Apache 2.0 https://central.sonatype.com/artifact/org.apache.commons/commons-lang3/3.13.0 -> Apache 2.0 https://central.sonatype.com/artifact/org.apache.commons/commons-lang3/3.18.0 -> Apache 2.0 https://central.sonatype.com/artifact/org.apache.commons/commons-pool2/2.0 -> Apache 2.0 https://central.sonatype.com/artifact/org.apache.commons/commons-text/1.10.0 -> Apache 2.0 - https://central.sonatype.com/artifact/org.apache.commons/commons-text/1.9 -> Apache 2.0 + https://central.sonatype.com/artifact/org.apache.commons/commons-text/1.11.0 -> Apache 2.0 https://central.sonatype.com/artifact/org.apache.fury/fury-core/0.9.0 -> Apache 2.0 https://central.sonatype.com/artifact/org.apache.hbase/hbase-client/2.6.5 -> Apache 2.0 https://central.sonatype.com/artifact/org.apache.hbase/hbase-common/2.6.5 -> Apache 2.0 @@ -514,7 +515,7 @@ non-Apache-2.0 license text is required for a bundled component. https://central.sonatype.com/artifact/org.apache.htrace/htrace-core4/4.2.0-incubating -> Apache 2.0 https://central.sonatype.com/artifact/org.apache.httpcomponents/httpclient/4.5.13 -> Apache 2.0 https://central.sonatype.com/artifact/org.apache.httpcomponents/httpcore/4.4.13 -> Apache 2.0 - https://central.sonatype.com/artifact/org.apache.ivy/ivy/2.4.0 -> Apache 2.0 + https://central.sonatype.com/artifact/org.apache.ivy/ivy/2.6.0 -> Apache 2.0 https://central.sonatype.com/artifact/org.apache.kerby/kerb-admin/2.0.0 -> Apache 2.0 https://central.sonatype.com/artifact/org.apache.kerby/kerb-client/2.0.0 -> Apache 2.0 https://central.sonatype.com/artifact/org.apache.kerby/kerb-common/2.0.0 -> Apache 2.0 @@ -574,15 +575,15 @@ non-Apache-2.0 license text is required for a bundled component. https://central.sonatype.com/artifact/org.apiguardian/apiguardian-api/1.1.0 -> Apache 2.0 https://central.sonatype.com/artifact/org.assertj/assertj-core/3.19.0 -> Apache 2.0 https://central.sonatype.com/artifact/org.caffinitas.ohc/ohc-core/0.7.4 -> Apache 2.0 - https://central.sonatype.com/artifact/org.codehaus.groovy/groovy-cli-picocli/2.5.14 -> Apache 2.0 - https://central.sonatype.com/artifact/org.codehaus.groovy/groovy-console/2.5.14 -> Apache 2.0 - https://central.sonatype.com/artifact/org.codehaus.groovy/groovy-groovysh/2.5.14 -> Apache 2.0 - https://central.sonatype.com/artifact/org.codehaus.groovy/groovy-json/2.5.14 -> Apache 2.0 - https://central.sonatype.com/artifact/org.codehaus.groovy/groovy-jsr223/2.5.14 -> Apache 2.0 - https://central.sonatype.com/artifact/org.codehaus.groovy/groovy-swing/2.5.14 -> Apache 2.0 - https://central.sonatype.com/artifact/org.codehaus.groovy/groovy-templates/2.5.14 -> Apache 2.0 - https://central.sonatype.com/artifact/org.codehaus.groovy/groovy-xml/2.5.14 -> Apache 2.0 - https://central.sonatype.com/artifact/org.codehaus.groovy/groovy/2.5.14 -> Apache 2.0 + https://central.sonatype.com/artifact/org.codehaus.groovy/groovy-cli-picocli/2.5.23 -> Apache 2.0 + https://central.sonatype.com/artifact/org.codehaus.groovy/groovy-console/2.5.23 -> Apache 2.0 + https://central.sonatype.com/artifact/org.codehaus.groovy/groovy-groovysh/2.5.23 -> Apache 2.0 + https://central.sonatype.com/artifact/org.codehaus.groovy/groovy-json/2.5.23 -> Apache 2.0 + https://central.sonatype.com/artifact/org.codehaus.groovy/groovy-jsr223/2.5.23 -> Apache 2.0 + https://central.sonatype.com/artifact/org.codehaus.groovy/groovy-swing/2.5.23 -> Apache 2.0 + https://central.sonatype.com/artifact/org.codehaus.groovy/groovy-templates/2.5.23 -> Apache 2.0 + https://central.sonatype.com/artifact/org.codehaus.groovy/groovy-xml/2.5.23 -> Apache 2.0 + https://central.sonatype.com/artifact/org.codehaus.groovy/groovy/2.5.23 -> Apache 2.0 https://central.sonatype.com/artifact/org.eclipse.jetty.websocket/javax-websocket-client-impl/9.4.46.v20220331 -> Apache 2.0 https://central.sonatype.com/artifact/org.eclipse.jetty.websocket/javax-websocket-server-impl/9.4.46.v20220331 -> Apache 2.0 https://central.sonatype.com/artifact/org.eclipse.jetty.websocket/websocket-api/9.4.46.v20220331 -> Apache 2.0 @@ -878,3 +879,7 @@ The text of each license is also included in licenses/LICENSE-[project].txt. # # This product includes software developed by the Jansi project (http://fusesource.github.io/jansi/). # See licenses/LICENSE-jansi-2.4.0.txt for license details (Apache License 2.0 or LGPL 3.0+). + +Groovy 2.5.23 modules share licenses/LICENSE-groovy-2.5.23.txt. + +Ivy 2.6.0 includes third-party BSD and MIT licensed code; see licenses/LICENSE-ivy-2.6.0.txt. diff --git a/install-dist/release-docs/NOTICE b/install-dist/release-docs/NOTICE index 99e69b7f11..c28bc67f1e 100644 --- a/install-dist/release-docs/NOTICE +++ b/install-dist/release-docs/NOTICE @@ -456,7 +456,7 @@ ivy NOTICE ======================================================================== Apache Ivy (TM) -Copyright 2007-2014 The Apache Software Foundation +Copyright 2007-2026 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). @@ -471,6 +471,11 @@ which is open source software, available under the terms of a BSD style license. The original software and related information is available at http://www.jcraft.com/jsch/. + +The Scp class is based on the SCPClient from the ganymed ssh +library. Which has been distributed under the BSD style license +included in the LICENSE file. + ======================================================================== jackson-core NOTICE @@ -1766,3 +1771,14 @@ Copyright 2020-2021 SmartBear Software Inc. # This product includes software developed by the Jansi project (http://fusesource.github.io/jansi/). # See licenses/LICENSE-jansi-2.4.0.txt for license details (Apache License 2.0 or LGPL 3.0+). +======================================================================== +Groovy 2.5.23 NOTICE +======================================================================== +Apache Groovy +Copyright 2003-2023 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This product includes/uses ANTLR (http://www.antlr2.org/) +developed by Terence Parr 1989-2006 diff --git a/install-dist/release-docs/licenses/LICENSE-groovy-2.5.23.txt b/install-dist/release-docs/licenses/LICENSE-groovy-2.5.23.txt new file mode 100644 index 0000000000..b301adac5e --- /dev/null +++ b/install-dist/release-docs/licenses/LICENSE-groovy-2.5.23.txt @@ -0,0 +1,215 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +------------------------------------------------------------------------ + +ANTLR 2 License + +Antlr2 is released in the public domain. +See licenses/antlr2-license.txt for details. + +------------------------------------------------------------------------ + +ASM 4 License + +ASM 4 uses a 3-clause BSD license. For details, see licenses/asm-license.txt. diff --git a/install-dist/release-docs/licenses/LICENSE-ivy-2.6.0.txt b/install-dist/release-docs/licenses/LICENSE-ivy-2.6.0.txt new file mode 100644 index 0000000000..d5d48cfdef --- /dev/null +++ b/install-dist/release-docs/licenses/LICENSE-ivy-2.6.0.txt @@ -0,0 +1,290 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +------------------------------------------------------------------------------ +License for JCraft JSch package +------------------------------------------------------------------------------ +Copyright (c) 2002,2003,2004,2005,2006,2007 Atsuhiko Yamanaka, JCraft,Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the distribution. + + 3. The names of the authors may not be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL JCRAFT, +INC. OR ANY CONTRIBUTORS TO THIS SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +------------------------------------------------------------------------------ +License for jQuery +------------------------------------------------------------------------------ +Copyright (c) 2007 John Resig, http://jquery.com/ + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +------------------------------------------------------------------------------ +License for ganymed ssh library +------------------------------------------------------------------------------ + +Copyright (c) 2005 - 2006 Swiss Federal Institute of Technology (ETH Zurich), + Department of Computer Science (http://www.inf.ethz.ch), + Christian Plattner. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +a.) Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +b.) Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +c.) Neither the name of ETH Zurich nor the names of its contributors may + be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE diff --git a/install-dist/scripts/dependency/known-dependencies.txt b/install-dist/scripts/dependency/known-dependencies.txt index 2534e259d4..b5ea8e4b00 100644 --- a/install-dist/scripts/dependency/known-dependencies.txt +++ b/install-dist/scripts/dependency/known-dependencies.txt @@ -1,3 +1,6 @@ +HdrHistogram-2.1.12.jar +LatencyUtils-2.0.3.jar +RoaringBitmap-0.9.38.jar accessors-smart-1.2.jar android-json-0.0.20131108.vaadin1.jar animal-sniffer-annotations-1.14.jar @@ -21,8 +24,8 @@ asm-tree-9.2.jar assertj-core-3.19.0.jar ast-9.0-9.0.20190305.jar audience-annotations-0.13.0.jar -automaton-1.11-8.jar auto-service-annotations-1.0.jar +automaton-1.11-8.jar bolt-1.6.2.jar bolt-1.6.4.jar byte-buddy-1.10.20.jar @@ -45,6 +48,7 @@ commons-collections-3.2.2.jar commons-collections4-4.4.jar commons-compress-1.21.jar commons-configuration-1.10.jar +commons-configuration2-2.10.1.jar commons-configuration2-2.8.0.jar commons-crypto-1.1.0.jar commons-io-2.12.0.jar @@ -58,7 +62,7 @@ commons-logging-1.1.1.jar commons-logging-1.2.jar commons-pool2-2.0.jar commons-text-1.10.0.jar -commons-text-1.9.jar +commons-text-1.11.0.jar cypher-gremlin-extensions-1.0.4.jar disruptor-3.3.7.jar disruptor-3.4.1.jar @@ -76,7 +80,7 @@ exp4j-0.4.8.jar expressions-9.0-9.0.20190305.jar failsafe-2.4.1.jar failureaccess-1.0.1.jar -fastjson-1.2.83.jar +fastjson-1.2.84.jar fastparse_2.12-2.0.4.jar fastutil-8.1.0.jar fastutil-8.5.9.jar @@ -96,15 +100,15 @@ grizzly-framework-3.0.1.jar grizzly-http-3.0.1.jar grizzly-http-server-3.0.1.jar grizzly-http-servlet-3.0.1.jar -groovy-2.5.14-indy.jar -groovy-cli-picocli-2.5.14.jar -groovy-console-2.5.14.jar -groovy-groovysh-2.5.14-indy.jar -groovy-json-2.5.14-indy.jar -groovy-jsr223-2.5.14-indy.jar -groovy-swing-2.5.14.jar -groovy-templates-2.5.14.jar -groovy-xml-2.5.14.jar +groovy-2.5.23-indy.jar +groovy-cli-picocli-2.5.23.jar +groovy-console-2.5.23.jar +groovy-groovysh-2.5.23-indy.jar +groovy-json-2.5.23-indy.jar +groovy-jsr223-2.5.23-indy.jar +groovy-swing-2.5.23.jar +groovy-templates-2.5.23.jar +groovy-xml-2.5.23.jar grpc-api-1.28.1.jar grpc-api-1.39.0.jar grpc-api-1.47.0.jar @@ -143,8 +147,8 @@ hanlp-portable-1.8.3.jar hbase-client-2.6.5.jar hbase-common-2.6.5.jar hbase-endpoint-2.6.5.jar -hbase-hadoop2-compat-2.6.5.jar hbase-hadoop-compat-2.6.5.jar +hbase-hadoop2-compat-2.6.5.jar hbase-logging-2.6.5.jar hbase-metrics-2.6.5.jar hbase-metrics-api-2.6.5.jar @@ -157,7 +161,6 @@ hbase-shaded-netty-4.1.13.jar hbase-shaded-protobuf-4.1.13.jar hbase-unsafe-4.1.13.jar hbase-zookeeper-2.6.5.jar -HdrHistogram-2.1.12.jar hessian-3.3.6.jar hessian-3.3.7.jar hk2-api-3.0.1.jar @@ -168,7 +171,7 @@ htrace-core4-4.1.0-incubating.jar httpclient-4.5.13.jar httpcore-4.4.13.jar ikanalyzer-2012_u6.jar -ivy-2.4.0.jar +ivy-2.6.0.jar j2objc-annotations-1.1.jar j2objc-annotations-1.3.jar j2objc-annotations-2.8.jar @@ -220,11 +223,11 @@ javassist-3.24.0-GA.jar javassist-3.25.0-GA.jar javassist-3.28.0-GA.jar javatuples-1.2.jar +javax-websocket-client-impl-9.4.46.v20220331.jar +javax-websocket-server-impl-9.4.46.v20220331.jar javax.activation-api-1.2.0.jar javax.annotation-api-1.3.2.jar javax.json-1.0.jar -javax-websocket-client-impl-9.4.46.v20220331.jar -javax-websocket-server-impl-9.4.46.v20220331.jar jaxb-api-2.3.1.jar jaxb-core-3.0.2.jar jaxb-impl-3.0.2.jar @@ -283,9 +286,9 @@ jna-5.7.0.jar joda-time-2.10.8.jar joni-2.2.1.jar jraft-core-1.3.14.jar -jsonassert-1.5.0.jar json-path-2.5.0.jar json-smart-2.3.jar +jsonassert-1.5.0.jar jsr305-3.0.1.jar jsr305-3.0.2.jar jul-to-slf4j-1.7.36.jar @@ -337,7 +340,6 @@ kubernetes-model-policy-5.6.0.jar kubernetes-model-rbac-5.6.0.jar kubernetes-model-scheduling-5.6.0.jar kubernetes-model-storageclass-5.6.0.jar -LatencyUtils-2.0.3.jar listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar log4j-api-2.15.0.jar log4j-api-2.17.0.jar @@ -392,10 +394,10 @@ netty-buffer-4.1.52.Final.jar netty-buffer-4.1.72.Final.jar netty-codec-4.1.52.Final.jar netty-codec-4.1.72.Final.jar -netty-codec-http2-4.1.52.Final.jar -netty-codec-http2-4.1.72.Final.jar netty-codec-http-4.1.52.Final.jar netty-codec-http-4.1.72.Final.jar +netty-codec-http2-4.1.52.Final.jar +netty-codec-http2-4.1.72.Final.jar netty-codec-socks-4.1.52.Final.jar netty-codec-socks-4.1.72.Final.jar netty-common-4.1.52.Final.jar @@ -450,33 +452,32 @@ powermock-module-junit4-2.0.0-RC.3.jar powermock-module-junit4-common-2.0.0-RC.3.jar powermock-module-junit4-rule-2.0.0-RC.3.jar powermock-reflect-2.0.0-RC.3.jar +proto-google-common-protos-1.17.0.jar +proto-google-common-protos-2.0.1.jar protobuf-java-3.11.0.jar protobuf-java-3.17.2.jar protobuf-java-3.21.7.jar protobuf-java-3.5.1.jar protobuf-java-util-3.17.2.jar -proto-google-common-protos-1.17.0.jar -proto-google-common-protos-2.0.1.jar protostuff-api-1.6.0.jar protostuff-collectionschema-1.6.0.jar protostuff-core-1.6.0.jar protostuff-runtime-1.6.0.jar rewriting-9.0-9.0.20190305.jar -RoaringBitmap-0.9.38.jar rocksdbjni-8.10.2.jar scala-java8-compat_2.12-0.8.0.jar scala-library-2.12.7.jar scala-reflect-2.12.7.jar shims-0.9.38.jar -simpleclient_common-0.10.0.jar simpleclient-0.10.0.jar +simpleclient_common-0.10.0.jar sjk-agent-0.22.jar sjk-cli-0.22.jar sjk-core-0.22.jar sjk-hflame-0.22.jar +sjk-jfr-standalone-0.7.jar sjk-jfr5-0.5.jar sjk-jfr6-0.7.jar -sjk-jfr-standalone-0.7.jar sjk-json-0.22.jar sjk-nps-0.9.jar sjk-stacktrace-0.22.jar diff --git a/pom.xml b/pom.xml index 8f23a18043..699dd97fc6 100644 --- a/pom.xml +++ b/pom.xml @@ -86,14 +86,26 @@ + 2.5.23 5.6.0 1.7.0 1.18.30 1.3.14 hugegraph - 11 - 11 + 17 + + [17,18) + 3.15.0 + 3.5.6 + + false + 0.8.15 + 3.6.2 + UTF-8 bash 1.5.0 @@ -112,6 +124,45 @@ + + + org.codehaus.groovy + groovy-bom + ${groovy.compat.version} + pom + import + + + org.codehaus.groovy + groovy + ${groovy.compat.version} + indy + + + org.codehaus.groovy + groovy-groovysh + ${groovy.compat.version} + indy + + + org.codehaus.groovy + groovy-json + ${groovy.compat.version} + indy + + + org.codehaus.groovy + groovy-jsr223 + ${groovy.compat.version} + indy + + + + org.apache.ivy + ivy + 2.6.0 + org.projectlombok lombok @@ -228,35 +279,13 @@ org.apache.maven.plugins maven-enforcer-plugin - - - enforce-version - - enforce - - - false - - - - - - [11,) - - - [3.5.0,) - - - - - + ${maven.enforcer.plugin.version} maven-compiler-plugin - 3.1 + ${maven.compiler.plugin.version} - ${maven.compiler.source} - ${maven.compiler.target} + ${maven.compiler.release} 500 @@ -265,6 +294,21 @@ + + org.apache.maven.plugins + maven-surefire-plugin + ${maven.surefire.plugin.version} + + + ${surefire.failIfNoSpecifiedTests} + + + + + org.jacoco + jacoco-maven-plugin + ${jacoco.maven.plugin.version} + maven-remote-resources-plugin 3.3.0 @@ -277,6 +321,30 @@ + + org.apache.maven.plugins + maven-enforcer-plugin + + + enforce-version + + enforce + + + + + + + ${java.supported.version.range} + + + [3.6.3,) + + + + + + org.apache.rat apache-rat-plugin