From a04fdb35622afa0aff5f25c00d3db32ca8ca265d Mon Sep 17 00:00:00 2001 From: imbajin Date: Sat, 15 Nov 2025 00:54:35 +0800 Subject: [PATCH 1/8] chore: update release workflow and .gitignore entries Bumped default release version to 1.7.0 and updated default GPG user in the release workflow. Added installation step for subversion on Ubuntu. Appended WARP.md to .gitignore. --- .github/workflows/validate-release.yml | 7 +++++-- .gitignore | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index e25a44043..747f0d78e 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -6,11 +6,11 @@ on: release_version: required: true description: svn release version - default: '1.5.0' + default: '1.7.0' gpg_user: required: true description: current release manager (gpg username) - default: 'vgalaxies' + default: 'Junzhi Peng' push: branches: @@ -67,6 +67,9 @@ jobs: if [[ ${{ matrix.os }} =~ "macos" ]]; then brew install svn fi + if [[ ${{ matrix.os }} =~ "ubuntu" ]]; then + sudo apt-get install -y subversion + fi rm -rf dist/${{ inputs.release_version }} svn co ${URL_PREFIX}/${{ inputs.release_version }} dist/${{ inputs.release_version }} diff --git a/.gitignore b/.gitignore index 39893e594..86f2983b1 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,4 @@ GEMINI.md .vscode/settings.json .aider* .gemini/ +WARP.md From 8a8acf8716f0f990f207c7f407f686fdb229f7a1 Mon Sep 17 00:00:00 2001 From: imbajin Date: Sat, 15 Nov 2025 01:35:20 +0800 Subject: [PATCH 2/8] refactor: unify release validation script and add local path support Deleted validate-release-in-local.sh and enhanced validate-release.sh to support both SVN and local directory validation. Added color-coded output, improved argument handling, and included Java version checks for better usability and error reporting. --- dist/validate-release-in-local.sh | 356 ------------------------------ dist/validate-release.sh | 161 +++++++++----- 2 files changed, 101 insertions(+), 416 deletions(-) delete mode 100755 dist/validate-release-in-local.sh diff --git a/dist/validate-release-in-local.sh b/dist/validate-release-in-local.sh deleted file mode 100755 index e935be693..000000000 --- a/dist/validate-release-in-local.sh +++ /dev/null @@ -1,356 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# This script is used to validate the release package, including: -# 1. Check the release package name & content -# 2. Check the release package sha512 & gpg signature -# 3. Compile the source package & run server & toolchain -# 4. Run server & toolchain in binary package - -# exit when any error occurs -set -e - -# release version (input by committer) -RELEASE_VERSION=$1 # like 1.2.0 -JAVA_VERSION=$2 # like 11 -USER=$3 -LOCAL_DIST_PATH=$4 # local directory path containing release files - -# this URL is only valid during the release process -SVN_URL_PREFIX="https://dist.apache.org/repos/dist/dev/incubator/hugegraph" - -# git release branch (check it carefully) -#GIT_BRANCH="release-${RELEASE_VERSION}" - -RELEASE_VERSION=${RELEASE_VERSION:?"Please input the release version, like 1.2.0"} -USER=${USER:-"imbajin"} -WORK_DIR=$( - cd "$(dirname "$0")" - pwd -) - -# Use local directory if provided, otherwise use default dist path -if [[ -n "${LOCAL_DIST_PATH}" ]]; then - DIST_DIR="${LOCAL_DIST_PATH}" - echo "Using local directory: ${DIST_DIR}" -else - DIST_DIR="${WORK_DIR}/dist/${RELEASE_VERSION}" - echo "Using default directory: ${DIST_DIR}" -fi - -# Validate local directory exists -if [[ ! -d "${DIST_DIR}" ]]; then - echo "Error: Directory ${DIST_DIR} does not exist" - exit 1 -fi - -cd "${WORK_DIR}" -echo "Current work dir: $(pwd)" -echo "Release files directory: ${DIST_DIR}" - -################################ -# Step 1: Validate Local Directory # -################################ -cd "${DIST_DIR}" -echo "Contents of ${DIST_DIR}:" -ls -lh - -################################################## -# Step 2: Check Environment & Import Public Keys # -################################################## -shasum --version 1>/dev/null -gpg --version 1>/dev/null - -wget https://downloads.apache.org/incubator/hugegraph/KEYS -echo "Import KEYS:" && gpg --import KEYS -# TODO: how to trust all public keys in gpg list, currently only trust the first one -echo -e "5\ny\n" | gpg --batch --command-fd 0 --edit-key $USER trust - -echo "trust all pk" -for key in $(gpg --no-tty --list-keys --with-colons | awk -F: '/^pub/ {print $5}'); do - echo -e "5\ny\n" | gpg --batch --command-fd 0 --edit-key "$key" trust -done - -######################################## -# Step 3: Check SHA512 & GPG Signature # -######################################## -cd "${DIST_DIR}" - -for i in *.tar.gz; do - echo "$i" - shasum -a 512 --check "$i".sha512 - eval gpg "${GPG_OPT}" --verify "$i".asc "$i" -done - -#################################### -# Step 4: Validate Source Packages # -#################################### -cd "${DIST_DIR}" - -CATEGORY_X="\bGPL|\bLGPL|Sleepycat License|BSD-4-Clause|\bBCL\b|JSR-275|Amazon Software License|\bRSAL\b|\bQPL\b|\bSSPL|\bCPOL|\bNPL1|Creative Commons Non-Commercial|JSON\.org" -CATEGORY_B="\bCDDL1|\bCPL|\bEPL|\bIPL|\bMPL|\bSPL|OSL-3.0|UnRAR License|Erlang Public License|\bOFL\b|Ubuntu Font License Version 1.0|IPA Font License Agreement v1.0|EPL2.0|CC-BY" -ls -lh ./*.tar.gz -for i in *src.tar.gz; do - echo "$i" - - # 4.1: check the directory name include "incubating" - if [[ ! "$i" =~ "incubating" ]]; then - echo "The package name $i should include incubating" && exit 1 - fi - - MODULE_DIR=$(basename "$i" .tar.gz) - rm -rf ${MODULE_DIR} - tar -xzvf "$i" - pushd ${MODULE_DIR} - echo "Start to check the package content: ${MODULE_DIR}" - - # 4.2: check the directory include "NOTICE" and "LICENSE" file and "DISCLAIMER" file - if [[ ! -f "LICENSE" ]]; then - echo "The package $i should include LICENSE file" && exit 1 - fi - if [[ ! -f "NOTICE" ]]; then - echo "The package $i should include NOTICE file" && exit 1 - fi - if [[ ! -f "DISCLAIMER" ]]; then - echo "The package $i should include DISCLAIMER file" && exit 1 - fi - - # 4.3: ensure doesn't contains ASF CATEGORY X License dependencies in LICENSE and NOTICE files - COUNT=$(grep -E "$CATEGORY_X" LICENSE NOTICE | wc -l) - if [[ $COUNT -ne 0 ]]; then - grep -E "$CATEGORY_X" LICENSE NOTICE - echo "The package $i shouldn't include invalid ASF category X dependencies, but get $COUNT" && exit 1 - fi - - # 4.4: ensure doesn't contains ASF CATEGORY B License dependencies in LICENSE and NOTICE files - COUNT=$(grep -E "$CATEGORY_B" LICENSE NOTICE | wc -l) - if [[ $COUNT -ne 0 ]]; then - grep -E "$CATEGORY_B" LICENSE NOTICE - echo "The package $i shouldn't include invalid ASF category B dependencies, but get $COUNT" && exit 1 - fi - - # 4.5: ensure doesn't contains empty directory or file - find . -type d -empty | while read -r EMPTY_DIR; do - find . -type d -empty - echo "The package $i shouldn't include empty directory: $EMPTY_DIR is empty" && exit 1 - done - find . -type f -empty | while read -r EMPTY_FILE; do - find . -type f -empty - echo "The package $i shouldn't include empty file: $EMPTY_FILE is empty" && exit 1 - done - - # 4.6: ensure any file should less than 800kb - find . -type f -size +800k | while read -r FILE; do - find . -type f -size +800k - echo "The package $i shouldn't include file larger than 800kb: $FILE is larger than 800kb" && exit 1 - done - - # 4.7: ensure all binary files are documented in LICENSE - find . -type f | perl -lne 'print if -B' | while read -r BINARY_FILE; do - FILE_NAME=$(basename "$BINARY_FILE") - if grep -q "$FILE_NAME" LICENSE; then - echo "Binary file $BINARY_FILE is documented in LICENSE, please check manually" - else - echo "Error: Binary file $BINARY_FILE is not documented in LICENSE" && exit 1 - fi - done - - # 4.8: test compile the packages - if [[ ($JAVA_VERSION == 8 && "$i" =~ "hugegraph-computer") ]]; then - echo "Skip compile $i module in java8" - elif [[ "$i" =~ 'hugegraph-ai' ]]; then - echo "Skip compile $i module in all versions" - elif [[ "$i" =~ "hugegraph-commons" ]]; then - mvn install -DskipTests -Papache-release -ntp -e - elif [[ "$i" =~ "hugegraph-computer" ]]; then - cd computer - mvn install -DskipTests -Papache-release -ntp -e - else - # TODO: consider using commands that are entirely consistent with building binary packages - mvn package -DskipTests -Papache-release -ntp -e - ls -lh - fi - popd -done - -########################################### -# Step 5: Run Compiled Packages of Server # -########################################### -cd "${DIST_DIR}" - -ls -lh -pushd ./*hugegraph-incubating*src/hugegraph-server/*hugegraph*"${RELEASE_VERSION}" -bin/init-store.sh -sleep 3 -bin/start-hugegraph.sh -popd - -####################################################################### -# Step 6: Run Compiled Packages of ToolChain (Loader & Tool & Hubble) # -####################################################################### -cd "${DIST_DIR}" - -pushd ./*toolchain*src -ls -lh -pushd ./*toolchain*"${RELEASE_VERSION}" -ls -lh - -# 6.1: load some data first -echo "test loader" -pushd ./*loader*"${RELEASE_VERSION}" -bin/hugegraph-loader.sh -f ./example/file/struct.json -s ./example/file/schema.groovy \ - -g hugegraph -popd - -# 6.2: try some gremlin query & api in tool -echo "test tool" -pushd ./*tool*"${RELEASE_VERSION}" -bin/hugegraph gremlin-execute --script 'g.V().count()' -bin/hugegraph task-list -bin/hugegraph backup -t all --directory ./backup-test -popd - -# 6.3: start hubble and connect to server -echo "test hubble" -pushd ./*hubble*"${RELEASE_VERSION}" -# TODO: add hubble doc & test it -cat conf/hugegraph-hubble.properties -bin/start-hubble.sh -bin/stop-hubble.sh -popd - -popd -popd -# stop server -pushd ./*hugegraph-incubating*src/hugegraph-server/*hugegraph*"${RELEASE_VERSION}" -bin/stop-hugegraph.sh -popd - -# clear source packages -#rm -rf ./*src* -#ls -lh - -#################################### -# Step 7: Validate Binary Packages # -#################################### -cd "${DIST_DIR}" - -for i in *.tar.gz; do - if [[ "$i" == *-src.tar.gz ]]; then - # skip source packages - continue - fi - - echo "$i" - - # 7.1: check the directory name include "incubating" - if [[ ! "$i" =~ "incubating" ]]; then - echo "The package name $i should include incubating" && exit 1 - fi - - MODULE_DIR=$(basename "$i" .tar.gz) - rm -rf ${MODULE_DIR} - tar -xzvf "$i" - pushd ${MODULE_DIR} - ls -lh - echo "Start to check the package content: ${MODULE_DIR}" - - # 7.2: check root dir include "NOTICE"/"LICENSE"/"DISCLAIMER" files & "licenses" dir - if [[ ! -f "LICENSE" ]]; then - echo "The package $i should include LICENSE file" && exit 1 - fi - if [[ ! -f "NOTICE" ]]; then - echo "The package $i should include NOTICE file" && exit 1 - fi - if [[ ! -f "DISCLAIMER" ]]; then - echo "The package $i should include DISCLAIMER file" && exit 1 - fi - if [[ ! -d "licenses" ]]; then - echo "The package $i should include licenses dir" && exit 1 - fi - - # 7.3: ensure doesn't contains ASF CATEGORY X License dependencies in LICENSE/NOTICE and licenses/* files - COUNT=$(grep -r -E "$CATEGORY_X" LICENSE NOTICE licenses | wc -l) - if [[ $COUNT -ne 0 ]]; then - grep -r -E "$CATEGORY_X" LICENSE NOTICE licenses - echo "The package $i shouldn't include invalid ASF category X dependencies, but get $COUNT" && exit 1 - fi - - # 7.4: ensure doesn't contains empty directory or file - find . -type d -empty | while read -r EMPTY_DIR; do - find . -type d -empty - echo "The package $i shouldn't include empty directory: $EMPTY_DIR is empty" && exit 1 - done - find . -type f -empty | while read -r EMPTY_FILE; do - find . -type f -empty - echo "The package $i shouldn't include empty file: $EMPTY_FILE is empty" && exit 1 - done - - popd -done - -# TODO: skip the following steps by comparing the artifacts built from source packages with binary packages -######################################### -# Step 8: Run Binary Packages of Server # -######################################### -cd "${DIST_DIR}" - -# TODO: run pd & store -pushd ./*hugegraph-incubating*"${RELEASE_VERSION}"/*hugegraph-server-incubating*"${RELEASE_VERSION}" -bin/init-store.sh -sleep 3 -bin/start-hugegraph.sh -popd - -##################################################################### -# Step 9: Run Binary Packages of ToolChain (Loader & Tool & Hubble) # -##################################################################### -cd "${DIST_DIR}" - -pushd ./*toolchain*"${RELEASE_VERSION}" -ls -lh - -# 9.1: load some data first -echo "test loader" -pushd ./*loader*"${RELEASE_VERSION}" -bin/hugegraph-loader.sh -f ./example/file/struct.json -s ./example/file/schema.groovy -g hugegraph -popd - -# 9.2: try some gremlin query & api in tool -echo "test tool" -pushd ./*tool*"${RELEASE_VERSION}" -bin/hugegraph gremlin-execute --script 'g.V().count()' -bin/hugegraph task-list -bin/hugegraph backup -t all --directory ./backup-test -popd - -# 9.3: start hubble and connect to server -echo "test hubble" -pushd ./*hubble*"${RELEASE_VERSION}" -# TODO: add hubble doc & test it -cat conf/hugegraph-hubble.properties -bin/start-hubble.sh -bin/stop-hubble.sh -popd - -popd -# stop server -pushd ./*hugegraph-incubating*"${RELEASE_VERSION}"/*hugegraph-server-incubating*"${RELEASE_VERSION}" -bin/stop-hugegraph.sh -popd - -echo "Finish validate, please check all steps manually again!" diff --git a/dist/validate-release.sh b/dist/validate-release.sh index 8df1645f9..d9fdb186a 100755 --- a/dist/validate-release.sh +++ b/dist/validate-release.sh @@ -1,33 +1,36 @@ #!/usr/bin/env bash # -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# # This script is used to validate the release package, including: # 1. Check the release package name & content # 2. Check the release package sha512 & gpg signature # 3. Compile the source package & run server & toolchain # 4. Run server & toolchain in binary package +# +# Usage: +# 1. Validate from Apache SVN (default): +# ./validate-release.sh [local-path] [java-version] +# Example: ./validate-release.sh 1.7.0 pengjunzhi +# Example: ./validate-release.sh 1.7.0 pengjunzhi "" 11 +# +# 2. Validate from local directory: +# ./validate-release.sh [java-version] +# Example: ./validate-release.sh 1.7.0 pengjunzhi /path/to/dist +# Example: ./validate-release.sh 1.7.0 pengjunzhi /path/to/dist 11 # exit when any error occurs set -e +# Color definitions +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[0;33m' +NC='\033[0m' # No Color + # release version (input by committer) -RELEASE_VERSION=$1 # like 1.2.0 -JAVA_VERSION=$2 # like 11 -USER=$3 +RELEASE_VERSION=$1 # like 1.7.0 +USER=$2 +LOCAL_DIST_PATH=$3 # optional: local directory path containing release files +JAVA_VERSION=${4:-11} # optional: default to 11 # this URL is only valid during the release process SVN_URL_PREFIX="https://dist.apache.org/repos/dist/dev/incubator/hugegraph" @@ -35,8 +38,8 @@ SVN_URL_PREFIX="https://dist.apache.org/repos/dist/dev/incubator/hugegraph" # git release branch (check it carefully) #GIT_BRANCH="release-${RELEASE_VERSION}" -RELEASE_VERSION=${RELEASE_VERSION:?"Please input the release version, like 1.2.0"} -USER=${USER:-"imbajin"} +RELEASE_VERSION=${RELEASE_VERSION:?"Please input the release version, like 1.7.0"} +USER=${USER:?"Please input the user name"} WORK_DIR=$( cd "$(dirname "$0")" pwd @@ -45,13 +48,35 @@ WORK_DIR=$( cd "${WORK_DIR}" echo "Current work dir: $(pwd)" -################################ -# Step 1: Download SVN Sources # -################################ -rm -rf "${WORK_DIR}/dist/${RELEASE_VERSION}" -mkdir -p "${WORK_DIR}/dist/${RELEASE_VERSION}" -cd "${WORK_DIR}/dist/${RELEASE_VERSION}" -svn co "${SVN_URL_PREFIX}/${RELEASE_VERSION}" . +#################################################### +# Step 1: Prepare Release Files (SVN or Local) # +#################################################### +if [[ -n "${LOCAL_DIST_PATH}" ]]; then + # Use local directory + DIST_DIR="${LOCAL_DIST_PATH}" + echo "Using local directory: ${DIST_DIR}" + + # Validate local directory exists + if [[ ! -d "${DIST_DIR}" ]]; then + echo -e "${RED}Error: Directory ${DIST_DIR} does not exist${NC}" + exit 1 + fi + + echo "Contents of ${DIST_DIR}:" + ls -lh "${DIST_DIR}" +else + # Download from SVN + DIST_DIR="${WORK_DIR}/dist/${RELEASE_VERSION}" + echo "Downloading from SVN to: ${DIST_DIR}" + + rm -rf "${DIST_DIR}" + mkdir -p "${DIST_DIR}" + cd "${DIST_DIR}" + svn co "${SVN_URL_PREFIX}/${RELEASE_VERSION}" . +fi + +cd "${DIST_DIR}" +echo "Release files directory: ${DIST_DIR}" ################################################## # Step 2: Check Environment & Import Public Keys # @@ -59,6 +84,26 @@ svn co "${SVN_URL_PREFIX}/${RELEASE_VERSION}" . shasum --version 1>/dev/null gpg --version 1>/dev/null +# Check Java version +echo "Checking Java version..." +if ! command -v java &> /dev/null; then + echo -e "${RED}Error: Java is not installed or not in PATH${NC}" + exit 1 +fi + +CURRENT_JAVA_VERSION=$(java -version 2>&1 | head -n 1 | awk -F '"' '{print $2}' | awk -F '.' '{print $1}') +echo "Current Java version: $CURRENT_JAVA_VERSION (Required: ${JAVA_VERSION})" + +if [[ "$CURRENT_JAVA_VERSION" != "$JAVA_VERSION" ]]; then + echo -e "${RED}Error: Java version mismatch!${NC}" + echo -e "${RED} Current: Java $CURRENT_JAVA_VERSION${NC}" + echo -e "${RED} Required: Java ${JAVA_VERSION}${NC}" + echo -e "${RED} Please switch to Java ${JAVA_VERSION} before running this script${NC}" + exit 1 +fi + +echo -e "${GREEN}Java version check passed: Java $CURRENT_JAVA_VERSION${NC}" + wget https://downloads.apache.org/incubator/hugegraph/KEYS echo "Import KEYS:" && gpg --import KEYS # TODO: how to trust all public keys in gpg list, currently only trust the first one @@ -72,7 +117,7 @@ done ######################################## # Step 3: Check SHA512 & GPG Signature # ######################################## -cd "${WORK_DIR}/dist/${RELEASE_VERSION}" +cd "${DIST_DIR}" for i in *.tar.gz; do echo "$i" @@ -83,7 +128,7 @@ done #################################### # Step 4: Validate Source Packages # #################################### -cd "${WORK_DIR}/dist/${RELEASE_VERSION}" +cd "${DIST_DIR}" CATEGORY_X="\bGPL|\bLGPL|Sleepycat License|BSD-4-Clause|\bBCL\b|JSR-275|Amazon Software License|\bRSAL\b|\bQPL\b|\bSSPL|\bCPOL|\bNPL1|Creative Commons Non-Commercial|JSON\.org" CATEGORY_B="\bCDDL1|\bCPL|\bEPL|\bIPL|\bMPL|\bSPL|OSL-3.0|UnRAR License|Erlang Public License|\bOFL\b|Ubuntu Font License Version 1.0|IPA Font License Agreement v1.0|EPL2.0|CC-BY" @@ -93,7 +138,7 @@ for i in *src.tar.gz; do # 4.1: check the directory name include "incubating" if [[ ! "$i" =~ "incubating" ]]; then - echo "The package name $i should include incubating" && exit 1 + echo -e "${RED}The package name $i should include incubating${NC}" && exit 1 fi MODULE_DIR=$(basename "$i" .tar.gz) @@ -104,65 +149,61 @@ for i in *src.tar.gz; do # 4.2: check the directory include "NOTICE" and "LICENSE" file and "DISCLAIMER" file if [[ ! -f "LICENSE" ]]; then - echo "The package $i should include LICENSE file" && exit 1 + echo -e "${RED}The package $i should include LICENSE file${NC}" && exit 1 fi if [[ ! -f "NOTICE" ]]; then - echo "The package $i should include NOTICE file" && exit 1 + echo -e "${RED}The package $i should include NOTICE file${NC}" && exit 1 fi if [[ ! -f "DISCLAIMER" ]]; then - echo "The package $i should include DISCLAIMER file" && exit 1 + echo -e "${RED}The package $i should include DISCLAIMER file${NC}" && exit 1 fi # 4.3: ensure doesn't contains ASF CATEGORY X License dependencies in LICENSE and NOTICE files COUNT=$(grep -E "$CATEGORY_X" LICENSE NOTICE | wc -l) if [[ $COUNT -ne 0 ]]; then grep -E "$CATEGORY_X" LICENSE NOTICE - echo "The package $i shouldn't include invalid ASF category X dependencies, but get $COUNT" && exit 1 + echo -e "${RED}The package $i shouldn't include invalid ASF category X dependencies, but get $COUNT${NC}" && exit 1 fi # 4.4: ensure doesn't contains ASF CATEGORY B License dependencies in LICENSE and NOTICE files COUNT=$(grep -E "$CATEGORY_B" LICENSE NOTICE | wc -l) if [[ $COUNT -ne 0 ]]; then grep -E "$CATEGORY_B" LICENSE NOTICE - echo "The package $i shouldn't include invalid ASF category B dependencies, but get $COUNT" && exit 1 + echo -e "${RED}The package $i shouldn't include invalid ASF category B dependencies, but get $COUNT${NC}" && exit 1 fi # 4.5: ensure doesn't contains empty directory or file find . -type d -empty | while read -r EMPTY_DIR; do find . -type d -empty - echo "The package $i shouldn't include empty directory: $EMPTY_DIR is empty" && exit 1 + echo -e "${RED}The package $i shouldn't include empty directory: $EMPTY_DIR is empty${NC}" && exit 1 done find . -type f -empty | while read -r EMPTY_FILE; do find . -type f -empty - echo "The package $i shouldn't include empty file: $EMPTY_FILE is empty" && exit 1 + echo -e "${RED}The package $i shouldn't include empty file: $EMPTY_FILE is empty${NC}" && exit 1 done # 4.6: ensure any file should less than 800kb find . -type f -size +800k | while read -r FILE; do find . -type f -size +800k - echo "The package $i shouldn't include file larger than 800kb: $FILE is larger than 800kb" && exit 1 + echo -e "${RED}The package $i shouldn't include file larger than 800kb: $FILE is larger than 800kb${NC}" && exit 1 done # 4.7: ensure all binary files are documented in LICENSE find . -type f | perl -lne 'print if -B' | while read -r BINARY_FILE; do FILE_NAME=$(basename "$BINARY_FILE") if grep -q "$FILE_NAME" LICENSE; then - echo "Binary file $BINARY_FILE is documented in LICENSE, please check manually" + echo -e "${YELLOW}Binary file $BINARY_FILE is documented in LICENSE, please check manually${NC}" else - echo "Error: Binary file $BINARY_FILE is not documented in LICENSE" && exit 1 + echo -e "${RED}Error: Binary file $BINARY_FILE is not documented in LICENSE${NC}" && exit 1 fi done # 4.8: test compile the packages - if [[ ($JAVA_VERSION == 8 && "$i" =~ "hugegraph-computer") ]]; then - echo "Skip compile $i module in java8" - elif [[ "$i" =~ 'hugegraph-ai' ]]; then + if [[ "$i" =~ 'hugegraph-ai' ]]; then echo "Skip compile $i module in all versions" - elif [[ "$i" =~ "hugegraph-commons" ]]; then - mvn install -DskipTests -Papache-release -ntp -e elif [[ "$i" =~ "hugegraph-computer" ]]; then cd computer - mvn install -DskipTests -Papache-release -ntp -e + mvn package -DskipTests -Papache-release -ntp -e else # TODO: consider using commands that are entirely consistent with building binary packages mvn package -DskipTests -Papache-release -ntp -e @@ -174,7 +215,7 @@ done ########################################### # Step 5: Run Compiled Packages of Server # ########################################### -cd "${WORK_DIR}/dist/${RELEASE_VERSION}" +cd "${DIST_DIR}" ls -lh pushd ./*hugegraph-incubating*src/hugegraph-server/*hugegraph*"${RELEASE_VERSION}" @@ -186,7 +227,7 @@ popd ####################################################################### # Step 6: Run Compiled Packages of ToolChain (Loader & Tool & Hubble) # ####################################################################### -cd "${WORK_DIR}/dist/${RELEASE_VERSION}" +cd "${DIST_DIR}" pushd ./*toolchain*src ls -lh @@ -231,7 +272,7 @@ popd #################################### # Step 7: Validate Binary Packages # #################################### -cd "${WORK_DIR}/dist/${RELEASE_VERSION}" +cd "${DIST_DIR}" for i in *.tar.gz; do if [[ "$i" == *-src.tar.gz ]]; then @@ -243,7 +284,7 @@ for i in *.tar.gz; do # 7.1: check the directory name include "incubating" if [[ ! "$i" =~ "incubating" ]]; then - echo "The package name $i should include incubating" && exit 1 + echo -e "${RED}The package name $i should include incubating${NC}" && exit 1 fi MODULE_DIR=$(basename "$i" .tar.gz) @@ -255,33 +296,33 @@ for i in *.tar.gz; do # 7.2: check root dir include "NOTICE"/"LICENSE"/"DISCLAIMER" files & "licenses" dir if [[ ! -f "LICENSE" ]]; then - echo "The package $i should include LICENSE file" && exit 1 + echo -e "${RED}The package $i should include LICENSE file${NC}" && exit 1 fi if [[ ! -f "NOTICE" ]]; then - echo "The package $i should include NOTICE file" && exit 1 + echo -e "${RED}The package $i should include NOTICE file${NC}" && exit 1 fi if [[ ! -f "DISCLAIMER" ]]; then - echo "The package $i should include DISCLAIMER file" && exit 1 + echo -e "${RED}The package $i should include DISCLAIMER file${NC}" && exit 1 fi if [[ ! -d "licenses" ]]; then - echo "The package $i should include licenses dir" && exit 1 + echo -e "${RED}The package $i should include licenses dir${NC}" && exit 1 fi # 7.3: ensure doesn't contains ASF CATEGORY X License dependencies in LICENSE/NOTICE and licenses/* files COUNT=$(grep -r -E "$CATEGORY_X" LICENSE NOTICE licenses | wc -l) if [[ $COUNT -ne 0 ]]; then grep -r -E "$CATEGORY_X" LICENSE NOTICE licenses - echo "The package $i shouldn't include invalid ASF category X dependencies, but get $COUNT" && exit 1 + echo -e "${RED}The package $i shouldn't include invalid ASF category X dependencies, but get $COUNT${NC}" && exit 1 fi # 7.4: ensure doesn't contains empty directory or file find . -type d -empty | while read -r EMPTY_DIR; do find . -type d -empty - echo "The package $i shouldn't include empty directory: $EMPTY_DIR is empty" && exit 1 + echo -e "${RED}The package $i shouldn't include empty directory: $EMPTY_DIR is empty${NC}" && exit 1 done find . -type f -empty | while read -r EMPTY_FILE; do find . -type f -empty - echo "The package $i shouldn't include empty file: $EMPTY_FILE is empty" && exit 1 + echo -e "${RED}The package $i shouldn't include empty file: $EMPTY_FILE is empty${NC}" && exit 1 done popd @@ -291,7 +332,7 @@ done ######################################### # Step 8: Run Binary Packages of Server # ######################################### -cd "${WORK_DIR}/dist/${RELEASE_VERSION}" +cd "${DIST_DIR}" # TODO: run pd & store pushd ./*hugegraph-incubating*"${RELEASE_VERSION}"/*hugegraph-server-incubating*"${RELEASE_VERSION}" @@ -303,7 +344,7 @@ popd ##################################################################### # Step 9: Run Binary Packages of ToolChain (Loader & Tool & Hubble) # ##################################################################### -cd "${WORK_DIR}/dist/${RELEASE_VERSION}" +cd "${DIST_DIR}" pushd ./*toolchain*"${RELEASE_VERSION}" ls -lh @@ -337,4 +378,4 @@ pushd ./*hugegraph-incubating*"${RELEASE_VERSION}"/*hugegraph-server-incubating* bin/stop-hugegraph.sh popd -echo "Finish validate, please check all steps manually again!" +echo -e "${GREEN}Finish validate, please check all steps manually again!${NC}" From b9e896cc68d294e1f9dfec25df161c09d6db4ba4 Mon Sep 17 00:00:00 2001 From: imbajin Date: Sat, 15 Nov 2025 02:28:05 +0800 Subject: [PATCH 3/8] refactor: revamp release validation script with enhanced checks V2 Major rewrite of validate-release.sh for Apache HugeGraph, adding modular structure, improved logging, error/warning collection, colorized output, and comprehensive validation steps for source and binary packages. New features include dependency checks, GPG key management, license compliance, file size and binary checks, version consistency, and automated server/toolchain testing. Usage instructions and help output are expanded for clarity. --- dist/validate-release.sh | 1548 +++++++++++++++++++++++++++++--------- 1 file changed, 1173 insertions(+), 375 deletions(-) diff --git a/dist/validate-release.sh b/dist/validate-release.sh index d9fdb186a..06dcdbbd9 100755 --- a/dist/validate-release.sh +++ b/dist/validate-release.sh @@ -1,381 +1,1179 @@ #!/usr/bin/env bash # -# This script is used to validate the release package, including: -# 1. Check the release package name & content -# 2. Check the release package sha512 & gpg signature -# 3. Compile the source package & run server & toolchain -# 4. Run server & toolchain in binary package +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +# Apache HugeGraph Release Validation Script +################################################################################ +# +# This script validates Apache HugeGraph (Incubating) release packages: +# 1. Check package integrity (SHA512, GPG signatures) +# 2. Validate package names and required files +# 3. Check license compliance (ASF categories) +# 4. Validate package contents +# 5. Compile source packages +# 6. Run server and toolchain tests # # Usage: -# 1. Validate from Apache SVN (default): -# ./validate-release.sh [local-path] [java-version] -# Example: ./validate-release.sh 1.7.0 pengjunzhi -# Example: ./validate-release.sh 1.7.0 pengjunzhi "" 11 +# validate-release.sh [local-path] [java-version] +# validate-release.sh --help +# +# Arguments: +# version Release version (e.g., 1.7.0) +# user Apache username for GPG key trust +# local-path (Optional) Local directory containing release files +# If omitted, downloads from Apache SVN +# java-version (Optional) Java version to validate (default: 11) # -# 2. Validate from local directory: -# ./validate-release.sh [java-version] -# Example: ./validate-release.sh 1.7.0 pengjunzhi /path/to/dist -# Example: ./validate-release.sh 1.7.0 pengjunzhi /path/to/dist 11 - -# exit when any error occurs -set -e - -# Color definitions -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[0;33m' -NC='\033[0m' # No Color - -# release version (input by committer) -RELEASE_VERSION=$1 # like 1.7.0 -USER=$2 -LOCAL_DIST_PATH=$3 # optional: local directory path containing release files -JAVA_VERSION=${4:-11} # optional: default to 11 - -# this URL is only valid during the release process -SVN_URL_PREFIX="https://dist.apache.org/repos/dist/dev/incubator/hugegraph" - -# git release branch (check it carefully) -#GIT_BRANCH="release-${RELEASE_VERSION}" - -RELEASE_VERSION=${RELEASE_VERSION:?"Please input the release version, like 1.7.0"} -USER=${USER:?"Please input the user name"} -WORK_DIR=$( - cd "$(dirname "$0")" - pwd -) - -cd "${WORK_DIR}" -echo "Current work dir: $(pwd)" - -#################################################### -# Step 1: Prepare Release Files (SVN or Local) # -#################################################### -if [[ -n "${LOCAL_DIST_PATH}" ]]; then - # Use local directory - DIST_DIR="${LOCAL_DIST_PATH}" - echo "Using local directory: ${DIST_DIR}" - - # Validate local directory exists - if [[ ! -d "${DIST_DIR}" ]]; then - echo -e "${RED}Error: Directory ${DIST_DIR} does not exist${NC}" - exit 1 - fi - - echo "Contents of ${DIST_DIR}:" - ls -lh "${DIST_DIR}" -else - # Download from SVN - DIST_DIR="${WORK_DIR}/dist/${RELEASE_VERSION}" - echo "Downloading from SVN to: ${DIST_DIR}" - - rm -rf "${DIST_DIR}" - mkdir -p "${DIST_DIR}" - cd "${DIST_DIR}" - svn co "${SVN_URL_PREFIX}/${RELEASE_VERSION}" . -fi - -cd "${DIST_DIR}" -echo "Release files directory: ${DIST_DIR}" - -################################################## -# Step 2: Check Environment & Import Public Keys # -################################################## -shasum --version 1>/dev/null -gpg --version 1>/dev/null - -# Check Java version -echo "Checking Java version..." -if ! command -v java &> /dev/null; then - echo -e "${RED}Error: Java is not installed or not in PATH${NC}" - exit 1 -fi - -CURRENT_JAVA_VERSION=$(java -version 2>&1 | head -n 1 | awk -F '"' '{print $2}' | awk -F '.' '{print $1}') -echo "Current Java version: $CURRENT_JAVA_VERSION (Required: ${JAVA_VERSION})" - -if [[ "$CURRENT_JAVA_VERSION" != "$JAVA_VERSION" ]]; then - echo -e "${RED}Error: Java version mismatch!${NC}" - echo -e "${RED} Current: Java $CURRENT_JAVA_VERSION${NC}" - echo -e "${RED} Required: Java ${JAVA_VERSION}${NC}" - echo -e "${RED} Please switch to Java ${JAVA_VERSION} before running this script${NC}" - exit 1 -fi - -echo -e "${GREEN}Java version check passed: Java $CURRENT_JAVA_VERSION${NC}" - -wget https://downloads.apache.org/incubator/hugegraph/KEYS -echo "Import KEYS:" && gpg --import KEYS -# TODO: how to trust all public keys in gpg list, currently only trust the first one -echo -e "5\ny\n" | gpg --batch --command-fd 0 --edit-key $USER trust - -echo "trust all pk" -for key in $(gpg --no-tty --list-keys --with-colons | awk -F: '/^pub/ {print $5}'); do - echo -e "5\ny\n" | gpg --batch --command-fd 0 --edit-key "$key" trust -done - -######################################## -# Step 3: Check SHA512 & GPG Signature # -######################################## -cd "${DIST_DIR}" - -for i in *.tar.gz; do - echo "$i" - shasum -a 512 --check "$i".sha512 - eval gpg "${GPG_OPT}" --verify "$i".asc "$i" -done - -#################################### -# Step 4: Validate Source Packages # -#################################### -cd "${DIST_DIR}" - -CATEGORY_X="\bGPL|\bLGPL|Sleepycat License|BSD-4-Clause|\bBCL\b|JSR-275|Amazon Software License|\bRSAL\b|\bQPL\b|\bSSPL|\bCPOL|\bNPL1|Creative Commons Non-Commercial|JSON\.org" -CATEGORY_B="\bCDDL1|\bCPL|\bEPL|\bIPL|\bMPL|\bSPL|OSL-3.0|UnRAR License|Erlang Public License|\bOFL\b|Ubuntu Font License Version 1.0|IPA Font License Agreement v1.0|EPL2.0|CC-BY" -ls -lh ./*.tar.gz -for i in *src.tar.gz; do - echo "$i" - - # 4.1: check the directory name include "incubating" - if [[ ! "$i" =~ "incubating" ]]; then - echo -e "${RED}The package name $i should include incubating${NC}" && exit 1 - fi - - MODULE_DIR=$(basename "$i" .tar.gz) - rm -rf ${MODULE_DIR} - tar -xzvf "$i" - pushd ${MODULE_DIR} - echo "Start to check the package content: ${MODULE_DIR}" - - # 4.2: check the directory include "NOTICE" and "LICENSE" file and "DISCLAIMER" file - if [[ ! -f "LICENSE" ]]; then - echo -e "${RED}The package $i should include LICENSE file${NC}" && exit 1 - fi - if [[ ! -f "NOTICE" ]]; then - echo -e "${RED}The package $i should include NOTICE file${NC}" && exit 1 - fi - if [[ ! -f "DISCLAIMER" ]]; then - echo -e "${RED}The package $i should include DISCLAIMER file${NC}" && exit 1 - fi - - # 4.3: ensure doesn't contains ASF CATEGORY X License dependencies in LICENSE and NOTICE files - COUNT=$(grep -E "$CATEGORY_X" LICENSE NOTICE | wc -l) - if [[ $COUNT -ne 0 ]]; then - grep -E "$CATEGORY_X" LICENSE NOTICE - echo -e "${RED}The package $i shouldn't include invalid ASF category X dependencies, but get $COUNT${NC}" && exit 1 - fi - - # 4.4: ensure doesn't contains ASF CATEGORY B License dependencies in LICENSE and NOTICE files - COUNT=$(grep -E "$CATEGORY_B" LICENSE NOTICE | wc -l) - if [[ $COUNT -ne 0 ]]; then - grep -E "$CATEGORY_B" LICENSE NOTICE - echo -e "${RED}The package $i shouldn't include invalid ASF category B dependencies, but get $COUNT${NC}" && exit 1 - fi - - # 4.5: ensure doesn't contains empty directory or file - find . -type d -empty | while read -r EMPTY_DIR; do - find . -type d -empty - echo -e "${RED}The package $i shouldn't include empty directory: $EMPTY_DIR is empty${NC}" && exit 1 - done - find . -type f -empty | while read -r EMPTY_FILE; do - find . -type f -empty - echo -e "${RED}The package $i shouldn't include empty file: $EMPTY_FILE is empty${NC}" && exit 1 - done - - # 4.6: ensure any file should less than 800kb - find . -type f -size +800k | while read -r FILE; do - find . -type f -size +800k - echo -e "${RED}The package $i shouldn't include file larger than 800kb: $FILE is larger than 800kb${NC}" && exit 1 - done - - # 4.7: ensure all binary files are documented in LICENSE - find . -type f | perl -lne 'print if -B' | while read -r BINARY_FILE; do - FILE_NAME=$(basename "$BINARY_FILE") - if grep -q "$FILE_NAME" LICENSE; then - echo -e "${YELLOW}Binary file $BINARY_FILE is documented in LICENSE, please check manually${NC}" +# Examples: +# # Validate from Apache SVN +# ./validate-release.sh 1.7.0 pengjunzhi +# +# # Validate from local directory +# ./validate-release.sh 1.7.0 pengjunzhi /path/to/dist +# +# # Specify Java version +# ./validate-release.sh 1.7.0 pengjunzhi "" 11 +# ./validate-release.sh 1.7.0 pengjunzhi /path/to/dist 11 +# +################################################################################ + +# Strict mode - but don't exit on error yet (we collect all errors) +set -o pipefail +set -o nounset + +################################################################################ +# Configuration Constants +################################################################################ + +readonly SCRIPT_VERSION="2.0.0" +readonly SCRIPT_NAME=$(basename "$0") + +# URLs +readonly SVN_URL_PREFIX="https://dist.apache.org/repos/dist/dev/incubator/hugegraph" +readonly KEYS_URL="https://downloads.apache.org/incubator/hugegraph/KEYS" + +# Validation Rules +readonly MAX_FILE_SIZE="800k" +readonly SERVER_START_DELAY=3 +readonly SERVICE_HEALTH_TIMEOUT=30 + +# License Patterns (ASF Category X - Prohibited) +readonly CATEGORY_X="\bGPL|\bLGPL|Sleepycat License|BSD-4-Clause|\bBCL\b|JSR-275|Amazon Software License|\bRSAL\b|\bQPL\b|\bSSPL|\bCPOL|\bNPL1|Creative Commons Non-Commercial|JSON\.org" + +# License Patterns (ASF Category B - Must be documented) +readonly CATEGORY_B="\bCDDL1|\bCPL|\bEPL|\bIPL|\bMPL|\bSPL|OSL-3.0|UnRAR License|Erlang Public License|\bOFL\b|Ubuntu Font License Version 1.0|IPA Font License Agreement v1.0|EPL2.0|CC-BY" + +# Color Definitions +readonly RED='\033[0;31m' +readonly GREEN='\033[0;32m' +readonly YELLOW='\033[0;33m' +readonly BLUE='\033[0;34m' +readonly NC='\033[0m' # No Color + +################################################################################ +# Global Variables +################################################################################ + +# Script state +WORK_DIR="" +LOG_FILE="" +DIST_DIR="" +RELEASE_VERSION="" +USER="" +LOCAL_DIST_PATH="" +JAVA_VERSION=11 +NON_INTERACTIVE=0 + +# Error tracking +declare -a VALIDATION_ERRORS=() +declare -a VALIDATION_WARNINGS=() +TOTAL_CHECKS=0 +PASSED_CHECKS=0 +FAILED_CHECKS=0 + +# Service tracking for cleanup +SERVER_STARTED=0 +HUBBLE_STARTED=0 + +################################################################################ +# Helper Functions - Output & Logging +################################################################################ + +show_usage() { + cat << EOF +Apache HugeGraph Release Validation Script v${SCRIPT_VERSION} + +Usage: ${SCRIPT_NAME} [local-path] [java-version] + ${SCRIPT_NAME} --help | -h + ${SCRIPT_NAME} --version | -v + +Validates Apache HugeGraph release packages including: + - Package integrity (SHA512, GPG signatures) + - License compliance (ASF categories) + - Package contents and structure + - Compilation and runtime testing + +Arguments: + version Release version (e.g., 1.7.0) + user Apache username for GPG key trust + local-path (Optional) Local directory path containing release files + If omitted, downloads from Apache SVN + java-version (Optional) Java version to validate (default: 11) + +Options: + --help, -h Show this help message + --version, -v Show script version + --non-interactive Run without prompts (for CI/CD) + +Examples: + # Validate from Apache SVN (downloads files) + ${SCRIPT_NAME} 1.7.0 pengjunzhi + + # Validate from local directory + ${SCRIPT_NAME} 1.7.0 pengjunzhi /path/to/dist + + # Specify Java version + ${SCRIPT_NAME} 1.7.0 pengjunzhi "" 11 + ${SCRIPT_NAME} 1.7.0 pengjunzhi /path/to/dist 11 + + # Non-interactive mode for CI + ${SCRIPT_NAME} --non-interactive 1.7.0 pengjunzhi + +For more information, visit: + https://hugegraph.apache.org/docs/contribution-guidelines/validate-release/ + +EOF +} + +log() { + local level=$1 + shift + local message="$*" + local timestamp=$(date '+%Y-%m-%d %H:%M:%S') + echo "[${timestamp}] [${level}] ${message}" | tee -a "${LOG_FILE:-/dev/null}" +} + +info() { + echo -e "$*" + log "INFO" "$*" +} + +success() { + echo -e "${GREEN}✓ $*${NC}" + log "SUCCESS" "$*" +} + +warn() { + echo -e "${YELLOW}⚠ $*${NC}" >&2 + log "WARN" "$*" +} + +error() { + echo -e "${RED}✗ $*${NC}" >&2 + log "ERROR" "$*" +} + +print_step() { + local step=$1 + local total=$2 + local description=$3 + echo "" + echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" + echo -e "${BLUE}Step [$step/$total]: $description${NC}" + echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" + log "STEP" "[$step/$total] $description" +} + +print_progress() { + local current=$1 + local total=$2 + local item=$3 + echo -e " [${current}/${total}] ${item}" +} + +collect_error() { + local error_msg="$1" + VALIDATION_ERRORS+=("$error_msg") + FAILED_CHECKS=$((FAILED_CHECKS + 1)) + error "$error_msg" +} + +collect_warning() { + local warning_msg="$1" + VALIDATION_WARNINGS+=("$warning_msg") + warn "$warning_msg" +} + +mark_check_passed() { + PASSED_CHECKS=$((PASSED_CHECKS + 1)) + TOTAL_CHECKS=$((TOTAL_CHECKS + 1)) +} + +mark_check_failed() { + FAILED_CHECKS=$((FAILED_CHECKS + 1)) + TOTAL_CHECKS=$((TOTAL_CHECKS + 1)) +} + +################################################################################ +# Helper Functions - System & Environment +################################################################################ + +setup_logging() { + local log_dir="${WORK_DIR}/logs" + mkdir -p "$log_dir" + LOG_FILE="$log_dir/validate-${RELEASE_VERSION}-$(date +%Y%m%d-%H%M%S).log" + + info "Logging to: ${LOG_FILE}" + log "INIT" "Starting validation for HugeGraph ${RELEASE_VERSION}" + log "INIT" "User: ${USER}, Java: ${JAVA_VERSION}" +} + +check_dependencies() { + local missing_deps=() + local required_commands=("svn" "gpg" "shasum" "mvn" "java" "wget" "tar" "curl" "awk" "grep" "find" "perl") + + info "Checking required dependencies..." + + for cmd in "${required_commands[@]}"; do + if ! command -v "$cmd" &> /dev/null; then + missing_deps+=("$cmd") + error "Missing: $cmd" + else + local version_info + case "$cmd" in + java) + version_info=$(java -version 2>&1 | head -n1 | cut -d'"' -f2) + ;; + mvn) + version_info=$(mvn --version 2>&1 | head -n1 | awk '{print $3}') + ;; + *) + version_info=$($cmd --version 2>&1 | head -n1 || echo "installed") + ;; + esac + success "$cmd: $version_info" + fi + done + + if [[ ${#missing_deps[@]} -gt 0 ]]; then + error "Missing required dependencies: ${missing_deps[*]}" + echo "" + echo "Please install missing dependencies:" + echo " Ubuntu/Debian: sudo apt-get install ${missing_deps[*]}" + echo " macOS: brew install ${missing_deps[*]}" + exit 1 + fi + + success "All dependencies are installed" +} + +check_java_version() { + local required_version=$1 + + info "Checking Java version..." + + if ! command -v java &> /dev/null; then + collect_error "Java is not installed or not in PATH" + return 1 + fi + + local current_version=$(java -version 2>&1 | head -n 1 | awk -F '"' '{print $2}' | awk -F '.' '{print $1}') + info "Current Java version: $current_version (Required: ${required_version})" + + if [[ "$current_version" != "$required_version" ]]; then + collect_error "Java version mismatch! Current: Java $current_version, Required: Java ${required_version}" + collect_error "Please switch to Java ${required_version} before running this script" + return 1 + fi + + success "Java version check passed: Java $current_version" + mark_check_passed + return 0 +} + +find_package_dir() { + local pattern=$1 + local base_dir=${2:-"${DIST_DIR}"} + + local found=$(find "$base_dir" -maxdepth 3 -type d -path "$pattern" 2>/dev/null | head -n1) + + if [[ -z "$found" ]]; then + collect_error "Could not find directory matching pattern: $pattern" + return 1 + fi + + echo "$found" +} + +################################################################################ +# Helper Functions - GPG & Signatures +################################################################################ + +import_and_trust_gpg_keys() { + local user=$1 + + info "Downloading KEYS file from ${KEYS_URL}..." + if ! wget -q "${KEYS_URL}" -O KEYS; then + collect_error "Failed to download KEYS file from ${KEYS_URL}" + return 1 + fi + success "KEYS file downloaded" + + info "Importing GPG keys..." + local import_output=$(gpg --import KEYS 2>&1) + local imported_count=$(echo "$import_output" | grep -c "imported" || echo "0") + + if [[ "$imported_count" == "0" ]]; then + warn "No new keys imported (may already exist in keyring)" + else + success "Imported GPG keys" + fi + + # Trust specific user key + if ! gpg --list-keys "$user" &>/dev/null; then + collect_error "User '$user' key not found in imported keys. Please verify the username." + return 1 + fi + + info "Trusting GPG key for user: $user" + echo -e "5\ny\n" | gpg --batch --command-fd 0 --edit-key "$user" trust 2>/dev/null + success "Trusted key for $user" + + # Trust all imported keys + info "Trusting all imported public keys..." + local trusted=0 + for key in $(gpg --no-tty --list-keys --with-colons | awk -F: '/^pub/ {print $5}'); do + echo -e "5\ny\n" | gpg --batch --command-fd 0 --edit-key "$key" trust 2>/dev/null + trusted=$((trusted + 1)) + done + success "Trusted $trusted GPG keys" + + mark_check_passed + return 0 +} + +################################################################################ +# Validation Functions - Package Checks +################################################################################ + +check_incubating_name() { + local package=$1 + TOTAL_CHECKS=$((TOTAL_CHECKS + 1)) + + if [[ ! "$package" =~ "incubating" ]]; then + collect_error "Package name '$package' should include 'incubating'" + return 1 + fi + + mark_check_passed + return 0 +} + +check_required_files() { + local package=$1 + local require_disclaimer=${2:-true} + local has_error=0 + + if [[ ! -f "LICENSE" ]]; then + collect_error "Package '$package' missing LICENSE file" + has_error=1 + else + mark_check_passed + fi + + if [[ ! -f "NOTICE" ]]; then + collect_error "Package '$package' missing NOTICE file" + has_error=1 + else + mark_check_passed + fi + + if [[ "$require_disclaimer" == "true" ]] && [[ ! -f "DISCLAIMER" ]]; then + collect_error "Package '$package' missing DISCLAIMER file" + has_error=1 + else + mark_check_passed + fi + + return $has_error +} + +check_license_categories() { + local package=$1 + local files=$2 + local has_error=0 + + # Check Category X (Prohibited) + TOTAL_CHECKS=$((TOTAL_CHECKS + 1)) + local cat_x_count=$(grep -r -E "$CATEGORY_X" $files 2>/dev/null | wc -l | tr -d ' ') + if [[ $cat_x_count -ne 0 ]]; then + collect_error "Package '$package' contains $cat_x_count prohibited ASF Category X license(s)" + grep -r -E "$CATEGORY_X" $files + has_error=1 + else + mark_check_passed + fi + + # Check Category B (Must be documented - warning only) + TOTAL_CHECKS=$((TOTAL_CHECKS + 1)) + local cat_b_count=$(grep -r -E "$CATEGORY_B" $files 2>/dev/null | wc -l | tr -d ' ') + if [[ $cat_b_count -ne 0 ]]; then + collect_warning "Package '$package' contains $cat_b_count ASF Category B license(s) - please verify documentation" + grep -r -E "$CATEGORY_B" $files + else + mark_check_passed + fi + + return $has_error +} + +check_empty_files_and_dirs() { + local package=$1 + local has_error=0 + + TOTAL_CHECKS=$((TOTAL_CHECKS + 1)) + + # Find empty directories + local empty_dirs=() + while IFS= read -r empty_dir; do + empty_dirs+=("$empty_dir") + done < <(find . -type d -empty 2>/dev/null) + + # Find empty files + local empty_files=() + while IFS= read -r empty_file; do + empty_files+=("$empty_file") + done < <(find . -type f -empty 2>/dev/null) + + if [[ ${#empty_dirs[@]} -gt 0 ]]; then + collect_error "Package '$package' contains ${#empty_dirs[@]} empty director(y/ies):" + printf ' %s\n' "${empty_dirs[@]}" + has_error=1 + fi + + if [[ ${#empty_files[@]} -gt 0 ]]; then + collect_error "Package '$package' contains ${#empty_files[@]} empty file(s):" + printf ' %s\n' "${empty_files[@]}" + has_error=1 + fi + + if [[ $has_error -eq 0 ]]; then + mark_check_passed + fi + + return $has_error +} + +check_file_sizes() { + local package=$1 + local max_size=$2 + local has_error=0 + + TOTAL_CHECKS=$((TOTAL_CHECKS + 1)) + + local large_files=() + while IFS= read -r large_file; do + large_files+=("$large_file") + done < <(find . -type f -size "+${max_size}" 2>/dev/null) + + if [[ ${#large_files[@]} -gt 0 ]]; then + collect_error "Package '$package' contains ${#large_files[@]} file(s) larger than ${max_size}:" + for file in "${large_files[@]}"; do + local size=$(du -h "$file" | awk '{print $1}') + echo " $file ($size)" + done + has_error=1 else - echo -e "${RED}Error: Binary file $BINARY_FILE is not documented in LICENSE${NC}" && exit 1 - fi - done - - # 4.8: test compile the packages - if [[ "$i" =~ 'hugegraph-ai' ]]; then - echo "Skip compile $i module in all versions" - elif [[ "$i" =~ "hugegraph-computer" ]]; then - cd computer - mvn package -DskipTests -Papache-release -ntp -e - else - # TODO: consider using commands that are entirely consistent with building binary packages - mvn package -DskipTests -Papache-release -ntp -e - ls -lh - fi - popd -done - -########################################### -# Step 5: Run Compiled Packages of Server # -########################################### -cd "${DIST_DIR}" - -ls -lh -pushd ./*hugegraph-incubating*src/hugegraph-server/*hugegraph*"${RELEASE_VERSION}" -bin/init-store.sh -sleep 3 -bin/start-hugegraph.sh -popd - -####################################################################### -# Step 6: Run Compiled Packages of ToolChain (Loader & Tool & Hubble) # -####################################################################### -cd "${DIST_DIR}" - -pushd ./*toolchain*src -ls -lh -pushd ./*toolchain*"${RELEASE_VERSION}" -ls -lh - -# 6.1: load some data first -echo "test loader" -pushd ./*loader*"${RELEASE_VERSION}" -bin/hugegraph-loader.sh -f ./example/file/struct.json -s ./example/file/schema.groovy \ - -g hugegraph -popd - -# 6.2: try some gremlin query & api in tool -echo "test tool" -pushd ./*tool*"${RELEASE_VERSION}" -bin/hugegraph gremlin-execute --script 'g.V().count()' -bin/hugegraph task-list -bin/hugegraph backup -t all --directory ./backup-test -popd - -# 6.3: start hubble and connect to server -echo "test hubble" -pushd ./*hubble*"${RELEASE_VERSION}" -# TODO: add hubble doc & test it -cat conf/hugegraph-hubble.properties -bin/start-hubble.sh -bin/stop-hubble.sh -popd - -popd -popd -# stop server -pushd ./*hugegraph-incubating*src/hugegraph-server/*hugegraph*"${RELEASE_VERSION}" -bin/stop-hugegraph.sh -popd - -# clear source packages -#rm -rf ./*src* -#ls -lh - -#################################### -# Step 7: Validate Binary Packages # -#################################### -cd "${DIST_DIR}" - -for i in *.tar.gz; do - if [[ "$i" == *-src.tar.gz ]]; then - # skip source packages - continue - fi - - echo "$i" - - # 7.1: check the directory name include "incubating" - if [[ ! "$i" =~ "incubating" ]]; then - echo -e "${RED}The package name $i should include incubating${NC}" && exit 1 - fi - - MODULE_DIR=$(basename "$i" .tar.gz) - rm -rf ${MODULE_DIR} - tar -xzvf "$i" - pushd ${MODULE_DIR} - ls -lh - echo "Start to check the package content: ${MODULE_DIR}" - - # 7.2: check root dir include "NOTICE"/"LICENSE"/"DISCLAIMER" files & "licenses" dir - if [[ ! -f "LICENSE" ]]; then - echo -e "${RED}The package $i should include LICENSE file${NC}" && exit 1 - fi - if [[ ! -f "NOTICE" ]]; then - echo -e "${RED}The package $i should include NOTICE file${NC}" && exit 1 - fi - if [[ ! -f "DISCLAIMER" ]]; then - echo -e "${RED}The package $i should include DISCLAIMER file${NC}" && exit 1 - fi - if [[ ! -d "licenses" ]]; then - echo -e "${RED}The package $i should include licenses dir${NC}" && exit 1 - fi - - # 7.3: ensure doesn't contains ASF CATEGORY X License dependencies in LICENSE/NOTICE and licenses/* files - COUNT=$(grep -r -E "$CATEGORY_X" LICENSE NOTICE licenses | wc -l) - if [[ $COUNT -ne 0 ]]; then - grep -r -E "$CATEGORY_X" LICENSE NOTICE licenses - echo -e "${RED}The package $i shouldn't include invalid ASF category X dependencies, but get $COUNT${NC}" && exit 1 - fi - - # 7.4: ensure doesn't contains empty directory or file - find . -type d -empty | while read -r EMPTY_DIR; do - find . -type d -empty - echo -e "${RED}The package $i shouldn't include empty directory: $EMPTY_DIR is empty${NC}" && exit 1 - done - find . -type f -empty | while read -r EMPTY_FILE; do - find . -type f -empty - echo -e "${RED}The package $i shouldn't include empty file: $EMPTY_FILE is empty${NC}" && exit 1 - done - - popd -done - -# TODO: skip the following steps by comparing the artifacts built from source packages with binary packages -######################################### -# Step 8: Run Binary Packages of Server # -######################################### -cd "${DIST_DIR}" - -# TODO: run pd & store -pushd ./*hugegraph-incubating*"${RELEASE_VERSION}"/*hugegraph-server-incubating*"${RELEASE_VERSION}" -bin/init-store.sh -sleep 3 -bin/start-hugegraph.sh -popd - -##################################################################### -# Step 9: Run Binary Packages of ToolChain (Loader & Tool & Hubble) # -##################################################################### -cd "${DIST_DIR}" - -pushd ./*toolchain*"${RELEASE_VERSION}" -ls -lh - -# 9.1: load some data first -echo "test loader" -pushd ./*loader*"${RELEASE_VERSION}" -bin/hugegraph-loader.sh -f ./example/file/struct.json -s ./example/file/schema.groovy -g hugegraph -popd - -# 9.2: try some gremlin query & api in tool -echo "test tool" -pushd ./*tool*"${RELEASE_VERSION}" -bin/hugegraph gremlin-execute --script 'g.V().count()' -bin/hugegraph task-list -bin/hugegraph backup -t all --directory ./backup-test -popd - -# 9.3: start hubble and connect to server -echo "test hubble" -pushd ./*hubble*"${RELEASE_VERSION}" -# TODO: add hubble doc & test it -cat conf/hugegraph-hubble.properties -bin/start-hubble.sh -bin/stop-hubble.sh -popd - -popd -# stop server -pushd ./*hugegraph-incubating*"${RELEASE_VERSION}"/*hugegraph-server-incubating*"${RELEASE_VERSION}" -bin/stop-hugegraph.sh -popd - -echo -e "${GREEN}Finish validate, please check all steps manually again!${NC}" + mark_check_passed + fi + + return $has_error +} + +check_binary_files() { + local package=$1 + local has_error=0 + + TOTAL_CHECKS=$((TOTAL_CHECKS + 1)) + + info "Checking for undocumented binary files..." + + local binary_count=0 + local undocumented_count=0 + + # Find binary files using perl + while IFS= read -r binary_file; do + binary_count=$((binary_count + 1)) + local file_name=$(basename "$binary_file") + + # Check if documented in LICENSE + if grep -q "$file_name" LICENSE 2>/dev/null; then + success "Binary file '$binary_file' is documented in LICENSE" + else + collect_error "Undocumented binary file: $binary_file" + undocumented_count=$((undocumented_count + 1)) + has_error=1 + fi + done < <(find . -type f -exec perl -lne 'print if -B' {} \; 2>/dev/null) + + if [[ $binary_count -eq 0 ]]; then + success "No binary files found" + mark_check_passed + elif [[ $undocumented_count -eq 0 ]]; then + success "All $binary_count binary file(s) are documented" + mark_check_passed + fi + + return $has_error +} + +check_license_headers() { + local package=$1 + + TOTAL_CHECKS=$((TOTAL_CHECKS + 1)) + + info "Checking for ASF license headers in source files..." + + # Check Java files for Apache license headers + local files_without_license=() + while IFS= read -r java_file; do + if ! head -n 20 "$java_file" | grep -q "Licensed to the Apache Software Foundation"; then + files_without_license+=("$java_file") + fi + done < <(find . -name "*.java" -type f 2>/dev/null) + + if [[ ${#files_without_license[@]} -gt 0 ]]; then + collect_warning "Found ${#files_without_license[@]} Java file(s) without ASF license headers" + collect_warning "Run 'mvn apache-rat:check' for detailed license header analysis" + # Note: This is a warning, not an error + else + success "All Java source files have ASF license headers" + mark_check_passed + fi +} + +check_version_consistency() { + local package=$1 + local expected_version=$2 + + TOTAL_CHECKS=$((TOTAL_CHECKS + 1)) + + info "Checking version consistency in pom.xml files..." + + # Find inconsistent versions in pom.xml files + local inconsistent=() + while IFS= read -r pom_file; do + # Extract version tags (exclude parent versions and SNAPSHOT) + while IFS= read -r version_line; do + if [[ ! "$version_line" =~ "" ]] && \ + [[ ! "$version_line" =~ "SNAPSHOT" ]] && \ + [[ ! "$version_line" =~ "$expected_version" ]]; then + inconsistent+=("$pom_file: $version_line") + fi + done < <(grep "" "$pom_file" 2>/dev/null) + done < <(find . -name "pom.xml" -type f 2>/dev/null) + + if [[ ${#inconsistent[@]} -gt 0 ]]; then + collect_error "Found version inconsistencies in pom.xml files:" + printf ' %s\n' "${inconsistent[@]}" + return 1 + else + success "Version consistency check passed" + mark_check_passed + fi + + return 0 +} + +check_notice_year() { + local package=$1 + + TOTAL_CHECKS=$((TOTAL_CHECKS + 1)) + + if [[ ! -f "NOTICE" ]]; then + return 0 # Already checked in check_required_files + fi + + local current_year=$(date +%Y) + if ! grep -q "$current_year" NOTICE; then + collect_warning "NOTICE file may not contain current year ($current_year). Please verify copyright dates." + else + mark_check_passed + fi +} + +################################################################################ +# Main Validation Functions +################################################################################ + +validate_source_package() { + local package_file=$1 + local package_dir=$(basename "$package_file" .tar.gz) + + info "Validating source package: $package_file" + + # Extract package + rm -rf "$package_dir" + tar -xzf "$package_file" + + if [[ ! -d "$package_dir" ]]; then + collect_error "Failed to extract package: $package_file" + return 1 + fi + + pushd "$package_dir" > /dev/null + + # Run all checks + check_incubating_name "$package_file" + check_required_files "$package_file" true + check_license_categories "$package_file" "LICENSE NOTICE" + check_empty_files_and_dirs "$package_file" + check_file_sizes "$package_file" "$MAX_FILE_SIZE" + check_binary_files "$package_file" + check_license_headers "$package_file" + check_version_consistency "$package_file" "$RELEASE_VERSION" + check_notice_year "$package_file" + + # Compile check + info "Compiling source package: $package_file" + TOTAL_CHECKS=$((TOTAL_CHECKS + 1)) + + if [[ "$package_file" =~ 'hugegraph-ai' ]]; then + warn "Skipping compilation for AI module (not required)" + mark_check_passed + elif [[ "$package_file" =~ "hugegraph-computer" ]]; then + if cd computer 2>/dev/null && mvn package -DskipTests -Papache-release -ntp -e; then + success "Compilation successful: $package_file" + mark_check_passed + else + collect_error "Compilation failed: $package_file" + fi + cd .. + else + if mvn package -DskipTests -Papache-release -ntp -e; then + success "Compilation successful: $package_file" + mark_check_passed + else + collect_error "Compilation failed: $package_file" + fi + fi + + popd > /dev/null + + info "Finished validating source package: $package_file" +} + +validate_binary_package() { + local package_file=$1 + local package_dir=$(basename "$package_file" .tar.gz) + + info "Validating binary package: $package_file" + + # Extract package + rm -rf "$package_dir" + tar -xzf "$package_file" + + if [[ ! -d "$package_dir" ]]; then + collect_error "Failed to extract package: $package_file" + return 1 + fi + + pushd "$package_dir" > /dev/null + + # Run checks + check_incubating_name "$package_file" + check_required_files "$package_file" true + + # Binary packages should have licenses directory + TOTAL_CHECKS=$((TOTAL_CHECKS + 1)) + if [[ ! -d "licenses" ]]; then + collect_error "Package '$package_file' missing licenses directory" + else + mark_check_passed + fi + + check_license_categories "$package_file" "LICENSE NOTICE licenses" + check_empty_files_and_dirs "$package_file" + + popd > /dev/null + + info "Finished validating binary package: $package_file" +} + +################################################################################ +# Cleanup Function +################################################################################ + +cleanup() { + local exit_code=$? + + log "CLEANUP" "Starting cleanup (exit code: $exit_code)" + + # Stop running services + if [[ $SERVER_STARTED -eq 1 ]]; then + info "Stopping HugeGraph server..." + local server_dir=$(find_package_dir "*hugegraph-incubating*src/hugegraph-server/*hugegraph*${RELEASE_VERSION}" 2>/dev/null || echo "") + if [[ -n "$server_dir" ]] && [[ -d "$server_dir" ]]; then + pushd "$server_dir" > /dev/null 2>&1 + bin/stop-hugegraph.sh || true + popd > /dev/null 2>&1 + fi + fi + + if [[ $HUBBLE_STARTED -eq 1 ]]; then + info "Stopping Hubble..." + # Hubble stop is handled in the test flow + fi + + # Show final report + echo "" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo " VALIDATION SUMMARY " + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo "" + echo "Total Checks: $TOTAL_CHECKS" + echo -e "${GREEN}Passed: $PASSED_CHECKS${NC}" + echo -e "${RED}Failed: $FAILED_CHECKS${NC}" + echo -e "${YELLOW}Warnings: ${#VALIDATION_WARNINGS[@]}${NC}" + echo "" + + if [[ ${#VALIDATION_ERRORS[@]} -gt 0 ]]; then + echo -e "${RED}━━━ ERRORS ━━━${NC}" + for err in "${VALIDATION_ERRORS[@]}"; do + echo -e "${RED} ✗ $err${NC}" + done + echo "" + fi + + if [[ ${#VALIDATION_WARNINGS[@]} -gt 0 ]]; then + echo -e "${YELLOW}━━━ WARNINGS ━━━${NC}" + for warn in "${VALIDATION_WARNINGS[@]}"; do + echo -e "${YELLOW} ⚠ $warn${NC}" + done + echo "" + fi + + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo "" + + if [[ ${#VALIDATION_ERRORS[@]} -gt 0 ]]; then + echo -e "${RED}VALIDATION FAILED${NC}" + echo -e "Log file: ${LOG_FILE}" + echo "" + exit 1 + else + echo -e "${GREEN}✓ VALIDATION PASSED${NC}" + echo -e "Log file: ${LOG_FILE}" + echo "" + echo "Please review the validation results and provide feedback in the" + echo "release voting thread on the mailing list." + echo "" + exit 0 + fi +} + +# Set trap for cleanup +trap cleanup EXIT +trap 'echo -e "${RED}Script interrupted${NC}"; exit 130' INT TERM + +################################################################################ +# Main Execution +################################################################################ + +main() { + # Parse command line arguments + while [[ $# -gt 0 ]]; do + case $1 in + --help|-h) + show_usage + exit 0 + ;; + --version|-v) + echo "Apache HugeGraph Release Validation Script v${SCRIPT_VERSION}" + exit 0 + ;; + --non-interactive) + NON_INTERACTIVE=1 + shift + ;; + *) + break + ;; + esac + done + + # Parse positional arguments + RELEASE_VERSION=${1:-} + USER=${2:-} + LOCAL_DIST_PATH=${3:-} + JAVA_VERSION=${4:-11} + + # Validate required arguments + if [[ -z "$RELEASE_VERSION" ]]; then + error "Missing required argument: version" + echo "" + show_usage + exit 1 + fi + + if [[ -z "$USER" ]]; then + error "Missing required argument: user" + echo "" + show_usage + exit 1 + fi + + # Initialize + WORK_DIR=$(cd "$(dirname "$0")" && pwd) + cd "${WORK_DIR}" + + setup_logging + + echo "" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo " Apache HugeGraph Release Validation v${SCRIPT_VERSION}" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo "" + echo " Version: ${RELEASE_VERSION}" + echo " User: ${USER}" + echo " Java: ${JAVA_VERSION}" + echo " Mode: $([ -n "${LOCAL_DIST_PATH}" ] && echo "Local (${LOCAL_DIST_PATH})" || echo "SVN Download")" + echo " Log: ${LOG_FILE}" + echo "" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo "" + + #################################################### + # Step 1: Check Dependencies + #################################################### + print_step 1 9 "Check Dependencies" + check_dependencies + check_java_version "$JAVA_VERSION" + + #################################################### + # Step 2: Prepare Release Files + #################################################### + print_step 2 9 "Prepare Release Files" + + if [[ -n "${LOCAL_DIST_PATH}" ]]; then + # Use local directory + DIST_DIR="${LOCAL_DIST_PATH}" + info "Using local directory: ${DIST_DIR}" + + if [[ ! -d "${DIST_DIR}" ]]; then + collect_error "Directory ${DIST_DIR} does not exist" + exit 1 + fi + + info "Contents of ${DIST_DIR}:" + ls -lh "${DIST_DIR}" + else + # Download from SVN + DIST_DIR="${WORK_DIR}/dist/${RELEASE_VERSION}" + info "Downloading from SVN to: ${DIST_DIR}" + + rm -rf "${DIST_DIR}" + mkdir -p "${DIST_DIR}" + + if ! svn co "${SVN_URL_PREFIX}/${RELEASE_VERSION}" "${DIST_DIR}"; then + collect_error "Failed to download from SVN: ${SVN_URL_PREFIX}/${RELEASE_VERSION}" + exit 1 + fi + + success "Downloaded release files from SVN" + fi + + cd "${DIST_DIR}" + + #################################################### + # Step 3: Import GPG Keys + #################################################### + print_step 3 9 "Import & Trust GPG Keys" + import_and_trust_gpg_keys "$USER" + + #################################################### + # Step 4: Check SHA512 & GPG Signatures + #################################################### + print_step 4 9 "Verify SHA512 & GPG Signatures" + + local package_count=0 + local packages=() + for pkg in *.tar.gz; do + if [[ -f "$pkg" ]]; then + packages+=("$pkg") + package_count=$((package_count + 1)) + fi + done + + local current=0 + for pkg in "${packages[@]}"; do + current=$((current + 1)) + print_progress $current $package_count "$pkg" + + # Check SHA512 + TOTAL_CHECKS=$((TOTAL_CHECKS + 1)) + if shasum -a 512 --check "${pkg}.sha512"; then + success "SHA512 verified: $pkg" + mark_check_passed + else + collect_error "SHA512 verification failed: $pkg" + fi + + # Check GPG signature + TOTAL_CHECKS=$((TOTAL_CHECKS + 1)) + if gpg --verify "${pkg}.asc" "$pkg" 2>&1 | grep -q "Good signature"; then + success "GPG signature verified: $pkg" + mark_check_passed + else + collect_error "GPG signature verification failed: $pkg" + fi + done + + #################################################### + # Step 5: Validate Source Packages + #################################################### + print_step 5 9 "Validate Source Packages" + + local src_packages=() + for pkg in *-src.tar.gz; do + if [[ -f "$pkg" ]]; then + src_packages+=("$pkg") + fi + done + + info "Found ${#src_packages[@]} source package(s)" + + for src_pkg in "${src_packages[@]}"; do + validate_source_package "$src_pkg" + done + + #################################################### + # Step 6: Run Compiled Packages (Server) + #################################################### + print_step 6 9 "Test Compiled Server Package" + + local server_dir=$(find_package_dir "*hugegraph-incubating*src/hugegraph-server/*hugegraph*${RELEASE_VERSION}") + if [[ -n "$server_dir" ]]; then + info "Starting HugeGraph server from: $server_dir" + pushd "$server_dir" > /dev/null + + if bin/init-store.sh; then + success "Store initialized" + else + collect_error "Failed to initialize store" + fi + + sleep $SERVER_START_DELAY + + if bin/start-hugegraph.sh; then + success "Server started" + SERVER_STARTED=1 + else + collect_error "Failed to start server" + fi + + popd > /dev/null + else + collect_error "Could not find compiled server directory" + fi + + #################################################### + # Step 7: Test Toolchain (Loader, Tool, Hubble) + #################################################### + print_step 7 9 "Test Compiled Toolchain Packages" + + local toolchain_src=$(find_package_dir "*toolchain*src") + if [[ -n "$toolchain_src" ]]; then + pushd "$toolchain_src" > /dev/null + + local toolchain_dir=$(find . -maxdepth 1 -type d -name "*toolchain*${RELEASE_VERSION}" | head -n1) + if [[ -n "$toolchain_dir" ]]; then + pushd "$toolchain_dir" > /dev/null + + # Test Loader + info "Testing HugeGraph Loader..." + local loader_dir=$(find . -maxdepth 1 -type d -name "*loader*${RELEASE_VERSION}" | head -n1) + if [[ -n "$loader_dir" ]]; then + pushd "$loader_dir" > /dev/null + if bin/hugegraph-loader.sh -f ./example/file/struct.json -s ./example/file/schema.groovy -g hugegraph; then + success "Loader test passed" + else + collect_error "Loader test failed" + fi + popd > /dev/null + fi + + # Test Tool + info "Testing HugeGraph Tool..." + local tool_dir=$(find . -maxdepth 1 -type d -name "*tool*${RELEASE_VERSION}" | head -n1) + if [[ -n "$tool_dir" ]]; then + pushd "$tool_dir" > /dev/null + if bin/hugegraph gremlin-execute --script 'g.V().count()' && \ + bin/hugegraph task-list && \ + bin/hugegraph backup -t all --directory ./backup-test; then + success "Tool test passed" + else + collect_error "Tool test failed" + fi + popd > /dev/null + fi + + # Test Hubble + info "Testing HugeGraph Hubble..." + local hubble_dir=$(find . -maxdepth 1 -type d -name "*hubble*${RELEASE_VERSION}" | head -n1) + if [[ -n "$hubble_dir" ]]; then + pushd "$hubble_dir" > /dev/null + if bin/start-hubble.sh; then + HUBBLE_STARTED=1 + success "Hubble started" + sleep 2 + bin/stop-hubble.sh + HUBBLE_STARTED=0 + success "Hubble stopped" + else + collect_error "Hubble test failed" + fi + popd > /dev/null + fi + + popd > /dev/null + fi + + popd > /dev/null + fi + + # Stop server after toolchain tests + if [[ $SERVER_STARTED -eq 1 ]] && [[ -n "$server_dir" ]]; then + info "Stopping server..." + pushd "$server_dir" > /dev/null + bin/stop-hugegraph.sh + SERVER_STARTED=0 + success "Server stopped" + popd > /dev/null + fi + + #################################################### + # Step 8: Validate Binary Packages + #################################################### + print_step 8 9 "Validate Binary Packages" + + cd "${DIST_DIR}" + + local bin_packages=() + for pkg in *.tar.gz; do + if [[ "$pkg" != *-src.tar.gz ]]; then + bin_packages+=("$pkg") + fi + done + + info "Found ${#bin_packages[@]} binary package(s)" + + for bin_pkg in "${bin_packages[@]}"; do + validate_binary_package "$bin_pkg" + done + + #################################################### + # Step 9: Test Binary Packages + #################################################### + print_step 9 9 "Test Binary Server & Toolchain" + + # Test binary server + local bin_server_dir=$(find_package_dir "*hugegraph-incubating*${RELEASE_VERSION}/*hugegraph-server-incubating*${RELEASE_VERSION}") + if [[ -n "$bin_server_dir" ]]; then + info "Testing binary server package..." + pushd "$bin_server_dir" > /dev/null + + if bin/init-store.sh && sleep $SERVER_START_DELAY && bin/start-hugegraph.sh; then + success "Binary server started" + SERVER_STARTED=1 + else + collect_error "Failed to start binary server" + fi + + popd > /dev/null + fi + + # Test binary toolchain + local bin_toolchain=$(find_package_dir "*toolchain*${RELEASE_VERSION}" "${DIST_DIR}") + if [[ -n "$bin_toolchain" ]]; then + pushd "$bin_toolchain" > /dev/null + + # Test binary loader + local bin_loader=$(find . -maxdepth 1 -type d -name "*loader*${RELEASE_VERSION}" | head -n1) + if [[ -n "$bin_loader" ]]; then + pushd "$bin_loader" > /dev/null + if bin/hugegraph-loader.sh -f ./example/file/struct.json -s ./example/file/schema.groovy -g hugegraph; then + success "Binary loader test passed" + else + collect_error "Binary loader test failed" + fi + popd > /dev/null + fi + + # Test binary tool + local bin_tool=$(find . -maxdepth 1 -type d -name "*tool*${RELEASE_VERSION}" | head -n1) + if [[ -n "$bin_tool" ]]; then + pushd "$bin_tool" > /dev/null + if bin/hugegraph gremlin-execute --script 'g.V().count()' && \ + bin/hugegraph task-list && \ + bin/hugegraph backup -t all --directory ./backup-test; then + success "Binary tool test passed" + else + collect_error "Binary tool test failed" + fi + popd > /dev/null + fi + + # Test binary hubble + local bin_hubble=$(find . -maxdepth 1 -type d -name "*hubble*${RELEASE_VERSION}" | head -n1) + if [[ -n "$bin_hubble" ]]; then + pushd "$bin_hubble" > /dev/null + if bin/start-hubble.sh; then + HUBBLE_STARTED=1 + success "Binary hubble started" + sleep 2 + bin/stop-hubble.sh + HUBBLE_STARTED=0 + success "Binary hubble stopped" + else + collect_error "Binary hubble test failed" + fi + popd > /dev/null + fi + + popd > /dev/null + fi + + # Stop binary server + if [[ $SERVER_STARTED -eq 1 ]] && [[ -n "$bin_server_dir" ]]; then + pushd "$bin_server_dir" > /dev/null + bin/stop-hugegraph.sh + SERVER_STARTED=0 + success "Binary server stopped" + popd > /dev/null + fi + + #################################################### + # Validation Complete + #################################################### + success "All validation steps completed!" + + # Cleanup function will show the final report +} + +# Run main function +main "$@" From 97dc1b56ee0f42ccc6a3a617bf74c59ca65699f2 Mon Sep 17 00:00:00 2001 From: imbajin Date: Sat, 15 Nov 2025 02:57:57 +0800 Subject: [PATCH 4/8] refactor: enhance release validation for multi-arch and license checks V3 Updated CI workflow to support additional OS and architectures (arm64, macOS 14). Improved documentation and script usage instructions. The license header check now covers more file types and excludes generated/vendor files. Maven build commands in docs and scripts now use '-DskipTests' and '-Dcheckstyle.skip=true' for consistency. Added a detailed README for the release validation script. --- .github/workflows/validate-release.yml | 4 +- .../validate-release.md | 4 +- .../validate-release.md | 4 +- dist/README.md | 231 ++++++++++++++++++ dist/validate-release.sh | 134 +++++++--- 5 files changed, 342 insertions(+), 35 deletions(-) create mode 100644 dist/README.md diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index 747f0d78e..bbd1506f2 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -350,5 +350,5 @@ jobs: matrix: # disable java8 because of server java_version: ['11'] - # TODO: support windows-latest or other OS in future - os: [ubuntu-latest, macos-latest] + # Support multiple OS and architectures (x64 and arm64) + os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest, macos-14] diff --git a/content/cn/docs/contribution-guidelines/validate-release.md b/content/cn/docs/contribution-guidelines/validate-release.md index 86f6a22e5..7794a4ba2 100644 --- a/content/cn/docs/contribution-guidelines/validate-release.md +++ b/content/cn/docs/contribution-guidelines/validate-release.md @@ -125,8 +125,8 @@ PMC 同学请特别注意认真检查 `LICENSE` + `NOTICE` 文件，确保文件 # 请优先使用/切换到 `java 11` 版本进行后序的编译和运行操作 (注:`Computer` 仅支持 `java >= 11`) # java --version -# 尝试在 Unix 环境下编译测试是否正常 (stage 表示从 stage 仓库拉取依赖) -mvn clean package -P stage -Dmaven.test.skip=true -Dcheckstyle.skip=true +# 尝试在 Unix 环境下编译测试是否正常 +mvn clean package -DskipTests -Dcheckstyle.skip=true -P stage ``` ##### B. 二进制包 diff --git a/content/en/docs/contribution-guidelines/validate-release.md b/content/en/docs/contribution-guidelines/validate-release.md index 398d79c14..83613d13c 100644 --- a/content/en/docs/contribution-guidelines/validate-release.md +++ b/content/en/docs/contribution-guidelines/validate-release.md @@ -136,8 +136,8 @@ After decompressing `*hugegraph*src.tar.gz`, Do the following checks: # prefer to use/switch to `java 11` for the following operations (compiling/running) (Note: `Computer` only supports `java >= 11`) # java --version -# try to compile in the Unix env to check if it works well -mvn clean package -P stage -Dmaven.test.skip=true -Dcheckstyle.skip=true +# try to compile in the Unix env to check if it works well (-P is optional) +mvn clean package -P stage -DskipTests -Dcheckstyle.skip=true ``` ##### B. binary package diff --git a/dist/README.md b/dist/README.md new file mode 100644 index 000000000..b4d43d8ef --- /dev/null +++ b/dist/README.md @@ -0,0 +1,231 @@ +# Apache HugeGraph 发版验证脚本 + +Apache HugeGraph (Incubating) 发布包的自动化验证脚本。 + +## 概述 + +`validate-release.sh` 脚本对 Apache HugeGraph 发布包进行全面验证，自动执行 [Apache 发布政策](https://www.apache.org/legal/release-policy.html) 和 [孵化器发布检查清单](https://cwiki.apache.org/confluence/display/INCUBATOR/Incubator+Release+Checklist) 要求的大部分检查。 + +## 功能特性 + +- ✅ **自动依赖检查** - 验证所有必需工具（svn、gpg、java、maven 等） +- ✅ **SHA512 和 GPG 签名验证** - 确保包的完整性和真实性 +- ✅ **许可证合规性验证** - 检查禁止的 ASF Category X 和 B 类许可证 +- ✅ **包内容验证** - 验证必需文件（LICENSE、NOTICE、DISCLAIMER） +- ✅ **ASF 许可证头检查** - 验证所有源文件中的许可证头（Java、Python、Go、Shell 等） +- ✅ **版本一致性验证** - 确保 pom.xml 版本与预期发布版本匹配 +- ✅ **源码包编译** - 编译源码包以验证构建正确性 +- ✅ **运行时测试** - 测试服务器和工具链（loader、tool、hubble）功能 +- ✅ **进度跟踪** - 显示实时进度和分步指示器 +- ✅ **详细日志记录** - 将所有输出保存到带时间戳的日志文件 +- ✅ **全面的错误报告** - 收集所有错误并在最后显示摘要 + +## 环境要求 + +- Java 11（HugeGraph 1.5.0+ 必需） +- Maven 3.x +- svn（Subversion 客户端） +- gpg（用于签名验证的 GnuPG） +- wget 或 curl +- 标准 Unix 工具（bash、find、grep、awk 等） + +脚本会自动检查所有依赖项，如果缺少任何内容会提供安装说明。 + +## 使用方法 + +### 基本用法 + +```bash +# 查看帮助信息 +./validate-release.sh --help + +# 从 Apache SVN 验证（自动下载发布文件） +./validate-release.sh <版本号> + +# 示例 +./validate-release.sh 1.7.0 pengjunzhi +``` + +### 高级用法 + +```bash +# 从本地目录验证（如果已经下载了文件） +./validate-release.sh <版本号> <本地路径> + +# 示例 +./validate-release.sh 1.7.0 pengjunzhi /path/to/downloaded/dist + +# 指定 Java 版本（默认：11） +./validate-release.sh <版本号> <本地路径> + +# 示例 - 使用 Java 11 +./validate-release.sh 1.7.0 pengjunzhi /path/to/dist 11 + +# 示例 - SVN 模式使用 Java 11 +./validate-release.sh 1.7.0 pengjunzhi "" 11 + +# 非交互模式（用于 CI/CD） +./validate-release.sh --non-interactive 1.7.0 pengjunzhi +``` + +### 命令行选项 + +- `--help`, `-h` - 显示帮助信息并退出 +- `--version`, `-v` - 显示脚本版本并退出 +- `--non-interactive` - 无提示运行（用于 CI/CD 管道） + +## 验证步骤 + +脚本执行以下 9 个验证步骤： + +1. **检查依赖项** - 验证所有必需工具已安装 +2. **准备发布文件** - 从 Apache SVN 下载或使用本地目录 +3. **导入并信任 GPG 密钥** - 导入 KEYS 文件并信任发布管理员的 GPG 密钥 +4. **验证 SHA512 和 GPG 签名** - 验证所有包的校验和和签名 +5. **验证源码包** - 对源码包进行全面检查： + - 包命名（包含 "incubating"） + - 必需文件（LICENSE、NOTICE、DISCLAIMER） + - 许可证合规性（无 Category X，已记录 Category B） + - 无空文件或目录 + - 文件大小限制（无文件 > 800KB） + - 二进制文件文档 + - 所有源文件中的许可证头 + - pom.xml 文件之间的版本一致性 + - NOTICE 文件版权年份 + - 源码编译 +6. **测试编译的服务器** - 启动并测试编译的 HugeGraph 服务器 +7. **测试编译的工具链** - 从编译包测试 loader、tool 和 hubble +8. **验证二进制包** - 检查二进制包的必需文件和结构 +9. **测试二进制包** - 从二进制包测试服务器和工具链 + +## 输出结果 + +### 进度指示器 + +脚本提供实时进度信息： + +``` +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + Apache HugeGraph Release Validation v2.0.0 +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + + 版本: 1.7.0 + 用户: pengjunzhi + Java: 11 + 模式: SVN 下载 + 日志: logs/validate-1.7.0-20251115-021742.log + +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +步骤 [1/9]: 检查依赖项 +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +✓ svn: version 1.14.1 +✓ gpg: gpg (GnuPG) 2.2.41 +✓ java: 11.0.21 +... +``` + +### 彩色结果 + +- ✓ **绿色** - 成功的检查 +- ✗ **红色** - 需要修复的错误 +- ⚠ **黄色** - 需要审查的警告 +- **蓝色** - 步骤标题和进度信息 + +### 日志文件 + +所有输出都保存到 `logs/validate--.log` 以供后续查看。 + +### 最终摘要 + +验证结束时，会显示一个全面的摘要： + +``` +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + 验证摘要 +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +总检查数: 127 +通过: 125 +失败: 2 +警告: 3 + +━━━ 错误 ━━━ + ✗ 包 'xyz' 缺少 LICENSE 文件 + ✗ 二进制文件 'logo.png' 未在 LICENSE 中记录 + +━━━ 警告 ━━━ + ⚠ NOTICE 文件可能不包含当前年份 (2025) + +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +验证失败 +日志文件: logs/validate-1.7.0-20251115-021742.log +``` + +## 错误处理 + +脚本使用**"继续并报告"**方式： + +- 不会在第一个错误时退出 +- 收集所有验证错误 +- 在最后显示全面摘要 +- 退出码 0 = 所有检查通过 +- 退出码 1 = 一个或多个检查失败 + +这允许你一次看到所有问题，而不是逐个修复。 + +## 故障排除 + +### Java 版本不匹配 + +如果看到 Java 版本错误： + +```bash +# 检查你的 Java 版本 +java -version + +# 使用 JAVA_HOME 指定 Java 11 +export JAVA_HOME=/path/to/java11 +export PATH=$JAVA_HOME/bin:$PATH + +# 或使用 jenv 切换 Java 版本 +jenv global 11 +``` + +### GPG 密钥问题 + +如果 GPG 密钥导入失败： + +```bash +# 手动下载并导入 KEYS +curl https://downloads.apache.org/incubator/hugegraph/KEYS > KEYS +gpg --import KEYS + +# 信任特定密钥 +gpg --edit-key +# 在 GPG 提示符中，输入: trust, 然后 5, 然后 y, 然后 quit +``` + +### 权限被拒绝 + +确保脚本可执行： + +```bash +chmod +x validate-release.sh +``` + +## 参考文档 + +- [Apache 发布政策](https://www.apache.org/legal/release-policy.html) +- [孵化器发布检查清单](https://cwiki.apache.org/confluence/display/INCUBATOR/Incubator+Release+Checklist) +- [HugeGraph 验证发布指南](../content/cn/docs/contribution-guidelines/validate-release.md) + +## 贡献 + +如果发现问题或有改进建议，请： + +1. 查看现有问题：https://github.com/apache/incubator-hugegraph-doc/issues +2. 提交新问题或 pull request + diff --git a/dist/validate-release.sh b/dist/validate-release.sh index 06dcdbbd9..013694651 100755 --- a/dist/validate-release.sh +++ b/dist/validate-release.sh @@ -1,20 +1,4 @@ #!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ################################################################################ # Apache HugeGraph Release Validation Script ################################################################################ @@ -46,7 +30,6 @@ # ./validate-release.sh 1.7.0 pengjunzhi /path/to/dist # # # Specify Java version -# ./validate-release.sh 1.7.0 pengjunzhi "" 11 # ./validate-release.sh 1.7.0 pengjunzhi /path/to/dist 11 # ################################################################################ @@ -154,7 +137,7 @@ Examples: ${SCRIPT_NAME} --non-interactive 1.7.0 pengjunzhi For more information, visit: - https://hugegraph.apache.org/docs/contribution-guidelines/validate-release/ + https://github.com/apache/incubator-hugegraph-doc/tree/master/dist EOF } @@ -548,21 +531,114 @@ check_license_headers() { info "Checking for ASF license headers in source files..." - # Check Java files for Apache license headers + # Define file patterns to check for license headers + # Including: Java, Shell scripts, Python, Go, JavaScript, TypeScript, C/C++, Scala, Groovy, etc. + local -a file_patterns=( + "*.java" # Java files + "*.sh" # Shell scripts + "*.py" # Python files + "*.go" # Go files + "*.js" # JavaScript files + "*.ts" # TypeScript files + "*.jsx" # React JSX files + "*.tsx" # React TypeScript files + "*.c" # C files + "*.h" # C header files + "*.cpp" # C++ files + "*.cc" # C++ files + "*.cxx" # C++ files + "*.hpp" # C++ header files + "*.scala" # Scala files + "*.groovy" # Groovy files + "*.gradle" # Gradle build files + "*.rs" # Rust files + "*.kt" # Kotlin files + "*.proto" # Protocol buffer files + ) + + # Files to exclude from license header check + local -a exclude_patterns=( + "*.min.js" # Minified JavaScript + "*.min.css" # Minified CSS + "*node_modules*" # Node.js dependencies + "*target*" # Maven build output + "*build*" # Build directories + "*.pb.go" # Generated protobuf files + "*generated*" # Generated code + "*third_party*" # Third party code + "*vendor*" # Vendor dependencies + ) + local files_without_license=() - while IFS= read -r java_file; do - if ! head -n 20 "$java_file" | grep -q "Licensed to the Apache Software Foundation"; then - files_without_license+=("$java_file") + local total_checked=0 + local excluded_count=0 + + # Build find command with all patterns + local find_cmd="find . -type f \\(" + local first=1 + for pattern in "${file_patterns[@]}"; do + if [[ $first -eq 1 ]]; then + find_cmd="$find_cmd -name \"$pattern\"" + first=0 + else + find_cmd="$find_cmd -o -name \"$pattern\"" fi - done < <(find . -name "*.java" -type f 2>/dev/null) + done + find_cmd="$find_cmd \\) 2>/dev/null" + + # Check each source file for ASF license header + while IFS= read -r source_file; do + # Skip if file matches exclude patterns + local should_exclude=0 + for exclude_pattern in "${exclude_patterns[@]}"; do + if [[ "$source_file" == $exclude_pattern ]]; then + should_exclude=1 + excluded_count=$((excluded_count + 1)) + break + fi + done + + if [[ $should_exclude -eq 1 ]]; then + continue + fi + + total_checked=$((total_checked + 1)) + + # Check first 30 lines for Apache license header + # Looking for the standard ASF license header text + if ! head -n 30 "$source_file" | grep -q "Licensed to the Apache Software Foundation"; then + files_without_license+=("$source_file") + fi + done < <(eval "$find_cmd") + + # Report results + info "Checked $total_checked source file(s) for ASF license headers (excluded $excluded_count generated/vendored files)" if [[ ${#files_without_license[@]} -gt 0 ]]; then - collect_warning "Found ${#files_without_license[@]} Java file(s) without ASF license headers" - collect_warning "Run 'mvn apache-rat:check' for detailed license header analysis" - # Note: This is a warning, not an error + collect_error "Found ${#files_without_license[@]} source file(s) without ASF license headers:" + + # Show first 20 files without headers (to avoid overwhelming output) + local show_count=${#files_without_license[@]} + if [[ $show_count -gt 20 ]]; then + show_count=20 + fi + + for ((i=0; i/dev/null && mvn package -DskipTests -Papache-release -ntp -e; then + if cd computer 2>/dev/null && mvn clean package -DskipTests -Dcheckstyle.skip=true -ntp -e; then success "Compilation successful: $package_file" mark_check_passed else @@ -664,7 +740,7 @@ validate_source_package() { fi cd .. else - if mvn package -DskipTests -Papache-release -ntp -e; then + if mvn clean package -DskipTests -Dcheckstyle.skip=true -ntp -e; then success "Compilation successful: $package_file" mark_check_passed else From dfd69a7e7c756fedf9d473f58a5da6bb351a88d4 Mon Sep 17 00:00:00 2001 From: imbajin Date: Sat, 15 Nov 2025 04:07:28 +0800 Subject: [PATCH 5/8] refactor: improve validation script error context and reporting V4 Adds contextual error and warning reporting with step and package information, enhances license category and header checks, improves version consistency logic, and refines summary output with execution time and clearer formatting. These changes make validation results more actionable and easier to interpret, especially for multi-package and multi-step validations. --- dist/validate-release.sh | 209 +++++++++++++++++++++++++++++++-------- 1 file changed, 169 insertions(+), 40 deletions(-) diff --git a/dist/validate-release.sh b/dist/validate-release.sh index 013694651..cf39fdc6f 100755 --- a/dist/validate-release.sh +++ b/dist/validate-release.sh @@ -87,11 +87,16 @@ declare -a VALIDATION_WARNINGS=() TOTAL_CHECKS=0 PASSED_CHECKS=0 FAILED_CHECKS=0 +CURRENT_STEP="" +CURRENT_PACKAGE="" # Service tracking for cleanup SERVER_STARTED=0 HUBBLE_STARTED=0 +# Script execution time tracking +SCRIPT_START_TIME=0 + ################################################################################ # Helper Functions - Output & Logging ################################################################################ @@ -174,6 +179,7 @@ print_step() { local step=$1 local total=$2 local description=$3 + CURRENT_STEP="Step $step: $description" echo "" echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo -e "${BLUE}Step [$step/$total]: $description${NC}" @@ -190,25 +196,65 @@ print_progress() { collect_error() { local error_msg="$1" - VALIDATION_ERRORS+=("$error_msg") + local context="" + + # Build context string + if [[ -n "$CURRENT_STEP" ]]; then + context="[$CURRENT_STEP]" + fi + + if [[ -n "$CURRENT_PACKAGE" ]]; then + if [[ -n "$context" ]]; then + context="$context [$CURRENT_PACKAGE]" + else + context="[$CURRENT_PACKAGE]" + fi + fi + + # Store error with context + if [[ -n "$context" ]]; then + VALIDATION_ERRORS+=("$context $error_msg") + else + VALIDATION_ERRORS+=("$error_msg") + fi + FAILED_CHECKS=$((FAILED_CHECKS + 1)) error "$error_msg" } collect_warning() { local warning_msg="$1" - VALIDATION_WARNINGS+=("$warning_msg") + local context="" + + # Build context string + if [[ -n "$CURRENT_STEP" ]]; then + context="[$CURRENT_STEP]" + fi + + if [[ -n "$CURRENT_PACKAGE" ]]; then + if [[ -n "$context" ]]; then + context="$context [$CURRENT_PACKAGE]" + else + context="[$CURRENT_PACKAGE]" + fi + fi + + # Store warning with context + if [[ -n "$context" ]]; then + VALIDATION_WARNINGS+=("$context $warning_msg") + else + VALIDATION_WARNINGS+=("$warning_msg") + fi + warn "$warning_msg" } mark_check_passed() { PASSED_CHECKS=$((PASSED_CHECKS + 1)) - TOTAL_CHECKS=$((TOTAL_CHECKS + 1)) } mark_check_failed() { FAILED_CHECKS=$((FAILED_CHECKS + 1)) - TOTAL_CHECKS=$((TOTAL_CHECKS + 1)) } ################################################################################ @@ -402,10 +448,28 @@ check_license_categories() { # Check Category X (Prohibited) TOTAL_CHECKS=$((TOTAL_CHECKS + 1)) - local cat_x_count=$(grep -r -E "$CATEGORY_X" $files 2>/dev/null | wc -l | tr -d ' ') + local cat_x_matches=$(grep -r -E "$CATEGORY_X" $files 2>/dev/null) + local cat_x_count=$(echo "$cat_x_matches" | grep -v '^$' | wc -l | tr -d ' ') + if [[ $cat_x_count -ne 0 ]]; then - collect_error "Package '$package' contains $cat_x_count prohibited ASF Category X license(s)" - grep -r -E "$CATEGORY_X" $files + # Build detailed error message with license information + local error_details="Package '$package' contains $cat_x_count prohibited ASF Category X license(s):" + + # Extract and format each violation + while IFS= read -r match_line; do + if [[ -n "$match_line" ]]; then + # Parse file:content format + local file_name=$(echo "$match_line" | cut -d':' -f1) + local license_info=$(echo "$match_line" | cut -d':' -f2-) + + # Try to extract specific license name + local license_name=$(echo "$license_info" | grep -oE "$CATEGORY_X" | head -n1) + + error_details="${error_details}\n - File: ${file_name}\n License: ${license_name}\n Context: ${license_info}" + fi + done <<< "$cat_x_matches" + + collect_error "$error_details" has_error=1 else mark_check_passed @@ -416,7 +480,6 @@ check_license_categories() { local cat_b_count=$(grep -r -E "$CATEGORY_B" $files 2>/dev/null | wc -l | tr -d ' ') if [[ $cat_b_count -ne 0 ]]; then collect_warning "Package '$package' contains $cat_b_count ASF Category B license(s) - please verify documentation" - grep -r -E "$CATEGORY_B" $files else mark_check_passed fi @@ -511,7 +574,7 @@ check_binary_files() { undocumented_count=$((undocumented_count + 1)) has_error=1 fi - done < <(find . -type f -exec perl -lne 'print if -B' {} \; 2>/dev/null) + done < <(find . -type f 2>/dev/null | perl -lne 'print if -B $_') if [[ $binary_count -eq 0 ]]; then success "No binary files found" @@ -587,6 +650,7 @@ check_license_headers() { find_cmd="$find_cmd \\) 2>/dev/null" # Check each source file for ASF license header + local documented_count=0 while IFS= read -r source_file; do # Skip if file matches exclude patterns local should_exclude=0 @@ -607,13 +671,28 @@ check_license_headers() { # Check first 30 lines for Apache license header # Looking for the standard ASF license header text if ! head -n 30 "$source_file" | grep -q "Licensed to the Apache Software Foundation"; then - files_without_license+=("$source_file") + # No ASF header found - check if it's documented in LICENSE file as third-party code + local file_name=$(basename "$source_file") + local file_path_relative=$(echo "$source_file" | sed 's|^\./||') + + # Check if file name or path is mentioned in LICENSE file + if [[ -f "LICENSE" ]] && (grep -q "$file_name" LICENSE 2>/dev/null || grep -q "$file_path_relative" LICENSE 2>/dev/null); then + # File is documented in LICENSE as third-party code - this is allowed + documented_count=$((documented_count + 1)) + else + # Not documented - this is an error + files_without_license+=("$source_file") + fi fi done < <(eval "$find_cmd") # Report results info "Checked $total_checked source file(s) for ASF license headers (excluded $excluded_count generated/vendored files)" + if [[ $documented_count -gt 0 ]]; then + info "Found $documented_count source file(s) documented in LICENSE as third-party code (allowed)" + fi + if [[ ${#files_without_license[@]} -gt 0 ]]; then collect_error "Found ${#files_without_license[@]} source file(s) without ASF license headers:" @@ -632,11 +711,11 @@ check_license_headers() { fi echo "" - collect_error "All source files must include the Apache License header" + collect_error "All source files must include the Apache License header or be documented in LICENSE file" collect_error "You can use 'mvn apache-rat:check' for detailed license header analysis" return 1 else - success "All $total_checked source file(s) have ASF license headers" + success "All $total_checked source file(s) have ASF license headers or are documented in LICENSE" mark_check_passed return 0 fi @@ -648,30 +727,45 @@ check_version_consistency() { TOTAL_CHECKS=$((TOTAL_CHECKS + 1)) - info "Checking version consistency in pom.xml files..." + # Skip version check for Python projects (hugegraph-ai) + if [[ "$package" =~ 'hugegraph-ai' ]]; then + info "Skipping version check for Python project: $package" + mark_check_passed + return 0 + fi + + info "Checking version consistency (revision property)..." - # Find inconsistent versions in pom.xml files - local inconsistent=() + # Find the parent/root pom.xml that defines the revision property + local root_pom="" + local revision_value="" + + # Look for pom.xml files that define the revision property while IFS= read -r pom_file; do - # Extract version tags (exclude parent versions and SNAPSHOT) - while IFS= read -r version_line; do - if [[ ! "$version_line" =~ "" ]] && \ - [[ ! "$version_line" =~ "SNAPSHOT" ]] && \ - [[ ! "$version_line" =~ "$expected_version" ]]; then - inconsistent+=("$pom_file: $version_line") - fi - done < <(grep "" "$pom_file" 2>/dev/null) + if grep -q "" "$pom_file" 2>/dev/null; then + # Extract the revision value + revision_value=$(grep "" "$pom_file" | head -1 | sed 's/.*\(.*\)<\/revision>.*/\1/') + root_pom="$pom_file" + break + fi done < <(find . -name "pom.xml" -type f 2>/dev/null) - if [[ ${#inconsistent[@]} -gt 0 ]]; then - collect_error "Found version inconsistencies in pom.xml files:" - printf ' %s\n' "${inconsistent[@]}" - return 1 - else - success "Version consistency check passed" + if [[ -z "$root_pom" ]]; then + collect_warning "No property found in pom.xml files - skipping version check" mark_check_passed + return 0 fi + info "Found revision property in $root_pom: $revision_value" + + # Check if revision matches expected version + if [[ "$revision_value" != "$expected_version" ]]; then + collect_error "Version mismatch: $revision_value in $root_pom (expected: $expected_version)" + return 1 + fi + + success "Version consistency check passed: revision=$revision_value" + mark_check_passed return 0 } @@ -686,7 +780,7 @@ check_notice_year() { local current_year=$(date +%Y) if ! grep -q "$current_year" NOTICE; then - collect_warning "NOTICE file may not contain current year ($current_year). Please verify copyright dates." + collect_warning "Package '$package': NOTICE file may not contain current year ($current_year). Please verify copyright dates." else mark_check_passed fi @@ -700,6 +794,9 @@ validate_source_package() { local package_file=$1 local package_dir=$(basename "$package_file" .tar.gz) + # Set current package context for error reporting + CURRENT_PACKAGE="$package_file" + info "Validating source package: $package_file" # Extract package @@ -708,6 +805,7 @@ validate_source_package() { if [[ ! -d "$package_dir" ]]; then collect_error "Failed to extract package: $package_file" + CURRENT_PACKAGE="" return 1 fi @@ -750,6 +848,9 @@ validate_source_package() { popd > /dev/null + # Clear package context + CURRENT_PACKAGE="" + info "Finished validating source package: $package_file" } @@ -757,6 +858,9 @@ validate_binary_package() { local package_file=$1 local package_dir=$(basename "$package_file" .tar.gz) + # Set current package context for error reporting + CURRENT_PACKAGE="$package_file" + info "Validating binary package: $package_file" # Extract package @@ -765,6 +869,7 @@ validate_binary_package() { if [[ ! -d "$package_dir" ]]; then collect_error "Failed to extract package: $package_file" + CURRENT_PACKAGE="" return 1 fi @@ -787,6 +892,9 @@ validate_binary_package() { popd > /dev/null + # Clear package context + CURRENT_PACKAGE="" + info "Finished validating binary package: $package_file" } @@ -821,26 +929,44 @@ cleanup() { echo " VALIDATION SUMMARY " echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "" - echo "Total Checks: $TOTAL_CHECKS" - echo -e "${GREEN}Passed: $PASSED_CHECKS${NC}" - echo -e "${RED}Failed: $FAILED_CHECKS${NC}" - echo -e "${YELLOW}Warnings: ${#VALIDATION_WARNINGS[@]}${NC}" + + # Calculate execution time + local script_end_time=$(date +%s) + local execution_seconds=$((script_end_time - SCRIPT_START_TIME)) + local execution_minutes=$((execution_seconds / 60)) + local execution_seconds_remainder=$((execution_seconds % 60)) + + echo "Execution Time: ${execution_minutes}m ${execution_seconds_remainder}s" + echo "Total Checks: $TOTAL_CHECKS" + echo -e "${GREEN}Passed: $PASSED_CHECKS${NC}" + echo -e "${RED}Failed: $FAILED_CHECKS${NC}" + echo -e "${YELLOW}Warnings: ${#VALIDATION_WARNINGS[@]}${NC}" echo "" if [[ ${#VALIDATION_ERRORS[@]} -gt 0 ]]; then - echo -e "${RED}━━━ ERRORS ━━━${NC}" + echo -e "${RED}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" + echo -e "${RED} ERRORS ${NC}" + echo -e "${RED}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" + echo "" + local err_index=1 for err in "${VALIDATION_ERRORS[@]}"; do - echo -e "${RED} ✗ $err${NC}" + echo -e "${RED}[E${err_index}] $err${NC}" + echo "" # Blank line between errors for readability + err_index=$((err_index + 1)) done - echo "" fi if [[ ${#VALIDATION_WARNINGS[@]} -gt 0 ]]; then - echo -e "${YELLOW}━━━ WARNINGS ━━━${NC}" + echo -e "${YELLOW}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" + echo -e "${YELLOW} WARNINGS ${NC}" + echo -e "${YELLOW}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" + echo "" + local warn_index=1 for warn in "${VALIDATION_WARNINGS[@]}"; do - echo -e "${YELLOW} ⚠ $warn${NC}" + echo -e "${YELLOW}[W${warn_index}] $warn${NC}" + echo "" # Blank line between warnings for readability + warn_index=$((warn_index + 1)) done - echo "" fi echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" @@ -871,6 +997,9 @@ trap 'echo -e "${RED}Script interrupted${NC}"; exit 130' INT TERM ################################################################################ main() { + # Record script start time + SCRIPT_START_TIME=$(date +%s) + # Parse command line arguments while [[ $# -gt 0 ]]; do case $1 in From a86370e0ec161c461d37a31046b60636f8ce5688 Mon Sep 17 00:00:00 2001 From: imbajin Date: Sat, 15 Nov 2025 04:13:26 +0800 Subject: [PATCH 6/8] fix: add JSON to CATEGORY_X license validation The JSON license was added to the CATEGORY_X regex in the binary package validation step to ensure packages with this license are properly flagged during release validation. --- .github/workflows/validate-release.yml | 2 +- dist/README.md | 204 ++++++++++++++++++++----- 2 files changed, 164 insertions(+), 42 deletions(-) diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index bbd1506f2..6b7cb043c 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -244,7 +244,7 @@ jobs: - name: 7. Validate Binary Packages run: | cd dist/${{ inputs.release_version }} || exit - CATEGORY_X="\bGPL|\bLGPL|Sleepycat License|BSD-4-Clause|\bBCL\b|JSR-275|Amazon Software License|\bRSAL\b|\bQPL\b|\bSSPL|\bCPOL|\bNPL1|Creative Commons Non-Commercial" + CATEGORY_X="\bGPL|\bLGPL|Sleepycat License|BSD-4-Clause|\bBCL\b|JSR-275|Amazon Software License|\bRSAL\b|\bQPL\b|\bSSPL|\bCPOL|\bNPL1|Creative Commons Non-Commercial|JSON" for i in *.tar.gz; do if [[ "$i" == *-src.tar.gz ]]; then # skip source packages diff --git a/dist/README.md b/dist/README.md index b4d43d8ef..0ccdb8ce4 100644 --- a/dist/README.md +++ b/dist/README.md @@ -10,15 +10,18 @@ Apache HugeGraph (Incubating) 发布包的自动化验证脚本。 - ✅ **自动依赖检查** - 验证所有必需工具（svn、gpg、java、maven 等） - ✅ **SHA512 和 GPG 签名验证** - 确保包的完整性和真实性 -- ✅ **许可证合规性验证** - 检查禁止的 ASF Category X 和 B 类许可证 +- ✅ **许可证合规性验证** - 检查禁止的 ASF Category X 和需要文档化的 Category B 许可证 +- ✅ **详细的许可证错误报告** - 对 Category X 违规显示文件路径、许可证名称和上下文 - ✅ **包内容验证** - 验证必需文件（LICENSE、NOTICE、DISCLAIMER） -- ✅ **ASF 许可证头检查** - 验证所有源文件中的许可证头（Java、Python、Go、Shell 等） -- ✅ **版本一致性验证** - 确保 pom.xml 版本与预期发布版本匹配 +- ✅ **ASF 许可证头检查** - 验证所有源文件中的许可证头，支持第三方代码文档化 +- ✅ **版本一致性验证** - 验证 Maven `` 属性与预期发布版本匹配 +- ✅ **多语言项目支持** - 自动跳过 Python 项目（hugegraph-ai）的 Maven 版本检查 - ✅ **源码包编译** - 编译源码包以验证构建正确性 - ✅ **运行时测试** - 测试服务器和工具链（loader、tool、hubble）功能 -- ✅ **进度跟踪** - 显示实时进度和分步指示器 +- ✅ **智能进度跟踪** - 显示实时进度、步骤指示器和执行时间 +- ✅ **上下文化错误报告** - 错误和警告包含步骤、包名和索引编号 - ✅ **详细日志记录** - 将所有输出保存到带时间戳的日志文件 -- ✅ **全面的错误报告** - 收集所有错误并在最后显示摘要 +- ✅ **全面的错误摘要** - 收集所有错误并在最后显示格式化摘要 ## 环境要求 @@ -27,7 +30,7 @@ Apache HugeGraph (Incubating) 发布包的自动化验证脚本。 - svn（Subversion 客户端） - gpg（用于签名验证的 GnuPG） - wget 或 curl -- 标准 Unix 工具（bash、find、grep、awk 等） +- 标准 Unix 工具（bash、find、grep、awk、perl 等） 脚本会自动检查所有依赖项，如果缺少任何内容会提供安装说明。 @@ -78,25 +81,26 @@ Apache HugeGraph (Incubating) 发布包的自动化验证脚本。 脚本执行以下 9 个验证步骤： -1. **检查依赖项** - 验证所有必需工具已安装 +1. **检查依赖项** - 验证所有必需工具已安装并显示版本信息 2. **准备发布文件** - 从 Apache SVN 下载或使用本地目录 -3. **导入并信任 GPG 密钥** - 导入 KEYS 文件并信任发布管理员的 GPG 密钥 +3. **导入并信任 GPG 密钥** - 导入 KEYS 文件并信任所有公钥 4. **验证 SHA512 和 GPG 签名** - 验证所有包的校验和和签名 5. **验证源码包** - 对源码包进行全面检查： - 包命名（包含 "incubating"） - 必需文件（LICENSE、NOTICE、DISCLAIMER） - - 许可证合规性（无 Category X，已记录 Category B） + - 许可证合规性（禁止 Category X，记录 Category B） + - 详细的许可证违规报告（文件路径、许可证名称、上下文） - 无空文件或目录 - 文件大小限制（无文件 > 800KB） - - 二进制文件文档 - - 所有源文件中的许可证头 - - pom.xml 文件之间的版本一致性 + - 二进制文件文档化（在 LICENSE 中声明） + - 所有源文件的许可证头（支持第三方代码文档化） + - Maven `` 属性版本一致性（跳过 Python 项目） - NOTICE 文件版权年份 - - 源码编译 -6. **测试编译的服务器** - 启动并测试编译的 HugeGraph 服务器 + - 源码编译测试 +6. **测试编译的服务器** - 初始化并启动编译的 HugeGraph 服务器 7. **测试编译的工具链** - 从编译包测试 loader、tool 和 hubble -8. **验证二进制包** - 检查二进制包的必需文件和结构 -9. **测试二进制包** - 从二进制包测试服务器和工具链 +8. **验证二进制包** - 检查二进制包的必需文件、licenses 目录和许可证合规性 +9. **测试二进制包** - 从二进制包测试服务器和工具链功能 ## 输出结果 @@ -109,20 +113,21 @@ Apache HugeGraph (Incubating) 发布包的自动化验证脚本。 Apache HugeGraph Release Validation v2.0.0 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - 版本: 1.7.0 - 用户: pengjunzhi + Version: 1.7.0 + User: pengjunzhi Java: 11 - 模式: SVN 下载 - 日志: logs/validate-1.7.0-20251115-021742.log + Mode: SVN Download + Log: logs/validate-1.7.0-20251115-021742.log ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -步骤 [1/9]: 检查依赖项 +Step [1/9]: Check Dependencies ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ ✓ svn: version 1.14.1 ✓ gpg: gpg (GnuPG) 2.2.41 ✓ java: 11.0.21 +✓ mvn: Apache Maven 3.9.5 ... ``` @@ -139,42 +144,127 @@ Apache HugeGraph (Incubating) 发布包的自动化验证脚本。 ### 最终摘要 -验证结束时，会显示一个全面的摘要： +验证结束时，会显示一个全面的摘要，包含执行时间和详细的错误/警告信息： ``` ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - 验证摘要 + VALIDATION SUMMARY ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -总检查数: 127 -通过: 125 -失败: 2 -警告: 3 +Execution Time: 6m 34s +Total Checks: 139 +Passed: 134 +Failed: 3 +Warnings: 2 -━━━ 错误 ━━━ - ✗ 包 'xyz' 缺少 LICENSE 文件 - ✗ 二进制文件 'logo.png' 未在 LICENSE 中记录 +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + ERRORS +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -━━━ 警告 ━━━ - ⚠ NOTICE 文件可能不包含当前年份 (2025) +[E1] [Step 8: Validate Binary Packages] [xxxx] contains 1 prohibited ASF Category X license(s): +xxxxx ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + WARNINGS +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +xxxx +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +VALIDATION FAILED +Log file: logs/validate-1.7.0-20251115-021742.log +``` -验证失败 -日志文件: logs/validate-1.7.0-20251115-021742.log +## 许可证检查说明 + +### Category X 许可证（禁止使用） + +脚本会严格检查以下 ASF Category X 许可证，发现后会报错并提供详细信息： + +- GPL, LGPL 系列 +- Sleepycat License +- BSD-4-Clause +- BCL (Binary Code License) +- JSR-275 +- Amazon Software License +- RSAL (Reciprocal Public License) +- QPL (Q Public License) +- SSPL (Server Side Public License) +- CPOL (Code Project Open License) +- NPL1 (Netscape Public License) +- Creative Commons Non-Commercial +- **JSON.org** (JSON License) + +**错误报告格式：** +``` +Package 'xxx.tar.gz' contains 1 prohibited ASF Category X license(s): + - File: licenses/LICENSE-json.txt + License: JSON.org + Context: Copyright (c) 2002 JSON.org ``` +### Category B 许可证（需要文档化） + +以下许可证会触发警告，提醒检查是否在 LICENSE 文件中正确记录： + +- CDDL1, CPL, EPL, IPL, MPL, SPL +- OSL-3.0 +- UnRAR License +- Erlang Public License +- OFL (SIL Open Font License) +- Ubuntu Font License Version 1.0 +- IPA Font License Agreement v1.0 +- EPL2.0 +- CC-BY (Creative Commons Attribution) + +**警告报告格式（简洁）：** +``` +Package 'xxx.tar.gz' contains 2 ASF Category B license(s) - please verify documentation +``` + +### 许可证头检查 + +脚本会检查所有源代码文件（Java、Shell、Python、Go、JavaScript、TypeScript、C/C++、Scala、Groovy、Rust、Kotlin、Proto 等）是否包含 ASF 许可证头。 + +**第三方代码处理：** +- 如果源文件没有 ASF 许可证头，脚本会检查该文件是否在 LICENSE 文件中被文档化 +- 支持通过文件名或相对路径匹配 +- 已文档化的第三方代码会被标记为合法并单独统计 +- 只有未文档化且缺少 ASF 头的文件才会报错 + ## 错误处理 脚本使用**"继续并报告"**方式： - 不会在第一个错误时退出 -- 收集所有验证错误 -- 在最后显示全面摘要 +- 收集所有验证错误和警告 +- 在最后显示全面摘要，包含： + - 执行总时间 + - 检查统计（总数、通过、失败、警告） + - 带编号和上下文的错误列表 + - 带编号和上下文的警告列表 - 退出码 0 = 所有检查通过 - 退出码 1 = 一个或多个检查失败 -这允许你一次看到所有问题，而不是逐个修复。 +每个错误和警告都包含： +- 编号索引（[E1], [E2], [W1], [W2] 等） +- 步骤上下文（哪个验证步骤） +- 包名上下文（哪个包） +- 详细的错误描述 + +这允许你一次看到所有问题，并能快速定位到具体的失败点。 + +## 特殊处理 + +### Python 项目（hugegraph-ai） + +- 自动跳过编译步骤 +- 自动跳过 Maven `` 版本检查 +- 仍然执行其他所有验证（许可证、文件结构等） + +### Computer 模块 + +- 在特殊目录结构下编译（`cd computer && mvn package`） +- 支持 Java 8 和 Java 11 ## 故障排除 @@ -189,9 +279,6 @@ java -version # 使用 JAVA_HOME 指定 Java 11 export JAVA_HOME=/path/to/java11 export PATH=$JAVA_HOME/bin:$PATH - -# 或使用 jenv 切换 Java 版本 -jenv global 11 ``` ### GPG 密钥问题 @@ -206,6 +293,11 @@ gpg --import KEYS # 信任特定密钥 gpg --edit-key # 在 GPG 提示符中，输入: trust, 然后 5, 然后 y, 然后 quit + +# 或者信任所有导入的密钥 +for key in $(gpg --no-tty --list-keys --with-colons | awk -F: '/^pub/ {print $5}'); do + echo -e "5\ny\n" | gpg --batch --command-fd 0 --edit-key "$key" trust +done ``` ### 权限被拒绝 @@ -216,10 +308,41 @@ gpg --edit-key chmod +x validate-release.sh ``` +### 许可证检查误报 + +如果合法的第三方代码被标记为缺少许可证头： + +1. 确保在根目录的 `LICENSE` 文件中记录了该文件 +2. 记录格式可以是文件名或相对路径 +3. 重新运行验证脚本 + +示例 LICENSE 文件条目： +``` +This product bundles ThirdParty.java from XYZ project, +which is available under a "MIT License". +For details, see licenses/LICENSE-mit.txt +``` + +### 查看详细日志 + +如果需要更多调试信息： + +```bash +# 查看完整日志 +cat logs/validate--.log + +# 搜索特定错误 +grep "ERROR" logs/validate-*.log + +# 查看特定步骤 +grep "Step \[5/9\]" logs/validate-*.log +``` + ## 参考文档 - [Apache 发布政策](https://www.apache.org/legal/release-policy.html) - [孵化器发布检查清单](https://cwiki.apache.org/confluence/display/INCUBATOR/Incubator+Release+Checklist) +- [Apache 许可证分类](https://www.apache.org/legal/resolved.html) - [HugeGraph 验证发布指南](../content/cn/docs/contribution-guidelines/validate-release.md) ## 贡献 @@ -228,4 +351,3 @@ chmod +x validate-release.sh 1. 查看现有问题：https://github.com/apache/incubator-hugegraph-doc/issues 2. 提交新问题或 pull request - From 606a92884e24f9f2d60d8b5a884f4cff51b56a65 Mon Sep 17 00:00:00 2001 From: Peng Junzhi <78788603+Pengzna@users.noreply.github.com> Date: Fri, 14 Nov 2025 20:03:47 -0600 Subject: [PATCH 7/8] Update .github/workflows/validate-release.yml --- .github/workflows/validate-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index 6b7cb043c..1de2ea399 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -244,7 +244,7 @@ jobs: - name: 7. Validate Binary Packages run: | cd dist/${{ inputs.release_version }} || exit - CATEGORY_X="\bGPL|\bLGPL|Sleepycat License|BSD-4-Clause|\bBCL\b|JSR-275|Amazon Software License|\bRSAL\b|\bQPL\b|\bSSPL|\bCPOL|\bNPL1|Creative Commons Non-Commercial|JSON" + CATEGORY_X="\bGPL|\bLGPL|Sleepycat License|BSD-4-Clause|\bBCL\b|JSR-275|Amazon Software License|\bRSAL\b|\bQPL\b|\bSSPL|\bCPOL|\bNPL1|Creative Commons Non-Commercial|JSON\.org" for i in *.tar.gz; do if [[ "$i" == *-src.tar.gz ]]; then # skip source packages From 9ccdada1c4dbb59b87bdaa8773bfa4106827bcc3 Mon Sep 17 00:00:00 2001 From: Peng Junzhi <78788603+Pengzna@users.noreply.github.com> Date: Fri, 14 Nov 2025 20:10:51 -0600 Subject: [PATCH 8/8] Update .github/workflows/validate-release.yml --- .github/workflows/validate-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index 1de2ea399..414602208 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -10,7 +10,7 @@ on: gpg_user: required: true description: current release manager (gpg username) - default: 'Junzhi Peng' + default: 'pengjunzhi' push: branches: