diff --git a/dist/validate-release-in-local.sh b/dist/validate-release-in-local.sh new file mode 100755 index 000000000..e935be693 --- /dev/null +++ b/dist/validate-release-in-local.sh @@ -0,0 +1,356 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This script is used to validate the release package, including: +# 1. Check the release package name & content +# 2. Check the release package sha512 & gpg signature +# 3. Compile the source package & run server & toolchain +# 4. Run server & toolchain in binary package + +# exit when any error occurs +set -e + +# release version (input by committer) +RELEASE_VERSION=$1 # like 1.2.0 +JAVA_VERSION=$2 # like 11 +USER=$3 +LOCAL_DIST_PATH=$4 # local directory path containing release files + +# this URL is only valid during the release process +SVN_URL_PREFIX="https://dist.apache.org/repos/dist/dev/incubator/hugegraph" + +# git release branch (check it carefully) +#GIT_BRANCH="release-${RELEASE_VERSION}" + +RELEASE_VERSION=${RELEASE_VERSION:?"Please input the release version, like 1.2.0"} +USER=${USER:-"imbajin"} +WORK_DIR=$( + cd "$(dirname "$0")" + pwd +) + +# Use local directory if provided, otherwise use default dist path +if [[ -n "${LOCAL_DIST_PATH}" ]]; then + DIST_DIR="${LOCAL_DIST_PATH}" + echo "Using local directory: ${DIST_DIR}" +else + DIST_DIR="${WORK_DIR}/dist/${RELEASE_VERSION}" + echo "Using default directory: ${DIST_DIR}" +fi + +# Validate local directory exists +if [[ ! -d "${DIST_DIR}" ]]; then + echo "Error: Directory ${DIST_DIR} does not exist" + exit 1 +fi + +cd "${WORK_DIR}" +echo "Current work dir: $(pwd)" +echo "Release files directory: ${DIST_DIR}" + +################################ +# Step 1: Validate Local Directory # +################################ +cd "${DIST_DIR}" +echo "Contents of ${DIST_DIR}:" +ls -lh + +################################################## +# Step 2: Check Environment & Import Public Keys # +################################################## +shasum --version 1>/dev/null +gpg --version 1>/dev/null + +wget https://downloads.apache.org/incubator/hugegraph/KEYS +echo "Import KEYS:" && gpg --import KEYS +# TODO: how to trust all public keys in gpg list, currently only trust the first one +echo -e "5\ny\n" | gpg --batch --command-fd 0 --edit-key $USER trust + +echo "trust all pk" +for key in $(gpg --no-tty --list-keys --with-colons | awk -F: '/^pub/ {print $5}'); do + echo -e "5\ny\n" | gpg --batch --command-fd 0 --edit-key "$key" trust +done + +######################################## +# Step 3: Check SHA512 & GPG Signature # +######################################## +cd "${DIST_DIR}" + +for i in *.tar.gz; do + echo "$i" + shasum -a 512 --check "$i".sha512 + eval gpg "${GPG_OPT}" --verify "$i".asc "$i" +done + +#################################### +# Step 4: Validate Source Packages # +#################################### +cd "${DIST_DIR}" + +CATEGORY_X="\bGPL|\bLGPL|Sleepycat License|BSD-4-Clause|\bBCL\b|JSR-275|Amazon Software License|\bRSAL\b|\bQPL\b|\bSSPL|\bCPOL|\bNPL1|Creative Commons Non-Commercial|JSON\.org" +CATEGORY_B="\bCDDL1|\bCPL|\bEPL|\bIPL|\bMPL|\bSPL|OSL-3.0|UnRAR License|Erlang Public License|\bOFL\b|Ubuntu Font License Version 1.0|IPA Font License Agreement v1.0|EPL2.0|CC-BY" +ls -lh ./*.tar.gz +for i in *src.tar.gz; do + echo "$i" + + # 4.1: check the directory name include "incubating" + if [[ ! "$i" =~ "incubating" ]]; then + echo "The package name $i should include incubating" && exit 1 + fi + + MODULE_DIR=$(basename "$i" .tar.gz) + rm -rf ${MODULE_DIR} + tar -xzvf "$i" + pushd ${MODULE_DIR} + echo "Start to check the package content: ${MODULE_DIR}" + + # 4.2: check the directory include "NOTICE" and "LICENSE" file and "DISCLAIMER" file + if [[ ! -f "LICENSE" ]]; then + echo "The package $i should include LICENSE file" && exit 1 + fi + if [[ ! -f "NOTICE" ]]; then + echo "The package $i should include NOTICE file" && exit 1 + fi + if [[ ! -f "DISCLAIMER" ]]; then + echo "The package $i should include DISCLAIMER file" && exit 1 + fi + + # 4.3: ensure doesn't contains ASF CATEGORY X License dependencies in LICENSE and NOTICE files + COUNT=$(grep -E "$CATEGORY_X" LICENSE NOTICE | wc -l) + if [[ $COUNT -ne 0 ]]; then + grep -E "$CATEGORY_X" LICENSE NOTICE + echo "The package $i shouldn't include invalid ASF category X dependencies, but get $COUNT" && exit 1 + fi + + # 4.4: ensure doesn't contains ASF CATEGORY B License dependencies in LICENSE and NOTICE files + COUNT=$(grep -E "$CATEGORY_B" LICENSE NOTICE | wc -l) + if [[ $COUNT -ne 0 ]]; then + grep -E "$CATEGORY_B" LICENSE NOTICE + echo "The package $i shouldn't include invalid ASF category B dependencies, but get $COUNT" && exit 1 + fi + + # 4.5: ensure doesn't contains empty directory or file + find . -type d -empty | while read -r EMPTY_DIR; do + find . -type d -empty + echo "The package $i shouldn't include empty directory: $EMPTY_DIR is empty" && exit 1 + done + find . -type f -empty | while read -r EMPTY_FILE; do + find . -type f -empty + echo "The package $i shouldn't include empty file: $EMPTY_FILE is empty" && exit 1 + done + + # 4.6: ensure any file should less than 800kb + find . -type f -size +800k | while read -r FILE; do + find . -type f -size +800k + echo "The package $i shouldn't include file larger than 800kb: $FILE is larger than 800kb" && exit 1 + done + + # 4.7: ensure all binary files are documented in LICENSE + find . -type f | perl -lne 'print if -B' | while read -r BINARY_FILE; do + FILE_NAME=$(basename "$BINARY_FILE") + if grep -q "$FILE_NAME" LICENSE; then + echo "Binary file $BINARY_FILE is documented in LICENSE, please check manually" + else + echo "Error: Binary file $BINARY_FILE is not documented in LICENSE" && exit 1 + fi + done + + # 4.8: test compile the packages + if [[ ($JAVA_VERSION == 8 && "$i" =~ "hugegraph-computer") ]]; then + echo "Skip compile $i module in java8" + elif [[ "$i" =~ 'hugegraph-ai' ]]; then + echo "Skip compile $i module in all versions" + elif [[ "$i" =~ "hugegraph-commons" ]]; then + mvn install -DskipTests -Papache-release -ntp -e + elif [[ "$i" =~ "hugegraph-computer" ]]; then + cd computer + mvn install -DskipTests -Papache-release -ntp -e + else + # TODO: consider using commands that are entirely consistent with building binary packages + mvn package -DskipTests -Papache-release -ntp -e + ls -lh + fi + popd +done + +########################################### +# Step 5: Run Compiled Packages of Server # +########################################### +cd "${DIST_DIR}" + +ls -lh +pushd ./*hugegraph-incubating*src/hugegraph-server/*hugegraph*"${RELEASE_VERSION}" +bin/init-store.sh +sleep 3 +bin/start-hugegraph.sh +popd + +####################################################################### +# Step 6: Run Compiled Packages of ToolChain (Loader & Tool & Hubble) # +####################################################################### +cd "${DIST_DIR}" + +pushd ./*toolchain*src +ls -lh +pushd ./*toolchain*"${RELEASE_VERSION}" +ls -lh + +# 6.1: load some data first +echo "test loader" +pushd ./*loader*"${RELEASE_VERSION}" +bin/hugegraph-loader.sh -f ./example/file/struct.json -s ./example/file/schema.groovy \ + -g hugegraph +popd + +# 6.2: try some gremlin query & api in tool +echo "test tool" +pushd ./*tool*"${RELEASE_VERSION}" +bin/hugegraph gremlin-execute --script 'g.V().count()' +bin/hugegraph task-list +bin/hugegraph backup -t all --directory ./backup-test +popd + +# 6.3: start hubble and connect to server +echo "test hubble" +pushd ./*hubble*"${RELEASE_VERSION}" +# TODO: add hubble doc & test it +cat conf/hugegraph-hubble.properties +bin/start-hubble.sh +bin/stop-hubble.sh +popd + +popd +popd +# stop server +pushd ./*hugegraph-incubating*src/hugegraph-server/*hugegraph*"${RELEASE_VERSION}" +bin/stop-hugegraph.sh +popd + +# clear source packages +#rm -rf ./*src* +#ls -lh + +#################################### +# Step 7: Validate Binary Packages # +#################################### +cd "${DIST_DIR}" + +for i in *.tar.gz; do + if [[ "$i" == *-src.tar.gz ]]; then + # skip source packages + continue + fi + + echo "$i" + + # 7.1: check the directory name include "incubating" + if [[ ! "$i" =~ "incubating" ]]; then + echo "The package name $i should include incubating" && exit 1 + fi + + MODULE_DIR=$(basename "$i" .tar.gz) + rm -rf ${MODULE_DIR} + tar -xzvf "$i" + pushd ${MODULE_DIR} + ls -lh + echo "Start to check the package content: ${MODULE_DIR}" + + # 7.2: check root dir include "NOTICE"/"LICENSE"/"DISCLAIMER" files & "licenses" dir + if [[ ! -f "LICENSE" ]]; then + echo "The package $i should include LICENSE file" && exit 1 + fi + if [[ ! -f "NOTICE" ]]; then + echo "The package $i should include NOTICE file" && exit 1 + fi + if [[ ! -f "DISCLAIMER" ]]; then + echo "The package $i should include DISCLAIMER file" && exit 1 + fi + if [[ ! -d "licenses" ]]; then + echo "The package $i should include licenses dir" && exit 1 + fi + + # 7.3: ensure doesn't contains ASF CATEGORY X License dependencies in LICENSE/NOTICE and licenses/* files + COUNT=$(grep -r -E "$CATEGORY_X" LICENSE NOTICE licenses | wc -l) + if [[ $COUNT -ne 0 ]]; then + grep -r -E "$CATEGORY_X" LICENSE NOTICE licenses + echo "The package $i shouldn't include invalid ASF category X dependencies, but get $COUNT" && exit 1 + fi + + # 7.4: ensure doesn't contains empty directory or file + find . -type d -empty | while read -r EMPTY_DIR; do + find . -type d -empty + echo "The package $i shouldn't include empty directory: $EMPTY_DIR is empty" && exit 1 + done + find . -type f -empty | while read -r EMPTY_FILE; do + find . -type f -empty + echo "The package $i shouldn't include empty file: $EMPTY_FILE is empty" && exit 1 + done + + popd +done + +# TODO: skip the following steps by comparing the artifacts built from source packages with binary packages +######################################### +# Step 8: Run Binary Packages of Server # +######################################### +cd "${DIST_DIR}" + +# TODO: run pd & store +pushd ./*hugegraph-incubating*"${RELEASE_VERSION}"/*hugegraph-server-incubating*"${RELEASE_VERSION}" +bin/init-store.sh +sleep 3 +bin/start-hugegraph.sh +popd + +##################################################################### +# Step 9: Run Binary Packages of ToolChain (Loader & Tool & Hubble) # +##################################################################### +cd "${DIST_DIR}" + +pushd ./*toolchain*"${RELEASE_VERSION}" +ls -lh + +# 9.1: load some data first +echo "test loader" +pushd ./*loader*"${RELEASE_VERSION}" +bin/hugegraph-loader.sh -f ./example/file/struct.json -s ./example/file/schema.groovy -g hugegraph +popd + +# 9.2: try some gremlin query & api in tool +echo "test tool" +pushd ./*tool*"${RELEASE_VERSION}" +bin/hugegraph gremlin-execute --script 'g.V().count()' +bin/hugegraph task-list +bin/hugegraph backup -t all --directory ./backup-test +popd + +# 9.3: start hubble and connect to server +echo "test hubble" +pushd ./*hubble*"${RELEASE_VERSION}" +# TODO: add hubble doc & test it +cat conf/hugegraph-hubble.properties +bin/start-hubble.sh +bin/stop-hubble.sh +popd + +popd +# stop server +pushd ./*hugegraph-incubating*"${RELEASE_VERSION}"/*hugegraph-server-incubating*"${RELEASE_VERSION}" +bin/stop-hugegraph.sh +popd + +echo "Finish validate, please check all steps manually again!" diff --git a/dist/validate-release.sh b/dist/validate-release.sh index 8573b12fb..8df1645f9 100755 --- a/dist/validate-release.sh +++ b/dist/validate-release.sh @@ -85,7 +85,7 @@ done #################################### cd "${WORK_DIR}/dist/${RELEASE_VERSION}" -CATEGORY_X="\bGPL|\bLGPL|Sleepycat License|BSD-4-Clause|\bBCL\b|JSR-275|Amazon Software License|\bRSAL\b|\bQPL\b|\bSSPL|\bCPOL|\bNPL1|Creative Commons Non-Commercial|JSON" +CATEGORY_X="\bGPL|\bLGPL|Sleepycat License|BSD-4-Clause|\bBCL\b|JSR-275|Amazon Software License|\bRSAL\b|\bQPL\b|\bSSPL|\bCPOL|\bNPL1|Creative Commons Non-Commercial|JSON\.org" CATEGORY_B="\bCDDL1|\bCPL|\bEPL|\bIPL|\bMPL|\bSPL|OSL-3.0|UnRAR License|Erlang Public License|\bOFL\b|Ubuntu Font License Version 1.0|IPA Font License Agreement v1.0|EPL2.0|CC-BY" ls -lh ./*.tar.gz for i in *src.tar.gz; do