diff --git a/.github/actions/install-flux/install-flux.sh b/.github/actions/install-flux/install-flux.sh index a9010389f1..f813918ed2 100755 --- a/.github/actions/install-flux/install-flux.sh +++ b/.github/actions/install-flux/install-flux.sh @@ -26,6 +26,7 @@ if [ -z "$FLUX_VERSION" ]; then fi for i in 1 2 3; do + echo "Attempt $i to install Flux version $FLUX_VERSION..." flux install --namespace=flux-system --version=v"$FLUX_VERSION" --components=source-controller --network-policy=false && \ kubectl wait --for=condition=available deployment -l app.kubernetes.io/component=source-controller -n flux-system --timeout=120s && break echo "Attempt $i failed, retrying in 10 seconds..." diff --git a/.github/actions/install-gitea/install-gitea.sh b/.github/actions/install-gitea/install-gitea.sh index 9206599391..bd3c126bcf 100755 --- a/.github/actions/install-gitea/install-gitea.sh +++ b/.github/actions/install-gitea/install-gitea.sh @@ -73,4 +73,8 @@ echo "$output" output=$(kubectl exec -it "$gitea_pod" -n gitea -- gitea admin user generate-access-token --username "$GITEA_USERNAME" --token-name "$GITEA_ACCESS_TOKEN_NAME" --scopes "write:repository,write:user" --raw) echo "$output" -echo "gitea-access-token=$output" >> "$GITHUB_OUTPUT" +if [ -n "$GITHUB_OUTPUT" ]; then + echo "gitea-access-token=$output" >> "$GITHUB_OUTPUT" +else + export GITEA_ACCESS_TOKEN="$output" +fi diff --git a/.github/workflows/long-running-test-deploy.yaml b/.github/workflows/long-running-test-deploy.yaml new file mode 100644 index 0000000000..4772a53d9c --- /dev/null +++ b/.github/workflows/long-running-test-deploy.yaml @@ -0,0 +1,34 @@ +# ------------------------------------------------------------ +# Copyright 2023 The Radius Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------ + +name: Long-running test on Azure + +permissions: + contents: read + packages: read + +on: + # Enable manual trigger to deploy the latest changes from main. + workflow_dispatch: + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Check Environment + run: make test-check-environment \ No newline at end of file diff --git a/build/.gitignore b/build/.gitignore new file mode 100644 index 0000000000..bb5e46a28a --- /dev/null +++ b/build/.gitignore @@ -0,0 +1,2 @@ +# Exclude environment files +*.env diff --git a/build/install.mk b/build/install.mk index c239720a63..1b3de58163 100644 --- a/build/install.mk +++ b/build/install.mk @@ -21,4 +21,26 @@ RAD_LOCATION := /usr/local/bin/rad .PHONY: install install: build-binaries ## Installs a local build for development @echo "$(ARROW) Installing rad" - cp $(OUT_DIR)/$(GOOS)_$(GOARCH)/$(BUILDTYPE_DIR)/rad$(BINARY_EXT) $(RAD_LOCATION) \ No newline at end of file + sudo cp $(OUT_DIR)/$(GOOS)_$(GOARCH)/$(BUILDTYPE_DIR)/rad$(BINARY_EXT) $(RAD_LOCATION) + +.PHONY: install-latest +install-latest: ## Installs the latest release from GitHub + @echo "$(ARROW) Installing latest rad release" + @bash ./deploy/install.sh + +FLUX_VERSION ?= 2.5.1 +.PHONY: install-flux +install-flux: ## Installs flux using the install script + @echo "$(ARROW) Installing flux" + @export FLUX_VERSION=$(FLUX_VERSION) && curl -s https://fluxcd.io/install.sh | sudo -E bash + @./.github/actions/install-flux/install-flux.sh $(FLUX_VERSION) + +GITEA_VERSION ?= v11.0.0 +GITEA_USERNAME ?= "testuser" +GITEA_EMAIL ?= "testuser@radapp.io" +GITEA_ACCESS_TOKEN_NAME ?= "radius-functional-test" +GITEA_PASSWORD ?= "" +.PHONY: install-gitea +install-gitea: ## Installs gitea + @echo "$(ARROW) Installing gitea" + @export GITEA_PASSWORD=$(GITEA_PASSWORD) && .github/actions/install-gitea/install-gitea.sh $(GITEA_VERSION) $(GITEA_USERNAME) $(GITEA_EMAIL) $(GITEA_ACCESS_TOKEN_NAME) \ No newline at end of file diff --git a/build/test.mk b/build/test.mk index bd02074c23..5a00edcd84 100644 --- a/build/test.mk +++ b/build/test.mk @@ -154,6 +154,18 @@ test-ucp-spec-examples: oav-installed ## Validates UCP examples conform to UCP O # @echo "$(ARROW) Testing x-ms-examples conform to ucp spec..." # oav validate-example swagger/specification/ucp/resource-manager/UCP/preview/2023-10-01-preview/openapi.json -.PHONY: test-deploy-lrt-cluster -test-deploy-aks-cluster: ## Deploys an AKS cluster to Azure for the long-running tests. Optional parameters: [TEST_AKS_AZURE_LOCATION=] [TEST_AKS_RG=] - @bash ./build/test.sh +.PHONY: test-deploy-aks-cluster +test-deploy-aks-cluster: ## Deploys an AKS cluster to Azure for the long-running tests. Optional parameters: [LRT_AZURE_LOCATION=] [LRT_RG=] + @bash ./build/test.sh deploy-aks-cluster + +.PHONY: test-lrt +test-lrt: ## Runs the long-running tests against the AKS cluster deployed by test-deploy-lrt-cluster using the currently installed version of the Radius CLI and the current az authenticated subscription. + @bash ./build/test.sh run-lrt + +.PHONY: test-create-env-file +test-create-env-file: ## Creates a test.env file with environment variables for testing + ./build/test.sh create-env-file + +.PHONY: test-check-environment +test-check-environment: ## Checks the environment for required tools and configurations for testing + @bash ./build/test.sh check-environment \ No newline at end of file diff --git a/build/test.sh b/build/test.sh index 1d3bda079b..177f4ebc1d 100755 --- a/build/test.sh +++ b/build/test.sh @@ -56,14 +56,166 @@ check_azure_cli() { fi } +# All environment variables required for the tests +get_env_vars() { + + local env_vars=( + "AZURE_SUBSCRIPTION_ID" + "AZURE_SP_TESTS_APPID" + "AZURE_SP_TESTS_TENANTID" + "TEST_BICEP_TYPES_REGISTRY" + "RAD_VERSION" + "BICEP_RECIPE_TAG_VERSION" + "BICEP_RECIPE_REGISTRY" + "TEST_RESOURCE_GROUP" + "TEST_RESOURCE_GROUP_LOCATION" + "AZURE_COSMOS_MONGODB_ACCOUNT_ID" + ) + + printf '%s\n' "${env_vars[@]}" +} + +# Set default values for environment variables if not already set +set_env_defaults(){ + export AZURE_SUBSCRIPTION_ID="${AZURE_SUBSCRIPTION_ID:-$(az account show --query id -o tsv)}" + export AZURE_SP_TESTS_TENANTID="${AZURE_SP_TESTS_TENANTID:-$(az account show --query tenantId -o tsv)}" + export TEST_BICEP_TYPES_REGISTRY="${TEST_BICEP_TYPES_REGISTRY:-testuserdefinedbiceptypes.azurecr.io}" + RAD_VERSION=$(gh release list -R radius-project/radius --limit 1 --exclude-drafts --exclude-pre-releases --json tagName --jq ".[0].tagName") + export RAD_VERSION + if [[ -n "${RAD_VERSION:-}" ]]; then + export BICEP_RECIPE_TAG_VERSION="${BICEP_RECIPE_TAG_VERSION:-$RAD_VERSION}" # Use the RAD version as the tag for recipes by default + fi + export BICEP_RECIPE_REGISTRY="${BICEP_RECIPE_REGISTRY:-ghcr.io/radius-project/dev/test/recipes}" + # Resource group where tests will deploy resources + local id="${GITHUB_RUN_NUMBER:-$(whoami)}" + TEST_RESOURCE_GROUP="${TEST_RESOURCE_GROUP:-$(whoami)-test-$(echo "$id" | sha1sum | head -c 5)}" + export TEST_RESOURCE_GROUP + export TEST_RESOURCE_GROUP_LOCATION="${TEST_RESOURCE_GROUP_LOCATION:-westus3}" +} + +# Add a secret to kubernetes using kubectl +add_secret() { + local key="$1" + local value="$2" + local secret_name="lrt-secrets" + + if [[ -z "$key" || -z "$value" ]]; then + print_error "Both key and value parameters are required" + print_info "Usage: add_secret " + return 1 + fi + + print_info "Adding secret '$key' to Kubernetes secret '$secret_name'..." + + # Create or update the secret + if kubectl get secret "$secret_name" &>/dev/null; then + # Secret exists, patch it to add the new key + if kubectl patch secret "$secret_name" --type='merge' -p="{\"data\":{\"$key\":\"$(echo -n "$value" | base64 -w 0)\"}}"; then + print_success "Secret '$key' added to existing '$secret_name' successfully" + fi + else + # Secret doesn't exist, create it + if kubectl create secret generic "$secret_name" --from-literal="$key=$value"; then + print_success "Secret '$secret_name' created with '$key' successfully" + fi + fi +} + +# Load secrets from Kubernetes secret and export as environment variables +load_secrets() { + local secret_name="lrt-secrets" + + print_info "Loading secrets from Kubernetes secret '$secret_name'..." + + # Get all key-value pairs from the secret. This will fail if the secret does not exist. + local secret_data + secret_data=$(kubectl get secret "$secret_name" -o jsonpath='{.data}' 2>/dev/null) + + if [[ -z "$secret_data" || "$secret_data" == "{}" ]]; then + print_error "Secret '$secret_name' exists but contains no data." + exit 1 + fi + + # Parse and export each key-value pair + local keys + keys=$(echo "$secret_data" | jq -r 'keys[]' 2>/dev/null) + + for key in $keys; do + value=$(echo "$secret_data" | jq -r ".$key" | base64 -d 2>/dev/null) + print_success "$key" + export "$key"="$value" + done +} + +# Create a test.env file with the required environment variables from the get_env_vars function +create_env_file() { + print_info " Creating test.env file with default values. Check the file contents to ensure accuracy." + set_env_defaults + local env_file_path="./build/test.env" + local env_vars + mapfile -t env_vars < <(get_env_vars) + { + echo "# shellcheck disable=SC2148" + echo "# Environment variables for tests" + + for var in "${env_vars[@]}"; do + echo "export $var=\"${!var:-}\"" + done + } > $env_file_path || { + print_error "Failed to create test.env file." + exit 1 + } + + print_success "test.env file created successfully" + cat $env_file_path +} + +# Validate that all required environment variables have values. +check_environment() { + + # Source the test environment file if it exists + if [[ -f "./build/test.env" ]]; then + # shellcheck source=/dev/null + source "./build/test.env" + fi + + # Validate that all required environment variables have values + print_info "Validating environment variables..." + local env_vars + mapfile -t env_vars < <(get_env_vars) + + local missing_vars=() + for var in "${env_vars[@]}"; do + if [[ -z "${!var:-}" ]]; then + print_error "$var" + missing_vars+=("$var") + else + print_success "$var" + fi + done + + if [[ ${#missing_vars[@]} -gt 0 ]]; then + print_error "Missing ${#missing_vars[@]} required environment variable(s). Exiting." + exit 1 + fi + + print_success "All required environment variables are properly set" +} + # Deploy long-running test cluster deploy_aks_cluster() { + + check_environment + + # Install latest Radius CLI release + make install-latest + print_info "Deploying long-running test cluster to Azure..." # Default values (can be overridden by environment variables) - local location="${TEST_AKS_AZURE_LOCATION:-westus3}" - local resource_group="${TEST_AKS_RG:-$(whoami)-radius-lrt}" - + local location="${LRT_AZURE_LOCATION:-westus3}" + local resource_group="${LRT_RG:-$(whoami)-radius-lrt}" + print_info "Configuration:" print_info " Subscription: $(az account show --query "[name,id]" --output tsv | paste -sd,)" print_info " Resource Group: $resource_group" @@ -77,6 +229,13 @@ deploy_aks_cluster() { az provider register --namespace Microsoft.ContainerService fi + # Check if resource group exists and delete it if it does + if az group exists --name "$resource_group" --output tsv | grep -q "true"; then + print_warning "Resource group '$resource_group' already exists. Deleting it..." + az group delete --name "$resource_group" --yes + print_success "Resource group '$resource_group' deleted successfully." + fi + print_info "Creating resource group '$resource_group' in location '$location'..." az group create --location "$location" --resource-group "$resource_group" --output none print_success "Resource group created successfully." @@ -88,13 +247,16 @@ deploy_aks_cluster() { --template-file "$template_file"; then print_success "Deployment completed successfully!" - # Get AKS cluster name + # Connect to the AKS cluster, assumes one cluster in the resource group local aks_cluster aks_cluster=$(az aks list --resource-group "$resource_group" --query '[0].name' -o tsv 2>/dev/null || echo "") + az aks get-credentials --resource-group "$resource_group" --name "$aks_cluster" --admin --overwrite-existing --output none + if [[ -n "$aks_cluster" ]]; then - print_info "AKS Cluster: $aks_cluster" + # Connect to cluster + print_success "AKS cluster created and connected: '$aks_cluster'." print_info "To connect to the cluster, run:" - print_info " az aks get-credentials --resource-group $resource_group --name $aks_cluster" + print_info " az aks get-credentials --resource-group $resource_group --name $aks_cluster --admin" print_info "To delete the cluster, run:" print_info " az group delete --name $resource_group --yes --no-wait" fi @@ -102,7 +264,171 @@ deploy_aks_cluster() { print_error "Deployment failed!" exit 1 fi + + print_info "Publishing Bicep types to registry..." + rad bicep publish-extension -f ./test/functional-portable/dynamicrp/noncloud/resources/testdata/testresourcetypes.yaml --target types.tgz --force + make publish-test-bicep-recipes + + print_info "Creating resource group for tests '$TEST_RESOURCE_GROUP' in location '$TEST_RESOURCE_GROUP_LOCATION'..." + az group create --name "$TEST_RESOURCE_GROUP" --location "$TEST_RESOURCE_GROUP_LOCATION" --output none + + print_info "Installing Radius CLI..." + rad install kubernetes --reinstall + + # Ensure that all pods are running before proceeding + kubectl wait --for=condition=available --all deployments --namespace radius-system --timeout=120s + + rad workspace create kubernetes --force + rad group create default + rad group switch default + rad env create default --namespace default + rad env switch default + rad env update default --azure-subscription-id "$AZURE_SUBSCRIPTION_ID" --azure-resource-group "$TEST_RESOURCE_GROUP" + rad credential register azure wi --client-id "$AZURE_SP_TESTS_APPID" --tenant-id "$AZURE_SP_TESTS_TENANTID" + + # rad env update ${{ env.RADIUS_TEST_ENVIRONMENT_NAME }} --aws-region ${{ env.AWS_REGION }} --aws-account-id ${{ secrets.FUNCTEST_AWS_ACCOUNT_ID }} + # rad credential register aws access-key \ + # --access-key-id ${{ secrets.FUNCTEST_AWS_ACCESS_KEY_ID }} --secret-access-key ${{ secrets.FUNCTEST_AWS_SECRET_ACCESS_KEY }} + + make publish-test-terraform-recipes + + # FUNC_TEST_OIDC_ISSUER not used + # export FUNCTEST_OIDC_ISSUER=$(az aks show -n $AKS_CLUSTER_NAME -g $AZURE_RESOURCE_GROUP --query "oidcIssuerProfile.issuerUrl" -otsv) + + # Restore Radius Bicep types + bicep restore ./test/functional-portable/corerp/cloud/resources/testdata/corerp-azure-connection-database-service.bicep --force + # Restore AWS Bicep types + bicep restore ./test/functional-portable/corerp/cloud/resources/testdata/aws-s3-bucket.bicep --force + make install-flux + make install-gitea + print_success "Radius and test tools installed to cluster '$aks_cluster'." +} + +run_lrt() { + + print_info "Running long-running tests against the AKS cluster..." + + check_environment + + make test-functional-corerp + make test-functional-msgrp + make test-functional-daprrp + make test-functional-datastoresrp + + + # make test-functional-all + # make test-functional-ucp + # make test-functional-kubernetes + + # Time out and one failure + # make test-functional-corerp + + # No return + #make test-functional-cli + + # 4 tests, 4 failures + #make test-functional-msgrp + + # 8 failures + #make test-functional-daprrp + + # 6 failures + #make test-functional-datastoresrp + + # Needs path to samples repo + #make test-functional-samples + + # 4 failures + #make test-functional-dynamicrp-noncloud + + #Delete the resource group after tests + #print_info "Deleting resource group '$TEST_RESOURCE_GROUP'..." + #az group delete --name "$TEST_RESOURCE_GROUP" --yes --no-wait +} + +tear_down_aks_cluster() { + print_info "Tearing down AKS cluster and resource group..." + + check_environment + + if az group exists --name "$TEST_RESOURCE_GROUP" --output tsv | grep -q "true"; then + print_info "Deleting resource group '$TEST_RESOURCE_GROUP'..." + az group delete --name "$TEST_RESOURCE_GROUP" --yes --no-wait + print_success "Resource group '$TEST_RESOURCE_GROUP' deleted successfully." + else + print_warning "Resource group '$TEST_RESOURCE_GROUP' does not exist." + fi +} + +# Check if GitHub CLI is installed and authenticated +check_gh_auth_status() { + print_info "Checking GitHub CLI authentication status..." + if ! command -v gh &> /dev/null; then + print_error "GitHub CLI (gh) is required but not installed." + exit 1 + fi + + # Check if user is authenticated and has required scopes + if ! gh auth status &> /dev/null; then + print_error "GitHub CLI is not authenticated." + print_info "Please run 'gh auth login' to authenticate." + exit 1 + elif ! gh auth status 2>/dev/null | grep -q "write:packages"; then + print_warning "GitHub CLI does not have 'write:packages' scope. Please run 'gh auth refresh --scopes write:packages'." + exit 1 + fi + + # Docker must be available + if ! docker info &> /dev/null; then + print_error "Docker is not running or not accessible." + exit 1 + fi + + # Authenticate to ghcr.io in docker + gh_user=$(gh api user --jq .login) + print_info "Authenticating to ghcr.io as user: $gh_user" + docker_login_message=$(gh auth token | docker login ghcr.io -u "$gh_user" --password-stdin) + print_success "$docker_login_message" +} + +main() { + if [[ $# -eq 0 ]]; then + print_error "No command specified." + local available_commands="load-secrets, add-secret, load-environment, create-env-file, deploy-aks-cluster, run-lrt" + print_info "Available commands: $available_commands" + exit 1 + fi + + check_azure_cli + check_gh_auth_status + + local command="$1" + case "$command" in + "check-environment") + check_environment + ;; + "create-env-file") + create_env_file + ;; + "deploy-aks-cluster") + deploy_aks_cluster + ;; + "run-lrt") + run_lrt + ;; + "add-secret") + add_secret "$2" "$3" + ;; + "load-secrets") + load_secrets + ;; + *) + print_error "Unknown command: $command" + print_info "Available commands: $available_commands" + exit 1 + ;; + esac } -check_azure_cli -deploy_aks_cluster +# Run main function with all arguments +main "$@" diff --git a/test/infra/azure/main.bicep b/test/infra/azure/main.bicep index a28c0a3936..9d8165a2d3 100644 --- a/test/infra/azure/main.bicep +++ b/test/infra/azure/main.bicep @@ -47,6 +47,8 @@ param azureMonitorWorkspaceName string = '${prefix}-azm-workspace' param azureMonitorWorkspaceLocation string = 'westus2' @description('Specifies the name of aks cluster. Default is {prefix}-aks.') +@minLength(1) +@maxLength(63) param aksClusterName string = '${prefix}-aks' @description('Enables Azure Monitoring and Grafana Dashboard. Default is false.') @@ -170,7 +172,7 @@ module alertManagement './modules/alert-management.bicep' = if (grafanaEnabled) } // This is a workaround to get the AKS cluster resource created by aksCluster module -resource aks 'Microsoft.ContainerService/managedClusters@2023-05-01' existing = { +resource aks 'Microsoft.ContainerService/managedClusters@2023-10-01' existing = { name: aksCluster.name } @@ -181,7 +183,7 @@ module promConfigMap './modules/ama-metrics-setting-configmap.bicep' = if (grafa kubeConfig: aks.listClusterAdminCredential().kubeconfigs[0].value } dependsOn: [ - aks, aksCluster, dataCollection, alertManagement + aks, dataCollection, alertManagement ] } @@ -197,10 +199,16 @@ module deploymentScript './modules/deployment-script.bicep' = if (installKuberne location: location tags: defaultTags } - dependsOn: [ - aksCluster - ] } +module mongoDB './modules/mongodb.bicep' = { + name: 'mongodb' + params: { + name: '${prefix}-mongodb' + location: location + } +} + +output mongodbAccountID string = mongoDB.outputs.cosmosMongoAccountID output aksControlPlaneFQDN string = aksCluster.outputs.controlPlaneFQDN output grafanaDashboardFQDN string = grafanaEnabled ? grafanaDashboard.outputs.dashboardFQDN : '' diff --git a/test/infra/azure/modules/datacollection.bicep b/test/infra/azure/modules/datacollection.bicep index d96e8eadbf..e26d6f7e22 100644 --- a/test/infra/azure/modules/datacollection.bicep +++ b/test/infra/azure/modules/datacollection.bicep @@ -95,9 +95,6 @@ module azureMonitorMetricsDcraClusterResourceId './datacollection-dcra.bicep' = dcraName: dcraName clusterLocation: clusterLocation } - dependsOn: [ - dce - ] } // Output diff --git a/test/createAzureTestResources.bicep b/test/infra/azure/modules/mongodb.bicep similarity index 94% rename from test/createAzureTestResources.bicep rename to test/infra/azure/modules/mongodb.bicep index 2bb507fa2f..c214e3d16e 100644 --- a/test/createAzureTestResources.bicep +++ b/test/infra/azure/modules/mongodb.bicep @@ -1,7 +1,9 @@ +param name string + param location string = resourceGroup().location resource account 'Microsoft.DocumentDB/databaseAccounts@2020-04-01' = { - name: 'account-radiustest' + name: name location: location kind: 'MongoDB' tags: {