From 7bbdcb8136379102c99d2b6779b3a249d9b7c8e8 Mon Sep 17 00:00:00 2001 From: Michael Elder Date: Tue, 7 Sep 2021 11:00:31 -0400 Subject: [PATCH 1/3] Support CREATE/DESTROY on GKE --- .gitignore | 3 + Dockerfile | 12 +++ container-utils/prereqs.yaml | 5 ++ container-utils/provision_wrapper.sh | 18 ++-- gke/deploy/README.md | 32 +++++++ gke/deploy/bootstrap-ks.config.secret.example | 6 ++ gke/deploy/bootstrap-ks.job.yaml | 55 ++++++++++++ gke/deploy/cluster-admin-role.yaml | 18 ++++ gke/deploy/google-cloud-sdk.repo | 8 ++ gke/deploy/kustomization.yaml | 34 +++++++ gke/deploy/prereqs.yaml | 33 +++++++ gke/destroy.sh | 1 + gke/provision.sh | 90 +++++++------------ 13 files changed, 253 insertions(+), 62 deletions(-) create mode 100644 gke/deploy/README.md create mode 100644 gke/deploy/bootstrap-ks.config.secret.example create mode 100644 gke/deploy/bootstrap-ks.job.yaml create mode 100644 gke/deploy/cluster-admin-role.yaml create mode 100644 gke/deploy/google-cloud-sdk.repo create mode 100644 gke/deploy/kustomization.yaml create mode 100644 gke/deploy/prereqs.yaml diff --git a/.gitignore b/.gitignore index b9a6dd0..b881feb 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,6 @@ container-utils/bootstrap-ks.config.secret container-utils/bootstrap-ks.job.yaml .* bin/ +*.secret +*-credentials.json +pull-secret.txt \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index b7c767c..297f70c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,15 +19,27 @@ # chmod +x kubectl && mv kubectl /usr/local/bin/kubectl && rm openshift-client-linux.tar.gz #-----END OLD-----# +# FROM gcr.io/google.com/cloudsdktool/cloud-sdk as cloud-sdk + # Pull my own homemade base image that uses the above old code. FROM quay.io/gurnbenibm/cloudclisbase:latest ENV HOME=/bootstrap-ks WORKDIR $HOME +RUN microdnf install python3 && \ + pip3 install google-cloud + +ADD gke/deploy/google-cloud-sdk.repo /etc/yum.repos.d/ +RUN microdnf update -y && \ + microdnf install google-cloud-sdk google-cloud-sdk-app-engine-python \ + google-cloud-sdk-app-engine-python-extras + # Add bootstrap-ks modules ADD aro/ aro/ ADD aks/ aks/ ADD eks/ eks/ ADD rosa/ rosa/ +ADD gke/ gke/ ADD container-utils/provision_wrapper.sh provision_wrapper.sh + diff --git a/container-utils/prereqs.yaml b/container-utils/prereqs.yaml index 682df85..24a7dcf 100644 --- a/container-utils/prereqs.yaml +++ b/container-utils/prereqs.yaml @@ -1,3 +1,8 @@ +kind: Namespace +apiVersion: v1 +metadata: + name: bootstrap-ks +--- kind: ServiceAccount apiVersion: v1 metadata: diff --git a/container-utils/provision_wrapper.sh b/container-utils/provision_wrapper.sh index 28851f5..08eba01 100755 --- a/container-utils/provision_wrapper.sh +++ b/container-utils/provision_wrapper.sh @@ -2,10 +2,10 @@ # provision_wrapper.sh # This file wraps a call to the provision.sh in the target other-ks platform -# specifically for bootstrap-ks in a containerized form. -# This is the entrypoint for containerized bootstrap-ks. +# specifically for bootstrap-ks in a containerized form. +# This is the entrypoint for containerized bootstrap-ks. # This script also handles the creation of a kubernetes secret containing data -# on the provisioned cluster. +# on the provisioned cluster. # OPERATION=$(echo $OPERATION | tr '[:lower:]' '[:upper:]') @@ -35,6 +35,9 @@ if [[ "$OPERATION" == "DESTROY" ]]; then elif [[ "$TARGET_KS" == "eks" ]]; then echo "#### Destroying ${CLUSTER_NAME} on EKS" pushd eks + elif [[ "$TARGET_KS" == "gke" ]]; then + echo "#### Destroying ${CLUSTER_NAME} on GKE" + pushd gke mkdir ${OUTPUT_DEST}/${CLUSTER_NAME} oc extract secret/${CLUSTER_NAME} --keys=json --to=${OUTPUT_DEST}/${CLUSTER_NAME} ./destroy.sh ${OUTPUT_DEST}/${CLUSTER_NAME}/json @@ -102,13 +105,18 @@ elif [[ "$OPERATION" == "CREATE" ]]; then STATE_FILE=${OUTPUT_DEST}/${CLUSTER_NAME}.json KUBECONFIG_FILE=${OUTPUT_DEST}/${CLUSTER_NAME}.kubeconfig pushd eks + elif [[ "$TARGET_KS" == "gke" ]]; then + echo "#### Provisioning ${CLUSTER_NAME} on GKE" + STATE_FILE=${OUTPUT_DEST}/${CLUSTER_NAME}.json + KUBECONFIG_FILE=${OUTPUT_DEST}/${CLUSTER_NAME}.kubeconfig + pushd gke ./provision.sh \ && oc create secret generic ${CLUSTER_NAME} \ --from-file=json=${STATE_FILE} \ --from-file=kubeconfig=${KUBECONFIG_FILE} \ - --from-literal=cloud_platform=`cat ${STATE_FILE} | jq -r '.PLATFORM'` \ --from-literal=cluster_name=`cat ${STATE_FILE} | jq -r '.CLUSTER_NAME'` \ - --from-literal=region=`cat ${STATE_FILE} | jq -r '.REGION'`; + --from-literal=region=`cat ${STATE_FILE} | jq -r '.REGION'` \ + --from-literal=cloud_platform=`cat ${STATE_FILE} | jq -r '.PLATFORM'`; popd else echo "Platform ${TARGET} currently unsupported via image/kubernetes job. Exiting" diff --git a/gke/deploy/README.md b/gke/deploy/README.md new file mode 100644 index 0000000..0bbe24b --- /dev/null +++ b/gke/deploy/README.md @@ -0,0 +1,32 @@ + + +## bootstrap-ks.config.secret + +```bash +CLUSTER_NAME=gke-cluster +GCLOUD_REGION=us-east4-a +GCLOUD_NODE_COUNT=2 +OPERATION=CREATE +TARGET_KS=gke +``` + +## bootstrap-ks-creds.secret +```bash +GCLOUD_PROJECT_ID= +``` + +## pull-secret.txt + +Download from cloud.redhat.com. + +## gcp-credentials.json + +Download your GCP Service Account JSON file. + + +## Apply kustomization.yaml + +```bash +oc create -k gke/deploy +``` + diff --git a/gke/deploy/bootstrap-ks.config.secret.example b/gke/deploy/bootstrap-ks.config.secret.example new file mode 100644 index 0000000..6ed2568 --- /dev/null +++ b/gke/deploy/bootstrap-ks.config.secret.example @@ -0,0 +1,6 @@ +CLUSTER_NAME=gke-cluster +GCLOUD_PROJECT_ID=TBD +GCLOUD_CREDS_FILE=~/.gcp/osServiceAccount.jso +GCLOUD_REGION=us-east-1 +GCLOUD_NODE_COUNT=2 +OPERATION=CREATE \ No newline at end of file diff --git a/gke/deploy/bootstrap-ks.job.yaml b/gke/deploy/bootstrap-ks.job.yaml new file mode 100644 index 0000000..c018c9c --- /dev/null +++ b/gke/deploy/bootstrap-ks.job.yaml @@ -0,0 +1,55 @@ +apiVersion: batch/v1 +kind: Job +metadata: + #generateName: bootstrap-ks-provision- + name: bootstrap-ks-provision + namespace: bootstrap-ks +spec: + template: + spec: + serviceAccountName: bootstrap-ks-service-account + volumes: + - name: pull-secret + secret: + secretName: openshift-pull-secret + - name: ssh-dir + emptyDir: {} + - name: tmp + emptyDir: {} + - name: config + emptyDir: {} + - name: gcp-creds + secret: + secretName: bootstrap-ks-creds + items: + - key: gcp-credentials.json + path: osServiceAccount.json + containers: + - name: bootstrap + image: quay.io/gurnbenibm/bootstrapks:latest + command: ["/bin/bash", "-c", "./provision_wrapper.sh"] + envFrom: + - secretRef: + name: bootstrap-ks-config + - secretRef: + name: bootstrap-ks-creds + env: + - name: OCP_PULL_SECRET_FILE + value: /opt/pull-secret.txt + - name: OUTPUT_DEST + value: /tmp + - name: COLOR + value: "false" + volumeMounts: + - name: pull-secret + mountPath: "/opt" + readOnly: true + - name: config + mountPath: /tmp/.config + - name: gcp-creds + mountPath: /tmp/.gcp + - name: ssh-dir + mountPath: /.ssh + - name: tmp + mountPath: /tmp + restartPolicy: Never diff --git a/gke/deploy/cluster-admin-role.yaml b/gke/deploy/cluster-admin-role.yaml new file mode 100644 index 0000000..09e1648 --- /dev/null +++ b/gke/deploy/cluster-admin-role.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cluster-admin + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kube-system-cluster-admin +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: +- kind: ServiceAccount + name: cluster-admin + namespace: kube-system \ No newline at end of file diff --git a/gke/deploy/google-cloud-sdk.repo b/gke/deploy/google-cloud-sdk.repo new file mode 100644 index 0000000..adf5271 --- /dev/null +++ b/gke/deploy/google-cloud-sdk.repo @@ -0,0 +1,8 @@ +[google-cloud-sdk] +name=Google Cloud SDK +baseurl=https://packages.cloud.google.com/yum/repos/cloud-sdk-el8-x86_64 +enabled=1 +gpgcheck=1 +repo_gpgcheck=0 +gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg + https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg \ No newline at end of file diff --git a/gke/deploy/kustomization.yaml b/gke/deploy/kustomization.yaml new file mode 100644 index 0000000..d1b0d0c --- /dev/null +++ b/gke/deploy/kustomization.yaml @@ -0,0 +1,34 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + + + +# Disable random hash suffix +generatorOptions: + disableNameSuffixHash: true + +# namespace to deploy all Resources to +namespace: bootstrap-ks + +# Point to the target job image +images: + - name: quay.io/gurnbenibm/bootstrapks + newName: quay.io/mdelder/bootstrapks + newTag: latest + +# list of Resource Config to be Applied +resources: + - prereqs.yaml + - bootstrap-ks.job.yaml + +namePrefix: gke-cluster- +secretGenerator: +- name: bootstrap-ks-creds + env: bootstrap-ks-creds.secret + files: + - gcp-credentials.json +- name: openshift-pull-secret + files: + - pull-secret.txt +- name: bootstrap-ks-config + env: bootstrap-ks.config.secret diff --git a/gke/deploy/prereqs.yaml b/gke/deploy/prereqs.yaml new file mode 100644 index 0000000..682df85 --- /dev/null +++ b/gke/deploy/prereqs.yaml @@ -0,0 +1,33 @@ +kind: ServiceAccount +apiVersion: v1 +metadata: + name: bootstrap-ks-service-account + namespace: bootstrap-ks +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: bootstrap-ks-service-account-role + namespace: bootstrap-ks +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: bootstrap-ks-service-account-binding + namespace: bootstrap-ks +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: bootstrap-ks-service-account-role +subjects: +- kind: ServiceAccount + name: bootstrap-ks-service-account + namespace: bootstrap-ks diff --git a/gke/destroy.sh b/gke/destroy.sh index ecbd81a..a5c4619 100755 --- a/gke/destroy.sh +++ b/gke/destroy.sh @@ -24,6 +24,7 @@ if [ ! -f "$1" ]; then fi CLUSTER_NAME=$(cat $1 | jq -r '.CLUSTER_NAME') REGION=$(cat $1 | jq -r '.REGION') +GCLOUD_CREDS_FILE=${GCLOUD_CREDS_FILE:-"$HOME/.gcp/osServiceAccount.json"} #----VALIDATE ENV VARS----# # Validate that we have all required env vars and exit with a failure if any are missing diff --git a/gke/provision.sh b/gke/provision.sh index 1b9a976..551a134 100755 --- a/gke/provision.sh +++ b/gke/provision.sh @@ -37,7 +37,6 @@ GCLOUD_CREDS_FILE=${GCLOUD_CREDS_FILE:-"$HOME/.gcp/osServiceAccount.json"} GCLOUD_REGION=${GCLOUD_REGION:-"us-east1"} GCLOUD_NODE_COUNT=${GCLOUD_NODE_COUNT:-"3"} - #----VALIDATE ENV VARS----# # Validate that we have all required env vars and exit with a failure if any are missing missing=0 @@ -63,13 +62,19 @@ else printf "${BLUE}Using $RESOURCE_NAME to identify all created resources.${CLEAR}\n" fi - #----VERIFY GCLOUD CLI----# if [ -z "$(which gcloud)" ]; then printf "${RED}Could not find the gcloud cli, exiting. Try running ./install.sh.${CLEAR}\n" exit 1 fi +export WORKDIR=$(pwd) +if [ -w ${WORKDIR} ]; then + printf "${BLUE}Writing to current directory.${CLEAR}\n" +else + export WORKDIR="/tmp" +fi +printf "${BLUE}Writing output files to ${WORKDIR}${CLEAR}\n" #----LOG IN----# # Log in and optionally choose a specific subscription @@ -77,7 +82,7 @@ printf "${BLUE}Logging in to the gcloud cli.${CLEAR}\n" #gcloud auth activate-service-account --key-file ~/.secrets/gc-acm-cicd.json gcloud auth activate-service-account --key-file $GCLOUD_CREDS_FILE if [ "$?" -ne 0 ]; then - printf "${RED}ibmcloud cli login failed, check credentials. Exiting.${CLEAR}\n" + printf "${RED}gcloud cli login failed, check credentials. Exiting.${CLEAR}\n" exit 1 fi @@ -87,6 +92,7 @@ gcloud config set project ${GCLOUD_PROJECT_ID} #----CREATE GKE CLUSTER----# GKE_CLUSTER_NAME="${RESOURCE_NAME}-${NAME_SUFFIX}" +export KUBECONFIG=${WORKDIR}/${CLUSTER_NAME}.kubeconfig printf "${BLUE}Creating an GKE cluster named ${GKE_CLUSTER_NAME}.${CLEAR}\n" printf "${YELLOW}" gcloud container clusters create ${GKE_CLUSTER_NAME} --num-nodes=${GCLOUD_NODE_COUNT} --region="${GCLOUD_REGION}" @@ -100,18 +106,11 @@ printf "${GREEN}Successfully provisioned GKE cluster ${GKE_CLUSTER_NAME}.${CLEAR #----EXTRACTING KUBECONFIG----# printf "${BLUE}Getting Kubeconfig for the cluster named ${GKE_CLUSTER_NAME}.${CLEAR}\n" printf "${YELLOW}" -export KUBECONFIG=$PWD/${GKE_CLUSTER_NAME}.kubeconfig gcloud container clusters get-credentials ${GKE_CLUSTER_NAME} --region="${GCLOUD_REGION}" if [ "$?" -ne 0 ]; then printf "${RED}Failed to get credentials for GKE cluster ${GKE_CLUSTER_NAME}, complaining and continuing${CLEAR}\n" exit 1 fi -unset KUBECONFIG - - -#----Make KUBECONFIG that is useable from anywhere ----# -export KUBECONFIG_SAVED=$KUBECONFIG -export KUBECONFIG=$(pwd)/${GKE_CLUSTER_NAME}.kubeconfig # Check for which base64 command we have available so we can use the right option echo | base64 -w 0 > /dev/null 2>&1 @@ -123,75 +122,52 @@ else BASE64_OPTION=" " fi -echo | kubectl apply -f - &> /dev/null < /dev/null < /dev/null +# Give cluster time to recognize Service Account & generate token sleep 1 -#TMP - DEBUG - TODO -cp $(pwd)/${GKE_CLUSTER_NAME}.kubeconfig $(pwd)/${GKE_CLUSTER_NAME}.kubeconfig.orig +cp $KUBECONFIG $KUBECONFIG.${GKE_CLUSTER_NAME} -cat > "$(pwd)/${GKE_CLUSTER_NAME}.kubeconfig.portable" < "$KUBECONFIG.portable" < $(pwd)/${GKE_CLUSTER_NAME}.json < Date: Tue, 7 Sep 2021 22:41:20 -0400 Subject: [PATCH 2/3] Update to support auto-import after provisioning --- Dockerfile | 5 ++ container-utils/provision_wrapper.sh | 5 ++ gke/deploy/kustomization.yaml | 7 ++- gke/deploy/prereqs.yaml | 64 ++++++++++++++++++++++++++ import-cluster/import.sh | 68 ++++++++++++++++++++++++++++ 5 files changed, 145 insertions(+), 4 deletions(-) create mode 100755 import-cluster/import.sh diff --git a/Dockerfile b/Dockerfile index 297f70c..3f98beb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,11 +35,16 @@ RUN microdnf update -y && \ microdnf install google-cloud-sdk google-cloud-sdk-app-engine-python \ google-cloud-sdk-app-engine-python-extras +# RUN curl -LO https://github.com/open-cluster-management/cm-cli/releases/download/v1.0.0-beta.4/cm_linux_amd64.tar.gz && \ +# tar zxvf cm_linux_amd64.tar.gz -C /usr/bin && \ +# rm cm_linux_amd64.tar.gz + # Add bootstrap-ks modules ADD aro/ aro/ ADD aks/ aks/ ADD eks/ eks/ ADD rosa/ rosa/ ADD gke/ gke/ +ADD import-cluster/ import-cluster/ ADD container-utils/provision_wrapper.sh provision_wrapper.sh diff --git a/container-utils/provision_wrapper.sh b/container-utils/provision_wrapper.sh index 08eba01..04f261b 100755 --- a/container-utils/provision_wrapper.sh +++ b/container-utils/provision_wrapper.sh @@ -117,6 +117,11 @@ elif [[ "$OPERATION" == "CREATE" ]]; then --from-literal=cluster_name=`cat ${STATE_FILE} | jq -r '.CLUSTER_NAME'` \ --from-literal=region=`cat ${STATE_FILE} | jq -r '.REGION'` \ --from-literal=cloud_platform=`cat ${STATE_FILE} | jq -r '.PLATFORM'`; + #cm attach cluster --cluster ${CLUSTER_NAME} --cluster-kubeconfig ${KUBECONFIG_FILE} + popd + # Requires ${CLUSTER_NAME} and ${KUBECONFIG_FILE} to be defined + pushd import-cluster + ./import.sh popd else echo "Platform ${TARGET} currently unsupported via image/kubernetes job. Exiting" diff --git a/gke/deploy/kustomization.yaml b/gke/deploy/kustomization.yaml index d1b0d0c..2bc5167 100644 --- a/gke/deploy/kustomization.yaml +++ b/gke/deploy/kustomization.yaml @@ -1,8 +1,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization - - # Disable random hash suffix generatorOptions: disableNameSuffixHash: true @@ -18,10 +16,11 @@ images: # list of Resource Config to be Applied resources: - - prereqs.yaml + # Apply separately to reuse ServiceAccount, Role/Bindings across Job runs + # - prereqs.yaml - bootstrap-ks.job.yaml -namePrefix: gke-cluster- +namePrefix: gke-cluster5- secretGenerator: - name: bootstrap-ks-creds env: bootstrap-ks-creds.secret diff --git a/gke/deploy/prereqs.yaml b/gke/deploy/prereqs.yaml index 682df85..4ca5b5f 100644 --- a/gke/deploy/prereqs.yaml +++ b/gke/deploy/prereqs.yaml @@ -31,3 +31,67 @@ subjects: - kind: ServiceAccount name: bootstrap-ks-service-account namespace: bootstrap-ks +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: bootstrap-ks-service-account-clusterrole +rules: +- apiGroups: + - "" + resources: + - namespaces + - secrets + verbs: + - create + - get +- apiGroups: + - "cluster.open-cluster-management.io" + resources: + - managedclusters + verbs: + - create + - get +- apiGroups: + - "agent.open-cluster-management.io" + resources: + - klusterletaddonconfigs + verbs: + - create + - get +- apiGroups: + - "register.open-cluster-management.io" + resources: + - managedclusters/accept + verbs: + - create + - update + - patch +- apiGroups: + - cluster.open-cluster-management.io + resources: + - managedclustersets/join + resourceNames: + - all-clusters + verbs: + - get + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: bootstrap-ks-service-account-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: bootstrap-ks-service-account-clusterrole +subjects: +- kind: ServiceAccount + name: bootstrap-ks-service-account + namespace: bootstrap-ks +--- +apiVersion: cluster.open-cluster-management.io/v1alpha1 +kind: ManagedClusterSet +metadata: + name: all-clusters +spec: {} \ No newline at end of file diff --git a/import-cluster/import.sh b/import-cluster/import.sh new file mode 100755 index 0000000..85da0c8 --- /dev/null +++ b/import-cluster/import.sh @@ -0,0 +1,68 @@ +#!/bin/sh + +# Assumes ${CLUSTER_NAME} is defined + +oc create namespace ${CLUSTER_NAME} + +# Give the namespace time to populate +sleep 5 + +echo "Creating auto-import-secret for ${CLUSTER_NAME}..." +echo | kubectl apply -f - < Date: Thu, 9 Sep 2021 14:28:08 -0400 Subject: [PATCH 3/3] Support GKE node customization + cleanup --- .gitignore | 3 +- Dockerfile | 2 +- container-utils/provision_wrapper.sh | 28 +++++++++----- gke/README.md | 38 ++++++++++++++++++- gke/deploy/README.md | 32 ---------------- gke/deploy/bootstrap-ks-config.secret.example | 5 +++ gke/deploy/bootstrap-ks-creds.secret.example | 1 + gke/deploy/bootstrap-ks.config.secret.example | 6 --- gke/deploy/bootstrap-ks.job.yaml | 12 +----- gke/deploy/kustomization.yaml | 7 +--- gke/deploy/prereqs.yaml | 1 + gke/{deploy => files}/cluster-admin-role.yaml | 0 gke/{deploy => files}/google-cloud-sdk.repo | 0 gke/provision.sh | 4 +- import-cluster/import.sh | 4 +- 15 files changed, 74 insertions(+), 69 deletions(-) delete mode 100644 gke/deploy/README.md create mode 100644 gke/deploy/bootstrap-ks-config.secret.example create mode 100644 gke/deploy/bootstrap-ks-creds.secret.example delete mode 100644 gke/deploy/bootstrap-ks.config.secret.example rename gke/{deploy => files}/cluster-admin-role.yaml (100%) rename gke/{deploy => files}/google-cloud-sdk.repo (100%) diff --git a/.gitignore b/.gitignore index b881feb..e40c0b1 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ container-utils/bootstrap-ks.job.yaml bin/ *.secret *-credentials.json -pull-secret.txt \ No newline at end of file +pull-secret.txt +*.kubeconfig \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 3f98beb..3f2add3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,7 +30,7 @@ WORKDIR $HOME RUN microdnf install python3 && \ pip3 install google-cloud -ADD gke/deploy/google-cloud-sdk.repo /etc/yum.repos.d/ +ADD gke/files/google-cloud-sdk.repo /etc/yum.repos.d/ RUN microdnf update -y && \ microdnf install google-cloud-sdk google-cloud-sdk-app-engine-python \ google-cloud-sdk-app-engine-python-extras diff --git a/container-utils/provision_wrapper.sh b/container-utils/provision_wrapper.sh index 04f261b..72a6a5f 100755 --- a/container-utils/provision_wrapper.sh +++ b/container-utils/provision_wrapper.sh @@ -10,6 +10,10 @@ OPERATION=$(echo $OPERATION | tr '[:lower:]' '[:upper:]') +export STATE_FILE=${OUTPUT_DEST}/${CLUSTER_NAME}.json +export KUBECONFIG_FILE=${OUTPUT_DEST}/${CLUSTER_NAME}.kubeconfig + + if [[ "$OPERATION" == "DESTROY" ]]; then if [[ "$TARGET_KS" == "aro" ]]; then echo "#### Destroying ${CLUSTER_NAME} on ARO" @@ -33,8 +37,11 @@ if [[ "$OPERATION" == "DESTROY" ]]; then ./destroy.sh ${OUTPUT_DEST}/${CLUSTER_NAME}/json popd elif [[ "$TARGET_KS" == "eks" ]]; then - echo "#### Destroying ${CLUSTER_NAME} on EKS" pushd eks + mkdir ${OUTPUT_DEST}/${CLUSTER_NAME} + oc extract secret/${CLUSTER_NAME} --keys=json --to=${OUTPUT_DEST}/${CLUSTER_NAME} + ./destroy.sh ${OUTPUT_DEST}/${CLUSTER_NAME}/json + popd elif [[ "$TARGET_KS" == "gke" ]]; then echo "#### Destroying ${CLUSTER_NAME} on GKE" pushd gke @@ -42,6 +49,7 @@ if [[ "$OPERATION" == "DESTROY" ]]; then oc extract secret/${CLUSTER_NAME} --keys=json --to=${OUTPUT_DEST}/${CLUSTER_NAME} ./destroy.sh ${OUTPUT_DEST}/${CLUSTER_NAME}/json popd + oc delete managedcluster ${CLUSTER_NAME} else echo "Platform ${TARGET} currently unsupported via image/kubernetes job. Exiting" exit 0 @@ -49,7 +57,6 @@ if [[ "$OPERATION" == "DESTROY" ]]; then elif [[ "$OPERATION" == "CREATE" ]]; then if [[ "$TARGET_KS" == "aro" ]]; then echo "#### Provisioning ${CLUSTER_NAME} on ARO" - STATE_FILE=${OUTPUT_DEST}/${CLUSTER_NAME}.json pushd aro ./provision.sh \ && oc create secret generic ${CLUSTER_NAME} \ @@ -69,8 +76,6 @@ elif [[ "$OPERATION" == "CREATE" ]]; then popd elif [[ "$TARGET_KS" == "aks" ]]; then echo "#### Provisioning ${CLUSTER_NAME} on AKS" - STATE_FILE=${OUTPUT_DEST}/${CLUSTER_NAME}.json - KUBECONFIG_FILE=${OUTPUT_DEST}/${CLUSTER_NAME}.kubeconfig pushd aks ./provision.sh \ && oc create secret generic ${CLUSTER_NAME} \ @@ -85,7 +90,6 @@ elif [[ "$OPERATION" == "CREATE" ]]; then popd elif [[ "$TARGET_KS" == "rosa" ]]; then echo "#### Provisioning ${CLUSTER_NAME} on ROSA" - STATE_FILE=${OUTPUT_DEST}/${CLUSTER_NAME}.json pushd rosa ./provision.sh \ && oc create secret generic ${CLUSTER_NAME} \ @@ -102,14 +106,19 @@ elif [[ "$OPERATION" == "CREATE" ]]; then popd elif [[ "$TARGET_KS" == "eks" ]]; then echo "#### Provisioning ${CLUSTER_NAME} on EKS" - STATE_FILE=${OUTPUT_DEST}/${CLUSTER_NAME}.json - KUBECONFIG_FILE=${OUTPUT_DEST}/${CLUSTER_NAME}.kubeconfig pushd eks + ./provision.sh \ + && oc create secret generic ${CLUSTER_NAME} \ + --from-file=json=${STATE_FILE} \ + --from-file=kubeconfig=${KUBECONFIG_FILE} \ + --from-literal=cloud_platform=`cat ${STATE_FILE} | jq -r '.PLATFORM'` \ + --from-literal=cluster_name=`cat ${STATE_FILE} | jq -r '.CLUSTER_NAME'` \ + --from-literal=region=`cat ${STATE_FILE} | jq -r '.REGION'`; + popd elif [[ "$TARGET_KS" == "gke" ]]; then echo "#### Provisioning ${CLUSTER_NAME} on GKE" - STATE_FILE=${OUTPUT_DEST}/${CLUSTER_NAME}.json - KUBECONFIG_FILE=${OUTPUT_DEST}/${CLUSTER_NAME}.kubeconfig pushd gke + echo "Saving KUBECONFIG @ ${KUBECONFIG_FILE}" ./provision.sh \ && oc create secret generic ${CLUSTER_NAME} \ --from-file=json=${STATE_FILE} \ @@ -119,6 +128,7 @@ elif [[ "$OPERATION" == "CREATE" ]]; then --from-literal=cloud_platform=`cat ${STATE_FILE} | jq -r '.PLATFORM'`; #cm attach cluster --cluster ${CLUSTER_NAME} --cluster-kubeconfig ${KUBECONFIG_FILE} popd + echo "About to call import-cluster KUBECONFIG @ ${KUBECONFIG_FILE}" # Requires ${CLUSTER_NAME} and ${KUBECONFIG_FILE} to be defined pushd import-cluster ./import.sh diff --git a/gke/README.md b/gke/README.md index c9f4ce3..9033a58 100644 --- a/gke/README.md +++ b/gke/README.md @@ -1,9 +1,9 @@ ## Google Kubernetes Engine (GKE) -This module can be used to install dependencies for Google Kubernetes Engine (GKE) provisioning (the `gcloud` cli), provision a GKE cluster, and destroy an GKE cluster provisioned using this utility. +This module can be used to install dependencies for Google Kubernetes Engine (GKE) provisioning (the `gcloud` cli), provision a GKE cluster, and destroy an GKE cluster provisioned using this utility. ### Getting Started -*Optionally* run install.sh, this only works on MacOS and Fedora-based Linux instances. +*Optionally* run install.sh, this only works on MacOS and Fedora-based Linux instances. ### Provisioning 1. Set the following env vars before provisioning: @@ -25,3 +25,37 @@ export GCLOUD_REGION= # defaults to us-east1 ### Cleaning up a cluster 1. run `./destroy.sh <.json file of your cluster metadata>` + +## Provisioning through a Kubernetes Job + +Create the following configuration files with your desired settings: + +## bootstrap-ks-config.secret + +```bash +CLUSTER_NAME=gke-cluster +GCLOUD_REGION=us-east4-a +GCLOUD_NODE_COUNT=2 +OPERATION=CREATE +TARGET_KS=gke +``` + +## bootstrap-ks-creds.secret +```bash +GCLOUD_PROJECT_ID= +``` + +## gcp-credentials.json + +Download your GCP Service Account JSON file. + +## Apply kustomization.yaml + +Opetionally, edit the `namePrefix` within `gke/deploy/kustomization.yaml. + +```bash +oc apply -f gke/deploy/prereqs.yaml +oc create -k gke/deploy +``` + + diff --git a/gke/deploy/README.md b/gke/deploy/README.md deleted file mode 100644 index 0bbe24b..0000000 --- a/gke/deploy/README.md +++ /dev/null @@ -1,32 +0,0 @@ - - -## bootstrap-ks.config.secret - -```bash -CLUSTER_NAME=gke-cluster -GCLOUD_REGION=us-east4-a -GCLOUD_NODE_COUNT=2 -OPERATION=CREATE -TARGET_KS=gke -``` - -## bootstrap-ks-creds.secret -```bash -GCLOUD_PROJECT_ID= -``` - -## pull-secret.txt - -Download from cloud.redhat.com. - -## gcp-credentials.json - -Download your GCP Service Account JSON file. - - -## Apply kustomization.yaml - -```bash -oc create -k gke/deploy -``` - diff --git a/gke/deploy/bootstrap-ks-config.secret.example b/gke/deploy/bootstrap-ks-config.secret.example new file mode 100644 index 0000000..fc34b84 --- /dev/null +++ b/gke/deploy/bootstrap-ks-config.secret.example @@ -0,0 +1,5 @@ +CLUSTER_NAME=gke-cluster +GCLOUD_REGION=us-east4-a +GCLOUD_MACHINE_TYPE=e2-medium +GCLOUD_NODE_COUNT=2 +OPERATION=CREATE \ No newline at end of file diff --git a/gke/deploy/bootstrap-ks-creds.secret.example b/gke/deploy/bootstrap-ks-creds.secret.example new file mode 100644 index 0000000..7a322d8 --- /dev/null +++ b/gke/deploy/bootstrap-ks-creds.secret.example @@ -0,0 +1 @@ +GCLOUD_PROJECT_ID=TBD diff --git a/gke/deploy/bootstrap-ks.config.secret.example b/gke/deploy/bootstrap-ks.config.secret.example deleted file mode 100644 index 6ed2568..0000000 --- a/gke/deploy/bootstrap-ks.config.secret.example +++ /dev/null @@ -1,6 +0,0 @@ -CLUSTER_NAME=gke-cluster -GCLOUD_PROJECT_ID=TBD -GCLOUD_CREDS_FILE=~/.gcp/osServiceAccount.jso -GCLOUD_REGION=us-east-1 -GCLOUD_NODE_COUNT=2 -OPERATION=CREATE \ No newline at end of file diff --git a/gke/deploy/bootstrap-ks.job.yaml b/gke/deploy/bootstrap-ks.job.yaml index c018c9c..bac41f8 100644 --- a/gke/deploy/bootstrap-ks.job.yaml +++ b/gke/deploy/bootstrap-ks.job.yaml @@ -9,9 +9,6 @@ spec: spec: serviceAccountName: bootstrap-ks-service-account volumes: - - name: pull-secret - secret: - secretName: openshift-pull-secret - name: ssh-dir emptyDir: {} - name: tmp @@ -34,20 +31,15 @@ spec: - secretRef: name: bootstrap-ks-creds env: - - name: OCP_PULL_SECRET_FILE - value: /opt/pull-secret.txt - name: OUTPUT_DEST value: /tmp - name: COLOR value: "false" volumeMounts: - - name: pull-secret - mountPath: "/opt" - readOnly: true - name: config - mountPath: /tmp/.config + mountPath: /bootstrap-ks/.config - name: gcp-creds - mountPath: /tmp/.gcp + mountPath: /bootstrap-ks/.gcp - name: ssh-dir mountPath: /.ssh - name: tmp diff --git a/gke/deploy/kustomization.yaml b/gke/deploy/kustomization.yaml index 2bc5167..dafef83 100644 --- a/gke/deploy/kustomization.yaml +++ b/gke/deploy/kustomization.yaml @@ -20,14 +20,11 @@ resources: # - prereqs.yaml - bootstrap-ks.job.yaml -namePrefix: gke-cluster5- +namePrefix: gke-cluster10-create- secretGenerator: - name: bootstrap-ks-creds env: bootstrap-ks-creds.secret files: - gcp-credentials.json -- name: openshift-pull-secret - files: - - pull-secret.txt - name: bootstrap-ks-config - env: bootstrap-ks.config.secret + env: bootstrap-ks-config.secret diff --git a/gke/deploy/prereqs.yaml b/gke/deploy/prereqs.yaml index 4ca5b5f..d493f5f 100644 --- a/gke/deploy/prereqs.yaml +++ b/gke/deploy/prereqs.yaml @@ -52,6 +52,7 @@ rules: verbs: - create - get + - delete - apiGroups: - "agent.open-cluster-management.io" resources: diff --git a/gke/deploy/cluster-admin-role.yaml b/gke/files/cluster-admin-role.yaml similarity index 100% rename from gke/deploy/cluster-admin-role.yaml rename to gke/files/cluster-admin-role.yaml diff --git a/gke/deploy/google-cloud-sdk.repo b/gke/files/google-cloud-sdk.repo similarity index 100% rename from gke/deploy/google-cloud-sdk.repo rename to gke/files/google-cloud-sdk.repo diff --git a/gke/provision.sh b/gke/provision.sh index 551a134..129f040 100755 --- a/gke/provision.sh +++ b/gke/provision.sh @@ -95,7 +95,7 @@ GKE_CLUSTER_NAME="${RESOURCE_NAME}-${NAME_SUFFIX}" export KUBECONFIG=${WORKDIR}/${CLUSTER_NAME}.kubeconfig printf "${BLUE}Creating an GKE cluster named ${GKE_CLUSTER_NAME}.${CLEAR}\n" printf "${YELLOW}" -gcloud container clusters create ${GKE_CLUSTER_NAME} --num-nodes=${GCLOUD_NODE_COUNT} --region="${GCLOUD_REGION}" +gcloud container clusters create ${GKE_CLUSTER_NAME} --num-nodes=${GCLOUD_NODE_COUNT} --region="${GCLOUD_REGION}" --machine-type="${GCLOUD_MACHINE_TYPE}" if [ "$?" -ne 0 ]; then printf "${RED}Failed to provision GKE cluster. See error above. Exiting${CLEAR}\n" exit 1 @@ -122,7 +122,7 @@ else BASE64_OPTION=" " fi -kubectl apply -f deploy/cluster-admin-role.yaml &> /dev/null +kubectl apply -f files/cluster-admin-role.yaml &> /dev/null # Give cluster time to recognize Service Account & generate token sleep 1 diff --git a/import-cluster/import.sh b/import-cluster/import.sh index 85da0c8..265b418 100755 --- a/import-cluster/import.sh +++ b/import-cluster/import.sh @@ -8,6 +8,8 @@ oc create namespace ${CLUSTER_NAME} sleep 5 echo "Creating auto-import-secret for ${CLUSTER_NAME}..." + +# TODO Add error checking to make sure KUBECONFIG_FILE exists/has content echo | kubectl apply -f - <