Support running GKE as a Kubernetes Job with auto-import after creation

.gitignore

@@ -2,3 +2,7 @@ container-utils/bootstrap-ks.config.secret
 container-utils/bootstrap-ks.job.yaml
 .*
 bin/
+*.secret
+*-credentials.json
+pull-secret.txt
+*.kubeconfig

Dockerfile

@@ -19,15 +19,32 @@
 #     chmod +x kubectl && mv kubectl /usr/local/bin/kubectl && rm openshift-client-linux.tar.gz
 #-----END OLD-----#
 
+# FROM gcr.io/google.com/cloudsdktool/cloud-sdk as cloud-sdk
+
 # Pull my own homemade base image that uses the above old code.
 FROM quay.io/gurnbenibm/cloudclisbase:latest
 
 ENV HOME=/bootstrap-ks
 WORKDIR $HOME
 
+RUN microdnf install python3 && \
+    pip3 install google-cloud
+
+ADD gke/files/google-cloud-sdk.repo /etc/yum.repos.d/
+RUN microdnf update -y && \
+    microdnf install google-cloud-sdk google-cloud-sdk-app-engine-python \
+    google-cloud-sdk-app-engine-python-extras
+
+# RUN curl -LO https://github.com/open-cluster-management/cm-cli/releases/download/v1.0.0-beta.4/cm_linux_amd64.tar.gz && \
+#     tar zxvf cm_linux_amd64.tar.gz -C /usr/bin && \
+#     rm cm_linux_amd64.tar.gz
+
 # Add bootstrap-ks modules
 ADD aro/ aro/
 ADD aks/ aks/
 ADD eks/ eks/
 ADD rosa/ rosa/
+ADD gke/ gke/
+ADD import-cluster/ import-cluster/
 ADD container-utils/provision_wrapper.sh provision_wrapper.sh
+

container-utils/prereqs.yaml

@@ -1,3 +1,8 @@
+kind: Namespace
+apiVersion: v1
+metadata:
+  name: bootstrap-ks
+---
 kind: ServiceAccount
 apiVersion: v1
 metadata:

container-utils/provision_wrapper.sh

@@ -2,14 +2,18 @@
 
 # provision_wrapper.sh
 #   This file wraps a call to the provision.sh in the target other-ks platform
-#       specifically for bootstrap-ks in a containerized form.  
-#   This is the entrypoint for containerized bootstrap-ks.  
+#       specifically for bootstrap-ks in a containerized form.
+#   This is the entrypoint for containerized bootstrap-ks.
 #   This script also handles the creation of a kubernetes secret containing data
-#       on the provisioned cluster.  
+#       on the provisioned cluster.
 #
 
 OPERATION=$(echo $OPERATION | tr '[:lower:]' '[:upper:]')
 
+export STATE_FILE=${OUTPUT_DEST}/${CLUSTER_NAME}.json
+export KUBECONFIG_FILE=${OUTPUT_DEST}/${CLUSTER_NAME}.kubeconfig
+
+
 if [[ "$OPERATION" == "DESTROY" ]]; then
     if [[ "$TARGET_KS" == "aro" ]]; then
         echo "#### Destroying ${CLUSTER_NAME} on ARO"
@@ -33,20 +37,26 @@ if [[ "$OPERATION" == "DESTROY" ]]; then
         ./destroy.sh ${OUTPUT_DEST}/${CLUSTER_NAME}/json
         popd
     elif [[ "$TARGET_KS" == "eks" ]]; then
-        echo "#### Destroying ${CLUSTER_NAME} on EKS"
         pushd eks
         mkdir ${OUTPUT_DEST}/${CLUSTER_NAME}
         oc extract secret/${CLUSTER_NAME} --keys=json --to=${OUTPUT_DEST}/${CLUSTER_NAME}
         ./destroy.sh ${OUTPUT_DEST}/${CLUSTER_NAME}/json
         popd
+    elif [[ "$TARGET_KS" == "gke" ]]; then
+        echo "#### Destroying ${CLUSTER_NAME} on GKE"
+        pushd gke
+        mkdir ${OUTPUT_DEST}/${CLUSTER_NAME}
+        oc extract secret/${CLUSTER_NAME} --keys=json --to=${OUTPUT_DEST}/${CLUSTER_NAME}
+        ./destroy.sh ${OUTPUT_DEST}/${CLUSTER_NAME}/json
+        popd
+        oc delete managedcluster ${CLUSTER_NAME}
     else
         echo "Platform ${TARGET} currently unsupported via image/kubernetes job.  Exiting"
         exit 0
     fi
 elif [[ "$OPERATION" == "CREATE" ]]; then
     if [[ "$TARGET_KS" == "aro" ]]; then
         echo "#### Provisioning ${CLUSTER_NAME} on ARO"
-        STATE_FILE=${OUTPUT_DEST}/${CLUSTER_NAME}.json
         pushd aro
         ./provision.sh \
             && oc create secret generic ${CLUSTER_NAME} \
@@ -66,8 +76,6 @@ elif [[ "$OPERATION" == "CREATE" ]]; then
         popd
     elif [[ "$TARGET_KS" == "aks" ]]; then
         echo "#### Provisioning ${CLUSTER_NAME} on AKS"
-        STATE_FILE=${OUTPUT_DEST}/${CLUSTER_NAME}.json
-        KUBECONFIG_FILE=${OUTPUT_DEST}/${CLUSTER_NAME}.kubeconfig
         pushd aks
         ./provision.sh \
             && oc create secret generic ${CLUSTER_NAME} \
@@ -82,7 +90,6 @@ elif [[ "$OPERATION" == "CREATE" ]]; then
         popd
     elif [[ "$TARGET_KS" == "rosa" ]]; then
         echo "#### Provisioning ${CLUSTER_NAME} on ROSA"
-        STATE_FILE=${OUTPUT_DEST}/${CLUSTER_NAME}.json
         pushd rosa
         ./provision.sh \
             && oc create secret generic ${CLUSTER_NAME} \
@@ -99,8 +106,6 @@ elif [[ "$OPERATION" == "CREATE" ]]; then
         popd
     elif [[ "$TARGET_KS" == "eks" ]]; then
         echo "#### Provisioning ${CLUSTER_NAME} on EKS"
-        STATE_FILE=${OUTPUT_DEST}/${CLUSTER_NAME}.json
-        KUBECONFIG_FILE=${OUTPUT_DEST}/${CLUSTER_NAME}.kubeconfig
         pushd eks
         ./provision.sh \
             && oc create secret generic ${CLUSTER_NAME} \
@@ -110,6 +115,24 @@ elif [[ "$OPERATION" == "CREATE" ]]; then
                 --from-literal=cluster_name=`cat ${STATE_FILE} | jq -r '.CLUSTER_NAME'` \
                 --from-literal=region=`cat ${STATE_FILE} | jq -r '.REGION'`;
         popd
+    elif [[ "$TARGET_KS" == "gke" ]]; then
+        echo "#### Provisioning ${CLUSTER_NAME} on GKE"
+        pushd gke
+        echo "Saving KUBECONFIG @ ${KUBECONFIG_FILE}"
+        ./provision.sh \
+            && oc create secret generic ${CLUSTER_NAME} \
+                --from-file=json=${STATE_FILE} \
+                --from-file=kubeconfig=${KUBECONFIG_FILE} \
+                --from-literal=cluster_name=`cat ${STATE_FILE} | jq -r '.CLUSTER_NAME'` \
+                --from-literal=region=`cat ${STATE_FILE} | jq -r '.REGION'` \
+                --from-literal=cloud_platform=`cat ${STATE_FILE} | jq -r '.PLATFORM'`;
+        #cm attach cluster --cluster ${CLUSTER_NAME} --cluster-kubeconfig ${KUBECONFIG_FILE}
+        popd
+        echo "About to call import-cluster KUBECONFIG @ ${KUBECONFIG_FILE}"
+        # Requires ${CLUSTER_NAME} and ${KUBECONFIG_FILE} to be defined
+        pushd import-cluster
+        ./import.sh
+        popd
     else
         echo "Platform ${TARGET} currently unsupported via image/kubernetes job.  Exiting"
         exit 0

gke/README.md

@@ -1,9 +1,9 @@
 ## Google Kubernetes Engine (GKE)
 
-This module can be used to install dependencies for Google Kubernetes Engine (GKE) provisioning (the `gcloud` cli), provision a GKE cluster, and destroy an GKE cluster provisioned using this utility.  
+This module can be used to install dependencies for Google Kubernetes Engine (GKE) provisioning (the `gcloud` cli), provision a GKE cluster, and destroy an GKE cluster provisioned using this utility.
 
 ### Getting Started
-*Optionally* run install.sh, this only works on MacOS and Fedora-based Linux instances.  
+*Optionally* run install.sh, this only works on MacOS and Fedora-based Linux instances.
 
 ### Provisioning
 1. Set the following env vars before provisioning:
@@ -25,3 +25,37 @@ export GCLOUD_REGION=<desired region>         # defaults to us-east1
 
 ### Cleaning up a cluster
 1. run `./destroy.sh <.json file of your cluster metadata>`
+
+## Provisioning through a Kubernetes Job
+
+Create the following configuration files with your desired settings:
+
+## bootstrap-ks-config.secret
+
+```bash
+CLUSTER_NAME=gke-cluster
+GCLOUD_REGION=us-east4-a
+GCLOUD_NODE_COUNT=2
+OPERATION=CREATE
+TARGET_KS=gke
+```
+
+## bootstrap-ks-creds.secret
+```bash
+GCLOUD_PROJECT_ID=
+```
+
+## gcp-credentials.json
+
+Download your GCP Service Account JSON file.
+
+## Apply kustomization.yaml
+
+Opetionally, edit the `namePrefix` within `gke/deploy/kustomization.yaml.
+
+```bash
+oc apply -f gke/deploy/prereqs.yaml
+oc create -k gke/deploy
+```
+
+

gke/deploy/bootstrap-ks-config.secret.example

@@ -0,0 +1,5 @@
+CLUSTER_NAME=gke-cluster
+GCLOUD_REGION=us-east4-a
+GCLOUD_MACHINE_TYPE=e2-medium
+GCLOUD_NODE_COUNT=2
+OPERATION=CREATE

gke/deploy/bootstrap-ks-creds.secret.example

@@ -0,0 +1 @@
+GCLOUD_PROJECT_ID=TBD

gke/deploy/bootstrap-ks.job.yaml

@@ -0,0 +1,47 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  #generateName: bootstrap-ks-provision-
+  name: bootstrap-ks-provision
+  namespace: bootstrap-ks
+spec:
+  template:
+    spec:
+      serviceAccountName: bootstrap-ks-service-account
+      volumes:
+      - name: ssh-dir
+        emptyDir: {}
+      - name: tmp
+        emptyDir: {}
+      - name: config
+        emptyDir: {}
+      - name: gcp-creds
+        secret:
+          secretName: bootstrap-ks-creds
+          items:
+          - key: gcp-credentials.json
+            path: osServiceAccount.json
+      containers:
+      - name: bootstrap
+        image: quay.io/gurnbenibm/bootstrapks:latest
+        command: ["/bin/bash", "-c", "./provision_wrapper.sh"]
+        envFrom:
+        - secretRef:
+            name: bootstrap-ks-config
+        - secretRef:
+            name: bootstrap-ks-creds
+        env:
+        - name: OUTPUT_DEST
+          value: /tmp
+        - name: COLOR
+          value: "false"
+        volumeMounts:
+        - name: config
+          mountPath: /bootstrap-ks/.config
+        - name: gcp-creds
+          mountPath: /bootstrap-ks/.gcp
+        - name: ssh-dir
+          mountPath: /.ssh
+        - name: tmp
+          mountPath: /tmp
+      restartPolicy: Never

gke/deploy/kustomization.yaml

@@ -0,0 +1,30 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+# Disable random hash suffix
+generatorOptions:
+  disableNameSuffixHash: true
+
+# namespace to deploy all Resources to
+namespace: bootstrap-ks
+
+# Point to the target job image
+images:
+  - name: quay.io/gurnbenibm/bootstrapks
+    newName: quay.io/mdelder/bootstrapks
+    newTag: latest
+
+# list of Resource Config to be Applied
+resources:
+  # Apply separately to reuse ServiceAccount, Role/Bindings across Job runs
+  # - prereqs.yaml
+  - bootstrap-ks.job.yaml
+
+namePrefix: gke-cluster10-create-
+secretGenerator:
+- name: bootstrap-ks-creds
+  env: bootstrap-ks-creds.secret
+  files:
+  - gcp-credentials.json
+- name: bootstrap-ks-config
+  env: bootstrap-ks-config.secret

gke/deploy/prereqs.yaml

@@ -0,0 +1,98 @@
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+  name: bootstrap-ks-service-account
+  namespace: bootstrap-ks
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: bootstrap-ks-service-account-role
+  namespace: bootstrap-ks
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - create
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: bootstrap-ks-service-account-binding
+  namespace: bootstrap-ks
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: bootstrap-ks-service-account-role
+subjects:
+- kind: ServiceAccount
+  name: bootstrap-ks-service-account
+  namespace: bootstrap-ks
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: bootstrap-ks-service-account-clusterrole
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - namespaces
+  - secrets
+  verbs:
+  - create
+  - get
+- apiGroups:
+  - "cluster.open-cluster-management.io"
+  resources:
+  - managedclusters
+  verbs:
+  - create
+  - get
+  - delete
+- apiGroups:
+  - "agent.open-cluster-management.io"
+  resources:
+  - klusterletaddonconfigs
+  verbs:
+  - create
+  - get
+- apiGroups:
+  - "register.open-cluster-management.io"
+  resources:
+  - managedclusters/accept
+  verbs:
+  - create
+  - update
+  - patch
+- apiGroups:
+  - cluster.open-cluster-management.io
+  resources:
+  - managedclustersets/join
+  resourceNames:
+  - all-clusters
+  verbs:
+  - get
+  - create
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: bootstrap-ks-service-account-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: bootstrap-ks-service-account-clusterrole
+subjects:
+- kind: ServiceAccount
+  name: bootstrap-ks-service-account
+  namespace: bootstrap-ks
+---
+apiVersion: cluster.open-cluster-management.io/v1alpha1
+kind: ManagedClusterSet
+metadata:
+  name: all-clusters
+spec: {}

gke/destroy.sh

@@ -24,6 +24,7 @@ if [ ! -f "$1" ]; then
 fi
 CLUSTER_NAME=$(cat $1 | jq -r '.CLUSTER_NAME')
 REGION=$(cat $1 | jq -r '.REGION')
+GCLOUD_CREDS_FILE=${GCLOUD_CREDS_FILE:-"$HOME/.gcp/osServiceAccount.json"}
 
 #----VALIDATE ENV VARS----#
 # Validate that we have all required env vars and exit with a failure if any are missing