Merge pull request #142 from dasmeta/DMVP-7517-default-wait-5-second-before-termination

DMVP-7517: have default preStop lifecycle rule with 5 seconds sleep/await to allow standard web apps have graceful shutdown

Author: mrdntgrn

charts/base/Chart.yaml

@@ -15,10 +15,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.3.7
+version: 0.3.8
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "0.3.7"
+appVersion: "0.3.8"

charts/base/README.md

@@ -725,11 +725,22 @@ rolloutStrategy:
           namespace: ingress-nginx
 ```
 
-## Deprecations and incompatible changes
+## Deprecations and incompatible changes and release important notes
 
 ### Deprecations
  - 0.3.0: `deployment` option is deprecated and all underlying fields are now available as top level variables, so that `deployment.volumes` moved to `volumes`, `deployment.additionalvolumeMounts` to `additionalVolumeMounts` and `deployment.lifecycle` to `lifecycle`
  - 0.3.0: for mounting volume to extra container use `container: <extra-container-name>` or `container: [<extra-container-name>]` instead of `container: extra` config in new `volumes` listing.
 
 ### incompatible changes
  - 0.3.0: the `extraContainer.deployment.volumes` field have been removed, use `volumes` field with `container: <extra-container-name>` for having volume mounted to extra container
+
+### release important notes
+ - Version 0.3.8:
+   This release introduces a new feature that enhances the robustness of pod termination by implementing a default preStop lifecycle hook.
+    - Default preStop Hook: Version 0.3.8 introduces the `defaultLifecycle` variable, which, by default, automatically configures a preStop hook with a `sleep 5` command.
+    - Graceful Pod Termination: This 5-second delay before container shutdown provides sufficient time for the pod's IP address to be unregistered from service target lists. This proactive measure minimizes the occurrence of failed HTTP requests during pod termination, leading to improved application stability and user experience.
+    - sleep Command Dependency: While the absence of the sleep command within a container's operating system will prevent the 5-second delay from functioning, but will not produce any additional issues, it is strongly recommended to include or install the sleep utility in your container images to ensure the intended graceful termination behavior.
+    - Custom preStop Configurations: If your containers already have custom `preStop` lifecycle configurations defined, these existing configurations will take precedence, effectively overriding the default configuration introduced in this release. However, to benefit from the graceful termination feature, it is advisable to incorporate a sleep 5 command within your custom `preStop` configurations.
+    - Scope of Default Configuration: The default `preStop` configuration is applied uniformly to all containers within a pod, including the main application container and all extra containers.
+    - Default Timeout Considerations: The default 5-second sleep duration has been determined through testing on standard web applications within an EKS cluster and is generally sufficient for facilitating a graceful shutdown. However, applications with longer request processing times (exceeding 2 seconds) may require an increased timeout value. You can adjust this by defining custom `preStop` configurations with a longer sleep duration.
+    - Disabling the Default Configuration: The default `preStop` hook can be easily disabled by setting the Helm chart value `defaultLifecycle.enabled=false`.

charts/base/templates/deployment.yaml

@@ -1,7 +1,8 @@
 {{ $baseFullName := (include "base.fullname" .) }}
 {{ $volumes := (coalesce .Values.volumes (.Values.deployment).volumes) }}
 {{ $additionalVolumeMounts := (coalesce .Values.additionalVolumeMounts (.Values.deployment).additionalvolumeMounts) }}
-{{ $lifecycle := (coalesce .Values.lifecycle (.Values.deployment).lifecycle) }}
+{{ $defaultLifecycle := ternary (unset (deepCopy .Values.defaultLifecycle) "enabled" ) (dict) .Values.defaultLifecycle.enabled }}
+{{ $lifecycle := merge (coalesce .Values.lifecycle (.Values.deployment).lifecycle) $defaultLifecycle }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -300,9 +301,9 @@ spec:
             {{- toYaml $extraContainer.additionalVolumeMounts | nindent 12 }}
             {{- end }}
           {{- end }}
-          {{- if (coalesce $extraContainer.lifecycle ($extraContainer.deployment).lifecycle)  }}
+          {{- if (merge (coalesce $extraContainer.lifecycle ($extraContainer.deployment).lifecycle) $defaultLifecycle) }}
           lifecycle:
-            {{- toYaml (coalesce $extraContainer.lifecycle ($extraContainer.deployment).lifecycle) | nindent 12 }}
+            {{- toYaml (merge (coalesce $extraContainer.lifecycle ($extraContainer.deployment).lifecycle) $defaultLifecycle) | nindent 12 }}
           {{- end }}
         {{- end }}
         {{- end }}

charts/base/values.yaml

@@ -318,6 +318,16 @@ pdb:
   minAvailable: 1
   pdbName:
 
+# the default lifecycle rule to apply which allows to have some kind of graceful shutdown by waiting/sleep for 5 seconds, even if there is no sleep command in os this will not produce issues and container will get shuted down
+defaultLifecycle:
+  enabled: true # whether this default is enabled, by default it is enabled but can be disabled if needed by setting this to false
+  preStop:
+    exec:
+      command:
+        - sh
+        - -c
+        - sleep 5 # the sleep interval can vary for app, if you still get issues at pod termination consider increasing it little by little to get optimal one for you, based on tests the 5 seconds seems suitable for standard web app on eks to get the pod IP removed from service targets and to finish active client requests/connections to not have failed request at pod termination
+
 serviceMonitor:
   enabled: false
   interval: 30s

examples/base/basic.yaml

@@ -0,0 +1,49 @@
+# helm diff upgrade --install -n localhost basic ./charts/base/ -f ./examples/base/basic.yaml
+
+image:
+  repository: mendhak/http-https-echo
+  tag: 34
+
+containerPort: 8080
+
+config:
+  ROLLOUT_TRIGGER_VERSION: "8"
+
+service:
+  enabled: true
+  type: ClusterIP
+
+autoscaling:
+  enabled: true
+  minReplicas: 1
+  maxReplicas: 5
+  targetCPUUtilizationPercentage: 99
+
+pdb:
+  enabled: true
+  minAvailable: 1
+
+readinessProbe:
+  initialDelaySeconds: 5
+  failureThreshold: 1
+  httpGet:
+    path: /health
+    port: http
+livenessProbe:
+  initialDelaySeconds: 5
+  failureThreshold: 3
+  httpGet:
+    path: /health
+    port: http
+resources:
+  requests:
+    cpu: 100m
+
+setIngressClassByField: true
+ingress:
+  enabled: true
+  class: nginx
+  hosts:
+    - host: http-echo-basic.localhost
+      paths:
+        - path: "/ping"

examples/base/with-canary-deployment/canary-with-manual-approval.yaml

@@ -98,46 +98,52 @@ rolloutStrategy:
 #     const cookieVersion = "canaryNewWebsite"; // the cookie name which defines the version of website that user should see
 #     const cookieSwitchSource = "canarySwitchSource"; // this cookie allows to keep in browser if user have been switched automatically to new/old website version or manually(by button click)
 #     const cookieAutoSwitchPercent = "canaryAutoSwitchPercent"; // we store this data in cookie to have the data in user/client browser that this were been proceeded by this percent of auto switch
-#     const cookieDefaultExpire = "expires=Mon, 14 Jul 2025 07:58:08 GMT"; // by default this expire date will be set on cookies, it is about 3 months, this is for auto cleanup of above 3 used cookies after this period
+#     const cookieDefaultExpire = "expires=Mon, 14 Jul 2025 07:58:08 UTC"; // by default this expire date will be set on cookies, it is about 3 months, this is for auto cleanup of above 3 used cookies after this period
 #     const removeCookieExpire = "expires=Thu, 01 Jan 1970 00:00:00 UTC"; // this back-day expire date for cookies is just for cleanup/remove of cookies
 #     let cookies = [];
 #     switch(action) {
 #         case "switch-auto":
 #             if ( !document.cookie.includes(`${cookieSwitchSource}=manual`) // we do auto switch for ones who do not change to new/old manually
 #                   && (!document.cookie.includes(`${cookieVersion}=`) // if this is first load
-#                   || !document.cookie.includes(`${cookieAutoSwitchPercent}=${newVersionAutoSwitchUsersPercent}`)) // or if percent changed
+#                   || (!document.cookie.includes(`${cookieAutoSwitchPercent}=${newVersionAutoSwitchUsersPercent}`) && !document.cookie.includes(`${cookieVersion}=always`))) // or if percent changed and user is not using new version of website
 #                 ) {
 #                 cookies = [
-#                     `${cookieSwitchSource}=auto`,
-#                     `${cookieAutoSwitchPercent}=${newVersionAutoSwitchUsersPercent}`
+#                     `${cookieSwitchSource}=auto; ${cookieDefaultExpire}`,
+#                     `${cookieAutoSwitchPercent}=${newVersionAutoSwitchUsersPercent}; ${cookieDefaultExpire}`
 #                 ];
 #                 if ((Math.floor(Math.random() * 100) + 1) < newVersionAutoSwitchUsersPercent) {
-#                     cookies.push(`${cookieVersion}=always`);
+#                     cookies.push(`${cookieVersion}=always; ${cookieDefaultExpire}`);
 #                 } else {
-#                     cookies.push(`${cookieVersion}=no`);
+#                     cookies.push(`${cookieVersion}=no; ${cookieDefaultExpire}`);
 #                 }
 #             }
 #             break;
 #         case "switch-to-new-manual":
-#             cookies = [
-#                 `${cookieVersion}=always; ${cookieDefaultExpire}`,
-#                 `${cookieSwitchSource}=manual; ${cookieDefaultExpire}`,
-#                 `${cookieAutoSwitchPercent}=r; ${removeCookieExpire}`
-#             ];
+#             if (!document.cookie.includes(`${cookieVersion}=always`)) { // if this cookies were not been set yet
+#               cookies = [
+#                   `${cookieVersion}=always; ${cookieDefaultExpire}`,
+#                   `${cookieSwitchSource}=manual; ${cookieDefaultExpire}`,
+#                   `${cookieAutoSwitchPercent}=r; ${removeCookieExpire}`
+#               ];
+#             }
 #             break;
 #         case "switch-to-old-manual":
-#             cookies = [
-#                 `${cookieVersion}=no; ${cookieDefaultExpire}`,
-#                 `${cookieSwitchSource}=manual; ${cookieDefaultExpire}`,
-#                 `${cookieAutoSwitchPercent}=r; ${removeCookieExpire}`
-#             ];
+#             if (!document.cookie.includes(`${cookieVersion}=no`)) { // if this cookies were not been set yet
+#               cookies = [
+#                   `${cookieVersion}=no; ${cookieDefaultExpire}`,
+#                   `${cookieSwitchSource}=manual; ${cookieDefaultExpire}`,
+#                   `${cookieAutoSwitchPercent}=r; ${removeCookieExpire}`
+#               ];
+#             }
 #             break;
 #         case "reset":
-#             cookies = [
-#                 `${cookieVersion}=r; ${removeCookieExpire}`,
-#                 `${cookieSwitchSource}=r; ${removeCookieExpire}`,
-#                 `${cookieAutoSwitchPercent}=r; ${removeCookieExpire}`
-#             ];
+#             if (document.cookie.includes(`${cookieVersion}=`)) { // if this cookies were been set
+#               cookies = [
+#                   `${cookieVersion}=r; ${removeCookieExpire}`,
+#                   `${cookieSwitchSource}=r; ${removeCookieExpire}`,
+#                   `${cookieAutoSwitchPercent}=r; ${removeCookieExpire}`
+#               ];
+#             }
 #             break;
 #         default:
 #             console.log("undefined action");
@@ -155,7 +161,7 @@ rolloutStrategy:
 #     }
 
 #     const switchButton = document.createElement("button");
-#     switchButton.style.cssText = "position: fixed;z-index: 999;top: 1px";
+#     switchButton.style.cssText = "position: fixed;z-index: 999;bottom: 1px; right: 1px; color: black; background: white; border: 3px solid black; padding: 1px 6px;";
 #     switchButton.innerHTML = isWebsiteNewVersion() ? "Switch to old UI" : "Switch to New UI";
 #     switchButton.onclick = function () {
 #         if (isWebsiteNewVersion()) {
@@ -167,6 +173,6 @@ rolloutStrategy:
 
 #     document.body.appendChild(switchButton);
 # }
-# canaryAction("switch-auto"); // possible values "switch-auto", "reset", "switch-to-new-manual" and "switch-to-old-manual"
-# createSwitchButton(); // place/crate a switch button in UI for manual switching(should be seen bottom left corner)
+# document.location.href.includes("staging-canary") && canaryAction("switch-auto"); // possible values "switch-auto", "reset", "switch-to-new-manual" and "switch-to-old-manual"
+# document.location.href.includes("staging-canary") && createSwitchButton(); // place/crate a switch button in UI for manual switching(should be seen bottom left corner)
 # ```

examples/base/with-extra-container.yaml

@@ -0,0 +1,61 @@
+# helm diff upgrade --install -n localhost with-extra-container ./charts/base/ -f ./examples/base/with-extra-container.yaml
+
+image:
+  repository: mendhak/http-https-echo
+  tag: 34
+
+containerPort: 8080
+
+config:
+  ROLLOUT_TRIGGER_VERSION: "8"
+
+service:
+  enabled: true
+  type: ClusterIP
+
+autoscaling:
+  enabled: true
+  minReplicas: 1
+  maxReplicas: 5
+  targetCPUUtilizationPercentage: 99
+
+pdb:
+  enabled: true
+  minAvailable: 1
+
+readinessProbe:
+  initialDelaySeconds: 5
+  failureThreshold: 1
+  httpGet:
+    path: /health
+    port: http
+livenessProbe:
+  initialDelaySeconds: 5
+  failureThreshold: 3
+  httpGet:
+    path: /health
+    port: http
+resources:
+  requests:
+    cpu: 100m
+
+setIngressClassByField: true
+ingress:
+  enabled: true
+  class: nginx
+  hosts:
+    - host: with-extra-container.localhost
+      paths:
+        - path: "/ping"
+
+extraContainer:
+  - name: extra-http-echo
+    containerPort: 8081
+    service:
+      enabled: true
+    image:
+      repository: mendhak/http-https-echo
+      tag: 33
+    extraEnv:
+      HTTP_PORT: 8180
+      HTTPS_PORT: 8543

examples/base/with-linkerd.yaml

@@ -0,0 +1,56 @@
+# helm diff upgrade --install -n localhost with-linkerd ./charts/base/ -f ./examples/base/with-linkerd.yaml
+
+image:
+  repository: mendhak/http-https-echo
+  tag: 34
+
+containerPort: 8080
+
+config:
+  ROLLOUT_TRIGGER_VERSION: "8"
+
+podAnnotations:
+  linkerd.io/inject: enabled
+  config.linkerd.io/proxy-cpu-request: 50m
+  config.linkerd.io/proxy-cpu-limit: 50m
+
+service:
+  enabled: true
+  type: ClusterIP
+
+autoscaling:
+  enabled: true
+  minReplicas: 1
+  maxReplicas: 5
+  targetCPUUtilizationPercentage: 99
+
+pdb:
+  enabled: true
+  minAvailable: 1
+
+readinessProbe:
+  initialDelaySeconds: 5
+  failureThreshold: 1
+  httpGet:
+    path: /health
+    port: http
+livenessProbe:
+  initialDelaySeconds: 5
+  failureThreshold: 3
+  httpGet:
+    path: /health
+    port: http
+resources:
+  requests:
+    cpu: 100m
+  limits:
+    cpu: 100m
+
+setIngressClassByField: true
+ingress:
+  enabled: true
+  class: nginx
+  hosts:
+    - host: http-echo-with-linkerd.localhost
+      paths:
+        - path: "/ping"