From 2e6d459bfefc9250f28741af3cb6f95f7adcc4ba Mon Sep 17 00:00:00 2001 From: Joonas Somero Date: Thu, 12 Sep 2024 13:55:58 +0300 Subject: [PATCH 01/32] Update dependencies to match production --- requirements.txt | 77 ++++++++++++++++++++++-------------------------- 1 file changed, 35 insertions(+), 42 deletions(-) diff --git a/requirements.txt b/requirements.txt index b8dd964..86c0e39 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,51 +1,44 @@ -# Packages for preview -# (commented-out already satisfied in packages for static sites) -# GitPython==3.1.41 +# Additional packages for preview Flask==2.2.5 - -# Dependencies for preview -# click==8.1.3 -# gitdb==4.0.9 -# importlib-metadata==4.12.0 itsdangerous==2.1.2 -# Jinja2==3.1.4 -# MarkupSafe==2.1.1 -# smmap==5.0.0 werkzeug==3.0.3 -# zipp==3.8.1 - # Packages for building the static pages -# This is what we actually install -mkdocs==1.4.3 -mkdocs-git-revision-date-localized-plugin==1.1.0 -mkdocs-material==8.4.0 -mkdocs-material-extensions==1.0.3 -mkdocs-redirects==1.2.0 -mkdocs-section-index==0.3.4 -feedparser==6.0.10 - -# These are dependencies of the above -Babel==2.10.3 -click==8.1.3 +babel==2.16.0 +certifi==2024.7.4 +charset-normalizer==3.3.2 +click==8.1.7 +colorama==0.4.6 +feedparser==6.0.11 ghp-import==2.1.0 -gitdb==4.0.9 -GitPython==3.1.41 -importlib-metadata==4.12.0 -Jinja2==3.1.3 -Markdown==3.3.7 -MarkupSafe==2.1.1 +gitdb==4.0.11 +GitPython==3.1.43 +idna==3.7 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 mergedeep==1.3.4 -packaging==21.3 -Pygments==2.15.0 -pymdown-extensions==10.3.1 -pyparsing==3.0.9 -python-dateutil==2.8.2 -pytz==2022.2.1 -PyYAML==6.0 -pyyaml-env-tag==0.1 +mkdocs==1.6.0 +mkdocs-get-deps==0.2.0 +mkdocs-git-revision-date-localized-plugin==1.2.6 +mkdocs-material==9.5.31 +mkdocs-material-extensions==1.3.1 +mkdocs-redirects==1.2.1 +mkdocs-section-index==0.3.9 +packaging==24.1 +paginate==0.5.6 +pathspec==0.12.1 +platformdirs==4.2.2 +Pygments==2.18.0 +pymdown-extensions==10.9 +python-dateutil==2.9.0.post0 +pytz==2024.1 +PyYAML==6.0.2 +pyyaml_env_tag==0.1 +regex==2024.7.24 +requests==2.32.3 sgmllib3k==1.0.0 six==1.16.0 -smmap==5.0.0 -watchdog==2.1.9 -zipp==3.19.1 +smmap==5.0.1 +urllib3==2.2.2 +watchdog==4.0.2 From e6faeed07a3d1b66cdaa1e31155da7769354a65b Mon Sep 17 00:00:00 2001 From: "alvaro.gonzalez" Date: Fri, 8 Nov 2024 09:16:08 +0200 Subject: [PATCH 02/32] Add build() back --- app.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app.py b/app.py index d8b37e3..1863429 100755 --- a/app.py +++ b/app.py @@ -175,6 +175,8 @@ def listenBuild(secret): if not secret == config["secret"]: return "Access denied" + build() + response = Response('built started') response.call_on_close(build) From bd964d05694301725618c82162bbcb2dee0aff7a Mon Sep 17 00:00:00 2001 From: "alvaro.gonzalez" Date: Fri, 8 Nov 2024 10:22:27 +0200 Subject: [PATCH 03/32] NO need for that --- app.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/app.py b/app.py index 1863429..d8b37e3 100755 --- a/app.py +++ b/app.py @@ -175,8 +175,6 @@ def listenBuild(secret): if not secret == config["secret"]: return "Access denied" - build() - response = Response('built started') response.call_on_close(build) From 6cb1b679747ec578ca56f7a0fc03e5e98b9a9f1e Mon Sep 17 00:00:00 2001 From: "alvaro.gonzalez" Date: Fri, 8 Nov 2024 14:49:12 +0200 Subject: [PATCH 04/32] Revert "Update dependencies to match production (#44)" This reverts commit 079580275a9aadfa20e02e7b3803da24cbdfdb85. --- requirements.txt | 77 ++++++++++++++++++++++++++---------------------- 1 file changed, 42 insertions(+), 35 deletions(-) diff --git a/requirements.txt b/requirements.txt index 86c0e39..b8dd964 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,44 +1,51 @@ -# Additional packages for preview +# Packages for preview +# (commented-out already satisfied in packages for static sites) +# GitPython==3.1.41 Flask==2.2.5 + +# Dependencies for preview +# click==8.1.3 +# gitdb==4.0.9 +# importlib-metadata==4.12.0 itsdangerous==2.1.2 +# Jinja2==3.1.4 +# MarkupSafe==2.1.1 +# smmap==5.0.0 werkzeug==3.0.3 +# zipp==3.8.1 + # Packages for building the static pages -babel==2.16.0 -certifi==2024.7.4 -charset-normalizer==3.3.2 -click==8.1.7 -colorama==0.4.6 -feedparser==6.0.11 +# This is what we actually install +mkdocs==1.4.3 +mkdocs-git-revision-date-localized-plugin==1.1.0 +mkdocs-material==8.4.0 +mkdocs-material-extensions==1.0.3 +mkdocs-redirects==1.2.0 +mkdocs-section-index==0.3.4 +feedparser==6.0.10 + +# These are dependencies of the above +Babel==2.10.3 +click==8.1.3 ghp-import==2.1.0 -gitdb==4.0.11 -GitPython==3.1.43 -idna==3.7 -Jinja2==3.1.4 -Markdown==3.6 -MarkupSafe==2.1.5 +gitdb==4.0.9 +GitPython==3.1.41 +importlib-metadata==4.12.0 +Jinja2==3.1.3 +Markdown==3.3.7 +MarkupSafe==2.1.1 mergedeep==1.3.4 -mkdocs==1.6.0 -mkdocs-get-deps==0.2.0 -mkdocs-git-revision-date-localized-plugin==1.2.6 -mkdocs-material==9.5.31 -mkdocs-material-extensions==1.3.1 -mkdocs-redirects==1.2.1 -mkdocs-section-index==0.3.9 -packaging==24.1 -paginate==0.5.6 -pathspec==0.12.1 -platformdirs==4.2.2 -Pygments==2.18.0 -pymdown-extensions==10.9 -python-dateutil==2.9.0.post0 -pytz==2024.1 -PyYAML==6.0.2 -pyyaml_env_tag==0.1 -regex==2024.7.24 -requests==2.32.3 +packaging==21.3 +Pygments==2.15.0 +pymdown-extensions==10.3.1 +pyparsing==3.0.9 +python-dateutil==2.8.2 +pytz==2022.2.1 +PyYAML==6.0 +pyyaml-env-tag==0.1 sgmllib3k==1.0.0 six==1.16.0 -smmap==5.0.1 -urllib3==2.2.2 -watchdog==4.0.2 +smmap==5.0.0 +watchdog==2.1.9 +zipp==3.19.1 From 99fcb12ea5a4f5a1f8f809639b39b2681d1bb07f Mon Sep 17 00:00:00 2001 From: "alvaro.gonzalez" Date: Fri, 8 Nov 2024 14:50:54 +0200 Subject: [PATCH 05/32] Revert "Revert "Update dependencies to match production (#44)"" This reverts commit 6cb1b679747ec578ca56f7a0fc03e5e98b9a9f1e. --- requirements.txt | 77 ++++++++++++++++++++++-------------------------- 1 file changed, 35 insertions(+), 42 deletions(-) diff --git a/requirements.txt b/requirements.txt index b8dd964..86c0e39 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,51 +1,44 @@ -# Packages for preview -# (commented-out already satisfied in packages for static sites) -# GitPython==3.1.41 +# Additional packages for preview Flask==2.2.5 - -# Dependencies for preview -# click==8.1.3 -# gitdb==4.0.9 -# importlib-metadata==4.12.0 itsdangerous==2.1.2 -# Jinja2==3.1.4 -# MarkupSafe==2.1.1 -# smmap==5.0.0 werkzeug==3.0.3 -# zipp==3.8.1 - # Packages for building the static pages -# This is what we actually install -mkdocs==1.4.3 -mkdocs-git-revision-date-localized-plugin==1.1.0 -mkdocs-material==8.4.0 -mkdocs-material-extensions==1.0.3 -mkdocs-redirects==1.2.0 -mkdocs-section-index==0.3.4 -feedparser==6.0.10 - -# These are dependencies of the above -Babel==2.10.3 -click==8.1.3 +babel==2.16.0 +certifi==2024.7.4 +charset-normalizer==3.3.2 +click==8.1.7 +colorama==0.4.6 +feedparser==6.0.11 ghp-import==2.1.0 -gitdb==4.0.9 -GitPython==3.1.41 -importlib-metadata==4.12.0 -Jinja2==3.1.3 -Markdown==3.3.7 -MarkupSafe==2.1.1 +gitdb==4.0.11 +GitPython==3.1.43 +idna==3.7 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 mergedeep==1.3.4 -packaging==21.3 -Pygments==2.15.0 -pymdown-extensions==10.3.1 -pyparsing==3.0.9 -python-dateutil==2.8.2 -pytz==2022.2.1 -PyYAML==6.0 -pyyaml-env-tag==0.1 +mkdocs==1.6.0 +mkdocs-get-deps==0.2.0 +mkdocs-git-revision-date-localized-plugin==1.2.6 +mkdocs-material==9.5.31 +mkdocs-material-extensions==1.3.1 +mkdocs-redirects==1.2.1 +mkdocs-section-index==0.3.9 +packaging==24.1 +paginate==0.5.6 +pathspec==0.12.1 +platformdirs==4.2.2 +Pygments==2.18.0 +pymdown-extensions==10.9 +python-dateutil==2.9.0.post0 +pytz==2024.1 +PyYAML==6.0.2 +pyyaml_env_tag==0.1 +regex==2024.7.24 +requests==2.32.3 sgmllib3k==1.0.0 six==1.16.0 -smmap==5.0.0 -watchdog==2.1.9 -zipp==3.19.1 +smmap==5.0.1 +urllib3==2.2.2 +watchdog==4.0.2 From 68426711b62f60e3682dbd8ea2aa6db19d2cb210 Mon Sep 17 00:00:00 2001 From: "alvaro.gonzalez" Date: Fri, 8 Nov 2024 14:51:04 +0200 Subject: [PATCH 06/32] Revert "NO need for that" This reverts commit bd964d05694301725618c82162bbcb2dee0aff7a. --- app.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app.py b/app.py index d8b37e3..1863429 100755 --- a/app.py +++ b/app.py @@ -175,6 +175,8 @@ def listenBuild(secret): if not secret == config["secret"]: return "Access denied" + build() + response = Response('built started') response.call_on_close(build) From 96c8a3aba7dc4864600783da05bc75860890aeab Mon Sep 17 00:00:00 2001 From: "alvaro.gonzalez" Date: Fri, 8 Nov 2024 14:51:07 +0200 Subject: [PATCH 07/32] Revert "Revert "Use threads instead" (#45)" This reverts commit 256fad5b15c9f1d3586a74a623666779122ae248. --- app.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app.py b/app.py index 1863429..0616f45 100755 --- a/app.py +++ b/app.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -import git, os, shutil, json +import git, os, shutil, json, threading from flask import Flask, Response @@ -179,7 +179,8 @@ def listenBuild(secret): response = Response('built started') - response.call_on_close(build) + thread = threading.Thread(target=build) + thread.start() return response @@ -187,6 +188,7 @@ def build(): print("Start build") + repo, origin = initRepo(config["workPath"], config["remoteUrl"]) output = "" @@ -229,7 +231,6 @@ def build(): exit(1) listenBuild(config["secret"]) - build() app.run(debug=config["debug"]=="True", port=defaultPort, From 1e655f6957fc82da96307881453a20003c718e9d Mon Sep 17 00:00:00 2001 From: "alvaro.gonzalez" Date: Mon, 11 Nov 2024 16:14:15 +0200 Subject: [PATCH 08/32] Read and write state to a file, so we do not rebuild the whole this every time --- app.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/app.py b/app.py index 0616f45..8541b8c 100755 --- a/app.py +++ b/app.py @@ -5,6 +5,7 @@ # defaults +defaultStateFile='/tmp/build_state.json' defaultWorkPath = "work" defaultBuildRoot = "/tmp/preview-bot/builds" defaultRemoteUrl = "https://github.com/CSCfi/csc-user-guide" @@ -12,6 +13,11 @@ defaultSecret = "changeme" # we are using secret but we should be utilizing whitelists defaultPort = 8081 +try: + STATEFILE = os.environ["STATEFILE"] +except KeyError: + STATEFILE = '/tmp/build_state.json' + try: workPath = os.environ["WORKPATH"] except KeyError: @@ -90,6 +96,7 @@ def buildRef(repo, ref, state): print(str(ref.commit), state["built"]) buildpath = os.path.join(config["buildRoot"], str(ref)) + print('Checking: %s == %s' % (str(ref.commit), state["built"])) if not str(ref.commit) == state["built"] or not os.path.isdir(buildpath): print("re-building %s in %s" % (ref, ref.commit)) repo.git.reset('--hard',ref) @@ -188,6 +195,7 @@ def build(): print("Start build") + buildState = read_state() repo, origin = initRepo(config["workPath"], config["remoteUrl"]) @@ -208,9 +216,18 @@ def build(): # Refresh builds for ref in origin.refs: buildRef(repo, ref, buildState[str(ref)]) + write_state(buildState) cleanUpZombies() +def write_state(state): + with open(STATEFILE, 'w') as file: + json.dump(state, file) + +def read_state(): + with open(STATEFILE, 'r') as file: + return json.load(file) + ### Entry functions if __name__=="__main__": From f3cae2e5f54c3c2ef7258ba81000f8bfdaabcad6 Mon Sep 17 00:00:00 2001 From: "alvaro.gonzalez" Date: Tue, 12 Nov 2024 09:06:46 +0200 Subject: [PATCH 09/32] Add try/except so it does not die when the state file is missing --- app.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/app.py b/app.py index 8541b8c..56f9898 100755 --- a/app.py +++ b/app.py @@ -225,9 +225,11 @@ def write_state(state): json.dump(state, file) def read_state(): - with open(STATEFILE, 'r') as file: - return json.load(file) - + try: + with open(STATEFILE, 'r') as file: + return json.load(file) + except FileNotFoundError: + return {} ### Entry functions if __name__=="__main__": From de1e43eb6de75bf07f942656adfa565cf80b6481 Mon Sep 17 00:00:00 2001 From: "alvaro.gonzalez" Date: Fri, 8 Nov 2024 16:10:27 +0200 Subject: [PATCH 10/32] First version, allows a deployment of the repository and basic customization. --- app.py | 7 +- helm/.helmignore | 23 +++++ helm/Chart.yaml | 24 +++++ .../csc-guide-preview-hook-route.yaml | 17 ++++ helm/templates/csc-guide-preview-route.yaml | 19 ++++ helm/templates/docs-preview-build.yaml | 30 +++++++ helm/templates/docs-preview-deploy.yaml | 87 +++++++++++++++++++ helm/templates/docs-preview-hook-service.yaml | 19 ++++ helm/templates/docs-preview-is.yaml | 5 ++ helm/templates/docs-preview-service.yaml | 22 +++++ helm/templates/nginx-conf-configmap.yaml | 16 ++++ helm/values.yaml | 8 ++ 12 files changed, 276 insertions(+), 1 deletion(-) create mode 100644 helm/.helmignore create mode 100644 helm/Chart.yaml create mode 100644 helm/templates/csc-guide-preview-hook-route.yaml create mode 100644 helm/templates/csc-guide-preview-route.yaml create mode 100644 helm/templates/docs-preview-build.yaml create mode 100644 helm/templates/docs-preview-deploy.yaml create mode 100644 helm/templates/docs-preview-hook-service.yaml create mode 100644 helm/templates/docs-preview-is.yaml create mode 100644 helm/templates/docs-preview-service.yaml create mode 100644 helm/templates/nginx-conf-configmap.yaml create mode 100644 helm/values.yaml diff --git a/app.py b/app.py index 0616f45..a86fb53 100755 --- a/app.py +++ b/app.py @@ -32,6 +32,11 @@ except KeyError: buildSecret = defaultSecret +try: + remoteUrl = os.environ["REMOTEURL"] +except KeyError: + remoteUrl = "https://github.com/CSCfi/csc-user-guide" + # Configurations in CONFIGFILE will override other environment variables try: configFile = os.environ["CONFIGFILE"] @@ -42,7 +47,7 @@ config = { "workPath": workPath, - "remoteUrl": "https://github.com/CSCfi/csc-user-guide", + "remoteUrl": remoteUrl, "buildRoot": buildRoot, "debug": "False", "secret": buildSecret, diff --git a/helm/.helmignore b/helm/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/helm/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/helm/Chart.yaml b/helm/Chart.yaml new file mode 100644 index 0000000..fc38992 --- /dev/null +++ b/helm/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: helm +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.16.0" diff --git a/helm/templates/csc-guide-preview-hook-route.yaml b/helm/templates/csc-guide-preview-hook-route.yaml new file mode 100644 index 0000000..1383659 --- /dev/null +++ b/helm/templates/csc-guide-preview-hook-route.yaml @@ -0,0 +1,17 @@ +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: docs-preview-build-hook +spec: + host: {{ .Release.Namespace }}-hook.2.rahtiapp.fi + port: + targetPort: 8081 + tls: + insecureEdgeTerminationPolicy: Redirect + termination: edge + to: + kind: Service + name: docs-preview-hook + weight: 100 + wildcardPolicy: None +status: {} diff --git a/helm/templates/csc-guide-preview-route.yaml b/helm/templates/csc-guide-preview-route.yaml new file mode 100644 index 0000000..3cfedbc --- /dev/null +++ b/helm/templates/csc-guide-preview-route.yaml @@ -0,0 +1,19 @@ +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + labels: + app: docs-preview + name: docs-preview-route +spec: + host: {{ .Values.host }} + port: + targetPort: 8080-tcp + tls: + insecureEdgeTerminationPolicy: Redirect + termination: edge + to: + kind: Service + name: docs-preview + weight: 100 + wildcardPolicy: None +status: {} diff --git a/helm/templates/docs-preview-build.yaml b/helm/templates/docs-preview-build.yaml new file mode 100644 index 0000000..25ee0cd --- /dev/null +++ b/helm/templates/docs-preview-build.yaml @@ -0,0 +1,30 @@ +apiVersion: build.openshift.io/v1 +kind: BuildConfig +metadata: + labels: + app: docs-preview + name: docs-preview +spec: + failedBuildsHistoryLimit: 2 + output: + to: + kind: ImageStreamTag + name: docs-preview:latest + postCommit: {} + resources: {} + runPolicy: Serial + source: + git: + uri: {{ .Values.git.source }} + type: Git + strategy: + sourceStrategy: + from: + kind: DockerImage + name: registry.fedoraproject.org/f33/python3 + type: Source + successfulBuildsHistoryLimit: 1 + triggers: + - type: ImageChange + - type: ConfigChange +status: {} diff --git a/helm/templates/docs-preview-deploy.yaml b/helm/templates/docs-preview-deploy.yaml new file mode 100644 index 0000000..10c3127 --- /dev/null +++ b/helm/templates/docs-preview-deploy.yaml @@ -0,0 +1,87 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + image.openshift.io/triggers: '[{"from":{"kind":"ImageStreamTag","name":"docs-preview:latest"},"fieldPath":"spec.template.spec.containers[?(@.name==\"docs-preview\")].image"}]' + labels: + app: docs-preview + name: docs-preview +spec: + progressDeadlineSeconds: 600 + replicas: {{ .Values.replicas }} + revisionHistoryLimit: 10 + selector: + matchLabels: + deployment: docs-preview + strategy: + type: Recreate + template: + metadata: + labels: + deployment: docs-preview + spec: + containers: + - image: bitnami/nginx:1.16-centos-7 + imagePullPolicy: IfNotPresent + name: web-server + ports: + - containerPort: 8080 + protocol: TCP + resources: + limits: + cpu: "1" + memory: 1G + requests: + cpu: 200m + memory: 200M + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /app + name: content-vol + - mountPath: /opt/bitnami/nginx/conf/server_blocks + name: nginx-conf + - env: + - name: BUILDSECRET + value: {{ .Values.secret }} + - name: BUILDROOT + value: /builds + - name: WORKPATH + value: /work + - name: REMOTEURL + value: {{ .Values.git.docs }} + image: ' ' # image-registry.openshift-image-registry.svc:5000/{{ .Release.Namespace }}/docs-preview:latest + imagePullPolicy: IfNotPresent + name: docs-preview + ports: + - containerPort: 8081 + protocol: TCP + resources: + limits: + cpu: "1" + memory: 1G + requests: + cpu: 200m + memory: 200M + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /builds + name: content-vol + - mountPath: /work + name: workpath + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + terminationGracePeriodSeconds: 30 + volumes: + - emptyDir: {} + name: content-vol + - emptyDir: {} + name: workpath + - configMap: + defaultMode: 420 + name: nginx-config + name: nginx-conf +status: {} diff --git a/helm/templates/docs-preview-hook-service.yaml b/helm/templates/docs-preview-hook-service.yaml new file mode 100644 index 0000000..071756a --- /dev/null +++ b/helm/templates/docs-preview-hook-service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: docs-preview-hook +spec: + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - port: 8081 + protocol: TCP + targetPort: 8081 + selector: + deployment: docs-preview + sessionAffinity: None + type: ClusterIP +status: + loadBalancer: {} diff --git a/helm/templates/docs-preview-is.yaml b/helm/templates/docs-preview-is.yaml new file mode 100644 index 0000000..f118e77 --- /dev/null +++ b/helm/templates/docs-preview-is.yaml @@ -0,0 +1,5 @@ +apiVersion: image.openshift.io/v1 +kind: ImageStream +metadata: + name: docs-preview +status: {} diff --git a/helm/templates/docs-preview-service.yaml b/helm/templates/docs-preview-service.yaml new file mode 100644 index 0000000..13ddd90 --- /dev/null +++ b/helm/templates/docs-preview-service.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: docs-preview + name: docs-preview +spec: + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - name: 8080-tcp + port: 8080 + protocol: TCP + targetPort: 8080 + selector: + deployment: docs-preview + sessionAffinity: None + type: ClusterIP +status: + loadBalancer: {} diff --git a/helm/templates/nginx-conf-configmap.yaml b/helm/templates/nginx-conf-configmap.yaml new file mode 100644 index 0000000..4446992 --- /dev/null +++ b/helm/templates/nginx-conf-configmap.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +data: + my_server_block.conf: | + server { + listen 8080; + location / { + root /app; + index index.html; + autoindex on; + } + #rewrite ^/(.*[^/])$ $scheme://$http_host/$1/ permanent; + port_in_redirect off; + } +kind: ConfigMap +metadata: + name: nginx-config diff --git a/helm/values.yaml b/helm/values.yaml new file mode 100644 index 0000000..17caa68 --- /dev/null +++ b/helm/values.yaml @@ -0,0 +1,8 @@ +git: + source: https://github.com/cscfi/docs-preview + docs: https://github.com/cscfi/csc-user-guide +secret: + +host: csc-guide-preview.rahtiapp.fi + +replicas: 1 From c396cdb762816ce45750ee139c0a1aae8f95dcbe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Nov 2024 10:44:52 +0200 Subject: [PATCH 11/32] Bump werkzeug from 2.3.8 to 3.0.3 (#38) Bumps [werkzeug](https://github.com/pallets/werkzeug) from 2.3.8 to 3.0.3. - [Release notes](https://github.com/pallets/werkzeug/releases) - [Changelog](https://github.com/pallets/werkzeug/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/werkzeug/compare/2.3.8...3.0.3) --- updated-dependencies: - dependency-name: werkzeug dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Alvaro Gonzalez --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 86c0e39..b9a0495 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ # Additional packages for preview Flask==2.2.5 itsdangerous==2.1.2 -werkzeug==3.0.3 +werkzeug==3.0.6 # Packages for building the static pages babel==2.16.0 From b784d1922c771a84170d526da3c856e3a5ed9e07 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Nov 2024 11:01:44 +0200 Subject: [PATCH 12/32] Bump zipp from 3.8.1 to 3.19.1 (#40) Bumps [zipp](https://github.com/jaraco/zipp) from 3.8.1 to 3.19.1. - [Release notes](https://github.com/jaraco/zipp/releases) - [Changelog](https://github.com/jaraco/zipp/blob/main/NEWS.rst) - [Commits](https://github.com/jaraco/zipp/compare/v3.8.1...v3.19.1) --- updated-dependencies: - dependency-name: zipp dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Alvaro Gonzalez --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index b9a0495..e562399 100644 --- a/requirements.txt +++ b/requirements.txt @@ -42,3 +42,4 @@ six==1.16.0 smmap==5.0.1 urllib3==2.2.2 watchdog==4.0.2 + From 07b14aac28604397eef9e83ae3a6bca1fb15beec Mon Sep 17 00:00:00 2001 From: "alvaro.gonzalez" Date: Tue, 12 Nov 2024 16:15:32 +0200 Subject: [PATCH 13/32] Improve logging --- app.py | 65 ++++++++++++++++++++++++++++++---------------------------- 1 file changed, 34 insertions(+), 31 deletions(-) diff --git a/app.py b/app.py index 0616f45..01cc115 100755 --- a/app.py +++ b/app.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 import git, os, shutil, json, threading +import logging + from flask import Flask, Response # defaults @@ -44,7 +46,7 @@ "workPath": workPath, "remoteUrl": "https://github.com/CSCfi/csc-user-guide", "buildRoot": buildRoot, - "debug": "False", + "debug": "True", "secret": buildSecret, "prune": "True" } @@ -66,15 +68,15 @@ def initRepo(workPath, remote_url): try: origin = repo.remote('origin') except ValueError: - print("creating origin") + app.logger.info(f"Creating origin {remote_url} into {workPath}") origin = repo.create_remote('origin', remote_url) assert origin.exists() assert origin == repo.remotes.origin == repo.remotes['origin'] - + app.logger.info("* Fetching remote branches' content") for fetch_info in origin.fetch(None, None, prune=True): - print("Updated %s in %s" % (fetch_info.ref, fetch_info.commit)) + app.logger.info(" Branch [%s], commit [%s]" % (fetch_info.ref, fetch_info.commit)) return repo, origin @@ -87,23 +89,27 @@ def buildRef(repo, ref, state): """ global config - print(str(ref.commit), state["built"]) buildpath = os.path.join(config["buildRoot"], str(ref)) + app.logger.info('Checking [%s]: %s == %s' % (ref, str(ref.commit), state["built"])) if not str(ref.commit) == state["built"] or not os.path.isdir(buildpath): - print("re-building %s in %s" % (ref, ref.commit)) + app.logger.info(" [%s] re-building %s" % (ref, ref.commit)) repo.git.reset('--hard',ref) repo.git.checkout(ref) - print("buildpath = %s" % (buildpath)) + app.logger.debug(" [%s] buildpath = %s" % (ref, buildpath)) mkdirp(buildpath) scripts=["generate_alpha.sh","generate_by_system.sh","generate_new.sh","generate_glossary.sh"] for script in scripts: cmd = "sh -c 'cd %s && ./scripts/%s 2>&1'" % (config["workPath"],script) - print("Executing: %s" % (cmd)) cmdout = os.popen(cmd) - print(cmdout.read()) + line = cmdout.readline() + app.logger.info(f" [{ref}] # {cmd}") + if cmdout.close(): + app.logger.error(f" [{ref}] {line}") + else: + app.logger.info(f" [{ref}] {line}") # # WORKAROUND @@ -119,9 +125,9 @@ def buildRef(repo, ref, state): # cmd = "sh -c 'cd %s && mkdocs build --site-dir %s -f mkdocs.yml2 2>&1'" % (config["workPath"], buildpath) - print("Executing: %s" % (cmd)) + app.logger.info(f" [{ref}] # %s" % (cmd)) cmdout = os.popen(cmd) - print(cmdout.read()) + app.logger.debug(cmdout.read()) state["built"] = str(ref.commit) @@ -132,36 +138,34 @@ def cleanUpZombies(): * When ChildProcessError raises, it means that there are no children left """ - print("Cleaning up Zombies") + app.logger.info("* Cleaning up Zombies") spid = -1 while spid != 0: try: spid, status, rusage = os.wait3(os.WNOHANG) - print("Process %d with status %d" % (spid, status)) + app.logger.debug("* Process %d with status %d" % (spid, status)) except ChildProcessError: break def pruneBuilds(repo, origin): - repo, origin = initRepo(config["workPath"], config["remoteUrl"]) try: builtrefs = os.listdir(config["buildRoot"]+'/origin') except FileNotFoundError: - print("Clean buildRoot") + app.logger.debug("* Clean buildRoot") return srefs = [str(x) for x in origin.refs] builtrefs = ['origin/'+str(x) for x in builtrefs] - print("Pruning old builds.") + app.logger.debug("* Pruning old builds.") for bref in builtrefs: if not bref in srefs: - print('found stale preview: ' + bref) - remove_build = remove_build=config["buildRoot"] + '/' + bref - print('Removing ' + remove_build) + remove_build = config["buildRoot"] + '/' + bref + prinf(f" [{bref}] Removing {remove_build}") shutil.rmtree(remove_build) - print("Done pruning old builds.") + app.logger.debug("DONE pruning old builds.") ### Route functions ### @@ -175,9 +179,7 @@ def listenBuild(secret): if not secret == config["secret"]: return "Access denied" - build() - - response = Response('built started') + response = Response("Build started") thread = threading.Thread(target=build) thread.start() @@ -186,8 +188,7 @@ def listenBuild(secret): def build(): - print("Start build") - + app.logger.info("* Start build loop") repo, origin = initRepo(config["workPath"], config["remoteUrl"]) @@ -199,7 +200,7 @@ def build(): output = output + "Found %s (%s)
" % (sref, str(ref.commit)) if not sref in buildState: - print(sref + " not found in " + str(buildState)) + app.logger.debug(f"Adding {sref} branch to build state") buildState[sref] = {"sha": str(ref.commit), "status": "init", "built": None} # Prune nonexisting builds @@ -214,20 +215,22 @@ def build(): ### Entry functions if __name__=="__main__": + app.logger.setLevel(logging.INFO) + if not configFile == None: - print("Loading configuration from file: " + configFile) + app.logger.info("Loading configuration from file: " + configFile) with open(configFile) as config_file: config = json.load(config_file) buildSecret = config["secret"] workPath = config["workPath"] buildRoot = config["buildRoot"] - print("workPath: " + workPath) - print("buildRoot: " + buildRoot) - print("buildSecret: " + "******") + app.logger.info("workPath: " + workPath) + app.logger.info("buildRoot: " + buildRoot) + app.logger.info("buildSecret: " + buildSecret) if buildSecret == defaultSecret: - print("Don't use default secret since it's freely available in the internet") + app.logger.error("Don't use default secret since it's freely available in the internet") exit(1) listenBuild(config["secret"]) From 4c985e0cf1b810030e7fca877bf7d0b938894690 Mon Sep 17 00:00:00 2001 From: "alvaro.gonzalez" Date: Tue, 19 Nov 2024 15:01:16 +0200 Subject: [PATCH 14/32] Add parallel build of only the PR branch, this must be merged the last --- app.py | 104 +++++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 97 insertions(+), 7 deletions(-) diff --git a/app.py b/app.py index 0616f45..29c7c6c 100755 --- a/app.py +++ b/app.py @@ -1,7 +1,11 @@ #!/usr/bin/env python3 -import git, os, shutil, json, threading +import git, os, json, threading -from flask import Flask, Response +from shutil import copytree, rmtree + +from random import randint + +from flask import Flask, Response, request # defaults @@ -125,6 +129,66 @@ def buildRef(repo, ref, state): state["built"] = str(ref.commit) +def buildCommit(commit, branch): + + buildpath = os.path.join(config["buildRoot"], branch) + + tmpFolder = '/tmp/%s-%s' % (commit, randint(0,9999)) + + try: + rmtree(tmpFolder) + except Exception: + pass + + copytree(config["workPath"], tmpFolder) + + repo = git.Repo.init(tmpFolder) + + repo.git.reset('--hard', commit) + repo.git.checkout(commit) + + mkdirp(buildpath) + + scripts=["generate_alpha.sh","generate_by_system.sh","generate_new.sh","generate_glossary.sh"] + + for script in scripts: + cmd = "sh -c 'cd %s && ./scripts/%s 2>&1'" % (tmpFolder, script) + print("Executing: %s" % (cmd)) + cmdout = os.popen(cmd) + print(cmdout.read()) + + # + # WORKAROUND + with open('%s/mkdocs.yml' % tmpFolder, 'r') as file : + filedata = file.read() + + # Replace the target string + filedata = filedata.replace('site_url: "%s"' % siteURL, 'site_url: "%s%s"' % (siteURL, branch)) + + # Write the file out again + with open('%s/mkdocs.yml2' % tmpFolder, 'w') as file: + file.write(filedata) + # + + cmd = "sh -c 'cd %s && mkdocs build --site-dir %s -f mkdocs.yml2 2>&1'" % (tmpFolder, buildpath) + print("Executing: %s" % (cmd)) + cmdout = os.popen(cmd) + print(cmdout.read()) + + app.logger.info("Built branch {branch} in commit {commit}") + + #buildState = read_state() + + try: + buildState[str(branch)]["built"] = str(commit) + except KeyError: + buildState[str(branch)] = {"sha": str(commit), "status": "init", "built": str(commit)} + #write_state(buildState) + + #write_state(buildState) + + rmdir(tmpFolder) + def cleanUpZombies(): """ We want to clean all Zombies: @@ -159,10 +223,20 @@ def pruneBuilds(repo, origin): print('found stale preview: ' + bref) remove_build = remove_build=config["buildRoot"] + '/' + bref print('Removing ' + remove_build) - shutil.rmtree(remove_build) + rmtree(remove_build) print("Done pruning old builds.") +def getBranch(commit): + repo, origin = initRepo(config["workPath"], config["remoteUrl"]) + + # Get the branch name + for ref in repo.refs: + if str(ref.commit.hexsha) == str(commit): + return ref.name + + return None + ### Route functions ### app = Flask(__name__) @@ -175,14 +249,29 @@ def listenBuild(secret): if not secret == config["secret"]: return "Access denied" - build() + if request.headers.get('Content-Type') == 'application/json': + + commit = request.json['after'] + branch = getBranch(commit) + if branch is None: + commit = request.json['after'] + branch = getBranch(commit) + + if branch == None: + return Response(f"Branch not found for commit {commit}") + + print(branch) - response = Response('built started') + thread = threading.Thread(target=buildCommit, args=[commit, branch]) + thread.start() + + return Response(f"{{\"commit\":\"{commit}\",\"branch\":\"{branch}\"}}" ) thread = threading.Thread(target=build) thread.start() - return response + return Response('{{\"built\":\"started\"}}') + def build(): @@ -230,7 +319,8 @@ def build(): print("Don't use default secret since it's freely available in the internet") exit(1) - listenBuild(config["secret"]) + thread = threading.Thread(target=build) + thread.start() app.run(debug=config["debug"]=="True", port=defaultPort, From b61001210e880ad2ca7c8806f60a2e61d4b779a8 Mon Sep 17 00:00:00 2001 From: "alvaro.gonzalez" Date: Tue, 19 Nov 2024 15:09:02 +0200 Subject: [PATCH 15/32] Rename function --- app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.py b/app.py index 29c7c6c..694bd43 100755 --- a/app.py +++ b/app.py @@ -187,7 +187,7 @@ def buildCommit(commit, branch): #write_state(buildState) - rmdir(tmpFolder) + rmtree(tmpFolder) def cleanUpZombies(): """ From 0ada5d0c42e1adcb5fec40ac2ec86f308cbc2f2c Mon Sep 17 00:00:00 2001 From: "alvaro.gonzalez" Date: Tue, 19 Nov 2024 15:37:59 +0200 Subject: [PATCH 16/32] Small curly brances fix --- app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.py b/app.py index 694bd43..5434594 100755 --- a/app.py +++ b/app.py @@ -270,7 +270,7 @@ def listenBuild(secret): thread = threading.Thread(target=build) thread.start() - return Response('{{\"built\":\"started\"}}') + return Response('{\"built\":\"started\"}') def build(): From 46cf85dc0ea94e8d805418c62cb9fc2d4eefef37 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Nov 2024 10:44:52 +0200 Subject: [PATCH 17/32] Bump werkzeug from 2.3.8 to 3.0.3 (#38) Bumps [werkzeug](https://github.com/pallets/werkzeug) from 2.3.8 to 3.0.3. - [Release notes](https://github.com/pallets/werkzeug/releases) - [Changelog](https://github.com/pallets/werkzeug/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/werkzeug/compare/2.3.8...3.0.3) --- updated-dependencies: - dependency-name: werkzeug dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Alvaro Gonzalez --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 86c0e39..b9a0495 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ # Additional packages for preview Flask==2.2.5 itsdangerous==2.1.2 -werkzeug==3.0.3 +werkzeug==3.0.6 # Packages for building the static pages babel==2.16.0 From 2acf241a4272cd962ec208e7f60f2e195b315ced Mon Sep 17 00:00:00 2001 From: "alvaro.gonzalez" Date: Mon, 25 Nov 2024 15:12:05 +0200 Subject: [PATCH 18/32] Add some docs on the chart --- helm/Chart.yaml | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/helm/Chart.yaml b/helm/Chart.yaml index fc38992..2dd853a 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -1,24 +1,6 @@ apiVersion: v2 name: helm -description: A Helm chart for Kubernetes - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. +description: Deploys docs-preview in Kubernetes type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "1.16.0" +version: 0.2.0 +appVersion: "1.2.0" From 2a1c578d161c63ecc253c81c821d5cfb667849ef Mon Sep 17 00:00:00 2001 From: "alvaro.gonzalez" Date: Mon, 25 Nov 2024 15:13:45 +0200 Subject: [PATCH 19/32] Add some docs on the chart --- helm/README.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 helm/README.md diff --git a/helm/README.md b/helm/README.md new file mode 100644 index 0000000..79e1ed8 --- /dev/null +++ b/helm/README.md @@ -0,0 +1,30 @@ +# Helm chart to deploy `docs-preview` + +> Original repository is + +This chart deploys docs-preview, which will clone a given repository (by default ), +and build every single branch in the repository. The branches will be available via a web interface. It also offers a API that understands +GitHub hooks and rebuilds the given branch by the hook. + +## Parameters + +* `values.yaml`: +```yaml +git: + source: https://github.com/cscfi/docs-preview + docs: https://github.com/cscfi/csc-user-guide +secret: + +host: csc-guide-preview.rahtiapp.fi + +replicas: 1 +``` + +|Key|Description|Default| +|:-:|:-:|:-:| +|git.source|The source (including `app.py`) to deploy|https://github.com/cscfi/docs-preview| +|git.docs|The repository hosting the documentation to build|https://github.com/cscfi/csc-user-guide| +|secret|Secret to be used in the web-hook|| +|host|URL to deploy the public web interface|`csc-guide-review.rahtiapp.fi`| +|replicas|Number of replicas for the Pod|1| + From 9b07433da0740cacbe417ee476872232d7fcd36e Mon Sep 17 00:00:00 2001 From: "alvaro.gonzalez" Date: Thu, 28 Nov 2024 09:29:26 +0200 Subject: [PATCH 20/32] Fix spaces with 'reindent' --- app.py | 387 ++++++++++++++++++++++++++++----------------------------- 1 file changed, 193 insertions(+), 194 deletions(-) diff --git a/app.py b/app.py index ec351a3..d3a0fa7 100755 --- a/app.py +++ b/app.py @@ -22,40 +22,40 @@ STATEFILE = '/tmp/build_state.json' try: - workPath = os.environ["WORKPATH"] + workPath = os.environ["WORKPATH"] except KeyError: - workPath = defaultWorkPath + workPath = defaultWorkPath try: - buildRoot = os.environ["BUILDROOT"] + buildRoot = os.environ["BUILDROOT"] except KeyError: - buildRoot = defaultBuildRoot + buildRoot = defaultBuildRoot try: - siteURL = os.environ["SITEURL"] + siteURL = os.environ["SITEURL"] except KeyError: - siteURL = defaultSiteURL + siteURL = defaultSiteURL try: - buildSecret = os.environ["BUILDSECRET"] + buildSecret = os.environ["BUILDSECRET"] except KeyError: - buildSecret = defaultSecret + buildSecret = defaultSecret try: - remoteUrl = os.environ["REMOTEURL"] + remoteUrl = os.environ["REMOTEURL"] except KeyError: - remoteUrl = "https://github.com/CSCfi/csc-user-guide" + remoteUrl = "https://github.com/CSCfi/csc-user-guide" # Configurations in CONFIGFILE will override other environment variables try: - configFile = os.environ["CONFIGFILE"] + configFile = os.environ["CONFIGFILE"] except KeyError: - configFile = None + configFile = None -# Default configuration +# Default configuration config = { - "workPath": workPath, + "workPath": workPath, "remoteUrl": remoteUrl, "buildRoot": buildRoot, "debug": "True", @@ -68,141 +68,141 @@ ### non-route functions def initRepo(workPath, remote_url): - """ - Updates current repository to match `origin` remote. - Does pruning fetch. - """ + """ + Updates current repository to match `origin` remote. + Does pruning fetch. + """ - mkdirp(workPath) + mkdirp(workPath) - repo = git.Repo.init(workPath) + repo = git.Repo.init(workPath) - try: - origin = repo.remote('origin') - except ValueError: - app.logger.info(f"Creating origin {remote_url} into {workPath}") - origin = repo.create_remote('origin', remote_url) + try: + origin = repo.remote('origin') + except ValueError: + app.logger.info(f"Creating origin {remote_url} into {workPath}") + origin = repo.create_remote('origin', remote_url) - assert origin.exists() - assert origin == repo.remotes.origin == repo.remotes['origin'] + assert origin.exists() + assert origin == repo.remotes.origin == repo.remotes['origin'] - app.logger.info("* Fetching remote branches' content") - for fetch_info in origin.fetch(None, None, prune=True): - app.logger.info(" Branch [%s], commit [%s]" % (fetch_info.ref, fetch_info.commit)) + app.logger.info("* Fetching remote branches' content") + for fetch_info in origin.fetch(None, None, prune=True): + app.logger.info(" Branch [%s], commit [%s]" % (fetch_info.ref, fetch_info.commit)) - return repo, origin + return repo, origin def mkdirp(path): - os.makedirs(path, exist_ok=True) + os.makedirs(path, exist_ok=True) def buildRef(repo, ref, state): - """ - Builds and updates. - """ - global config + """ + Builds and updates. + """ + global config + + buildpath = os.path.join(config["buildRoot"], str(ref)) + + app.logger.info('Checking [%s]: %s == %s' % (ref, str(ref.commit), state["built"])) + + if not str(ref.commit) == state["built"] or not os.path.isdir(buildpath): + app.logger.info(" [%s] re-building %s" % (ref, ref.commit)) + repo.git.reset('--hard',ref) + repo.git.checkout(ref) + app.logger.debug(" [%s] buildpath = %s" % (ref, buildpath)) + mkdirp(buildpath) + + scripts=["generate_alpha.sh","generate_by_system.sh","generate_new.sh","generate_glossary.sh"] + + for script in scripts: + cmd = "sh -c 'cd %s && ./scripts/%s 2>&1'" % (config["workPath"],script) + cmdout = os.popen(cmd) + line = cmdout.readline() + app.logger.info(f" [{ref}] # {cmd}") + if cmdout.close(): + app.logger.error(f" [{ref}] {line}") + else: + app.logger.info(f" [{ref}] {line}") + + # + # WORKAROUND + with open('%s/mkdocs.yml' % config["workPath"], 'r') as file : + filedata = file.read() + + # Replace the target string + filedata = filedata.replace('site_url: "%s"' % siteURL, 'site_url: "%s%s"' % (siteURL, str(ref))) + + # Write the file out again + with open('%s/mkdocs.yml2' % config["workPath"], 'w') as file: + file.write(filedata) + # + + cmd = "sh -c 'cd %s && mkdocs build --site-dir %s -f mkdocs.yml2 2>&1'" % (config["workPath"], buildpath) + app.logger.info(f" [{ref}] # %s" % (cmd)) + cmdout = os.popen(cmd) + app.logger.debug(cmdout.read()) - buildpath = os.path.join(config["buildRoot"], str(ref)) + state["built"] = str(ref.commit) - app.logger.info('Checking [%s]: %s == %s' % (ref, str(ref.commit), state["built"])) +def buildCommit(commit, branch): + + buildpath = os.path.join(config["buildRoot"], branch) + + tmpFolder = '/tmp/%s-%s' % (commit, randint(0,9999)) + + try: + rmtree(tmpFolder) + except Exception: + pass + + copytree(config["workPath"], tmpFolder) + + repo = git.Repo.init(tmpFolder) + + repo.git.reset('--hard', commit) + repo.git.checkout(commit) - if not str(ref.commit) == state["built"] or not os.path.isdir(buildpath): - app.logger.info(" [%s] re-building %s" % (ref, ref.commit)) - repo.git.reset('--hard',ref) - repo.git.checkout(ref) - app.logger.debug(" [%s] buildpath = %s" % (ref, buildpath)) mkdirp(buildpath) scripts=["generate_alpha.sh","generate_by_system.sh","generate_new.sh","generate_glossary.sh"] for script in scripts: - cmd = "sh -c 'cd %s && ./scripts/%s 2>&1'" % (config["workPath"],script) + cmd = "sh -c 'cd %s && ./scripts/%s 2>&1'" % (tmpFolder, script) + print("Executing: %s" % (cmd)) cmdout = os.popen(cmd) - line = cmdout.readline() - app.logger.info(f" [{ref}] # {cmd}") - if cmdout.close(): - app.logger.error(f" [{ref}] {line}") - else: - app.logger.info(f" [{ref}] {line}") + print(cmdout.read()) # # WORKAROUND - with open('%s/mkdocs.yml' % config["workPath"], 'r') as file : - filedata = file.read() + with open('%s/mkdocs.yml' % tmpFolder, 'r') as file : + filedata = file.read() # Replace the target string - filedata = filedata.replace('site_url: "%s"' % siteURL, 'site_url: "%s%s"' % (siteURL, str(ref))) + filedata = filedata.replace('site_url: "%s"' % siteURL, 'site_url: "%s%s"' % (siteURL, branch)) # Write the file out again - with open('%s/mkdocs.yml2' % config["workPath"], 'w') as file: - file.write(filedata) - # + with open('%s/mkdocs.yml2' % tmpFolder, 'w') as file: + file.write(filedata) + # - cmd = "sh -c 'cd %s && mkdocs build --site-dir %s -f mkdocs.yml2 2>&1'" % (config["workPath"], buildpath) - app.logger.info(f" [{ref}] # %s" % (cmd)) + cmd = "sh -c 'cd %s && mkdocs build --site-dir %s -f mkdocs.yml2 2>&1'" % (tmpFolder, buildpath) + print("Executing: %s" % (cmd)) cmdout = os.popen(cmd) - app.logger.debug(cmdout.read()) + print(cmdout.read()) - state["built"] = str(ref.commit) + app.logger.info("Built branch {branch} in commit {commit}") -def buildCommit(commit, branch): + #buildState = read_state() - buildpath = os.path.join(config["buildRoot"], branch) + try: + buildState[str(branch)]["built"] = str(commit) + except KeyError: + buildState[str(branch)] = {"sha": str(commit), "status": "init", "built": str(commit)} + #write_state(buildState) - tmpFolder = '/tmp/%s-%s' % (commit, randint(0,9999)) + #write_state(buildState) - try: rmtree(tmpFolder) - except Exception: - pass - - copytree(config["workPath"], tmpFolder) - - repo = git.Repo.init(tmpFolder) - - repo.git.reset('--hard', commit) - repo.git.checkout(commit) - - mkdirp(buildpath) - - scripts=["generate_alpha.sh","generate_by_system.sh","generate_new.sh","generate_glossary.sh"] - - for script in scripts: - cmd = "sh -c 'cd %s && ./scripts/%s 2>&1'" % (tmpFolder, script) - print("Executing: %s" % (cmd)) - cmdout = os.popen(cmd) - print(cmdout.read()) - - # - # WORKAROUND - with open('%s/mkdocs.yml' % tmpFolder, 'r') as file : - filedata = file.read() - - # Replace the target string - filedata = filedata.replace('site_url: "%s"' % siteURL, 'site_url: "%s%s"' % (siteURL, branch)) - - # Write the file out again - with open('%s/mkdocs.yml2' % tmpFolder, 'w') as file: - file.write(filedata) - # - - cmd = "sh -c 'cd %s && mkdocs build --site-dir %s -f mkdocs.yml2 2>&1'" % (tmpFolder, buildpath) - print("Executing: %s" % (cmd)) - cmdout = os.popen(cmd) - print(cmdout.read()) - - app.logger.info("Built branch {branch} in commit {commit}") - - #buildState = read_state() - - try: - buildState[str(branch)]["built"] = str(commit) - except KeyError: - buildState[str(branch)] = {"sha": str(commit), "status": "init", "built": str(commit)} - #write_state(buildState) - - #write_state(buildState) - - rmtree(tmpFolder) def cleanUpZombies(): """ @@ -214,40 +214,40 @@ def cleanUpZombies(): app.logger.info("* Cleaning up Zombies") spid = -1 while spid != 0: - try: - spid, status, rusage = os.wait3(os.WNOHANG) - app.logger.debug("* Process %d with status %d" % (spid, status)) - except ChildProcessError: - break + try: + spid, status, rusage = os.wait3(os.WNOHANG) + app.logger.debug("* Process %d with status %d" % (spid, status)) + except ChildProcessError: + break def pruneBuilds(repo, origin): - try: - builtrefs = os.listdir(config["buildRoot"]+'/origin') - except FileNotFoundError: - app.logger.debug("* Clean buildRoot") - return + try: + builtrefs = os.listdir(config["buildRoot"]+'/origin') + except FileNotFoundError: + app.logger.debug("* Clean buildRoot") + return - srefs = [str(x) for x in origin.refs] - builtrefs = ['origin/'+str(x) for x in builtrefs] + srefs = [str(x) for x in origin.refs] + builtrefs = ['origin/'+str(x) for x in builtrefs] - app.logger.debug("* Pruning old builds.") + app.logger.debug("* Pruning old builds.") - for bref in builtrefs: - if not bref in srefs: - print('found stale preview: ' + bref) - remove_build = config["buildRoot"] + '/' + bref - print('Removing ' + remove_build) - rmtree(remove_build) + for bref in builtrefs: + if not bref in srefs: + print('found stale preview: ' + bref) + remove_build = config["buildRoot"] + '/' + bref + print('Removing ' + remove_build) + rmtree(remove_build) - app.logger.debug("DONE pruning old builds.") + app.logger.debug("DONE pruning old builds.") def getBranch(commit): repo, origin = initRepo(config["workPath"], config["remoteUrl"]) # Get the branch name for ref in repo.refs: - if str(ref.commit.hexsha) == str(commit): - return ref.name + if str(ref.commit.hexsha) == str(commit): + return ref.name return None @@ -257,64 +257,64 @@ def getBranch(commit): @app.route("/build/", methods=["GET", "POST"]) def listenBuild(secret): - global buildState - global config + global buildState + global config + + if not secret == config["secret"]: + return "Access denied" - if not secret == config["secret"]: - return "Access denied" - - if request.headers.get('Content-Type') == 'application/json': + if request.headers.get('Content-Type') == 'application/json': - commit = request.json['after'] - branch = getBranch(commit) - if branch is None: commit = request.json['after'] branch = getBranch(commit) + if branch is None: + commit = request.json['after'] + branch = getBranch(commit) - if branch == None: - return Response(f"Branch not found for commit {commit}") + if branch == None: + return Response(f"Branch not found for commit {commit}") - print(branch) + print(branch) - thread = threading.Thread(target=buildCommit, args=[commit, branch]) - thread.start() + thread = threading.Thread(target=buildCommit, args=[commit, branch]) + thread.start() - return Response(f"{{\"commit\":\"{commit}\",\"branch\":\"{branch}\"}}" ) + return Response(f"{{\"commit\":\"{commit}\",\"branch\":\"{branch}\"}}" ) - thread = threading.Thread(target=build) - thread.start() + thread = threading.Thread(target=build) + thread.start() - return Response('{\"built\":\"started\"}') + return Response('{\"built\":\"started\"}') def build(): - app.logger.info("* Start build loop") + app.logger.info("* Start build loop") - buildState = read_state() + buildState = read_state() - repo, origin = initRepo(config["workPath"], config["remoteUrl"]) + repo, origin = initRepo(config["workPath"], config["remoteUrl"]) - output = "" + output = "" - # Clean buildState - for ref in origin.refs: - sref = str(ref) - output = output + "Found %s (%s)
" % (sref, str(ref.commit)) + # Clean buildState + for ref in origin.refs: + sref = str(ref) + output = output + "Found %s (%s)
" % (sref, str(ref.commit)) - if not sref in buildState: - app.logger.debug(f"Adding {sref} branch to build state") - buildState[sref] = {"sha": str(ref.commit), "status": "init", "built": None} + if not sref in buildState: + app.logger.debug(f"Adding {sref} branch to build state") + buildState[sref] = {"sha": str(ref.commit), "status": "init", "built": None} - # Prune nonexisting builds - if "prune" in config and config["prune"]: - pruneBuilds(repo, origin) - # Refresh builds - for ref in origin.refs: - buildRef(repo, ref, buildState[str(ref)]) - write_state(buildState) + # Prune nonexisting builds + if "prune" in config and config["prune"]: + pruneBuilds(repo, origin) + # Refresh builds + for ref in origin.refs: + buildRef(repo, ref, buildState[str(ref)]) + write_state(buildState) - cleanUpZombies() + cleanUpZombies() def write_state(state): with open(STATEFILE, 'w') as file: @@ -329,28 +329,27 @@ def read_state(): ### Entry functions if __name__=="__main__": - app.logger.setLevel(logging.INFO) + app.logger.setLevel(logging.INFO) - if not configFile == None: - app.logger.info("Loading configuration from file: " + configFile) - with open(configFile) as config_file: - config = json.load(config_file) - buildSecret = config["secret"] - workPath = config["workPath"] - buildRoot = config["buildRoot"] + if not configFile == None: + app.logger.info("Loading configuration from file: " + configFile) + with open(configFile) as config_file: + config = json.load(config_file) + buildSecret = config["secret"] + workPath = config["workPath"] + buildRoot = config["buildRoot"] - app.logger.info("workPath: " + workPath) - app.logger.info("buildRoot: " + buildRoot) - app.logger.info("buildSecret: " + buildSecret) + app.logger.info("workPath: " + workPath) + app.logger.info("buildRoot: " + buildRoot) + app.logger.info("buildSecret: " + buildSecret) - if buildSecret == defaultSecret: - app.logger.error("Don't use default secret since it's freely available in the internet") - exit(1) + if buildSecret == defaultSecret: + app.logger.error("Don't use default secret since it's freely available in the internet") + exit(1) - thread = threading.Thread(target=build) - thread.start() - - app.run(debug=config["debug"]=="True", - port=defaultPort, - host='0.0.0.0') + thread = threading.Thread(target=build) + thread.start() + app.run(debug=config["debug"]=="True", + port=defaultPort, + host='0.0.0.0') From b15348c17cd5f8dc4a3c3f4179d68d046a2bbfc8 Mon Sep 17 00:00:00 2001 From: "alvaro.gonzalez" Date: Fri, 29 Nov 2024 10:52:27 +0200 Subject: [PATCH 21/32] Reorder imports and add docs to app.py --- app.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/app.py b/app.py index d3a0fa7..fe59eb6 100755 --- a/app.py +++ b/app.py @@ -1,11 +1,19 @@ #!/usr/bin/env python3 -import git, os, json, threading +''' +Builds and serves the documention sites of every branch +''' + +import logging +import threading +import os +import json +import sys from shutil import copytree, rmtree from random import randint from flask import Flask, Response, request -import logging +import git # defaults defaultStateFile='/tmp/build_state.json' From 163bac5d5b8d8a4894fee557724857c4690c61d3 Mon Sep 17 00:00:00 2001 From: "alvaro.gonzalez" Date: Fri, 29 Nov 2024 13:35:23 +0200 Subject: [PATCH 22/32] Add encoding to file open operations --- app.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/app.py b/app.py index fe59eb6..17b96ae 100755 --- a/app.py +++ b/app.py @@ -134,14 +134,14 @@ def buildRef(repo, ref, state): # # WORKAROUND - with open('%s/mkdocs.yml' % config["workPath"], 'r') as file : + with open(f"{config['workPath']}/mkdocs.yml", 'r', encoding="utf-8") as file : filedata = file.read() # Replace the target string filedata = filedata.replace('site_url: "%s"' % siteURL, 'site_url: "%s%s"' % (siteURL, str(ref))) # Write the file out again - with open('%s/mkdocs.yml2' % config["workPath"], 'w') as file: + with open(f"{config['workPath']}/mkdocs.yml2", 'w', encoding="utf-8") as file: file.write(filedata) # @@ -182,14 +182,14 @@ def buildCommit(commit, branch): # # WORKAROUND - with open('%s/mkdocs.yml' % tmpFolder, 'r') as file : + with open(f"{tmp_folder}/mkdocs.yml", 'r', encoding="utf-8") as file : filedata = file.read() # Replace the target string filedata = filedata.replace('site_url: "%s"' % siteURL, 'site_url: "%s%s"' % (siteURL, branch)) # Write the file out again - with open('%s/mkdocs.yml2' % tmpFolder, 'w') as file: + with open(f'{tmp_folder}/mkdocs.yml2', 'w', encoding="utf-8") as file: file.write(filedata) # @@ -325,12 +325,15 @@ def build(): cleanUpZombies() def write_state(state): - with open(STATEFILE, 'w') as file: + ''' + Writes the state of the brnaches and their commits into the JSON state file + ''' + with open(STATEFILE, 'w', encoding="utf-8") as file: json.dump(state, file) def read_state(): try: - with open(STATEFILE, 'r') as file: + with open(STATEFILE, 'r', encoding="utf-8") as file: return json.load(file) except FileNotFoundError: return {} From 6816bd2bca79b7690409942414e653e2c5d29493 Mon Sep 17 00:00:00 2001 From: "alvaro.gonzalez" Date: Fri, 29 Nov 2024 13:51:41 +0200 Subject: [PATCH 23/32] Rename function and variables to conform PEP8 --- app.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/app.py b/app.py index 17b96ae..cf92eab 100755 --- a/app.py +++ b/app.py @@ -75,20 +75,20 @@ ### non-route functions -def initRepo(workPath, remote_url): +def init_repo(init_path, remote_url): """ Updates current repository to match `origin` remote. Does pruning fetch. """ - mkdirp(workPath) + mkdirp(init_path) - repo = git.Repo.init(workPath) + repo = git.Repo.init(init_path) try: origin = repo.remote('origin') except ValueError: - app.logger.info(f"Creating origin {remote_url} into {workPath}") + app.logger.info(f"Creating origin {remote_url} into {init_path}") origin = repo.create_remote('origin', remote_url) assert origin.exists() @@ -249,8 +249,11 @@ def pruneBuilds(repo, origin): app.logger.debug("DONE pruning old builds.") -def getBranch(commit): - repo, origin = initRepo(config["workPath"], config["remoteUrl"]) +def get_branch(commit): + ''' + Given a commit, it returns the corresponding branch containing the commit + ''' + repo, _ = init_repo(config["workPath"], config["remoteUrl"]) # Get the branch name for ref in repo.refs: @@ -274,10 +277,10 @@ def listenBuild(secret): if request.headers.get('Content-Type') == 'application/json': commit = request.json['after'] - branch = getBranch(commit) + branch = get_branch(commit) if branch is None: commit = request.json['after'] - branch = getBranch(commit) + branch = get_branch(commit) if branch == None: return Response(f"Branch not found for commit {commit}") @@ -301,7 +304,7 @@ def build(): buildState = read_state() - repo, origin = initRepo(config["workPath"], config["remoteUrl"]) + repo, origin = init_repo(config["workPath"], config["remoteUrl"]) output = "" From ed422e08d70eaa182afd9c1dbfa9dabe9a86d734 Mon Sep 17 00:00:00 2001 From: "alvaro.gonzalez" Date: Fri, 29 Nov 2024 14:37:45 +0200 Subject: [PATCH 24/32] Rename function and variables to conform to PEP8 --- app.py | 79 +++++++++++++++++++++++++++++++--------------------------- 1 file changed, 42 insertions(+), 37 deletions(-) diff --git a/app.py b/app.py index cf92eab..99dc8b5 100755 --- a/app.py +++ b/app.py @@ -16,13 +16,13 @@ import git # defaults -defaultStateFile='/tmp/build_state.json' -defaultWorkPath = "work" -defaultBuildRoot = "/tmp/preview-bot/builds" -defaultRemoteUrl = "https://github.com/CSCfi/csc-user-guide" -defaultSiteURL = "https://csc-guide-preview.rahtiapp.fi/" -defaultSecret = "changeme" # we are using secret but we should be utilizing whitelists -defaultPort = 8081 +DEFAULT_STATE_FILE='/tmp/build_state.json' +DEFAULT_WORK_PATH = "work" +DEFAULT_BUILD_ROOT = "/tmp/preview-bot/builds" +DEFAULT_REMOTE_URL = "https://github.com/CSCfi/csc-user-guide" +DEFAULT_SITE_URL = "https://csc-guide-preview.rahtiapp.fi/" +DEFAULT_SECRET = "changeme" # we are using secret but we should be utilizing whitelists +DEFAULT_PORT = 8081 try: STATEFILE = os.environ["STATEFILE"] @@ -30,48 +30,53 @@ STATEFILE = '/tmp/build_state.json' try: - workPath = os.environ["WORKPATH"] + WORK_PATH = os.environ["WORKPATH"] except KeyError: - workPath = defaultWorkPath + WORK_PATH = DEFAULT_WORK_PATH try: - buildRoot = os.environ["BUILDROOT"] + BUILD_ROOT = os.environ["BUILDROOT"] except KeyError: - buildRoot = defaultBuildRoot + BUILD_ROOT = DEFAULT_BUILD_ROOT try: - siteURL = os.environ["SITEURL"] + SITE_URL = os.environ["SITEURL"] except KeyError: - siteURL = defaultSiteURL + SITE_URL = DEFAULT_SITE_URL try: - buildSecret = os.environ["BUILDSECRET"] + BUILD_SECRET = os.environ["BUILDSECRET"] except KeyError: - buildSecret = defaultSecret + BUILD_SECRET = DEFAULT_SECRET try: - remoteUrl = os.environ["REMOTEURL"] + PORT = os.environ["PORT"] except KeyError: - remoteUrl = "https://github.com/CSCfi/csc-user-guide" + PORT = DEFAULT_PORT + +try: + REMOTE_URL = os.environ["REMOTEURL"] +except KeyError: + REMOTE_URL = "https://github.com/CSCfi/csc-user-guide" # Configurations in CONFIGFILE will override other environment variables try: - configFile = os.environ["CONFIGFILE"] + CONFIG_FILE = os.environ["CONFIGFILE"] except KeyError: - configFile = None + CONFIG_FILE = None # Default configuration config = { - "workPath": workPath, - "remoteUrl": remoteUrl, - "buildRoot": buildRoot, + "workPath": WORK_PATH, + "remoteUrl": REMOTE_URL, + "buildRoot": BUILD_ROOT, "debug": "True", - "secret": buildSecret, + "secret": BUILD_SECRET, "prune": "True" } -buildState = {} +#build_state = {} ### non-route functions @@ -138,7 +143,7 @@ def buildRef(repo, ref, state): filedata = file.read() # Replace the target string - filedata = filedata.replace('site_url: "%s"' % siteURL, 'site_url: "%s%s"' % (siteURL, str(ref))) + filedata = filedata.replace(f'SITE_URL: "{SITE_URL}"', f'SITE_URL: "{SITE_URL}{str(ref)}"') # Write the file out again with open(f"{config['workPath']}/mkdocs.yml2", 'w', encoding="utf-8") as file: @@ -186,7 +191,7 @@ def buildCommit(commit, branch): filedata = file.read() # Replace the target string - filedata = filedata.replace('site_url: "%s"' % siteURL, 'site_url: "%s%s"' % (siteURL, branch)) + filedata = filedata.replace(f"SITE_URL: {SITE_URL}", f'SITE_URL: "{SITE_URL}{branch}"') # Write the file out again with open(f'{tmp_folder}/mkdocs.yml2', 'w', encoding="utf-8") as file: @@ -345,19 +350,19 @@ def read_state(): if __name__=="__main__": app.logger.setLevel(logging.INFO) - if not configFile == None: - app.logger.info("Loading configuration from file: " + configFile) - with open(configFile) as config_file: + if CONFIG_FILE is not None: + app.logger.info("Loading configuration from file: " + CONFIG_FILE) + with open(CONFIG_FILE, encoding="utf-8") as config_file: config = json.load(config_file) - buildSecret = config["secret"] - workPath = config["workPath"] - buildRoot = config["buildRoot"] + BUILD_SECRET = config["secret"] + WORK_PATH = config["workPath"] + BUILD_ROOT = config["buildRoot"] - app.logger.info("workPath: " + workPath) - app.logger.info("buildRoot: " + buildRoot) - app.logger.info("buildSecret: " + buildSecret) + app.logger.info("WORK_PATH: " + WORK_PATH) + app.logger.info("BUILD_ROOT: " + BUILD_ROOT) + app.logger.info("BUILD_SECRET: " + BUILD_SECRET) - if buildSecret == defaultSecret: + if BUILD_SECRET == DEFAULT_SECRET: app.logger.error("Don't use default secret since it's freely available in the internet") exit(1) @@ -365,5 +370,5 @@ def read_state(): thread.start() app.run(debug=config["debug"]=="True", - port=defaultPort, + port=PORT, host='0.0.0.0') From f40b91a3c436aeedca8b62ced5db74dc01863e73 Mon Sep 17 00:00:00 2001 From: "alvaro.gonzalez" Date: Fri, 29 Nov 2024 15:00:08 +0200 Subject: [PATCH 25/32] Use f-strings instead of other methods --- app.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/app.py b/app.py index 99dc8b5..0119cce 100755 --- a/app.py +++ b/app.py @@ -101,7 +101,7 @@ def init_repo(init_path, remote_url): app.logger.info("* Fetching remote branches' content") for fetch_info in origin.fetch(None, None, prune=True): - app.logger.info(" Branch [%s], commit [%s]" % (fetch_info.ref, fetch_info.commit)) + app.logger.info(f" Branch [{fetch_info.ref}], commit [{fetch_info.commit}]") return repo, origin @@ -116,19 +116,19 @@ def buildRef(repo, ref, state): buildpath = os.path.join(config["buildRoot"], str(ref)) - app.logger.info('Checking [%s]: %s == %s' % (ref, str(ref.commit), state["built"])) + app.logger.info(f"Checking [{ref}]: {str(ref.commit)} == {state['built']}") if not str(ref.commit) == state["built"] or not os.path.isdir(buildpath): - app.logger.info(" [%s] re-building %s" % (ref, ref.commit)) + app.logger.info(f" [{ref}] re-building {ref.commit}") repo.git.reset('--hard',ref) repo.git.checkout(ref) - app.logger.debug(" [%s] buildpath = %s" % (ref, buildpath)) + app.logger.debug(f" [{ref}] buildpath = {buildpath}") mkdirp(buildpath) scripts=["generate_alpha.sh","generate_by_system.sh","generate_new.sh","generate_glossary.sh"] for script in scripts: - cmd = "sh -c 'cd %s && ./scripts/%s 2>&1'" % (config["workPath"],script) + cmd = f"sh -c 'cd {config['workPath']} && ./scripts/{script} 2>&1'" cmdout = os.popen(cmd) line = cmdout.readline() app.logger.info(f" [{ref}] # {cmd}") @@ -150,7 +150,7 @@ def buildRef(repo, ref, state): file.write(filedata) # - cmd = "sh -c 'cd %s && mkdocs build --site-dir %s -f mkdocs.yml2 2>&1'" % (config["workPath"], buildpath) + cmd = f"sh -c 'cd {config['workPath']} && mkdocs build --site-dir {buildpath} -f mkdocs.yml2 2>&1'" app.logger.info(f" [{ref}] # %s" % (cmd)) cmdout = os.popen(cmd) app.logger.debug(cmdout.read()) @@ -161,7 +161,7 @@ def buildCommit(commit, branch): buildpath = os.path.join(config["buildRoot"], branch) - tmpFolder = '/tmp/%s-%s' % (commit, randint(0,9999)) + tmp_folder = f'/tmp/{commit}-{randint(0,9999)}' try: rmtree(tmpFolder) @@ -180,8 +180,8 @@ def buildCommit(commit, branch): scripts=["generate_alpha.sh","generate_by_system.sh","generate_new.sh","generate_glossary.sh"] for script in scripts: - cmd = "sh -c 'cd %s && ./scripts/%s 2>&1'" % (tmpFolder, script) - print("Executing: %s" % (cmd)) + cmd = f"sh -c 'cd {tmp_folder} && ./scripts/{script} 2>&1'" + print(f"Executing: {cmd}") cmdout = os.popen(cmd) print(cmdout.read()) @@ -198,8 +198,8 @@ def buildCommit(commit, branch): file.write(filedata) # - cmd = "sh -c 'cd %s && mkdocs build --site-dir %s -f mkdocs.yml2 2>&1'" % (tmpFolder, buildpath) - print("Executing: %s" % (cmd)) + cmd = f"sh -c 'cd {tmp_folder} && mkdocs build --site-dir {buildpath} -f mkdocs.yml2 2>&1'" + print(f"Executing: {cmd}") cmdout = os.popen(cmd) print(cmdout.read()) @@ -228,8 +228,8 @@ def cleanUpZombies(): spid = -1 while spid != 0: try: - spid, status, rusage = os.wait3(os.WNOHANG) - app.logger.debug("* Process %d with status %d" % (spid, status)) + spid, status, _ = os.wait3(os.WNOHANG) + app.logger.debug(f"* Process {spid} with status {status}") except ChildProcessError: break @@ -247,7 +247,7 @@ def pruneBuilds(repo, origin): for bref in builtrefs: if not bref in srefs: - print('found stale preview: ' + bref) + print(f'found stale preview: {bref}') remove_build = config["buildRoot"] + '/' + bref print('Removing ' + remove_build) rmtree(remove_build) @@ -316,7 +316,7 @@ def build(): # Clean buildState for ref in origin.refs: sref = str(ref) - output = output + "Found %s (%s)
" % (sref, str(ref.commit)) + output = output + f"Found {sref} ({str(ref.commit)})
" if not sref in buildState: app.logger.debug(f"Adding {sref} branch to build state") From 9cfa4fe0b03c4f6ba4906f09b3d8ccaa583815a6 Mon Sep 17 00:00:00 2001 From: "alvaro.gonzalez" Date: Fri, 29 Nov 2024 15:02:00 +0200 Subject: [PATCH 26/32] Add doc strings to functions --- app.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/app.py b/app.py index 0119cce..96e04ff 100755 --- a/app.py +++ b/app.py @@ -106,6 +106,9 @@ def init_repo(init_path, remote_url): return repo, origin def mkdirp(path): + ''' + Makes the dir and it does not complain if it exists + ''' os.makedirs(path, exist_ok=True) def buildRef(repo, ref, state): @@ -304,7 +307,9 @@ def listenBuild(secret): def build(): - + ''' + Clones the repo, and makes sure that every branch is built, and prunes the deleted branches + ''' app.logger.info("* Start build loop") buildState = read_state() @@ -340,6 +345,9 @@ def write_state(state): json.dump(state, file) def read_state(): + ''' + Read the current state of bes and commits from file + ''' try: with open(STATEFILE, 'r', encoding="utf-8") as file: return json.load(file) From aabe2a005aaea9a6beb65a52c46ffd012a585089 Mon Sep 17 00:00:00 2001 From: "alvaro.gonzalez" Date: Fri, 29 Nov 2024 15:03:27 +0200 Subject: [PATCH 27/32] Split line to make shorter and use sys.exit --- app.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/app.py b/app.py index 96e04ff..5a001ec 100755 --- a/app.py +++ b/app.py @@ -128,7 +128,10 @@ def buildRef(repo, ref, state): app.logger.debug(f" [{ref}] buildpath = {buildpath}") mkdirp(buildpath) - scripts=["generate_alpha.sh","generate_by_system.sh","generate_new.sh","generate_glossary.sh"] + scripts=["generate_alpha.sh", + "generate_by_system.sh", + "generate_new.sh", + "generate_glossary.sh"] for script in scripts: cmd = f"sh -c 'cd {config['workPath']} && ./scripts/{script} 2>&1'" @@ -240,7 +243,7 @@ def pruneBuilds(repo, origin): try: builtrefs = os.listdir(config["buildRoot"]+'/origin') except FileNotFoundError: - app.logger.debug("* Clean buildRoot") + app.logger.debug("* Clean BUILD_ROOT") return srefs = [str(x) for x in origin.refs] From 01a4f329679493faccc2c7981effa6596f9926be Mon Sep 17 00:00:00 2001 From: "alvaro.gonzalez" Date: Fri, 29 Nov 2024 15:04:47 +0200 Subject: [PATCH 28/32] Add content type to response, do not use ==, and use sys.exit --- app.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app.py b/app.py index 5a001ec..128495e 100755 --- a/app.py +++ b/app.py @@ -293,7 +293,7 @@ def listenBuild(secret): commit = request.json['after'] branch = get_branch(commit) - if branch == None: + if branch is None: return Response(f"Branch not found for commit {commit}") print(branch) @@ -306,7 +306,8 @@ def listenBuild(secret): thread = threading.Thread(target=build) thread.start() - return Response('{\"built\":\"started\"}') + return Response('{\"built\":\"started\"}', + content_type="application/json") def build(): @@ -375,7 +376,7 @@ def read_state(): if BUILD_SECRET == DEFAULT_SECRET: app.logger.error("Don't use default secret since it's freely available in the internet") - exit(1) + sys.exit(1) thread = threading.Thread(target=build) thread.start() From ad107c904b7c8ba8e0a57c088ae05eee644739d4 Mon Sep 17 00:00:00 2001 From: "alvaro.gonzalez" Date: Fri, 29 Nov 2024 15:06:19 +0200 Subject: [PATCH 29/32] Rename variables so they follow PEP8 snake_case --- app.py | 71 +++++++++++++++++++++++++++++++++------------------------- 1 file changed, 41 insertions(+), 30 deletions(-) diff --git a/app.py b/app.py index 128495e..b496bb0 100755 --- a/app.py +++ b/app.py @@ -111,11 +111,11 @@ def mkdirp(path): ''' os.makedirs(path, exist_ok=True) -def buildRef(repo, ref, state): +def build_ref(repo, ref, state): """ Builds and updates. """ - global config + #global config buildpath = os.path.join(config["buildRoot"], str(ref)) @@ -163,20 +163,23 @@ def buildRef(repo, ref, state): state["built"] = str(ref.commit) -def buildCommit(commit, branch): +def build_commit(commit, branch): + ''' + Builds the given commit into the given branch folder. Uses a random tmp fold for the git. + ''' buildpath = os.path.join(config["buildRoot"], branch) tmp_folder = f'/tmp/{commit}-{randint(0,9999)}' try: - rmtree(tmpFolder) - except Exception: + rmtree(tmp_folder) + except OSError: pass - copytree(config["workPath"], tmpFolder) + copytree(config["workPath"], tmp_folder) - repo = git.Repo.init(tmpFolder) + repo = git.Repo.init(tmp_folder) repo.git.reset('--hard', commit) repo.git.checkout(commit) @@ -211,19 +214,19 @@ def buildCommit(commit, branch): app.logger.info("Built branch {branch} in commit {commit}") - #buildState = read_state() + build_state = read_state() try: - buildState[str(branch)]["built"] = str(commit) + build_state[str(branch)]["built"] = str(commit) except KeyError: - buildState[str(branch)] = {"sha": str(commit), "status": "init", "built": str(commit)} - #write_state(buildState) + build_state[str(branch)] = {"sha": str(commit), "status": "init", "built": str(commit)} + write_state(build_state) - #write_state(buildState) + write_state(build_state) - rmtree(tmpFolder) + rmtree(tmp_folder) -def cleanUpZombies(): +def clean_up_zombies(): """ We want to clean all Zombies: * When spid is 0, child processes exist, but they are still alive @@ -239,7 +242,10 @@ def cleanUpZombies(): except ChildProcessError: break -def pruneBuilds(repo, origin): +def prune_builds(origin): + ''' + Deletes any branch folder that is no longer in the repository + ''' try: builtrefs = os.listdir(config["buildRoot"]+'/origin') except FileNotFoundError: @@ -278,9 +284,14 @@ def get_branch(commit): app = Flask(__name__) @app.route("/build/", methods=["GET", "POST"]) -def listenBuild(secret): - global buildState - global config +def listen_build(secret): + ''' + This method listens to the given URL: + - Check the secret matchs + - If GET, checks all brnaches and builds the ones missing + - If POST, reads the json and builds the update branch + ''' + #global config if not secret == config["secret"]: return "Access denied" @@ -298,13 +309,13 @@ def listenBuild(secret): print(branch) - thread = threading.Thread(target=buildCommit, args=[commit, branch]) - thread.start() + build_commit_thread = threading.Thread(target=build_commit, args=[commit, branch]) + build_commit_thread.start() return Response(f"{{\"commit\":\"{commit}\",\"branch\":\"{branch}\"}}" ) - thread = threading.Thread(target=build) - thread.start() + build_thread = threading.Thread(target=build) + build_thread.start() return Response('{\"built\":\"started\"}', content_type="application/json") @@ -316,30 +327,30 @@ def build(): ''' app.logger.info("* Start build loop") - buildState = read_state() + build_state = read_state() repo, origin = init_repo(config["workPath"], config["remoteUrl"]) output = "" - # Clean buildState + # Clean build_state for ref in origin.refs: sref = str(ref) output = output + f"Found {sref} ({str(ref.commit)})
" - if not sref in buildState: + if not sref in build_state: app.logger.debug(f"Adding {sref} branch to build state") - buildState[sref] = {"sha": str(ref.commit), "status": "init", "built": None} + build_state[sref] = {"sha": str(ref.commit), "status": "init", "built": None} # Prune nonexisting builds if "prune" in config and config["prune"]: - pruneBuilds(repo, origin) + prune_builds(origin) # Refresh builds for ref in origin.refs: - buildRef(repo, ref, buildState[str(ref)]) - write_state(buildState) + build_ref(repo, ref, build_state[str(ref)]) + write_state(build_state) - cleanUpZombies() + clean_up_zombies() def write_state(state): ''' From f72378746511f5522a5f6a956bf55d1f113f0f2a Mon Sep 17 00:00:00 2001 From: "alvaro.gonzalez" Date: Fri, 29 Nov 2024 15:21:40 +0200 Subject: [PATCH 30/32] Exit earlier, instead of nested checks --- app.py | 75 ++++++++++++++++++++++++++++++---------------------------- 1 file changed, 39 insertions(+), 36 deletions(-) diff --git a/app.py b/app.py index b496bb0..e848b48 100755 --- a/app.py +++ b/app.py @@ -121,47 +121,50 @@ def build_ref(repo, ref, state): app.logger.info(f"Checking [{ref}]: {str(ref.commit)} == {state['built']}") - if not str(ref.commit) == state["built"] or not os.path.isdir(buildpath): - app.logger.info(f" [{ref}] re-building {ref.commit}") - repo.git.reset('--hard',ref) - repo.git.checkout(ref) - app.logger.debug(f" [{ref}] buildpath = {buildpath}") - mkdirp(buildpath) - - scripts=["generate_alpha.sh", - "generate_by_system.sh", - "generate_new.sh", - "generate_glossary.sh"] - - for script in scripts: - cmd = f"sh -c 'cd {config['workPath']} && ./scripts/{script} 2>&1'" - cmdout = os.popen(cmd) - line = cmdout.readline() - app.logger.info(f" [{ref}] # {cmd}") - if cmdout.close(): - app.logger.error(f" [{ref}] {line}") - else: - app.logger.info(f" [{ref}] {line}") - - # - # WORKAROUND - with open(f"{config['workPath']}/mkdocs.yml", 'r', encoding="utf-8") as file : - filedata = file.read() + if os.path.isdir(buildpath) and str(ref.commit) == state["built"]: + app.logger.info(f" [{str(ref)}]: is up to date") + return - # Replace the target string - filedata = filedata.replace(f'SITE_URL: "{SITE_URL}"', f'SITE_URL: "{SITE_URL}{str(ref)}"') + app.logger.info(f" [{ref}] re-building {ref.commit}") + repo.git.reset('--hard',ref) + repo.git.checkout(ref) + app.logger.debug(f" [{ref}] buildpath = {buildpath}") + mkdirp(buildpath) - # Write the file out again - with open(f"{config['workPath']}/mkdocs.yml2", 'w', encoding="utf-8") as file: - file.write(filedata) - # + scripts=["generate_alpha.sh", + "generate_by_system.sh", + "generate_new.sh", + "generate_glossary.sh"] - cmd = f"sh -c 'cd {config['workPath']} && mkdocs build --site-dir {buildpath} -f mkdocs.yml2 2>&1'" - app.logger.info(f" [{ref}] # %s" % (cmd)) + for script in scripts: + cmd = f"sh -c 'cd {config['workPath']} && ./scripts/{script} 2>&1'" cmdout = os.popen(cmd) - app.logger.debug(cmdout.read()) + line = cmdout.readline() + app.logger.info(f" [{ref}] # {cmd}") + if cmdout.close(): + app.logger.error(f" [{ref}] {line}") + else: + app.logger.info(f" [{ref}] {line}") + + # + # WORKAROUND + with open(f"{config['workPath']}/mkdocs.yml", 'r', encoding="utf-8") as file : + filedata = file.read() + + # Replace the target string + filedata = filedata.replace(f'SITE_URL: "{SITE_URL}"', f'SITE_URL: "{SITE_URL}{str(ref)}"') + + # Write the file out again + with open(f"{config['workPath']}/mk2.yml", 'w', encoding="utf-8") as file: + file.write(filedata) + # + + cmd = f"sh -c 'cd {config['workPath']} && mkdocs build --site-dir {buildpath} -f mk2.yml 2>&1'" + app.logger.info(f" [{ref}] # %s" % (cmd)) + cmdout = os.popen(cmd) + app.logger.debug(cmdout.read()) - state["built"] = str(ref.commit) + state["built"] = str(ref.commit) def build_commit(commit, branch): ''' From b22385ef536fded2309423bb3fa1bcc85f31f035 Mon Sep 17 00:00:00 2001 From: "alvaro.gonzalez" Date: Tue, 7 Jan 2025 15:37:12 +0200 Subject: [PATCH 31/32] Try to solve zombies issue: - Adding a bit of debug - and clean up zombies when SIGCHLD is received --- app.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/app.py b/app.py index e848b48..de1cbaa 100755 --- a/app.py +++ b/app.py @@ -8,12 +8,14 @@ import os import json import sys +import signal from shutil import copytree, rmtree from random import randint from flask import Flask, Response, request import git +from git.exc import GitCommandError # defaults DEFAULT_STATE_FILE='/tmp/build_state.json' @@ -126,7 +128,11 @@ def build_ref(repo, ref, state): return app.logger.info(f" [{ref}] re-building {ref.commit}") - repo.git.reset('--hard',ref) + try: + repo.git.reset('--hard',ref) + except GitCommandError: + app.logger.error(f" [{str(ref)}]: cannot reset hard at this moment") + repo.git.checkout(ref) app.logger.debug(f" [{ref}] buildpath = {buildpath}") mkdirp(buildpath) @@ -164,6 +170,8 @@ def build_ref(repo, ref, state): cmdout = os.popen(cmd) app.logger.debug(cmdout.read()) + cmdout.close() + state["built"] = str(ref.commit) def build_commit(commit, branch): @@ -197,6 +205,8 @@ def build_commit(commit, branch): cmdout = os.popen(cmd) print(cmdout.read()) + cmdout.close() + # # WORKAROUND with open(f"{tmp_folder}/mkdocs.yml", 'r', encoding="utf-8") as file : @@ -214,6 +224,7 @@ def build_commit(commit, branch): print(f"Executing: {cmd}") cmdout = os.popen(cmd) print(cmdout.read()) + cmdout.close() app.logger.info("Built branch {branch} in commit {commit}") @@ -236,14 +247,15 @@ def clean_up_zombies(): * When ChildProcessError raises, it means that there are no children left """ - app.logger.info("* Cleaning up Zombies") + app.logger.info(f"* Cleaning up Zombies. This is {os.getpid()}" ) spid = -1 while spid != 0: try: spid, status, _ = os.wait3(os.WNOHANG) - app.logger.debug(f"* Process {spid} with status {status}") + app.logger.info(f"* Process {spid} with status {status}") except ChildProcessError: break + app.logger.info("Cleaning process done.") def prune_builds(origin): ''' @@ -353,8 +365,6 @@ def build(): build_ref(repo, ref, build_state[str(ref)]) write_state(build_state) - clean_up_zombies() - def write_state(state): ''' Writes the state of the brnaches and their commits into the JSON state file @@ -371,6 +381,10 @@ def read_state(): return json.load(file) except FileNotFoundError: return {} + +def signal_handler(sig, frame): + clean_up_zombies() + ### Entry functions if __name__=="__main__": @@ -395,6 +409,8 @@ def read_state(): thread = threading.Thread(target=build) thread.start() + signal.signal(signal.SIGCHLD, signal_handler) + app.run(debug=config["debug"]=="True", port=PORT, host='0.0.0.0') From fe3153ba82ca386ecd4a09d123cd4c1c6b121149 Mon Sep 17 00:00:00 2001 From: "alvaro.gonzalez" Date: Tue, 7 Jan 2025 15:48:34 +0200 Subject: [PATCH 32/32] Upgrade Jinja2 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e562399..e83ff3d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ ghp-import==2.1.0 gitdb==4.0.11 GitPython==3.1.43 idna==3.7 -Jinja2==3.1.4 +Jinja2==3.1.5 Markdown==3.6 MarkupSafe==2.1.5 mergedeep==1.3.4