From 0a5053bc358a8da2fbfa9e4f5512c46a8ebcc3cd Mon Sep 17 00:00:00 2001 From: Jan Brezina Date: Tue, 26 Jul 2022 09:50:11 +0200 Subject: [PATCH 1/9] Try to make YAML-CPP build compatible with shared flow123d lib --- dockerfiles/base-build/Dockerfile-gnu | 3 ++- dockerfiles/libs-gnu/libs-build-gnu/common_cmake.mk | 1 + dockerfiles/libs-gnu/libs-build-gnu/yamlcpp/Makefile | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/dockerfiles/base-build/Dockerfile-gnu b/dockerfiles/base-build/Dockerfile-gnu index 98c2975..87b07c1 100644 --- a/dockerfiles/base-build/Dockerfile-gnu +++ b/dockerfiles/base-build/Dockerfile-gnu @@ -81,7 +81,8 @@ RUN sudo apt-get update && sudo apt-get install -y --no-install-recommends \ texlive-pstricks \ texlive-science \ lmodern \ - imagemagick + imagemagick \ + ps2eps # some pip packages RUN pip3 install \ diff --git a/dockerfiles/libs-gnu/libs-build-gnu/common_cmake.mk b/dockerfiles/libs-gnu/libs-build-gnu/common_cmake.mk index 99a9977..d58b81f 100644 --- a/dockerfiles/libs-gnu/libs-build-gnu/common_cmake.mk +++ b/dockerfiles/libs-gnu/libs-build-gnu/common_cmake.mk @@ -48,6 +48,7 @@ configure: $(build_dir) $(sources_dir) -DCMAKE_BUILD_TYPE=$(build_type) \ -DCMAKE_INSTALL_PREFIX=/usr/local/$(library)_$(version) \ -DCMAKE_CXX_FLAGS=$(CXX_FLAGS) \ + $(cmake_options) \ $(sources_dir) .PHONY : build diff --git a/dockerfiles/libs-gnu/libs-build-gnu/yamlcpp/Makefile b/dockerfiles/libs-gnu/libs-build-gnu/yamlcpp/Makefile index cb66bf6..57bd3c7 100644 --- a/dockerfiles/libs-gnu/libs-build-gnu/yamlcpp/Makefile +++ b/dockerfiles/libs-gnu/libs-build-gnu/yamlcpp/Makefile @@ -3,9 +3,10 @@ library=yamlcpp url=https://flow.nti.tul.cz/libraries/yaml-cpp-yaml-cpp-$(version).zip base_name=yaml-cpp-yaml-cpp-$(version) +cmake_options="-DYAML_BUILD_SHARED_LIBS=ON" # In order to combine staticaly linked yamlcpp with dynamicaly linked res of the code. -CXX_FLAGS=-fPIE +CXX_FLAGS=-fPIC # TODO: shared yamplcpp From 09bf09dfc09fdb1061df51118d189d7da005f60a Mon Sep 17 00:00:00 2001 From: Jan Brezina Date: Tue, 26 Jul 2022 19:30:41 +0200 Subject: [PATCH 2/9] Update build_images.yml temporary disable tag and release --- .github/workflows/build_images.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build_images.yml b/.github/workflows/build_images.yml index 3ad6e77..36305d7 100644 --- a/.github/workflows/build_images.yml +++ b/.github/workflows/build_images.yml @@ -51,10 +51,10 @@ jobs: - name: Read images_version file id: getversion run: echo "::set-output name=version::$(cat images_version)" - - name: Release - uses: avakar/tag-and-release@v1 - with: - tag_name: v${{ steps.getversion.outputs.version }} - release_name: release_${{ steps.getversion.outputs.version }} - env: - GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN }} +# - name: Release +# uses: avakar/tag-and-release@v1 +# with: +# tag_name: v${{ steps.getversion.outputs.version }} +# release_name: release_${{ steps.getversion.outputs.version }} +# env: +# GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN }} From 22337b1c003a3228d004e71546eda1680bfde83c Mon Sep 17 00:00:00 2001 From: Jan Brezina Date: Fri, 29 Jul 2022 12:30:26 +0200 Subject: [PATCH 3/9] Jenkins server configuration, first atempt. - setup.sh script as the entry point: setup, start, stop - Docker compose configuration to setup volumes and ports - Docker image with used plugins installation. - Preliminary Jenkins configuration through Configuration as Code (WIP) --- images_version | 2 +- server/Dockerfile | 16 ++ server/README.md | 14 ++ server/docker-compose.yaml | 19 +++ server/jenkins_config.yaml | 62 ++++++++ server/plugins.txt | 18 +++ server/setup.sh | 42 +++++ server/ubuntu-16.04.0/Dockerfile | 39 ----- server/ubuntu-16.04.1/Dockerfile | 33 ---- server/ubuntu-16.04.2/Dockerfile | 47 ------ server/ubuntu-16.04.2/install-plugins.sh | 192 ----------------------- server/ubuntu-16.04.2/jenkins-support | 127 --------------- server/ubuntu-16.04.2/jenkins.sh | 14 -- server/ubuntu-16.04.2/plugins.sh | 126 --------------- 14 files changed, 172 insertions(+), 579 deletions(-) create mode 100644 server/Dockerfile create mode 100644 server/README.md create mode 100644 server/docker-compose.yaml create mode 100644 server/jenkins_config.yaml create mode 100644 server/plugins.txt create mode 100755 server/setup.sh delete mode 100644 server/ubuntu-16.04.0/Dockerfile delete mode 100644 server/ubuntu-16.04.1/Dockerfile delete mode 100755 server/ubuntu-16.04.2/Dockerfile delete mode 100755 server/ubuntu-16.04.2/install-plugins.sh delete mode 100755 server/ubuntu-16.04.2/jenkins-support delete mode 100755 server/ubuntu-16.04.2/jenkins.sh delete mode 100755 server/ubuntu-16.04.2/plugins.sh diff --git a/images_version b/images_version index 1454f6e..4d54dad 100644 --- a/images_version +++ b/images_version @@ -1 +1 @@ -4.0.1 +4.0.2 diff --git a/server/Dockerfile b/server/Dockerfile new file mode 100644 index 0000000..d69659c --- /dev/null +++ b/server/Dockerfile @@ -0,0 +1,16 @@ +# Customized Jenkins docker image. + +FROM jenkins/jenkins:lts-jdk11 + +# Skip setup wizard and protection. +ENV JAVA_OPTS "-Djenkins.install.runSetupWizard=false" +ENV CASC_JENKINS_CONFIG /var/jenkins_home/casc.yaml + +# Add Jenkins plugins from separate file +COPY --chown=jenkins:jenkins plugins.txt /usr/share/jenkins/ref/plugins.txt +RUN jenkins-plugin-cli -f /usr/share/jenkins/ref/plugins.txt + +# Setup Configuration as Code +#COPY plugins.txt /usr/share/jenkins/ref/plugins.txt +#RUN /usr/local/bin/install-plugins.sh < /usr/share/jenkins/ref/plugins.txt +COPY jenkins_config.yaml /var/jenkins_home/casc.yaml diff --git a/server/README.md b/server/README.md new file mode 100644 index 0000000..89c666b --- /dev/null +++ b/server/README.md @@ -0,0 +1,14 @@ +Folder with complete configuration of the Jenkins server for CI testing. +This includes: +- Docker image dockerfile. +- Docker compose for the particular container configuration with volumes and port redirection. +- Jenkins YAML configuration using the : https://github.com/jenkinsci/docker + + +Resources: + +- [Jenkins Docker](https://www.jenkins.io/doc/book/installing/docker/) +- [Jenkins Configuration as Code](https://www.jenkins.io/projects/jcasc/) +- [Get started with Docker Compose](https://docs.docker.com/compose/gettingstarted/) +- [Docker Compose Tutorial](https://www.educative.io/blog/docker-compose-tutorial) +- [Tutorial: Automate Jenkins Setup](https://www.digitalocean.com/community/tutorials/how-to-automate-jenkins-setup-with-docker-and-jenkins-configuration-as-code#step-1-disabling-the-setup-wizard) diff --git a/server/docker-compose.yaml b/server/docker-compose.yaml new file mode 100644 index 0000000..5cfe801 --- /dev/null +++ b/server/docker-compose.yaml @@ -0,0 +1,19 @@ +version: '2' + +volumes: + jenkins_data: + external: true + name: jenkins-ciflow + +services: + jenkins_casc: # derived Jenkins image using Configuration as code plugin for configuring from scratch + build: . + restart: unless-stopped + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - jenkins_data:/var/jenkins_home + ports: + - "8080:8080" + - "50000:50000" + +# Can use .env file for host dependent data diff --git a/server/jenkins_config.yaml b/server/jenkins_config.yaml new file mode 100644 index 0000000..219ccdf --- /dev/null +++ b/server/jenkins_config.yaml @@ -0,0 +1,62 @@ +# Configuration of the Jenkins server through the Configuration as Code plugin. +# TODO: +# - test setup of the Jenkins server with proper plugins +# - configuration of nodes +# - try to upload flow123d jobs + +jenkins: + systemMessage: "Jenkins configured automatically by Jenkins Configuration as Code plugin\n\n" + #globalNodeProperties: + #- envVars: + #env: + #- key: VARIABLE1 + #value: foo + #- key: VARIABLE2 + #value: bar + + securityRealm: + local: + allowsSignup: false + users: + - id: ${JENKINS_ADMIN_ID} + password: ${JENKINS_ADMIN_PASSWORD} + nodes: + - permanent: + name: "ci2runner" + remoteFS: "/home/builder/jenkins" + launcher: + ssh: + credentialsId: builder + host: ci2runner.nti.tul.cz + sshHostKeyVerificationStrategy: manuallyProvidedKeyVerificationStrategy + disabled: true + failIfWorkDirIsMissing: false + internalDir: "remoting" + workDirPath: "/tmp" + retentionStrategy: always + numExecutors: 5 + + #slaveAgentPort: 50000 +unclassified: + location: + url: http://server_ip:8080/ + +#tool: + #git: + #installations: + #- name: git + #home: /usr/local/bin/git + +#credentials: + #system: + #domainCredentials: + #- credentials: + #- basicSSHUserPrivateKey: + #scope: SYSTEM + #id: ssh_with_passphrase_provided + #username: ssh_root + #passphrase: ${SSH_KEY_PASSWORD} + #description: "SSH passphrase with private key file. Private key provided" + #privateKeySource: + #directEntry: + #privateKey: ${SSH_PRIVATE_KEY} diff --git a/server/plugins.txt b/server/plugins.txt new file mode 100644 index 0000000..ff2ce1c --- /dev/null +++ b/server/plugins.txt @@ -0,0 +1,18 @@ +configuration-as-code:latest +ssh-slaves +build-name-setter +build-metrics +console-column-plugin +embeddable-build-status +envinject +git-parameter +git-client +github +jobConfigHistory +junit +jenkins-multijob-plugin +publish-over-cifs + + + + diff --git a/server/setup.sh b/server/setup.sh new file mode 100755 index 0000000..cd642a6 --- /dev/null +++ b/server/setup.sh @@ -0,0 +1,42 @@ +# Script documenting server creation, start and stop +# can only be executed within its directory with docker-compose and other configuration files +# It uses docker compose V2 that works as a plugin to docker, providing the "compose" command. +# +# Usage: +# setup.sh setup admin-password +# start +# stop + + + +script_path="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/$(basename "${BASH_SOURCE[0]}")" + +function setup_server { + docker compose rm --stop --force --volumes + docker volume rm jenkins-ciflow + + docker volume create jenkins-ciflow + docker pull jenkins/jenkins + docker compose build + docker compose up # initial service start +} + + +export CASC_JENKINS_CONFIG="${script_path}" +cmd=$1 +if [ "$cmd" == "setup" ] +then + read -p "Setup of the Jenkins server will overwrite all previous configurations. Do you want to proceed? [Yy]" -n 1 -r + echo # (optional) move to a new line + if [[ $REPLY =~ ^[Yy]$ ]] + then + password=$2 + setup_server + fi +elif [ "$cmd" == "start" ] +then + docker compose up +elif [ "$cmd" == "stop" ] +then + docker compose stop +fi diff --git a/server/ubuntu-16.04.0/Dockerfile b/server/ubuntu-16.04.0/Dockerfile deleted file mode 100644 index 711b431..0000000 --- a/server/ubuntu-16.04.0/Dockerfile +++ /dev/null @@ -1,39 +0,0 @@ -FROM ubuntu:16.04 -MAINTAINER Jan Hybs - -# add link for bash -RUN rm /bin/sh && ln -s /bin/bash /bin/sh - -# install sudo (for 16.04) -RUN apt-get update && apt-get install -y sudo - - -# preq -RUN sudo apt-get update && sudo apt-get install -y - nano \ - wget \ - git \ - apt-transport-https \ - ca-certificates \ - apparmor \ - man \ - python \ - git \ - python-pip - - -# install docker -# alt way to install wget -qO- https://get.docker.com/ | sh -RUN sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D -RUN echo "deb https://apt.dockerproject.org/repo ubuntu-trusty main" > /etc/apt/sources.list.d/docker.list -RUN sudo apt-get update -RUN sudo apt-get purge lxc-docker -RUN sudo apt-cache policy docker-engine -RUN sudo apt-get install docker-engine - - -# install jenkins -RUN wget -q -O - https://jenkins-ci.org/debian/jenkins-ci.org.key | sudo apt-key add - -RUN sudo sh -c 'echo deb http://pkg.jenkins-ci.org/debian-stable binary/ > /etc/apt/sources.list.d/jenkins.list' -RUN sudo apt-get update -RUN sudo apt-get install -y jenkins \ No newline at end of file diff --git a/server/ubuntu-16.04.1/Dockerfile b/server/ubuntu-16.04.1/Dockerfile deleted file mode 100644 index 8fddf1b..0000000 --- a/server/ubuntu-16.04.1/Dockerfile +++ /dev/null @@ -1,33 +0,0 @@ -FROM ubuntu:16.04 -MAINTAINER Jan Hybs - -# add link for bash -RUN rm /bin/sh && ln -s /bin/bash /bin/sh - -# install sudo (for 16.04) -RUN apt-get update && apt-get install -y sudo - - -# preq -RUN sudo apt-get update && sudo apt-get install -y \ - nano \ - wget \ - git \ - apt-transport-https \ - ca-certificates \ - apparmor \ - man \ - python \ - git \ - python-pip - - -# install docker -RUN sudo apt-get install -y docker.io - - -# install jenkins -RUN wget -q -O - https://jenkins-ci.org/debian/jenkins-ci.org.key | sudo apt-key add - -RUN sudo sh -c 'echo deb http://pkg.jenkins-ci.org/debian-stable binary/ > /etc/apt/sources.list.d/jenkins.list' -RUN sudo apt-get update -RUN sudo apt-get install -y jenkins \ No newline at end of file diff --git a/server/ubuntu-16.04.2/Dockerfile b/server/ubuntu-16.04.2/Dockerfile deleted file mode 100755 index 12818fc..0000000 --- a/server/ubuntu-16.04.2/Dockerfile +++ /dev/null @@ -1,47 +0,0 @@ -FROM ubuntu:16.04 -MAINTAINER Jan Hybs - -# add link for bash -RUN rm /bin/sh && ln -s /bin/bash /bin/sh - -# install sudo (for 16.04) -RUN apt-get update && apt-get install -y sudo - - -# preq -RUN sudo apt-get update && sudo apt-get install -y \ - nano \ - wget \ - git \ - apt-transport-https \ - ca-certificates \ - apparmor \ - man \ - python \ - git \ - python-pip - - -# install docker -# alt way to install -# RUN wget -qO- https://get.docker.com/ | sh - - -# install jenkins -RUN wget -q -O - https://jenkins-ci.org/debian/jenkins-ci.org.key | sudo apt-key add - -RUN sudo sh -c 'echo deb http://pkg.jenkins-ci.org/debian-stable binary/ > /etc/apt/sources.list.d/jenkins.list' -RUN sudo apt-get update -RUN sudo apt-get install -y jenkins - -# copy files -# COPY plugins.sh /usr/local/bin/plugins.sh -# COPY install-plugins.sh /usr/local/bin/install-plugins.sh -# COPY jenkins-support /usr/local/bin/jenkins-support -# COPY jenkins.sh /usr/local/bin/jenkins.sh -# -# # install curl and zip -# RUN sudo apt-get install -y curl zip -# ENV JENKINS_UC https://updates.jenkins.io -# -# # install plugins -# RUN /usr/local/bin/install-plugins.sh jenkins-multijob view-job-filters github build-name-setter envinject cloudbees-folder \ No newline at end of file diff --git a/server/ubuntu-16.04.2/install-plugins.sh b/server/ubuntu-16.04.2/install-plugins.sh deleted file mode 100755 index d626a52..0000000 --- a/server/ubuntu-16.04.2/install-plugins.sh +++ /dev/null @@ -1,192 +0,0 @@ -#!/bin/bash - -# Resolve dependencies and download plugins given on the command line -# -# FROM jenkins -# RUN install-plugins.sh docker-slaves github-branch-source - -set -o pipefail - -REF_DIR=${REF:-/usr/share/jenkins/ref/plugins} -FAILED="$REF_DIR/failed-plugins.txt" - -. /usr/local/bin/jenkins-support - -function getLockFile() { - echo -n "$REF_DIR/${1}.lock" -} - -function getArchiveFilename() { - echo -n "$REF_DIR/${1}.jpi" -} - -function download() { - local plugin originalPlugin version lock ignoreLockFile - plugin="$1" - version="${2:-latest}" - ignoreLockFile="$3" - lock="$(getLockFile "$plugin")" - - if [[ $ignoreLockFile ]] || mkdir "$lock" &>/dev/null; then - if ! doDownload "$plugin" "$version"; then - # some plugin don't follow the rules about artifact ID - # typically: docker-plugin - originalPlugin="$plugin" - plugin="${plugin}-plugin" - if ! doDownload "$plugin" "$version"; then - echo "Failed to download plugin: $originalPlugin or $plugin" >&2 - echo "Not downloaded: ${originalPlugin}" >> "$FAILED" - return 1 - fi - fi - - if ! checkIntegrity "$plugin"; then - echo "Downloaded file is not a valid ZIP: $(getArchiveFilename "$plugin")" >&2 - echo "Download integrity: ${plugin}" >> "$FAILED" - return 1 - fi - - resolveDependencies "$plugin" - fi -} - -function doDownload() { - local plugin version url jpi - plugin="$1" - version="$2" - jpi="$(getArchiveFilename "$plugin")" - - # If plugin already exists and is the same version do not download - if test -f "$jpi" && unzip -p "$jpi" META-INF/MANIFEST.MF | tr -d '\r' | grep "^Plugin-Version: ${version}$" > /dev/null; then - echo "Using provided plugin: $plugin" - return 0 - fi - - url="$JENKINS_UC/download/plugins/$plugin/$version/${plugin}.hpi" - - echo "Downloading plugin: $plugin from $url" - curl --connect-timeout 5 --retry 5 --retry-delay 0 --retry-max-time 60 -s -f -L "$url" -o "$jpi" - return $? -} - -function checkIntegrity() { - local plugin jpi - plugin="$1" - jpi="$(getArchiveFilename "$plugin")" - - zip -T "$jpi" >/dev/null - return $? -} - -function resolveDependencies() { - local plugin jpi dependencies - plugin="$1" - jpi="$(getArchiveFilename "$plugin")" - - # ^M below is a control character, inserted by typing ctrl+v ctrl+m - dependencies="$(unzip -p "$jpi" META-INF/MANIFEST.MF | sed -e 's# -##g' | tr '\n' '|' | sed -e 's#| ##g' | tr '|' '\n' | grep "^Plugin-Dependencies: " | sed -e 's#^Plugin-Dependencies: ##')" - - if [[ ! $dependencies ]]; then - echo " > $plugin has no dependencies" - return - fi - - echo " > $plugin depends on $dependencies" - - IFS=',' read -a array <<< "$dependencies" - - for d in "${array[@]}" - do - plugin="$(cut -d':' -f1 - <<< "$d")" - if [[ $d == *"resolution:=optional"* ]]; then - echo "Skipping optional dependency $plugin" - else - pluginInstalled="$(echo "${bundledPlugins}" | grep "^${plugin}:")" - pluginInstalled="${pluginInstalled//[$'\r']}" - if ! [ -z "${pluginInstalled}" ]; then - versionInstalled=$(versionFromPlugin "${pluginInstalled}") - versionToInstall=$(versionFromPlugin "${d}") - if versionLT "${versionInstalled}" "${versionToInstall}"; then - echo "Upgrading bundled dependency $d ($versionToInstall > $versionInstalled)" - download "$plugin" "$versionToInstall" & - else - echo "Skipping already bundled dependency $d ($versionToInstall <= $versionInstalled)" - fi - else - download "$plugin" "$(versionFromPlugin "${d}")" & - fi - fi - done - wait -} - -function bundledPlugins() { - local JENKINS_WAR=/usr/share/jenkins/jenkins.war - if [ -f $JENKINS_WAR ] - then - TEMP_PLUGIN_DIR=/tmp/plugintemp.$$ - for i in `jar tf $JENKINS_WAR | egrep '[^detached-]plugins.*\..pi' | sort` - do - rm -fr $TEMP_PLUGIN_DIR - mkdir -p $TEMP_PLUGIN_DIR - PLUGIN=`basename $i|cut -f1 -d'.'` - (cd $TEMP_PLUGIN_DIR;jar xf $JENKINS_WAR "$i";jar xvf $TEMP_PLUGIN_DIR/$i META-INF/MANIFEST.MF >/dev/null 2>&1) - VER=`egrep -i Plugin-Version "$TEMP_PLUGIN_DIR/META-INF/MANIFEST.MF"|cut -d\: -f2|sed 's/ //'` - echo "$PLUGIN:$VER" - done - rm -fr $TEMP_PLUGIN_DIR - else - rm -f $TEMP_ALREADY_INSTALLED - echo "ERROR file not found: $JENKINS_WAR" - exit 1 - fi -} - -function versionFromPlugin() { - local plugin=$1 - if [[ $plugin =~ .*:.* ]]; then - echo "${plugin##*:}" - else - echo "latest" - fi - -} - -main() { - local plugin version - - mkdir -p "$REF_DIR" || exit 1 - - # Create lockfile manually before first run to make sure any explicit version set is used. - echo "Creating initial locks..." - for plugin in "$@"; do - mkdir "$(getLockFile "${plugin%%:*}")" - done - - echo -e "\nAnalyzing war..." - bundledPlugins="$(bundledPlugins)" - - echo -e "\nDownloading plugins..." - for plugin in "$@"; do - version="" - - if [[ $plugin =~ .*:.* ]]; then - version=$(versionFromPlugin "${plugin}") - plugin="${plugin%%:*}" - fi - - download "$plugin" "$version" "true" & - done - wait - - if [[ -f $FAILED ]]; then - echo -e "\nSome plugins failed to download!\n$(<"$FAILED")" >&2 - exit 1 - fi - - echo -e "\nCleaning up locks" - rm -r "$REF_DIR"/*.lock -} - -main "$@" \ No newline at end of file diff --git a/server/ubuntu-16.04.2/jenkins-support b/server/ubuntu-16.04.2/jenkins-support deleted file mode 100755 index 6174b86..0000000 --- a/server/ubuntu-16.04.2/jenkins-support +++ /dev/null @@ -1,127 +0,0 @@ -#!/bin/bash -e - -# compare if version1 < version2 -versionLT() { - local v1; v1=$(echo $1 | cut -d '-' -f 1 ) - local q1; q1=$(echo $1 | cut -s -d '-' -f 2- ) - local v2; v2=$(echo $2 | cut -d '-' -f 1 ) - local q2; q2=$(echo $2 | cut -s -d '-' -f 2- ) - if [ "$v1" = "$v2" ]; then - if [ "$q1" = "$q2" ]; then - return 1 - else - if [ -z "$q1" ]; then - return 1 - else - if [ -z "$q2" ]; then - return 0 - else - [ "$q1" = "`echo -e "$q1\n$q2" | sort -V | head -n1`" ] - fi - fi - fi - else - [ "$v1" = "`echo -e "$v1\n$v2" | sort -V | head -n1`" ] - fi -} - -# returns a plugin version from a plugin archive -get_plugin_version() { - local archive; archive=$1 - local version; version=$(unzip -p $archive META-INF/MANIFEST.MF | grep "^Plugin-Version: " | sed -e 's#^Plugin-Version: ##') - version=${version%%[[:space:]]} - echo $version -} - -# Copy files from /usr/share/jenkins/ref into $JENKINS_HOME -# So the initial JENKINS-HOME is set with expected content. -# Don't override, as this is just a reference setup, and use from UI -# can then change this, upgrade plugins, etc. -copy_reference_file() { - f="${1%/}" - b="${f%.override}" - rel="${b:23}" - version_marker="${rel}.version_from_image" - dir=$(dirname "${b}") - local action; - local reason; - local container_version; - local image_version; - local marker_version; - local log; log=false - if [[ ${rel} == plugins/*.jpi ]]; then - container_version=$(get_plugin_version $JENKINS_HOME/${rel}) - image_version=$(get_plugin_version ${f}) - if [[ -e $JENKINS_HOME/${version_marker} ]]; then - marker_version=$(cat $JENKINS_HOME/${version_marker}) - if versionLT $marker_version $container_version; then - action="SKIPPED" - reason="Installed version ($container_version) has been manually upgraded from initial version ($marker_version)" - log=true - else - if [[ "$image_version" == "$container_version" ]]; then - action="SKIPPED" - reason="Version from image is the same as the installed version $image_version" - else - if versionLT $image_version $container_version; then - action="SKIPPED" - log=true - reason="Image version ($image_version) is older than installed version ($container_version)" - else - action="UPGRADED" - log=true - reason="Image version ($image_version) is newer than installed version ($container_version)" - fi - fi - fi - else - if [[ -n "$TRY_UPGRADE_IF_NO_MARKER" ]]; then - if [[ "$image_version" == "$container_version" ]]; then - action="SKIPPED" - reason="Version from image is the same as the installed version $image_version (no marker found)" - # Add marker for next time - echo $image_version > $JENKINS_HOME/${version_marker} - else - if versionLT $image_version $container_version; then - action="SKIPPED" - log=true - reason="Image version ($image_version) is older than installed version ($container_version) (no marker found)" - else - action="UPGRADED" - log=true - reason="Image version ($image_version) is newer than installed version ($container_version) (no marker found)" - fi - fi - fi - fi - if [[ ! -e $JENKINS_HOME/${rel} || "$action" == "UPGRADED" || $f = *.override ]]; then - action=${action:-"INSTALLED"} - log=true - mkdir -p "$JENKINS_HOME/${dir:23}" - cp -r "${f}" "$JENKINS_HOME/${rel}"; - # pin plugins on initial copy - touch "$JENKINS_HOME/${rel}.pinned" - echo $image_version > $JENKINS_HOME/${version_marker} - reason=${reason:-$image_version} - else - action=${action:-"SKIPPED"} - fi - else - if [[ ! -e $JENKINS_HOME/${rel} || $f = *.override ]] - then - action="INSTALLED" - log=true - mkdir -p "$JENKINS_HOME/${dir:23}" - cp -r "${f}" "$JENKINS_HOME/${rel}"; - else - action="SKIPPED" - fi - fi - if [[ -n "$VERBOSE" || "$log" == "true" ]]; then - if [ -z "$reason" ]; then - echo "$action $rel" >> "$COPY_REFERENCE_FILE_LOG" - else - echo "$action $rel : $reason" >> "$COPY_REFERENCE_FILE_LOG" - fi - fi -} \ No newline at end of file diff --git a/server/ubuntu-16.04.2/jenkins.sh b/server/ubuntu-16.04.2/jenkins.sh deleted file mode 100755 index 49ec276..0000000 --- a/server/ubuntu-16.04.2/jenkins.sh +++ /dev/null @@ -1,14 +0,0 @@ -#! /bin/bash -e - -: ${JENKINS_HOME:="/var/jenkins_home"} -touch "${COPY_REFERENCE_FILE_LOG}" || (echo "Can not write to ${COPY_REFERENCE_FILE_LOG}. Wrong volume permissions?" && exit 1) -echo "--- Copying files at $(date)" >> "$COPY_REFERENCE_FILE_LOG" -find /usr/share/jenkins/ref/ -type f -exec bash -c ". /usr/local/bin/jenkins-support; copy_reference_file '{}'" \; - -# if `docker run` first argument start with `--` the user is passing jenkins launcher arguments -if [[ $# -lt 1 ]] || [[ "$1" == "--"* ]]; then - eval "exec java $JAVA_OPTS -jar /usr/share/jenkins/jenkins.war $JENKINS_OPTS \"\$@\"" -fi - -# As argument is not jenkins, assume user want to run his own process, for sample a `bash` shell to explore this image -exec "$@" \ No newline at end of file diff --git a/server/ubuntu-16.04.2/plugins.sh b/server/ubuntu-16.04.2/plugins.sh deleted file mode 100755 index 3c99ff2..0000000 --- a/server/ubuntu-16.04.2/plugins.sh +++ /dev/null @@ -1,126 +0,0 @@ -#! /bin/bash - -# Parse a support-core plugin -style txt file as specification for jenkins plugins to be installed -# in the reference directory, so user can define a derived Docker image with just : -# -# FROM jenkins -# COPY plugins.txt /plugins.txt -# RUN /usr/local/bin/plugins.sh /plugins.txt -# -# Note: Plugins already installed are skipped -# - -set -e - -echo "WARN: plugins.sh is deprecated, please switch to install-plugins.sh" - -if [ -z "$1" ] -then - echo " -USAGE: - Parse a support-core plugin -style txt file as specification for jenkins plugins to be installed - in the reference directory, so user can define a derived Docker image with just : - - FROM jenkins - COPY plugins.txt /plugins.txt - RUN /usr/local/bin/plugins.sh /plugins.txt - - Note: Plugins already installed are skipped - -" - exit 1 -else - JENKINS_INPUT_JOB_LIST=$1 - if [ ! -f $JENKINS_INPUT_JOB_LIST ] - then - echo "ERROR File not found: $JENKINS_INPUT_JOB_LIST" - exit 1 - fi -fi - -# the war includes a # of plugins, to make the build efficient filter out -# the plugins so we dont install 2x - there about 17! -if [ -d $JENKINS_HOME ] -then - TEMP_ALREADY_INSTALLED=$JENKINS_HOME/preinstalled.plugins.$$.txt -else - echo "ERROR $JENKINS_HOME not found" - exit 1 -fi - -JENKINS_PLUGINS_DIR=/var/jenkins_home/plugins -if [ -d $JENKINS_PLUGINS_DIR ] -then - echo "Analyzing: $JENKINS_PLUGINS_DIR" - for i in `ls -pd1 $JENKINS_PLUGINS_DIR/*|egrep '\/$'` - do - JENKINS_PLUGIN=`basename $i` - JENKINS_PLUGIN_VER=`egrep -i Plugin-Version "$i/META-INF/MANIFEST.MF"|cut -d\: -f2|sed 's/ //'` - echo "$JENKINS_PLUGIN:$JENKINS_PLUGIN_VER" - done > $TEMP_ALREADY_INSTALLED -else - JENKINS_WAR=/usr/share/jenkins/jenkins.war - if [ -f $JENKINS_WAR ] - then - echo "Analyzing war: $JENKINS_WAR" - TEMP_PLUGIN_DIR=/tmp/plugintemp.$$ - for i in `jar tf $JENKINS_WAR | egrep '[^detached-]plugins.*\..pi' | sort` - do - rm -fr $TEMP_PLUGIN_DIR - mkdir -p $TEMP_PLUGIN_DIR - PLUGIN=`basename $i|cut -f1 -d'.'` - (cd $TEMP_PLUGIN_DIR;jar xf $JENKINS_WAR "$i";jar xvf $TEMP_PLUGIN_DIR/$i META-INF/MANIFEST.MF >/dev/null 2>&1) - VER=`egrep -i Plugin-Version "$TEMP_PLUGIN_DIR/META-INF/MANIFEST.MF"|cut -d\: -f2|sed 's/ //'` - echo "$PLUGIN:$VER" - done > $TEMP_ALREADY_INSTALLED - rm -fr $TEMP_PLUGIN_DIR - else - rm -f $TEMP_ALREADY_INSTALLED - echo "ERROR file not found: $JENKINS_WAR" - exit 1 - fi -fi - -REF=/usr/share/jenkins/ref/plugins -mkdir -p $REF -COUNT_PLUGINS_INSTALLED=0 -while read spec || [ -n "$spec" ]; do - - plugin=(${spec//:/ }); - [[ ${plugin[0]} =~ ^# ]] && continue - [[ ${plugin[0]} =~ ^\s*$ ]] && continue - [[ -z ${plugin[1]} ]] && plugin[1]="latest" - - if [ -z "$JENKINS_UC_DOWNLOAD" ]; then - JENKINS_UC_DOWNLOAD=$JENKINS_UC/download - fi - - if ! grep -q "${plugin[0]}:${plugin[1]}" $TEMP_ALREADY_INSTALLED - then - echo "Downloading ${plugin[0]}:${plugin[1]}" - curl --retry 3 --retry-delay 5 -sSL -f ${JENKINS_UC_DOWNLOAD}/plugins/${plugin[0]}/${plugin[1]}/${plugin[0]}.hpi -o $REF/${plugin[0]}.jpi - unzip -qqt $REF/${plugin[0]}.jpi - COUNT_PLUGINS_INSTALLED=`expr $COUNT_PLUGINS_INSTALLED + 1` - else - echo " ... skipping already installed: ${plugin[0]}:${plugin[1]}" - fi -done < $JENKINS_INPUT_JOB_LIST - -echo "---------------------------------------------------" -if [ $COUNT_PLUGINS_INSTALLED -gt 0 ] -then - echo "INFO: Successfully installed $COUNT_PLUGINS_INSTALLED plugins." - - if [ -d $JENKINS_PLUGINS_DIR ] - then - echo "INFO: Please restart the container for changes to take effect!" - fi -else - echo "INFO: No changes, all plugins previously installed." - -fi -echo "---------------------------------------------------" - -#cleanup -rm $TEMP_ALREADY_INSTALLED -exit 0 \ No newline at end of file From e5fd5571c34715f286bce8134d05edf7922520a7 Mon Sep 17 00:00:00 2001 From: Jan Brezina Date: Fri, 29 Jul 2022 15:02:39 +0200 Subject: [PATCH 4/9] Credentials support WIP --- server/docker-compose.yaml | 3 ++- server/jenkins_config.yaml | 11 +++++++++++ server/plugins.txt | 1 + 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/server/docker-compose.yaml b/server/docker-compose.yaml index 5cfe801..1b5845d 100644 --- a/server/docker-compose.yaml +++ b/server/docker-compose.yaml @@ -15,5 +15,6 @@ services: ports: - "8080:8080" - "50000:50000" - + environment: + - SECRETS_FILE="test_secrets.properties" # Can use .env file for host dependent data diff --git a/server/jenkins_config.yaml b/server/jenkins_config.yaml index 219ccdf..43123d4 100644 --- a/server/jenkins_config.yaml +++ b/server/jenkins_config.yaml @@ -20,6 +20,17 @@ jenkins: users: - id: ${JENKINS_ADMIN_ID} password: ${JENKINS_ADMIN_PASSWORD} + securityRealm: + local: + allowsSignup: false + enableCaptcha: false + users: + - id: "jb" + name: "Jan Brezina" + password: ${jb_pass} + properties: + - mailer: + emailAddress: "jan.brezina@tul.cz" nodes: - permanent: name: "ci2runner" diff --git a/server/plugins.txt b/server/plugins.txt index ff2ce1c..1c389ba 100644 --- a/server/plugins.txt +++ b/server/plugins.txt @@ -1,5 +1,6 @@ configuration-as-code:latest ssh-slaves +credentials build-name-setter build-metrics console-column-plugin From bd9dcbe64e831acc01e08d34450327d029b39f21 Mon Sep 17 00:00:00 2001 From: Jan Brezina Date: Tue, 2 Aug 2022 10:14:42 +0200 Subject: [PATCH 5/9] initial generated jenkins config --- server/jenkins_config.yaml | 180 +++++++++++++++++++++++++------------ 1 file changed, 122 insertions(+), 58 deletions(-) diff --git a/server/jenkins_config.yaml b/server/jenkins_config.yaml index 43123d4..3a7f149 100644 --- a/server/jenkins_config.yaml +++ b/server/jenkins_config.yaml @@ -3,71 +3,135 @@ # - test setup of the Jenkins server with proper plugins # - configuration of nodes # - try to upload flow123d jobs - +credentials: + system: + domainCredentials: + - credentials: + - basicSSHUserPrivateKey: + id: "builder-key" + passphrase: ${SSH_passphrase} + privateKeySource: + directEntry: + privateKey: ${SSH_ci2runner_privatekey} + scope: SYSTEM + username: "builder" + usernameSecret: true jenkins: - systemMessage: "Jenkins configured automatically by Jenkins Configuration as Code plugin\n\n" - #globalNodeProperties: - #- envVars: - #env: - #- key: VARIABLE1 - #value: foo - #- key: VARIABLE2 - #value: bar - - securityRealm: - local: - allowsSignup: false - users: - - id: ${JENKINS_ADMIN_ID} - password: ${JENKINS_ADMIN_PASSWORD} + agentProtocols: + - "JNLP4-connect" + - "Ping" + crumbIssuer: + standard: + excludeClientIPFromCrumb: false + disableRememberMe: false + labelAtoms: + - name: "built-in" + - name: "ci2runner" + - name: "linux" + markupFormatter: "plainText" + mode: NORMAL + myViewsTabBar: "standard" + nodes: + - permanent: + labelString: "linux ci2runner" + launcher: + ssh: + credentialsId: "builder-key" + host: "ci2runner.nti.tul.cz" + port: 22 + sshHostKeyVerificationStrategy: "nonVerifyingKeyVerificationStrategy" + name: "ci2runner" + numExecutors: 5 + remoteFS: "/home/builder/jenkins" + retentionStrategy: "always" + numExecutors: 2 + primaryView: + all: + name: "all" + projectNamingStrategy: "standard" + quietPeriod: 5 + remotingSecurity: + enabled: true + scmCheckoutRetryCount: 0 securityRealm: local: allowsSignup: false enableCaptcha: false users: - - id: "jb" - name: "Jan Brezina" - password: ${jb_pass} - properties: + - id: "admin" + name: "admin" + properties: + - "apiToken" + - "myView" + - preferredProvider: + providerId: "default" + - "timezone" - mailer: - emailAddress: "jan.brezina@tul.cz" - nodes: - - permanent: - name: "ci2runner" - remoteFS: "/home/builder/jenkins" - launcher: - ssh: - credentialsId: builder - host: ci2runner.nti.tul.cz - sshHostKeyVerificationStrategy: manuallyProvidedKeyVerificationStrategy - disabled: true - failIfWorkDirIsMissing: false - internalDir: "remoting" - workDirPath: "/tmp" - retentionStrategy: always - numExecutors: 5 + emailAddress: "j@j.com" + slaveAgentPort: 50000 + systemMessage: |+ + Jenkins configured automatically by Jenkins Configuration as Code plugin - #slaveAgentPort: 50000 + updateCenter: + sites: + - id: "default" + url: "https://updates.jenkins.io/update-center.json" + views: + - all: + name: "all" + viewsTabBar: "standard" +globalCredentialsConfiguration: + configuration: + providerFilter: "none" + typeFilter: "none" +security: + apiToken: + creationOfLegacyTokenEnabled: false + tokenGenerationOnCreationEnabled: false + usageStatisticsEnabled: true + envInject: + enableLoadingFromMaster: false + enablePermissions: false + hideInjectedVars: false + gitHostKeyVerificationConfiguration: + sshHostKeyVerificationStrategy: "acceptFirstConnectionStrategy" + sSHD: + port: -1 unclassified: + buildDiscarders: + configuredBuildDiscarders: + - "jobBuildDiscarder" + fingerprints: + fingerprintCleanupDisabled: false + storage: "file" + gitHubPluginConfig: + hookUrl: "http://server_ip:8080/github-webhook/" + gitSCM: + addGitTagAction: false + allowSecondFetch: false + createAccountBasedOnEmail: false + disableGitToolChooser: false + hideCredentials: false + showEntireCommitSummaryInChanges: false + useExistingAccountWithSameEmail: false + junitTestResultStorage: + storage: "file" location: - url: http://server_ip:8080/ - -#tool: - #git: - #installations: - #- name: git - #home: /usr/local/bin/git - -#credentials: - #system: - #domainCredentials: - #- credentials: - #- basicSSHUserPrivateKey: - #scope: SYSTEM - #id: ssh_with_passphrase_provided - #username: ssh_root - #passphrase: ${SSH_KEY_PASSWORD} - #description: "SSH passphrase with private key file. Private key provided" - #privateKeySource: - #directEntry: - #privateKey: ${SSH_PRIVATE_KEY} + adminAddress: "address not configured yet " + url: "http://server_ip:8080/" + mailer: + charset: "UTF-8" + useSsl: false + useTls: false + mavenModuleSet: + localRepository: "default" + pollSCM: + pollingThreadCount: 10 +tool: + git: + installations: + - home: "git" + name: "Default" + mavenGlobalConfig: + globalSettingsProvider: "standard" + settingsProvider: "standard" From fbd8c798a32f417fd3b3a2f69972d0e86c3bc2c9 Mon Sep 17 00:00:00 2001 From: Jan Brezina Date: Tue, 2 Aug 2022 10:16:41 +0200 Subject: [PATCH 6/9] Fix, docker and dockerfile compose --- server/Dockerfile | 7 ++++++- server/docker-compose.yaml | 6 +++--- server/setup.sh | 11 +++++++++++ 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/server/Dockerfile b/server/Dockerfile index d69659c..23b10f4 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -1,7 +1,11 @@ # Customized Jenkins docker image. FROM jenkins/jenkins:lts-jdk11 - +USER root +RUN apt-get -y update && \ + apt-get -y install wget curl +USER jenkins + # Skip setup wizard and protection. ENV JAVA_OPTS "-Djenkins.install.runSetupWizard=false" ENV CASC_JENKINS_CONFIG /var/jenkins_home/casc.yaml @@ -10,6 +14,7 @@ ENV CASC_JENKINS_CONFIG /var/jenkins_home/casc.yaml COPY --chown=jenkins:jenkins plugins.txt /usr/share/jenkins/ref/plugins.txt RUN jenkins-plugin-cli -f /usr/share/jenkins/ref/plugins.txt +RUN ssh-keyscan -H ci2runner.nti.tul.cz >> ~/.ssh/known_hosts # Setup Configuration as Code #COPY plugins.txt /usr/share/jenkins/ref/plugins.txt #RUN /usr/local/bin/install-plugins.sh < /usr/share/jenkins/ref/plugins.txt diff --git a/server/docker-compose.yaml b/server/docker-compose.yaml index 1b5845d..560f1ea 100644 --- a/server/docker-compose.yaml +++ b/server/docker-compose.yaml @@ -14,7 +14,7 @@ services: - jenkins_data:/var/jenkins_home ports: - "8080:8080" - - "50000:50000" - environment: - - SECRETS_FILE="test_secrets.properties" + - "50001:50000" +# environment: +# - SECRETS_FILE="secrets.properties" # Can use .env file for host dependent data diff --git a/server/setup.sh b/server/setup.sh index cd642a6..f427181 100755 --- a/server/setup.sh +++ b/server/setup.sh @@ -11,7 +11,18 @@ script_path="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/$(basename "${BASH_SOURCE[0]}")" +# function instal_docker_compose { +# DOCKER_CONFIG=${DOCKER_CONFIG:-$HOME/.docker} +# mkdir -p $DOCKER_CONFIG/cli-plugins +# curl -SL https://github.com/docker/compose/releases/download/v2.7.0/docker-compose-linux-x86_64 -o $DOCKER_CONFIG/cli-plugins/docker-compose +# } + function setup_server { +# if ! docker compose --help >/dev/null; +# then install_docker_compose +# fi + + docker compose rm --stop --force --volumes docker volume rm jenkins-ciflow From 8ef304c0a4d5072e47c253974566f454b8fe1dde Mon Sep 17 00:00:00 2001 From: Jan Brezina Date: Tue, 2 Aug 2022 13:17:43 +0200 Subject: [PATCH 7/9] Working secrets for login an agent key --- server/Dockerfile | 2 +- server/README.md | 8 ++++++++ server/docker-compose.yaml | 23 ++++++++++++++++++++--- server/jenkins_config.yaml | 32 +++++++++++++++++--------------- server/setup.sh | 9 +++++++-- 5 files changed, 53 insertions(+), 21 deletions(-) diff --git a/server/Dockerfile b/server/Dockerfile index 23b10f4..fe5ca0a 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -14,7 +14,7 @@ ENV CASC_JENKINS_CONFIG /var/jenkins_home/casc.yaml COPY --chown=jenkins:jenkins plugins.txt /usr/share/jenkins/ref/plugins.txt RUN jenkins-plugin-cli -f /usr/share/jenkins/ref/plugins.txt -RUN ssh-keyscan -H ci2runner.nti.tul.cz >> ~/.ssh/known_hosts +#RUN ssh-keyscan -H ci2runner.nti.tul.cz >> ~/.ssh/known_hosts # Setup Configuration as Code #COPY plugins.txt /usr/share/jenkins/ref/plugins.txt #RUN /usr/local/bin/install-plugins.sh < /usr/share/jenkins/ref/plugins.txt diff --git a/server/README.md b/server/README.md index 89c666b..442aa74 100644 --- a/server/README.md +++ b/server/README.md @@ -9,6 +9,14 @@ Resources: - [Jenkins Docker](https://www.jenkins.io/doc/book/installing/docker/) - [Jenkins Configuration as Code](https://www.jenkins.io/projects/jcasc/) +- [JCasC - handling secrets](https://github.com/jenkinsci/configuration-as-code-plugin/blob/master/docs/features/secrets.adoc) - [Get started with Docker Compose](https://docs.docker.com/compose/gettingstarted/) - [Docker Compose Tutorial](https://www.educative.io/blog/docker-compose-tutorial) - [Tutorial: Automate Jenkins Setup](https://www.digitalocean.com/community/tutorials/how-to-automate-jenkins-setup-with-docker-and-jenkins-configuration-as-code#step-1-disabling-the-setup-wizard) + + +Decrypt Jenkins Secrets: + +``` +println(hudson.util.Secret.decrypt("{XXX=}")) +``` diff --git a/server/docker-compose.yaml b/server/docker-compose.yaml index 560f1ea..c1fa023 100644 --- a/server/docker-compose.yaml +++ b/server/docker-compose.yaml @@ -15,6 +15,23 @@ services: ports: - "8080:8080" - "50001:50000" -# environment: -# - SECRETS_FILE="secrets.properties" -# Can use .env file for host dependent data + secrets: + - jenkins_admin_pw + - ci2runner_ssh_key + - ci2runner_ssh_passphrase + + +# Docker secrets - here we use file secrets, the path is the file on the host containing the secret value +# Service (jenkins) then use defined secrets. +# In the container secrets are presented as files: /run/secrets/ + +secrets: #Be careful with newlines in your secrets, make sure to remove them before running docker-compose up. + #github: + # file: /var/deploy/secrets/github + jenkins_admin_pw: + file: ${SECRETS_PATH}/jenkins_admin_pw + ci2runner_ssh_key: #This should point to the private key you want your jenkins master to use when connecting to slaves. So in AWS for example this would be your .pem file + file: ${SECRETS_PATH}/ci2runner_ssh_key + # Can use .env file for host dependent data + ci2runner_ssh_passphrase: + file: ${SECRETS_PATH}/ci2runner_ssh_passphrase diff --git a/server/jenkins_config.yaml b/server/jenkins_config.yaml index 3a7f149..0fc2258 100644 --- a/server/jenkins_config.yaml +++ b/server/jenkins_config.yaml @@ -9,10 +9,10 @@ credentials: - credentials: - basicSSHUserPrivateKey: id: "builder-key" - passphrase: ${SSH_passphrase} + passphrase: ${ci2runner_ssh_passphrase} privateKeySource: directEntry: - privateKey: ${SSH_ci2runner_privatekey} + privateKey: ${ci2runner_ssh_key} scope: SYSTEM username: "builder" usernameSecret: true @@ -20,6 +20,7 @@ jenkins: agentProtocols: - "JNLP4-connect" - "Ping" + authorizationStrategy: "loggedInUsersCanDoAnything" crumbIssuer: standard: excludeClientIPFromCrumb: false @@ -58,20 +59,21 @@ jenkins: allowsSignup: false enableCaptcha: false users: - - id: "admin" - name: "admin" - properties: - - "apiToken" - - "myView" - - preferredProvider: - providerId: "default" - - "timezone" - - mailer: - emailAddress: "j@j.com" - slaveAgentPort: 50000 - systemMessage: |+ - Jenkins configured automatically by Jenkins Configuration as Code plugin + - id: Admin + password: ${jenkins_admin_pw} +# Generated: +# name: "admin" +# properties: +# - "apiToken" +# - "myView" +# - preferredProvider: +# providerId: "default" +# - "timezone" +# - mailer: +# emailAddress: "j@j.com" + slaveAgentPort: 50000 + systemMessage: Jenkins configured automatically by Jenkins Configuration as Code plugin updateCenter: sites: - id: "default" diff --git a/server/setup.sh b/server/setup.sh index f427181..38c4c22 100755 --- a/server/setup.sh +++ b/server/setup.sh @@ -21,19 +21,24 @@ function setup_server { # if ! docker compose --help >/dev/null; # then install_docker_compose # fi - + + # configuration in docker-compose.yaml, variables in .env + # set secrets path out of the git repository + echo SECRETS_PATH=../../secrets > .env docker compose rm --stop --force --volumes docker volume rm jenkins-ciflow docker volume create jenkins-ciflow + docker pull jenkins/jenkins docker compose build + docker compose up # initial service start } -export CASC_JENKINS_CONFIG="${script_path}" +#export CASC_JENKINS_CONFIG="${script_path}" cmd=$1 if [ "$cmd" == "setup" ] then From ebe5c781ce6aacb4f94550059c421f1a74a5726e Mon Sep 17 00:00:00 2001 From: Jan Brezina Date: Sat, 13 May 2023 11:52:14 +0200 Subject: [PATCH 8/9] remove deprecated ciflow server configuration --- server/Dockerfile | 21 ------ server/README.md | 22 ------ server/docker-compose.yaml | 37 ---------- server/jenkins_config.yaml | 139 ------------------------------------- server/plugins.txt | 19 ----- server/setup.sh | 58 ---------------- 6 files changed, 296 deletions(-) delete mode 100644 server/Dockerfile delete mode 100644 server/README.md delete mode 100644 server/docker-compose.yaml delete mode 100644 server/jenkins_config.yaml delete mode 100644 server/plugins.txt delete mode 100755 server/setup.sh diff --git a/server/Dockerfile b/server/Dockerfile deleted file mode 100644 index fe5ca0a..0000000 --- a/server/Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -# Customized Jenkins docker image. - -FROM jenkins/jenkins:lts-jdk11 -USER root -RUN apt-get -y update && \ - apt-get -y install wget curl -USER jenkins - -# Skip setup wizard and protection. -ENV JAVA_OPTS "-Djenkins.install.runSetupWizard=false" -ENV CASC_JENKINS_CONFIG /var/jenkins_home/casc.yaml - -# Add Jenkins plugins from separate file -COPY --chown=jenkins:jenkins plugins.txt /usr/share/jenkins/ref/plugins.txt -RUN jenkins-plugin-cli -f /usr/share/jenkins/ref/plugins.txt - -#RUN ssh-keyscan -H ci2runner.nti.tul.cz >> ~/.ssh/known_hosts -# Setup Configuration as Code -#COPY plugins.txt /usr/share/jenkins/ref/plugins.txt -#RUN /usr/local/bin/install-plugins.sh < /usr/share/jenkins/ref/plugins.txt -COPY jenkins_config.yaml /var/jenkins_home/casc.yaml diff --git a/server/README.md b/server/README.md deleted file mode 100644 index 442aa74..0000000 --- a/server/README.md +++ /dev/null @@ -1,22 +0,0 @@ -Folder with complete configuration of the Jenkins server for CI testing. -This includes: -- Docker image dockerfile. -- Docker compose for the particular container configuration with volumes and port redirection. -- Jenkins YAML configuration using the : https://github.com/jenkinsci/docker - - -Resources: - -- [Jenkins Docker](https://www.jenkins.io/doc/book/installing/docker/) -- [Jenkins Configuration as Code](https://www.jenkins.io/projects/jcasc/) -- [JCasC - handling secrets](https://github.com/jenkinsci/configuration-as-code-plugin/blob/master/docs/features/secrets.adoc) -- [Get started with Docker Compose](https://docs.docker.com/compose/gettingstarted/) -- [Docker Compose Tutorial](https://www.educative.io/blog/docker-compose-tutorial) -- [Tutorial: Automate Jenkins Setup](https://www.digitalocean.com/community/tutorials/how-to-automate-jenkins-setup-with-docker-and-jenkins-configuration-as-code#step-1-disabling-the-setup-wizard) - - -Decrypt Jenkins Secrets: - -``` -println(hudson.util.Secret.decrypt("{XXX=}")) -``` diff --git a/server/docker-compose.yaml b/server/docker-compose.yaml deleted file mode 100644 index c1fa023..0000000 --- a/server/docker-compose.yaml +++ /dev/null @@ -1,37 +0,0 @@ -version: '2' - -volumes: - jenkins_data: - external: true - name: jenkins-ciflow - -services: - jenkins_casc: # derived Jenkins image using Configuration as code plugin for configuring from scratch - build: . - restart: unless-stopped - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - jenkins_data:/var/jenkins_home - ports: - - "8080:8080" - - "50001:50000" - secrets: - - jenkins_admin_pw - - ci2runner_ssh_key - - ci2runner_ssh_passphrase - - -# Docker secrets - here we use file secrets, the path is the file on the host containing the secret value -# Service (jenkins) then use defined secrets. -# In the container secrets are presented as files: /run/secrets/ - -secrets: #Be careful with newlines in your secrets, make sure to remove them before running docker-compose up. - #github: - # file: /var/deploy/secrets/github - jenkins_admin_pw: - file: ${SECRETS_PATH}/jenkins_admin_pw - ci2runner_ssh_key: #This should point to the private key you want your jenkins master to use when connecting to slaves. So in AWS for example this would be your .pem file - file: ${SECRETS_PATH}/ci2runner_ssh_key - # Can use .env file for host dependent data - ci2runner_ssh_passphrase: - file: ${SECRETS_PATH}/ci2runner_ssh_passphrase diff --git a/server/jenkins_config.yaml b/server/jenkins_config.yaml deleted file mode 100644 index 0fc2258..0000000 --- a/server/jenkins_config.yaml +++ /dev/null @@ -1,139 +0,0 @@ -# Configuration of the Jenkins server through the Configuration as Code plugin. -# TODO: -# - test setup of the Jenkins server with proper plugins -# - configuration of nodes -# - try to upload flow123d jobs -credentials: - system: - domainCredentials: - - credentials: - - basicSSHUserPrivateKey: - id: "builder-key" - passphrase: ${ci2runner_ssh_passphrase} - privateKeySource: - directEntry: - privateKey: ${ci2runner_ssh_key} - scope: SYSTEM - username: "builder" - usernameSecret: true -jenkins: - agentProtocols: - - "JNLP4-connect" - - "Ping" - authorizationStrategy: "loggedInUsersCanDoAnything" - crumbIssuer: - standard: - excludeClientIPFromCrumb: false - disableRememberMe: false - labelAtoms: - - name: "built-in" - - name: "ci2runner" - - name: "linux" - markupFormatter: "plainText" - mode: NORMAL - myViewsTabBar: "standard" - nodes: - - permanent: - labelString: "linux ci2runner" - launcher: - ssh: - credentialsId: "builder-key" - host: "ci2runner.nti.tul.cz" - port: 22 - sshHostKeyVerificationStrategy: "nonVerifyingKeyVerificationStrategy" - name: "ci2runner" - numExecutors: 5 - remoteFS: "/home/builder/jenkins" - retentionStrategy: "always" - numExecutors: 2 - primaryView: - all: - name: "all" - projectNamingStrategy: "standard" - quietPeriod: 5 - remotingSecurity: - enabled: true - scmCheckoutRetryCount: 0 - securityRealm: - local: - allowsSignup: false - enableCaptcha: false - users: - - id: Admin - password: ${jenkins_admin_pw} -# Generated: -# name: "admin" -# properties: -# - "apiToken" -# - "myView" -# - preferredProvider: -# providerId: "default" -# - "timezone" -# - mailer: -# emailAddress: "j@j.com" - - slaveAgentPort: 50000 - systemMessage: Jenkins configured automatically by Jenkins Configuration as Code plugin - updateCenter: - sites: - - id: "default" - url: "https://updates.jenkins.io/update-center.json" - views: - - all: - name: "all" - viewsTabBar: "standard" -globalCredentialsConfiguration: - configuration: - providerFilter: "none" - typeFilter: "none" -security: - apiToken: - creationOfLegacyTokenEnabled: false - tokenGenerationOnCreationEnabled: false - usageStatisticsEnabled: true - envInject: - enableLoadingFromMaster: false - enablePermissions: false - hideInjectedVars: false - gitHostKeyVerificationConfiguration: - sshHostKeyVerificationStrategy: "acceptFirstConnectionStrategy" - sSHD: - port: -1 -unclassified: - buildDiscarders: - configuredBuildDiscarders: - - "jobBuildDiscarder" - fingerprints: - fingerprintCleanupDisabled: false - storage: "file" - gitHubPluginConfig: - hookUrl: "http://server_ip:8080/github-webhook/" - gitSCM: - addGitTagAction: false - allowSecondFetch: false - createAccountBasedOnEmail: false - disableGitToolChooser: false - hideCredentials: false - showEntireCommitSummaryInChanges: false - useExistingAccountWithSameEmail: false - junitTestResultStorage: - storage: "file" - location: - adminAddress: "address not configured yet " - url: "http://server_ip:8080/" - mailer: - charset: "UTF-8" - useSsl: false - useTls: false - mavenModuleSet: - localRepository: "default" - pollSCM: - pollingThreadCount: 10 -tool: - git: - installations: - - home: "git" - name: "Default" - mavenGlobalConfig: - globalSettingsProvider: "standard" - settingsProvider: "standard" diff --git a/server/plugins.txt b/server/plugins.txt deleted file mode 100644 index 1c389ba..0000000 --- a/server/plugins.txt +++ /dev/null @@ -1,19 +0,0 @@ -configuration-as-code:latest -ssh-slaves -credentials -build-name-setter -build-metrics -console-column-plugin -embeddable-build-status -envinject -git-parameter -git-client -github -jobConfigHistory -junit -jenkins-multijob-plugin -publish-over-cifs - - - - diff --git a/server/setup.sh b/server/setup.sh deleted file mode 100755 index 38c4c22..0000000 --- a/server/setup.sh +++ /dev/null @@ -1,58 +0,0 @@ -# Script documenting server creation, start and stop -# can only be executed within its directory with docker-compose and other configuration files -# It uses docker compose V2 that works as a plugin to docker, providing the "compose" command. -# -# Usage: -# setup.sh setup admin-password -# start -# stop - - - -script_path="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/$(basename "${BASH_SOURCE[0]}")" - -# function instal_docker_compose { -# DOCKER_CONFIG=${DOCKER_CONFIG:-$HOME/.docker} -# mkdir -p $DOCKER_CONFIG/cli-plugins -# curl -SL https://github.com/docker/compose/releases/download/v2.7.0/docker-compose-linux-x86_64 -o $DOCKER_CONFIG/cli-plugins/docker-compose -# } - -function setup_server { -# if ! docker compose --help >/dev/null; -# then install_docker_compose -# fi - - # configuration in docker-compose.yaml, variables in .env - # set secrets path out of the git repository - echo SECRETS_PATH=../../secrets > .env - - docker compose rm --stop --force --volumes - docker volume rm jenkins-ciflow - - docker volume create jenkins-ciflow - - docker pull jenkins/jenkins - docker compose build - - docker compose up # initial service start -} - - -#export CASC_JENKINS_CONFIG="${script_path}" -cmd=$1 -if [ "$cmd" == "setup" ] -then - read -p "Setup of the Jenkins server will overwrite all previous configurations. Do you want to proceed? [Yy]" -n 1 -r - echo # (optional) move to a new line - if [[ $REPLY =~ ^[Yy]$ ]] - then - password=$2 - setup_server - fi -elif [ "$cmd" == "start" ] -then - docker compose up -elif [ "$cmd" == "stop" ] -then - docker compose stop -fi From 45c19c5b081944aae86d3e4bee072b57b3a00fc7 Mon Sep 17 00:00:00 2001 From: Jan Brezina Date: Sat, 13 May 2023 11:53:45 +0200 Subject: [PATCH 9/9] unify GNU variant to a single multistage dockerfile --- dockerfiles/base-build/Dockerfile-gnu | 23 ++-- dockerfiles/base/Dockerfile-gnu | 15 +-- dockerfiles/flow-dev-gnu/Dockerfile | 111 ++++++++++++++++-- .../libs-build-gnu/armadillo/Makefile | 0 .../libs-build-gnu/bddcml/Makefile | 2 +- .../bddcml/bddcml_make_inc.common | 0 .../bddcml/blopex_make_inc.common | 0 .../libs-build-gnu/common_cmake.mk | 2 +- .../libs-build-gnu/common_configure.mk | 5 +- .../libs-build-gnu/mpich/Makefile | 4 + .../libs-build-gnu/permon/Makefile | 2 +- .../libs-build-gnu/petsc/Makefile | 2 +- .../libs-build-gnu/yamlcpp/Makefile | 5 +- dockerfiles/install-gnu/Dockerfile | 36 ------ dockerfiles/libs-gnu/Dockerfile | 41 ------- images_version | 2 +- makefile | 44 +++---- 17 files changed, 157 insertions(+), 137 deletions(-) rename dockerfiles/{libs-gnu => flow-dev-gnu}/libs-build-gnu/armadillo/Makefile (100%) rename dockerfiles/{libs-gnu => flow-dev-gnu}/libs-build-gnu/bddcml/Makefile (99%) rename dockerfiles/{libs-gnu => flow-dev-gnu}/libs-build-gnu/bddcml/bddcml_make_inc.common (100%) rename dockerfiles/{libs-gnu => flow-dev-gnu}/libs-build-gnu/bddcml/blopex_make_inc.common (100%) rename dockerfiles/{libs-gnu => flow-dev-gnu}/libs-build-gnu/common_cmake.mk (99%) rename dockerfiles/{libs-gnu => flow-dev-gnu}/libs-build-gnu/common_configure.mk (96%) rename dockerfiles/{libs-gnu => flow-dev-gnu}/libs-build-gnu/mpich/Makefile (74%) rename dockerfiles/{libs-gnu => flow-dev-gnu}/libs-build-gnu/permon/Makefile (99%) rename dockerfiles/{libs-gnu => flow-dev-gnu}/libs-build-gnu/petsc/Makefile (99%) rename dockerfiles/{libs-gnu => flow-dev-gnu}/libs-build-gnu/yamlcpp/Makefile (71%) delete mode 100644 dockerfiles/install-gnu/Dockerfile delete mode 100644 dockerfiles/libs-gnu/Dockerfile diff --git a/dockerfiles/base-build/Dockerfile-gnu b/dockerfiles/base-build/Dockerfile-gnu index 87b07c1..e45b19c 100644 --- a/dockerfiles/base-build/Dockerfile-gnu +++ b/dockerfiles/base-build/Dockerfile-gnu @@ -3,6 +3,7 @@ ARG images_version FROM flow123d/base-gnu:${images_version} MAINTAINER Jan Hybs ARG images_version +ARG boost_version # Environment variables are part of the images. # We will use the same base boost package when selectiong specific parts of the BOOST for the install image. @@ -14,7 +15,7 @@ ARG images_version # 2 /usr/bin/gcc-7 7 manual mode # 3 /usr/bin/gcc-8 8 manual mode # 4 /usr/bin/gcc-9 9 manual mode -ARG DEFAULT_GCC=/usr/bin/gcc-9 +ARG DEFAULT_GCC=/usr/bin/gcc-11 # [g++] # 0 /usr/bin/g++-9 9 auto mode @@ -22,13 +23,13 @@ ARG DEFAULT_GCC=/usr/bin/gcc-9 # 2 /usr/bin/g++-7 7 manual mode # 3 /usr/bin/g++-8 8 manual mode # 4 /usr/bin/g++-9 9 manual mode -ARG DEFAULT_GXX=/usr/bin/g++-9 +ARG DEFAULT_GXX=/usr/bin/g++-11 # [c++] # 0 /usr/bin/g++ 20 auto mode # 1 /usr/bin/clang++ 10 manual mode # 2 /usr/bin/g++ 20 manual mode -ARG DEFAULT_CXX=/usr/bin/g++-9 +ARG DEFAULT_CXX=/usr/bin/g++-11 # install basic set of tools and pandoc tools @@ -42,25 +43,23 @@ RUN sudo apt-get update && sudo apt-get install -y \ man \ tree \ gfortran \ - gcc-7 g++-7 \ - gcc-8 g++-8 \ - gcc-9 g++-9 \ + gcc-11 g++-11 \ clang \ gdbserver \ valgrind \ oprofile \ pandoc \ ccache \ - libboost1.71-dev \ - libboost-program-options1.71-dev \ - libboost-serialization1.71-dev \ - libboost-regex1.71-dev \ - libboost-filesystem1.71-dev \ + libboost${boost_version}-dev \ + libboost-program-options${boost_version}-dev \ + libboost-serialization${boost_version}-dev \ + libboost-regex${boost_version}-dev \ + libboost-filesystem${boost_version}-dev \ libblas-dev \ liblapack-dev \ lib32z1-dev \ libpugixml-dev \ - libpython3.9-dev + libpython3-dev # install doxygen and graphviz, latex diff --git a/dockerfiles/base/Dockerfile-gnu b/dockerfiles/base/Dockerfile-gnu index 49cb44c..a18f1f2 100644 --- a/dockerfiles/base/Dockerfile-gnu +++ b/dockerfiles/base/Dockerfile-gnu @@ -1,6 +1,7 @@ # BASE Dockerfile for running flow123d -FROM ubuntu:20.04 +FROM ubuntu:22.04 ARG images_version +ARG boost_version MAINTAINER Jan Březina, Jan Hybš @@ -26,19 +27,19 @@ RUN apt-get update && apt-get install -y \ less \ tzdata \ perl \ - python3.9 \ + python3 \ python3-pip \ openssh-client \ libpsm2-dev \ - libboost-program-options1.71.0 \ - libboost-serialization1.71.0 \ - libboost-regex1.71.0 \ - libboost-filesystem1.71.0 \ + libboost-program-options${boost_version} \ + libboost-serialization${boost_version} \ + libboost-regex${boost_version} \ + libboost-filesystem${boost_version} \ libblas3 \ liblapack3 \ lib32z1 \ libpugixml1v5 \ - libpython3.9 \ + libpython3-stdlib \ python3-numpy \ python3-venv \ libxinerama-dev \ diff --git a/dockerfiles/flow-dev-gnu/Dockerfile b/dockerfiles/flow-dev-gnu/Dockerfile index 637ae7a..492a4a0 100644 --- a/dockerfiles/flow-dev-gnu/Dockerfile +++ b/dockerfiles/flow-dev-gnu/Dockerfile @@ -1,13 +1,67 @@ -# FLOW-LIBS-DEV-DBG Dockerfile for running flow123d +# BASE-BUILD Dockerfile for running flow123d ARG images_version +FROM flow123d/base-build-gnu:${images_version} as libs-build ARG BUILD_TYPE -ARG source_image -FROM ${source_image} as builder +# TODO: +# COPY invalidates all layers even if the chance takes place only in configuration of a single library +# Resolution: +# - break into multiple stages one for each library, copy only relevant directory +# - merge with flow-dev-gnu +# auxiliary build image +# just build libraries Dbug or Release using ${BUILD_TYPE} variable + +RUN mkdir /build +COPY libs-build-gnu/common_cmake.mk /build/common_cmake.mk +COPY libs-build-gnu/common_configure.mk /build/common_configure.mk + + +### YAMLCPP +COPY libs-build-gnu/yamlcpp /build/yamlcpp +ARG ver_yamlcpp +# Need to copy these into ../libs-dbg/Docrefile +RUN echo "*** building YAMLCPP ***"; \ + make -C /build/yamlcpp build_type=${BUILD_TYPE} version=$ver_yamlcpp install + +### ARMADILO +COPY libs-build-gnu/armadillo /build/armadillo +ARG ver_armadillo +RUN echo "*** building Armadillo ***"; \ + make -C /build/armadillo build_type=${BUILD_TYPE} version=$ver_armadillo install + +### MPICH +COPY libs-build-gnu/mpich /build/mpich +ARG ver_mpich +RUN echo "*** building MPICH ***"; \ + make -C /build/mpich build_type=${BUILD_TYPE} version=$ver_mpich install + +### PETSC +COPY libs-build-gnu/petsc /build/petsc +ARG ver_petsc +ARG ver_hypre +RUN echo "*** building PETSC ***"; \ + make -C /build/petsc build_type=${BUILD_TYPE} mpich_lib=mpich_${ver_mpich} version=${ver_petsc} hypre_version=${ver_hypre} install + +### BDDCML +COPY libs-build-gnu/bddcml /build/bddcml +ARG ver_bddcml +RUN echo "*** building BDDCML ***"; \ + make -C /build/bddcml build_type=${BUILD_TYPE} version=${ver_bddcml} mpich_lib=mpich_${ver_mpich} petsc_lib=petsc_${ver_petsc} install + +### PERMON +COPY libs-build-gnu/permon /build/permon +ARG ver_permon +RUN echo "*** building PERMON ***"; \ + make -C /build/permon build_type=${BUILD_TYPE} version=${ver_permon} petsc_lib=petsc_${ver_petsc} install + + + +################################## +# Target flow-dev-gnu image ARG images_version -FROM flow123d/base-build-gnu:${images_version} -MAINTAINER Jan Hybs +FROM flow123d/base-build-gnu:${images_version} as flow-dev-gnu +MAINTAINER Jan Brezina ARG images_version ARG BUILD_TYPE @@ -20,15 +74,50 @@ ARG ver_bddcml ARG ver_permon # install libraries -COPY --from=builder /usr/local/yamlcpp_$ver_yamlcpp /usr/local/yamlcpp_$ver_yamlcpp -COPY --from=builder /usr/local/armadillo_${ver_armadillo} /usr/local/armadillo_${ver_armadillo} -COPY --from=builder /usr/local/mpich_${ver_mpich} /usr/local/mpich_${ver_mpich} -COPY --from=builder /usr/local/petsc_${ver_petsc} /usr/local/petsc_${ver_petsc} -COPY --from=builder /usr/local/bddcml_${ver_bddcml} /usr/local/bddcml_${ver_bddcml} -COPY --from=builder /usr/local/permon_${ver_permon} /usr/local/permon_${ver_permon} +COPY --from=libs-build /usr/local/yamlcpp_$ver_yamlcpp /usr/local/yamlcpp_$ver_yamlcpp +COPY --from=libs-build /usr/local/armadillo_${ver_armadillo} /usr/local/armadillo_${ver_armadillo} +COPY --from=libs-build /usr/local/mpich_${ver_mpich} /usr/local/mpich_${ver_mpich} +COPY --from=libs-build /usr/local/petsc_${ver_petsc} /usr/local/petsc_${ver_petsc} +COPY --from=libs-build /usr/local/bddcml_${ver_bddcml} /usr/local/bddcml_${ver_bddcml} +COPY --from=libs-build /usr/local/permon_${ver_permon} /usr/local/permon_${ver_permon} RUN sudo apt-get update && sudo apt-get install -y --no-install-recommends \ python3-vtk7 RUN echo ${BUILD_TYPE} ${images_version} >/.dockerversion + +################################### +# Target install-gnu image + + +ARG images_version +FROM flow123d/base-gnu:${images_version} as install-gnu +MAINTAINER Jan Brezina + +ARG images_version +ARG BUILD_TYPE=Release + + +ARG ver_yamlcpp +ARG ver_armadillo +ARG ver_mpich +ARG ver_petsc +ARG ver_bddcml +ARG ver_permon + +# install libraries +COPY --from=libs-build /usr/local/yamlcpp_$ver_yamlcpp /usr/local/yamlcpp_$ver_yamlcpp +COPY --from=libs-build /usr/local/armadillo_${ver_armadillo} /usr/local/armadillo_${ver_armadillo} +COPY --from=libs-build /usr/local/mpich_${ver_mpich} /usr/local/mpich_${ver_mpich} +COPY --from=libs-build /usr/local/petsc_${ver_petsc} /usr/local/petsc_${ver_petsc} +COPY --from=libs-build /usr/local/bddcml_${ver_bddcml} /usr/local/bddcml_${ver_bddcml} +COPY --from=libs-build /usr/local/permon_${ver_permon} /usr/local/permon_${ver_permon} + +RUN echo ${BUILD_TYPE} ${images_version} >/.dockerversion; \ + echo /usr/local/mpich_${ver_mpich}/bin/mpicc >/.mpiccpath; \ + ln -s /usr/local/${ver_mpich}/bin/mpirun /usr/bin/mpirun; \ + ln -s /usr/local/${ver_mpich}/bin/mpicc /usr/bin/mpicc; \ + ln -s /usr/local/${ver_mpich}/bin/mpicc /usr/bin/mpicxx; \ + ln -s /usr/local/${ver_mpich}/bin/mpicc /usr/bin/mpif90; \ + ln -s /usr/local/${ver_mpich}/bin/mpicc /usr/bin/mpifort diff --git a/dockerfiles/libs-gnu/libs-build-gnu/armadillo/Makefile b/dockerfiles/flow-dev-gnu/libs-build-gnu/armadillo/Makefile similarity index 100% rename from dockerfiles/libs-gnu/libs-build-gnu/armadillo/Makefile rename to dockerfiles/flow-dev-gnu/libs-build-gnu/armadillo/Makefile diff --git a/dockerfiles/libs-gnu/libs-build-gnu/bddcml/Makefile b/dockerfiles/flow-dev-gnu/libs-build-gnu/bddcml/Makefile similarity index 99% rename from dockerfiles/libs-gnu/libs-build-gnu/bddcml/Makefile rename to dockerfiles/flow-dev-gnu/libs-build-gnu/bddcml/Makefile index 0f43265..00258f8 100644 --- a/dockerfiles/libs-gnu/libs-build-gnu/bddcml/Makefile +++ b/dockerfiles/flow-dev-gnu/libs-build-gnu/bddcml/Makefile @@ -9,7 +9,7 @@ base_name=bddcml-$(version) clear=false -build_root=/libs-build-gnu +build_root=/build build_dir = $(build_root)/$(library)/build_$(build_type) # package_dir = $(build_root)/packages # package_name = $(library)_$(build_type) diff --git a/dockerfiles/libs-gnu/libs-build-gnu/bddcml/bddcml_make_inc.common b/dockerfiles/flow-dev-gnu/libs-build-gnu/bddcml/bddcml_make_inc.common similarity index 100% rename from dockerfiles/libs-gnu/libs-build-gnu/bddcml/bddcml_make_inc.common rename to dockerfiles/flow-dev-gnu/libs-build-gnu/bddcml/bddcml_make_inc.common diff --git a/dockerfiles/libs-gnu/libs-build-gnu/bddcml/blopex_make_inc.common b/dockerfiles/flow-dev-gnu/libs-build-gnu/bddcml/blopex_make_inc.common similarity index 100% rename from dockerfiles/libs-gnu/libs-build-gnu/bddcml/blopex_make_inc.common rename to dockerfiles/flow-dev-gnu/libs-build-gnu/bddcml/blopex_make_inc.common diff --git a/dockerfiles/libs-gnu/libs-build-gnu/common_cmake.mk b/dockerfiles/flow-dev-gnu/libs-build-gnu/common_cmake.mk similarity index 99% rename from dockerfiles/libs-gnu/libs-build-gnu/common_cmake.mk rename to dockerfiles/flow-dev-gnu/libs-build-gnu/common_cmake.mk index d58b81f..5f3b3a7 100644 --- a/dockerfiles/libs-gnu/libs-build-gnu/common_cmake.mk +++ b/dockerfiles/flow-dev-gnu/libs-build-gnu/common_cmake.mk @@ -10,7 +10,7 @@ package_name ?= $(library)_$(build_type) -build_root=/libs-build-gnu +build_root=/build build_dir = $(build_root)/$(library)/build_$(build_type) # package_dir = $(build_root)/packages # package=$(package_dir)/$(package_name).tar.gz diff --git a/dockerfiles/libs-gnu/libs-build-gnu/common_configure.mk b/dockerfiles/flow-dev-gnu/libs-build-gnu/common_configure.mk similarity index 96% rename from dockerfiles/libs-gnu/libs-build-gnu/common_configure.mk rename to dockerfiles/flow-dev-gnu/libs-build-gnu/common_configure.mk index 65b77e9..9f2416e 100644 --- a/dockerfiles/libs-gnu/libs-build-gnu/common_configure.mk +++ b/dockerfiles/flow-dev-gnu/libs-build-gnu/common_configure.mk @@ -9,7 +9,7 @@ # 'package-name' - optional, default is $(library) -build_root=/libs-build-gnu +build_root=/build package_name ?= $(library)_$(build_type) build_dir = $(build_root)/$(library)/build_$(build_type) @@ -39,13 +39,14 @@ $(CURDIR)/$(install_file): wget $(url) $(build_dir)/configure: clean_build $(CURDIR)/$(install_file) - if [ ! -d $(build_dir) ]; then \ + if [ ! -d $(build_dir) -o ! -x $(build_dir)/configure ]; then \ cmake -E tar x $(install_file); \ mv $(base_name) build_$(build_type); \ fi lib_makefile=$(build_dir)/Makefile $(lib_makefile): $(build_dir)/configure + ls $(build_dir) cd $(build_dir) && ./configure --prefix=$(install_prefix) $(configure_options) diff --git a/dockerfiles/libs-gnu/libs-build-gnu/mpich/Makefile b/dockerfiles/flow-dev-gnu/libs-build-gnu/mpich/Makefile similarity index 74% rename from dockerfiles/libs-gnu/libs-build-gnu/mpich/Makefile rename to dockerfiles/flow-dev-gnu/libs-build-gnu/mpich/Makefile index bae59ce..5beaa07 100644 --- a/dockerfiles/libs-gnu/libs-build-gnu/mpich/Makefile +++ b/dockerfiles/flow-dev-gnu/libs-build-gnu/mpich/Makefile @@ -5,9 +5,13 @@ url=http://flow.nti.tul.cz/libraries/mpich-${version}.tar.gz base_name=mpich-$(version) conf_opt_Debug= \ + FFLAGS=-fallow-argument-mismatch \ + FCFLAGS=-fallow-argument-mismatch \ --with-device=ch4:ofi conf_opt_Release= \ + FFLAGS=-fallow-argument-mismatch \ + FCFLAGS=-fallow-argument-mismatch \ --enable-fast=O3,ndebug \ --disable-error-checking \ --without-timing \ diff --git a/dockerfiles/libs-gnu/libs-build-gnu/permon/Makefile b/dockerfiles/flow-dev-gnu/libs-build-gnu/permon/Makefile similarity index 99% rename from dockerfiles/libs-gnu/libs-build-gnu/permon/Makefile rename to dockerfiles/flow-dev-gnu/libs-build-gnu/permon/Makefile index ed89a9f..257dd9f 100644 --- a/dockerfiles/libs-gnu/libs-build-gnu/permon/Makefile +++ b/dockerfiles/flow-dev-gnu/libs-build-gnu/permon/Makefile @@ -9,7 +9,7 @@ base_name=$(library)-$(version) clear=true -build_root=/libs-build-gnu +build_root=/build build_dir = $(build_root)/$(library)/build_$(build_type) # package_dir = $(build_root)/packages # package_name = $(library)_$(build_type) diff --git a/dockerfiles/libs-gnu/libs-build-gnu/petsc/Makefile b/dockerfiles/flow-dev-gnu/libs-build-gnu/petsc/Makefile similarity index 99% rename from dockerfiles/libs-gnu/libs-build-gnu/petsc/Makefile rename to dockerfiles/flow-dev-gnu/libs-build-gnu/petsc/Makefile index 660bd5e..b992361 100644 --- a/dockerfiles/libs-gnu/libs-build-gnu/petsc/Makefile +++ b/dockerfiles/flow-dev-gnu/libs-build-gnu/petsc/Makefile @@ -10,7 +10,7 @@ base_name=petsc-$(version) package_name = $(library)_$(build_type) clear=false -build_root=/libs-build-gnu +build_root=/build build_dir = $(build_root)/$(library)/build_$(build_type) package_dir = $(build_root)/packages n_jobs = 5 diff --git a/dockerfiles/libs-gnu/libs-build-gnu/yamlcpp/Makefile b/dockerfiles/flow-dev-gnu/libs-build-gnu/yamlcpp/Makefile similarity index 71% rename from dockerfiles/libs-gnu/libs-build-gnu/yamlcpp/Makefile rename to dockerfiles/flow-dev-gnu/libs-build-gnu/yamlcpp/Makefile index 57bd3c7..388ae30 100644 --- a/dockerfiles/libs-gnu/libs-build-gnu/yamlcpp/Makefile +++ b/dockerfiles/flow-dev-gnu/libs-build-gnu/yamlcpp/Makefile @@ -1,8 +1,8 @@ # Simple makefile for building and packaging project Yaml cpp library=yamlcpp -url=https://flow.nti.tul.cz/libraries/yaml-cpp-yaml-cpp-$(version).zip -base_name=yaml-cpp-yaml-cpp-$(version) +url=https://flow.nti.tul.cz/libraries/yaml-cpp-$(version).zip +base_name=yaml-cpp-$(version) cmake_options="-DYAML_BUILD_SHARED_LIBS=ON" # In order to combine staticaly linked yamlcpp with dynamicaly linked res of the code. @@ -10,7 +10,6 @@ CXX_FLAGS=-fPIC # TODO: shared yamplcpp - include ../common_cmake.mk diff --git a/dockerfiles/install-gnu/Dockerfile b/dockerfiles/install-gnu/Dockerfile deleted file mode 100644 index 50ceece..0000000 --- a/dockerfiles/install-gnu/Dockerfile +++ /dev/null @@ -1,36 +0,0 @@ -# FLOW-LIBS-DEV-DBG Dockerfile for running flow123d -ARG images_version -ARG source_image -FROM ${source_image} as builder - - -ARG images_version -FROM flow123d/base-gnu:${images_version} -MAINTAINER Jan Brezina - -ARG images_version -ARG BUILD_TYPE=Release - - -ARG ver_yamlcpp -ARG ver_armadillo -ARG ver_mpich -ARG ver_petsc -ARG ver_bddcml -ARG ver_permon - -# install libraries -COPY --from=builder /usr/local/yamlcpp_$ver_yamlcpp /usr/local/yamlcpp_$ver_yamlcpp -COPY --from=builder /usr/local/armadillo_${ver_armadillo} /usr/local/armadillo_${ver_armadillo} -COPY --from=builder /usr/local/mpich_${ver_mpich} /usr/local/mpich_${ver_mpich} -COPY --from=builder /usr/local/petsc_${ver_petsc} /usr/local/petsc_${ver_petsc} -COPY --from=builder /usr/local/bddcml_${ver_bddcml} /usr/local/bddcml_${ver_bddcml} -COPY --from=builder /usr/local/permon_${ver_permon} /usr/local/permon_${ver_permon} - -RUN echo ${BUILD_TYPE} ${images_version} >/.dockerversion; \ - echo /usr/local/mpich_${ver_mpich}/bin/mpicc >/.mpiccpath; \ - ln -s /usr/local/${ver_mpich}/bin/mpirun /usr/bin/mpirun; \ - ln -s /usr/local/${ver_mpich}/bin/mpicc /usr/bin/mpicc; \ - ln -s /usr/local/${ver_mpich}/bin/mpicc /usr/bin/mpicxx; \ - ln -s /usr/local/${ver_mpich}/bin/mpicc /usr/bin/mpif90; \ - ln -s /usr/local/${ver_mpich}/bin/mpicc /usr/bin/mpifort diff --git a/dockerfiles/libs-gnu/Dockerfile b/dockerfiles/libs-gnu/Dockerfile deleted file mode 100644 index 6ae58e0..0000000 --- a/dockerfiles/libs-gnu/Dockerfile +++ /dev/null @@ -1,41 +0,0 @@ -# BASE-BUILD Dockerfile for running flow123d -ARG images_version -FROM flow123d/base-build-gnu:${images_version} -MAINTAINER Jan Hybs -ARG BUILD_TYPE - -# auxiliary build image -# just build libraries Dbug or Release using ${BUILD_TYPE} variable -COPY libs-build-gnu /libs-build-gnu - -# Need to copy these into ../libs-dbg/Docrefile -ARG ver_yamlcpp -ARG ver_armadillo -ARG ver_mpich -ARG ver_petsc -ARG ver_hypre -ARG ver_bddcml -ARG ver_permon - -RUN echo "*** building YAMLCPP ***"; \ - make -C libs-build-gnu/yamlcpp build_type=${BUILD_TYPE} version=$ver_yamlcpp install - -RUN echo "*** building Armadillo ***"; \ - make -C libs-build-gnu/armadillo build_type=${BUILD_TYPE} version=$ver_armadillo install - - -RUN echo "*** building MPICH ***"; \ - make -C libs-build-gnu/mpich build_type=${BUILD_TYPE} version=$ver_mpich install - - -RUN echo "*** building PETSC ***"; \ - make -C libs-build-gnu/petsc build_type=${BUILD_TYPE} mpich_lib=mpich_${ver_mpich} version=${ver_petsc} hypre_version=${ver_hypre} install - - -RUN echo "*** building BDDCML ***"; \ - make -C libs-build-gnu/bddcml build_type=${BUILD_TYPE} version=${ver_bddcml} mpich_lib=mpich_${ver_mpich} petsc_lib=petsc_${ver_petsc} install - - -RUN echo "*** building Permon ***"; \ - make -C libs-build-gnu/permon build_type=${BUILD_TYPE} version=${ver_permon} petsc_lib=petsc_${ver_petsc} install -#make build_type=Debug version=3.15.0 petsc_lib=petsc_3.15.1 install diff --git a/images_version b/images_version index 4d54dad..c4e41f9 100644 --- a/images_version +++ b/images_version @@ -1 +1 @@ -4.0.2 +4.0.3 diff --git a/makefile b/makefile index 4ea1895..c24702b 100644 --- a/makefile +++ b/makefile @@ -12,14 +12,14 @@ build=docker build --build-arg images_version=$(images_version) build_gnu=$(build) -f Dockerfile-gnu build_intel=$(build) -f Dockerfile-intel run=docker run -v ${PWD}/$(build_dir):/build_dir -w /build_dir -versions=--build-arg ver_yamlcpp=0.6.3 \ - --build-arg ver_armadillo=10.5.2 \ - --build-arg ver_mpich=3.4.2 \ - --build-arg ver_petsc=3.17.0 \ - --build-arg ver_hypre=2.24.0 \ +versions=--build-arg ver_yamlcpp=0e6e28d \ + --build-arg ver_armadillo=12.2.0 \ + --build-arg ver_mpich=4.0.3 \ + --build-arg ver_petsc=v3.18.6 \ + --build-arg ver_hypre=2.25.0 \ --build-arg ver_bddcml=2.6 \ - --build-arg ver_permon=3.17.0 - + --build-arg ver_permon=3.18.0 +ver_boost=--build-arg boost_version=1.74 # TODO: write versions available into the image and use autoamticaly in Flow123d configuration @@ -32,11 +32,11 @@ versions=--build-arg ver_yamlcpp=0.6.3 \ .PHONY: img-base-intel img-base-intel: dockerfiles/base/Dockerfile-intel dockerfiles/base/entrypoint.sh - cd dockerfiles/base && $(build_intel) --tag flow123d/base-intel:$(images_version) . + cd dockerfiles/base && $(build_intel) $(ver_boost) --tag flow123d/base-intel:$(images_version) . .PHONY: img-base-build-intel img-base-build-intel: img-base-intel dockerfiles/base-build/Dockerfile-intel - cd dockerfiles/base-build && $(build_intel) --tag flow123d/base-build-intel:$(images_version) . + cd dockerfiles/base-build && $(build_intel) $(ver_boost) --tag flow123d/base-build-intel:$(images_version) . libs_dbg:=libs-build-intel-dbg .PHONY: libs-build-intel-dbg @@ -71,30 +71,37 @@ img-install-intel: img-base-intel $(libs_rel) .PHONY: img-base-gnu img-base-gnu: dockerfiles/base/Dockerfile-gnu dockerfiles/base/entrypoint.sh - cd dockerfiles/base && $(build_gnu) --tag flow123d/base-gnu:$(images_version) . + cd dockerfiles/base && $(build_gnu) $(ver_boost) --tag flow123d/base-gnu:$(images_version) . .PHONY: img-base-build-gnu img-base-build-gnu: img-base-gnu dockerfiles/base-build/Dockerfile-gnu - cd dockerfiles/base-build && $(build_gnu) --tag flow123d/base-build-gnu:$(images_version) . + cd dockerfiles/base-build && $(build_gnu) $(ver_boost) --tag flow123d/base-build-gnu:$(images_version) . +build_gnu_dbg:=cd dockerfiles/flow-dev-gnu && $(build) $(versions) --build-arg BUILD_TYPE=Debug libs_dbg:=libs-build-gnu-dbg .PHONY: $(libs_dbg) -$(libs_dbg): img-base-build-gnu dockerfiles/libs-gnu/Dockerfile - cd dockerfiles/libs-gnu && $(build) $(versions) --build-arg BUILD_TYPE=Debug --tag flow123d/$(libs_dbg):$(images_version) . +$(libs_dbg): img-base-build-gnu dockerfiles/flow-dev-gnu/Dockerfile + $(build_gnu_dbg) --target libs-build --tag flow123d/$(libs_dbg):$(images_version) . .PHONY: img-flow-dev-gnu-dbg img-flow-dev-gnu-dbg: $(libs_dbg) dockerfiles/flow-dev-gnu/Dockerfile - cd dockerfiles/flow-dev-gnu && $(build) $(versions) --build-arg BUILD_TYPE=Debug --build-arg source_image=flow123d/$(libs_dbg):$(images_version) --tag flow123d/flow-dev-gnu-dbg:$(images_version) . + $(build_gnu_dbg) --build-arg source_image=flow123d/$(libs_dbg):$(images_version) --target flow-dev-gnu --tag flow123d/flow-dev-gnu-dbg:$(images_version) . +build_gnu_rel:=cd dockerfiles/flow-dev-gnu && $(build) $(versions) --build-arg BUILD_TYPE=Release libs_rel:=libs-build-gnu-rel .PHONY: $(libs_rel) -$(libs_rel): img-base-build-gnu dockerfiles/libs-gnu/Dockerfile - cd dockerfiles/libs-gnu && $(build) $(versions) --build-arg BUILD_TYPE=Release --tag flow123d/$(libs_rel):$(images_version) . +$(libs_rel): img-base-build-gnu dockerfiles/flow-dev-gnu/Dockerfile + $(build_gnu_rel) --target libs-build --tag flow123d/$(libs_rel):$(images_version) . .PHONY: img-flow-dev-gnu-rel img-flow-dev-gnu-rel: $(libs_rel) dockerfiles/flow-dev-gnu/Dockerfile - cd dockerfiles/flow-dev-gnu && $(build) $(versions) --build-arg BUILD_TYPE=Release --build-arg source_image=flow123d/$(libs_rel):$(images_version) --tag flow123d/flow-dev-gnu-rel:$(images_version) . + $(build_gnu_rel) --build-arg source_image=flow123d/$(libs_rel):$(images_version) --target flow-dev-gnu --tag flow123d/flow-dev-gnu-rel:$(images_version) . + +.PHONY: img-install-gnu +img-install-gnu: img-base-gnu $(libs_rel) dockerfiles/flow-dev-gnu/Dockerfile + $(build_gnu_rel) --build-arg source_image=flow123d/$(libs_rel):$(images_version) --target install-gnu --tag flow123d/install-gnu:$(images_version) . + # .PHONY: img-flow-dev-gnu-profile # img-flow-dev-gnu-profile: img-flow-dev-gnu-rel # cd dockerfiles/flow-dev-profile && $(build) --build-arg source_image=flow123d/flow-dev-gnu-rel:$(images_version) --tag flow123d/flow-dev-gnu-profile:$(images_version) . @@ -103,9 +110,6 @@ img-flow-dev-gnu-rel: $(libs_rel) dockerfiles/flow-dev-gnu/Dockerfile .PHONY: flow-dev-gnu flow-dev-gnu: img-install-gnu img-flow-dev-gnu-dbg img-flow-dev-gnu-rel # img-flow-dev-gnu-profile --PHONY: img-install-gnu -img-install-gnu: img-base-gnu $(libs_rel) - cd dockerfiles/install-gnu && $(build) $(versions) --build-arg source_image=flow123d/$(libs_rel):$(images_version) --tag flow123d/install-gnu:$(images_version) . # Push all public images.