From 653bf4f21e582ec98ff3318bf79b864889a33754 Mon Sep 17 00:00:00 2001 From: Koshy John Date: Fri, 18 Apr 2025 15:39:04 -0700 Subject: [PATCH 01/10] Feature: Updated Packager with Auto-version update in Publish paths --- Makefile | 2 +- src/build.bat | 2 + src/build.sh | 35 ++++++ src/core/src/bootstrap/Bootstrapper.py | 2 +- ...chExtShim.sh => AzGPSLinuxPatchExtShim.sh} | 2 +- src/extension/src/HandlerManifest.json | 12 +- src/publish.bat | 3 + src/publish.sh | 35 ++++++ src/tools/misc/EnableVirtualTerminal.reg | Bin 214 -> 0 bytes src/tools/{ => packager}/Package-All.py | 85 +++++++------- src/tools/{ => packager}/Package-Core.py | 70 +++++++----- src/tools/packager/Publish.py | 108 ++++++++++++++++++ 12 files changed, 280 insertions(+), 76 deletions(-) create mode 100644 src/build.bat create mode 100644 src/build.sh rename src/extension/src/{MsftLinuxPatchExtShim.sh => AzGPSLinuxPatchExtShim.sh} (99%) create mode 100644 src/publish.bat create mode 100644 src/publish.sh delete mode 100644 src/tools/misc/EnableVirtualTerminal.reg rename src/tools/{ => packager}/Package-All.py (75%) rename src/tools/{ => packager}/Package-Core.py (74%) create mode 100644 src/tools/packager/Publish.py diff --git a/Makefile b/Makefile index b83b7b4e5..3989fec52 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -PACKAGER_PATH = src//tools/Package-All.py +PACKAGER_PATH = src//tools//packager//Package-All.py BUILD_PATH = build ZIP_SRC_PATH = src//out//LinuxPatchExtension.zip MANIFEST_PATH = src//extension//src//manifest.xml diff --git a/src/build.bat b/src/build.bat new file mode 100644 index 000000000..3816bfad5 --- /dev/null +++ b/src/build.bat @@ -0,0 +1,2 @@ +@echo off +python tools\packager\Package-All.py \ No newline at end of file diff --git a/src/build.sh b/src/build.sh new file mode 100644 index 000000000..50417df3e --- /dev/null +++ b/src/build.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# Copyright 2025 Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Requires Python 2.7+ + +COMMAND="tools/packager/Package-All.py" + +function find_python(){ + local python_exec_command=$1 + + # Check if there is python defined. + for p in python3 /usr/share/oem/python/bin/python3 python python2 /usr/libexec/platform-python /usr/share/oem/python/bin/python; do + if command -v "${p}" ; then + eval ${python_exec_command}=${p} + return + fi + done +} + +find_python PYTHON + +${PYTHON} "${COMMAND}" \ No newline at end of file diff --git a/src/core/src/bootstrap/Bootstrapper.py b/src/core/src/bootstrap/Bootstrapper.py index 50d66b032..1d1490d58 100644 --- a/src/core/src/bootstrap/Bootstrapper.py +++ b/src/core/src/bootstrap/Bootstrapper.py @@ -135,7 +135,7 @@ def build_core_components(self, container): return lifecycle_manager, status_handler def bootstrap_splash_text(self): - self.composite_logger.log("\n\n[%exec_name%] \t -- \t Copyright (c) Microsoft Corporation. All rights reserved. \nApplication version: 3.0.[%exec_sub_ver%]\n\n") + self.composite_logger.log("\n\n[%exec_name%] \t -- \t Copyright (c) Microsoft Corporation. All rights reserved. \nApplication version: 3.0.[%exec_build_timestamp%]\n\n") def basic_environment_health_check(self): self.composite_logger.log("Python version: " + " ".join(sys.version.splitlines())) diff --git a/src/extension/src/MsftLinuxPatchExtShim.sh b/src/extension/src/AzGPSLinuxPatchExtShim.sh similarity index 99% rename from src/extension/src/MsftLinuxPatchExtShim.sh rename to src/extension/src/AzGPSLinuxPatchExtShim.sh index 714fea85d..1d0e33313 100644 --- a/src/extension/src/MsftLinuxPatchExtShim.sh +++ b/src/extension/src/AzGPSLinuxPatchExtShim.sh @@ -17,7 +17,7 @@ # Requires Python 2.7+ # Keeping the default command -COMMAND="MsftLinuxPatchExt.py" +COMMAND="AzGPSLinuxPatchExt.py" PYTHON="" USAGE="$(basename "$0") [-h] [-i|--install] [-u|--uninstall] [-d|--disable] [-e|--enable] [-p|--update] [-r|--reset] diff --git a/src/extension/src/HandlerManifest.json b/src/extension/src/HandlerManifest.json index 3f08d65ed..6c5babd55 100644 --- a/src/extension/src/HandlerManifest.json +++ b/src/extension/src/HandlerManifest.json @@ -2,12 +2,12 @@ { "version": 1.0, "handlerManifest": { - "disableCommand": "MsftLinuxPatchExtShim.sh -d", - "enableCommand": "MsftLinuxPatchExtShim.sh -e", - "installCommand": "MsftLinuxPatchExtShim.sh -i", - "uninstallCommand": "MsftLinuxPatchExtShim.sh -u", - "updateCommand": "MsftLinuxPatchExtShim.sh -p", - "resetStateCommand": "MsftLinuxPatchExtShim.sh -r", + "disableCommand": "AzGPSLinuxPatchExtShim.sh -d", + "enableCommand": "AzGPSLinuxPatchExtShim.sh -e", + "installCommand": "AzGPSLinuxPatchExtShim.sh -i", + "uninstallCommand": "AzGPSLinuxPatchExtShim.sh -u", + "updateCommand": "AzGPSLinuxPatchExtShim.sh -p", + "resetStateCommand": "AzGPSLinuxPatchExtShim.sh -r", "rebootAfterInstall": false, "reportHeartbeat": false, "updateMode": "UpdateWithoutInstall" diff --git a/src/publish.bat b/src/publish.bat new file mode 100644 index 000000000..257b55c7f --- /dev/null +++ b/src/publish.bat @@ -0,0 +1,3 @@ +@echo off +python tools\packager\Publish.py +git status \ No newline at end of file diff --git a/src/publish.sh b/src/publish.sh new file mode 100644 index 000000000..5eae1f5fd --- /dev/null +++ b/src/publish.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# Copyright 2025 Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Requires Python 2.7+ + +COMMAND="tools/packager/Publish.py" + +function find_python(){ + local python_exec_command=$1 + + # Check if there is python defined. + for p in python3 /usr/share/oem/python/bin/python3 python python2 /usr/libexec/platform-python /usr/share/oem/python/bin/python; do + if command -v "${p}" ; then + eval ${python_exec_command}=${p} + return + fi + done +} + +find_python PYTHON + +${PYTHON} "${COMMAND}" \ No newline at end of file diff --git a/src/tools/misc/EnableVirtualTerminal.reg b/src/tools/misc/EnableVirtualTerminal.reg deleted file mode 100644 index 21291ee1b293fc67633a9deee0bff7a925d64592..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 214 zcmXAj!3x4a3`E~q@E=M)K=j~6@FaqucvxFSR79whg}Q}qwfOVuG!nAxn<0~YKL%VMYT9#FTauN#X&q}#8@?CV0~i6S~EwkEXbATL|Za(&Q5>F vSx!G4^;XKe*kr0~uk4m5H|d#edPob#8}UnBkuuj!on2Hgv;F-I|C;avk#r=+ diff --git a/src/tools/Package-All.py b/src/tools/packager/Package-All.py similarity index 75% rename from src/tools/Package-All.py rename to src/tools/packager/Package-All.py index b7729dac1..f99951321 100644 --- a/src/tools/Package-All.py +++ b/src/tools/packager/Package-All.py @@ -4,7 +4,7 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -14,9 +14,10 @@ # # Requires Python 2.7+ -""" Merges individual python modules from src to the MsftLinuxPatchExt files in the out directory. +""" Merges individual python modules from src to the AzGPSLinuxPatchExt files in the out directory. Relative source and destination paths for the extension are auto-detected if the optional src parameter is not present. -How to use: python Package.py """ +How to use: python Package-All.py +Note: Package-All.py internally invokes Package-Core.py to generate AzGPSLinuxPatchCore.py """ from __future__ import print_function import sys @@ -26,11 +27,13 @@ from shutil import copyfile from shutil import make_archive import subprocess +import xml.etree.ElementTree as et # imports in VERY_FIRST_IMPORTS, order should be kept VERY_FIRST_IMPORTS = [ 'from __future__ import print_function\n', - 'from abc import ABCMeta, abstractmethod\n'] + 'from abc import ABCMeta, abstractmethod\n', + 'from distutils.version import LooseVersion\n'] GLOBAL_IMPORTS = set() @@ -59,15 +62,16 @@ def write_merged_code(code, merged_file_full_path): def insert_copyright_notice(merged_file_full_path, merged_file_name): - notice = '# --------------------------------------------------------------------------------------------------------------------\n' + notice = '# coding=utf-8\n' + notice += '# --------------------------------------------------------------------------------------------------------------------\n' notice += '# \n' - notice += '# Copyright 2020 Microsoft Corporation\n' \ + notice += '# Copyright ' + str(datetime.date.today().year) + ' Microsoft Corporation\n' \ '#\n' \ '# Licensed under the Apache License, Version 2.0 (the "License");\n' \ '# you may not use this file except in compliance with the License.\n' \ '# You may obtain a copy of the License at\n' \ '#\n' \ - '# http://www.apache.org/licenses/LICENSE-2.0\n' \ + '# https://www.apache.org/licenses/LICENSE-2.0\n' \ '#\n' \ '# Unless required by applicable law or agreed to in writing, software\n' \ '# distributed under the License is distributed on an "AS IS" BASIS,\n' \ @@ -104,15 +108,15 @@ def prepend_content_to_file(content, file_name): os.rename(temp_file, file_name) -def generate_compiled_script(source_code_path, merged_file_full_path, merged_file_name, environment): +def generate_compiled_script(source_code_path, merged_file_full_path, merged_file_name, environment, new_version): try: - print('\n\n=============================== GENERATING ' + merged_file_name + '... =============================================================\n') + print('\n=============================== (2/3) GENERATING ' + merged_file_name + '... ================================\n') - print('========== Delete old extension file if it exists.') + print('------------- Deleting old extension file if it exists.') if os.path.exists(merged_file_full_path): os.remove(merged_file_full_path) - print('\n========== Merging modules: \n') + print('------------- Merging modules: ') modules_to_be_merged = [] for root, dirs, files in os.walk(source_code_path): for file_name in files: @@ -137,19 +141,20 @@ def generate_compiled_script(source_code_path, merged_file_full_path, merged_fil write_merged_code(codes, merged_file_full_path) print("") - print('\n========== Prepend all import statements\n') + print('------------- Prepend all import statements') insert_imports(GLOBAL_IMPORTS, merged_file_full_path) insert_imports(VERY_FIRST_IMPORTS, merged_file_full_path) - print('========== Set Copyright, Version and Environment. Also enforce UNIX-style line endings.\n') + print('------------- Set Copyright, Version and Environment. Also enforce UNIX-style line endings.') insert_copyright_notice(merged_file_full_path, merged_file_name) - timestamp = datetime.datetime.utcnow().strftime("%y%m%d-%H%M") - replace_text_in_file(merged_file_full_path, '[%exec_name%]', merged_file_name.split('.')[0]) - replace_text_in_file(merged_file_full_path, '[%exec_sub_ver%]', timestamp) + timestamp = datetime.datetime.now(datetime.UTC).strftime("%y%m%d-%H%M") + replace_text_in_file(merged_file_full_path, '[%exec_name%]', merged_file_name) + replace_text_in_file(merged_file_full_path, '[%exec_ver%]', str(new_version)) + replace_text_in_file(merged_file_full_path, '[%exec_build_timestamp%]', timestamp) replace_text_in_file(merged_file_full_path, 'Constants.UNKNOWN_ENV', environment) replace_text_in_file(merged_file_full_path, '\r\n', '\n') - print("========== Merged extension code was saved to:\n{0}\n".format(merged_file_full_path)) + print("------------- Merged extension code was saved to:\n{0}\n".format(merged_file_full_path)) except Exception as error: print('Exception during merge python modules: ' + repr(error)) @@ -165,7 +170,7 @@ def main(argv): # Determine code path if not specified if len(argv) < 2: # auto-detect src path - source_code_path = os.path.dirname(os.path.realpath(__file__)).replace("tools", os.path.join("extension", "src")) + source_code_path = os.path.dirname(os.path.realpath(__file__)).replace(os.path.join("tools", "packager"), os.path.join("extension", "src")) if os.path.exists(os.path.join(source_code_path, "__main__.py")) is False: print("Invalid extension source code path. Check enlistment.\n") return @@ -186,34 +191,32 @@ def main(argv): raise # Invoke core business logic code packager - exec_core_build_path = os.path.join(working_directory, 'tools', 'Package-Core.py') + exec_core_build_path = os.path.join(working_directory, 'tools', 'packager', 'Package-Core.py') subprocess.call('python ' + exec_core_build_path, shell=True) + # Get version from manifest for code + new_version = None + manifest_xml_file_path = os.path.join(working_directory, 'extension', 'src', 'manifest.xml') + manifest_tree = et.parse(manifest_xml_file_path) + manifest_root = manifest_tree.getroot() + for i in range(0, len(manifest_root)): + if 'Version' in str(manifest_root[i]): + new_version = manifest_root[i].text + if new_version is None: + raise Exception("Unable to determine target version.") + # Generated compiled scripts at the destination - merged_file_details = [('MsftLinuxPatchExt.py', 'Constants.PROD')] + merged_file_details = [('AzGPSLinuxPatchExt.py', 'Constants.PROD')] for merged_file_detail in merged_file_details: merged_file_destination = os.path.join(working_directory, 'out', merged_file_detail[0]) - generate_compiled_script(source_code_path, merged_file_destination, merged_file_detail[0], merged_file_detail[1]) + generate_compiled_script(source_code_path, merged_file_destination, merged_file_detail[0], merged_file_detail[1], new_version) # GENERATING EXTENSION - print('\n\n=============================== GENERATING LinuxPatchExtension.zip... =============================================================\n') - # Rev handler version - # print('\n========== Revising extension version.') - # manifest_xml_file_path = os.path.join(working_directory, 'extension', 'src', 'manifest.xml') - # manifest_tree = et.parse(manifest_xml_file_path) - # manifest_root = manifest_tree.getroot() - # for i in range(0, len(manifest_root)): - # if 'Version' in str(manifest_root[i]): - # current_version = manifest_root[i].text - # version_split = current_version.split('.') - # version_split[len(version_split)-1] = str(int(version_split[len(version_split)-1]) + 1) - # new_version = '.'.join(version_split) - # print("Changing extension version from {0} to {1}.".format(current_version, new_version)) - # replace_text_in_file(manifest_xml_file_path, current_version, new_version) + print('\n=============================== (3/3) GENERATING LinuxPatchExtension.zip... ==============================\n') # Copy extension files - print('\n========== Copying extension files + enforcing UNIX style line endings.\n') - ext_files = ['HandlerManifest.json', 'manifest.xml', 'MsftLinuxPatchExtShim.sh'] + print('------------- Copying extension files + enforcing UNIX style line endings.') + ext_files = ['HandlerManifest.json', 'manifest.xml', 'AzGPSLinuxPatchExtShim.sh'] for ext_file in ext_files: ext_file_src = os.path.join(working_directory, 'extension', 'src', ext_file) ext_file_destination = os.path.join(working_directory, 'out', ext_file) @@ -230,18 +233,18 @@ def main(argv): os.remove(ext_zip_file_path_dest) # Generate zip - print('\n========== Generating extension zip.\n') - make_archive(os.path.splitext(ext_zip_file_path_src)[0], 'zip', os.path.join(working_directory, 'out'), '.') + print('------------- Generating extension zip.') + make_archive(os.path.splitext(ext_zip_file_path_src)[0], 'zip', os.path.join(working_directory, 'out'), '..') copyfile(ext_zip_file_path_src, ext_zip_file_path_dest) os.remove(ext_zip_file_path_src) # Remove extension file copies - print('\n========== Cleaning up environment.\n') + print('------------- Cleaning up environment.') for ext_file in ext_files: ext_file_path = os.path.join(working_directory, 'out', ext_file) os.remove(ext_file_path) - print("========== Extension ZIP was saved to:\n{0}\n".format(ext_zip_file_path_dest)) + print("------------- Extension ZIP was saved to:\n{0}\n".format(ext_zip_file_path_dest)) except Exception as error: print('Exception during merge python modules: ' + repr(error)) diff --git a/src/tools/Package-Core.py b/src/tools/packager/Package-Core.py similarity index 74% rename from src/tools/Package-Core.py rename to src/tools/packager/Package-Core.py index bae1a0337..180a2581c 100644 --- a/src/tools/Package-Core.py +++ b/src/tools/packager/Package-Core.py @@ -4,7 +4,7 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -14,9 +14,9 @@ # # Requires Python 2.7+ -""" Merges individual python modules from src to the PatchMicrosoftOMSLinuxComputer.py and MsftLinuxPatchCore.py files in the out directory. -Relative source and destination paths for the patch runbook are auto-detected if the optional src parameter is not present. -How to use: python Package.py """ +""" Merges individual python modules from src to the AzGPSLinuxPatchCore.py files in the out directory. +Relative source and destination paths for the extension Core are auto-detected if the optional src parameter is not present. +How to use: python Package-Core.py """ from __future__ import print_function @@ -25,6 +25,7 @@ import os import errno import datetime +import xml.etree.ElementTree as et # imports in VERY_FIRST_IMPORTS, order should be kept @@ -62,15 +63,16 @@ def write_merged_code(code, merged_file_full_path): def insert_copyright_notice(merged_file_full_path, merged_file_name): - notice = '# --------------------------------------------------------------------------------------------------------------------\n' + notice = '# coding=utf-8\n' + notice += '# --------------------------------------------------------------------------------------------------------------------\n' notice += '# \n' - notice += '# Copyright 2020 Microsoft Corporation\n' \ + notice += '# Copyright ' + str(datetime.date.today().year) + ' Microsoft Corporation\n' \ '#\n' \ '# Licensed under the Apache License, Version 2.0 (the "License");\n' \ '# you may not use this file except in compliance with the License.\n' \ '# You may obtain a copy of the License at\n' \ '#\n' \ - '# http://www.apache.org/licenses/LICENSE-2.0\n' \ + '# https://www.apache.org/licenses/LICENSE-2.0\n' \ '#\n' \ '# Unless required by applicable law or agreed to in writing, software\n' \ '# distributed under the License is distributed on an "AS IS" BASIS,\n' \ @@ -107,15 +109,15 @@ def prepend_content_to_file(content, file_name): os.rename(temp_file, file_name) -def generate_compiled_script(source_code_path, merged_file_full_path, merged_file_name, environment): +def generate_compiled_script(source_code_path, merged_file_full_path, merged_file_name, environment, new_version): try: - print('\n\n=============================== GENERATING ' + merged_file_name + '... =============================================================\n') + print('\n=============================== (1/3) GENERATING ' + merged_file_name + '... ===============================\n') - print('========== Delete old core file if it exists.') + print('------------- Delete old core file if it exists.') if os.path.exists(merged_file_full_path): os.remove(merged_file_full_path) - print('\n========== Merging modules: \n') + print('------------- Merging modules:') modules_to_be_merged = [] for root, dirs, files in os.walk(source_code_path): for file_name in files: @@ -128,7 +130,7 @@ def generate_compiled_script(source_code_path, merged_file_full_path, merged_fil continue elif 'external_dependencies' in file_path: continue - elif os.path.basename(file_path) in ('PackageManager.py', 'Constants.py', 'LifecycleManager.py', 'SystemctlManager.py'): + elif os.path.basename(file_path) in ('PatchOperator.py', 'PackageManager.py', 'Constants.py', 'LifecycleManager.py', 'SystemctlManager.py'): modules_to_be_merged.insert(0, file_path) else: if len(modules_to_be_merged) > 0 and '__main__.py' in modules_to_be_merged[-1]: @@ -142,18 +144,19 @@ def generate_compiled_script(source_code_path, merged_file_full_path, merged_fil write_merged_code(codes, merged_file_full_path) print("") - print('\n========== Prepend all import statements\n') + print('------------- Prepend all import statements') insert_imports(GLOBAL_IMPORTS, merged_file_full_path) insert_imports(VERY_FIRST_IMPORTS, merged_file_full_path) - print('========== Set Copyright, Version and Environment. Also enforce UNIX-style line endings.\n') + print('------------- Set Copyright, Version and Environment. Also enforce UNIX-style line endings.') insert_copyright_notice(merged_file_full_path, merged_file_name) - timestamp = datetime.datetime.utcnow().strftime("%y%m%d-%H%M") - replace_text_in_file(merged_file_full_path, '[%exec_name%]', merged_file_name.split('.')[0]) - replace_text_in_file(merged_file_full_path, '[%exec_sub_ver%]', timestamp) + timestamp = datetime.datetime.now(datetime.UTC).strftime("%y%m%d-%H%M") + replace_text_in_file(merged_file_full_path, '[%exec_name%]', merged_file_name) + replace_text_in_file(merged_file_full_path, '[%exec_ver%]', str(new_version)) + replace_text_in_file(merged_file_full_path, '[%exec_build_timestamp%]', timestamp) replace_text_in_file(merged_file_full_path, '\r\n', '\n') - print("========== Merged core code was saved to:\n{0}\n".format(merged_file_full_path)) + print('------------- Merged core code was saved to:\n{0}\n'.format(merged_file_full_path)) except Exception as error: print('Exception during merge python modules: ' + repr(error)) @@ -162,13 +165,13 @@ def generate_compiled_script(source_code_path, merged_file_full_path, merged_fil def add_external_dependencies(external_dependencies_destination, external_dependencies_source_code_path): try: - print('\n========= ADDING EXTERNAL DEPENDENCIES\n') + print('\n------------- ADDING EXTERNAL DEPENDENCIES') - print('========== Deleting old dependencies if they exists.') + print('------------- Deleting old dependencies if they exist.') if os.path.exists(external_dependencies_destination): shutil.rmtree(external_dependencies_destination) - print('\n========== Adding all dependencies to external_dependencies directory: \n') + print('------------- Adding all dependencies to external_dependencies directory: ') dependencies_to_be_added = [] for root, dirs, files in os.walk(external_dependencies_source_code_path): for file_name in files: @@ -181,7 +184,7 @@ def add_external_dependencies(external_dependencies_destination, external_depend print(format(os.path.basename(dependency)), end=', ') shutil.copyfile(dependency, os.path.join(external_dependencies_destination, os.path.basename(dependency))) - print("\n\n========== External dependencies saved to:\n{0}\n".format(external_dependencies_destination)) + print('\n------------- External dependencies saved to:\n{0}\n'.format(external_dependencies_destination)) except Exception as error: print('Exception during adding external dependencies: ' + repr(error)) @@ -194,10 +197,15 @@ def main(argv): # Clear os.system('cls' if os.name == 'nt' else 'clear') + # Pro packager branding + print("==========================================================================================================\n") + print(" * AzGPS LINUX PATCH EXTENSION PACKAGER") + print(" * Microsoft Azure \\ Compute Platform \\ Azure Guest Patching Service") + # Determine code path if not specified if len(argv) < 2: # auto-detect src path - source_code_path = os.path.dirname(os.path.realpath(__file__)).replace("tools", os.path.join("core","src")) + source_code_path = os.path.dirname(os.path.realpath(__file__)).replace(os.path.join("tools", "packager"), os.path.join("core","src")) if os.path.exists(os.path.join(source_code_path, "__main__.py")) is False: print("Invalid core source code path. Check enlistment.\n") return @@ -217,11 +225,22 @@ def main(argv): if e.errno != errno.EEXIST: raise + # Get version from manifest for code + new_version = None + manifest_xml_file_path = os.path.join(working_directory, 'extension', 'src', 'manifest.xml') + manifest_tree = et.parse(manifest_xml_file_path) + manifest_root = manifest_tree.getroot() + for i in range(0, len(manifest_root)): + if 'Version' in str(manifest_root[i]): + new_version = manifest_root[i].text + if new_version is None: + raise Exception("Unable to determine target version.") + # Generated compiled scripts at the destination - merged_file_details = [('MsftLinuxPatchCore.py', 'Constants.PROD')] + merged_file_details = [('AzGPSLinuxPatchCore.py', 'Constants.PROD')] for merged_file_detail in merged_file_details: merged_file_destination = os.path.join(working_directory, 'out', merged_file_detail[0]) - generate_compiled_script(source_code_path, merged_file_destination, merged_file_detail[0], merged_file_detail[1]) + generate_compiled_script(source_code_path, merged_file_destination, merged_file_detail[0], merged_file_detail[1], new_version) # add all dependencies under core/src/external_dependencies to destination directory external_dependencies_destination = os.path.join(merge_file_directory, 'external_dependencies') @@ -235,4 +254,3 @@ def main(argv): if __name__ == "__main__": main(sys.argv) - diff --git a/src/tools/packager/Publish.py b/src/tools/packager/Publish.py new file mode 100644 index 000000000..3c896ed38 --- /dev/null +++ b/src/tools/packager/Publish.py @@ -0,0 +1,108 @@ +# Copyright 2025 Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Requires Python 2.7+ + +""" Merges individual python modules from src to the AzGPSLinuxPatchExt files in the out directory. +Relative source and destination paths for the extension are auto-detected if the optional src parameter is not present. +How to use: python Package-All.py +Note: Package-All.py internally invokes Package-Core.py to generate AzGPSLinuxPatchCore.py """ + +from __future__ import print_function +import sys +import os +import errno +import datetime +from shutil import copyfile +from shutil import make_archive +import subprocess +import xml.etree.ElementTree as et + +# noinspection PyPep8 +def replace_text_in_file(file_path, old_text, new_text): + with open(file_path, 'rb') as file_handle: text = file_handle.read() + text = text.replace(old_text.encode(encoding='UTF-8'), new_text.encode(encoding='UTF-8')) + with open(file_path, 'wb') as file_handle: file_handle.write(text) + + + +def main(argv): + try: + # Clear + os.system('cls' if os.name == 'nt' else 'clear') + + # Determine code path if not specified + if len(argv) < 2: + # auto-detect src path + source_code_path = os.path.dirname(os.path.realpath(__file__)).replace(os.path.join("tools", "packager"), os.path.join("extension", "src")) + if os.path.exists(os.path.join(source_code_path, "__main__.py")) is False: + print("Invalid extension source code path. Check enlistment.\n") + return + else: + # explicit src path parameter + source_code_path = argv[1] + if os.path.exists(os.path.join(source_code_path, "ActionHandler.py")) is False: + print("Invalid extension source code path. Check src parameter.\n") + return + + # Prepare destination for compiled scripts + working_directory = os.path.abspath(os.path.join(source_code_path, os.pardir, os.pardir)) + merge_file_directory = os.path.join(working_directory, 'out') + try: + os.makedirs(merge_file_directory) + except OSError as e: + if e.errno != errno.EEXIST: + raise + + # Get version from manifest for code + new_version = None + manifest_xml_file_path = os.path.join(working_directory, 'extension', 'src', 'manifest.xml') + manifest_tree = et.parse(manifest_xml_file_path) + manifest_root = manifest_tree.getroot() + for i in range(0, len(manifest_root)): + if 'Version' in str(manifest_root[i]): + new_version = manifest_root[i].text + if new_version is None: + raise Exception("Unable to determine target version.") + + # Rev handler version + current_version = "Unknown" + manifest_xml_file_path = os.path.join(working_directory, 'extension', 'src', 'manifest.xml') + manifest_tree = et.parse(manifest_xml_file_path) + manifest_root = manifest_tree.getroot() + for i in range(0, len(manifest_root)): + if 'Version' in str(manifest_root[i]): + current_version = manifest_root[i].text + version_split = current_version.split('.') + version_split[len(version_split)-1] = str(int(version_split[len(version_split)-1]) + 1) + new_version = '.'.join(version_split) + replace_text_in_file(manifest_xml_file_path, current_version, new_version) + + # Invoke core business logic code packager + exec_core_build_path = os.path.join(working_directory, 'tools', 'packager', 'Package-All.py') + subprocess.call('python ' + exec_core_build_path, shell=True) + + # Report extension version change + print("==========================================================================================================\n") + print("! PUBLISHER > THE EXTENSION VERSION WAS CHANGED FROM {0} to {1}. DO NOT RE-RUN.".format(current_version, new_version)) + print("! > This is only meant to be run once prior to extension publish and pushed as a PR. Not for automation.") + print("! > If this was an error, revert the extension manifest, and only use the build script instead of publish.\n") + + except Exception as error: + print('Exception during merge python modules: ' + repr(error)) + raise + + +if __name__ == "__main__": + main(sys.argv) From 61701ba12b20afb9a841e116354dd3d0cbfb13dd Mon Sep 17 00:00:00 2001 From: Koshy John Date: Fri, 18 Apr 2025 15:52:44 -0700 Subject: [PATCH 02/10] Feature: Updated Packager with Auto-version update in Publish paths --- src/build.bat | 2 +- src/build.sh | 2 +- src/publish.bat | 2 +- src/publish.sh | 2 +- src/tools/packager/Publish.py | 9 +-------- 5 files changed, 5 insertions(+), 12 deletions(-) diff --git a/src/build.bat b/src/build.bat index 3816bfad5..3f2d661a5 100644 --- a/src/build.bat +++ b/src/build.bat @@ -1,2 +1,2 @@ @echo off -python tools\packager\Package-All.py \ No newline at end of file +python tools\packager\Package-All.py diff --git a/src/build.sh b/src/build.sh index 50417df3e..74eaad6bf 100644 --- a/src/build.sh +++ b/src/build.sh @@ -32,4 +32,4 @@ function find_python(){ find_python PYTHON -${PYTHON} "${COMMAND}" \ No newline at end of file +${PYTHON} "${COMMAND}" diff --git a/src/publish.bat b/src/publish.bat index 257b55c7f..cb0469ecc 100644 --- a/src/publish.bat +++ b/src/publish.bat @@ -1,3 +1,3 @@ @echo off python tools\packager\Publish.py -git status \ No newline at end of file +git status diff --git a/src/publish.sh b/src/publish.sh index 5eae1f5fd..95a815a14 100644 --- a/src/publish.sh +++ b/src/publish.sh @@ -32,4 +32,4 @@ function find_python(){ find_python PYTHON -${PYTHON} "${COMMAND}" \ No newline at end of file +${PYTHON} "${COMMAND}" diff --git a/src/tools/packager/Publish.py b/src/tools/packager/Publish.py index 3c896ed38..90fb7ad1e 100644 --- a/src/tools/packager/Publish.py +++ b/src/tools/packager/Publish.py @@ -14,18 +14,12 @@ # # Requires Python 2.7+ -""" Merges individual python modules from src to the AzGPSLinuxPatchExt files in the out directory. -Relative source and destination paths for the extension are auto-detected if the optional src parameter is not present. -How to use: python Package-All.py -Note: Package-All.py internally invokes Package-Core.py to generate AzGPSLinuxPatchCore.py """ +""" Publishes a new extension version by incrementing the version number in the manifest.xml file.""" from __future__ import print_function import sys import os import errno -import datetime -from shutil import copyfile -from shutil import make_archive import subprocess import xml.etree.ElementTree as et @@ -36,7 +30,6 @@ def replace_text_in_file(file_path, old_text, new_text): with open(file_path, 'wb') as file_handle: file_handle.write(text) - def main(argv): try: # Clear From c829ccce585699c70c63409b7a1b9929d59b59c7 Mon Sep 17 00:00:00 2001 From: Koshy John Date: Mon, 21 Apr 2025 09:12:14 -0700 Subject: [PATCH 03/10] Additional required changes and test updates --- .../src/bootstrap/ConfigurationFactory.py | 19 +++++++++++++++++ src/core/src/bootstrap/Constants.py | 6 ++++-- .../core_logic/ConfigurePatchingProcessor.py | 21 +++++++++++++++---- src/core/src/core_logic/ServiceManager.py | 4 ++-- src/extension/src/Constants.py | 4 ++-- src/extension/tests/Test_HandlerManifest.py | 10 ++++----- .../updatecenter_troubleshooter.py | 12 +++++------ 7 files changed, 55 insertions(+), 21 deletions(-) diff --git a/src/core/src/bootstrap/ConfigurationFactory.py b/src/core/src/bootstrap/ConfigurationFactory.py index 6609d64c2..084c4eba3 100644 --- a/src/core/src/bootstrap/ConfigurationFactory.py +++ b/src/core/src/bootstrap/ConfigurationFactory.py @@ -234,6 +234,25 @@ def new_prod_configuration(self, package_manager_name, package_manager_component 'component_args': ['env_layer', 'execution_config', 'composite_logger', 'telemetry_writer', 'service_info'], 'component_kwargs': {} }, + 'service_info_legacy': { + 'component': ServiceInfo, + 'component_args': [], + 'component_kwargs': { + 'service_name': Constants.AUTO_ASSESSMENT_SERVICE_NAME_LEGACY, + 'service_desc': Constants.AUTO_ASSESSMENT_SERVICE_DESC, + 'service_exec_path': os.path.join(os.path.dirname(os.path.realpath(__file__)), Constants.CORE_AUTO_ASSESS_SH_FILE_NAME_LEGACY) + } + }, + 'auto_assess_service_manager_legacy': { + 'component': ServiceManager, + 'component_args': ['env_layer', 'execution_config', 'composite_logger', 'telemetry_writer', 'service_info'], + 'component_kwargs': {} + }, + 'auto_assess_timer_manager_legacy': { + 'component': TimerManager, + 'component_args': ['env_layer', 'execution_config', 'composite_logger', 'telemetry_writer', 'service_info'], + 'component_kwargs': {} + }, 'configure_patching_processor': { 'component': ConfigurePatchingProcessor, 'component_args': ['env_layer', 'execution_config', 'composite_logger', 'telemetry_writer', 'status_handler', 'package_manager', 'auto_assess_service_manager', 'auto_assess_timer_manager', 'lifecycle_manager'], diff --git a/src/core/src/bootstrap/Constants.py b/src/core/src/bootstrap/Constants.py index b3831b4df..88b90cd1a 100644 --- a/src/core/src/bootstrap/Constants.py +++ b/src/core/src/bootstrap/Constants.py @@ -94,8 +94,10 @@ class EulaSettings(EnumBackport): IMAGE_DEFAULT_PATCH_CONFIGURATION_BACKUP_PATH = "ImageDefaultPatchConfiguration.bak" # Auto assessment shell script name - CORE_AUTO_ASSESS_SH_FILE_NAME = "MsftLinuxPatchAutoAssess.sh" - AUTO_ASSESSMENT_SERVICE_NAME = "MsftLinuxPatchAutoAssess" + CORE_AUTO_ASSESS_SH_FILE_NAME = "AzGPSLinuxPatchAutoAssess.sh" + AUTO_ASSESSMENT_SERVICE_NAME = "AzGPSLinuxPatchAutoAssess" + CORE_AUTO_ASSESS_SH_FILE_NAME_LEGACY = "MsftLinuxPatchAutoAssess.sh" + AUTO_ASSESSMENT_SERVICE_NAME_LEGACY = "MsftLinuxPatchAutoAssess" AUTO_ASSESSMENT_SERVICE_DESC = "Microsoft Azure Linux Patch Extension - Auto Assessment" # Operations diff --git a/src/core/src/core_logic/ConfigurePatchingProcessor.py b/src/core/src/core_logic/ConfigurePatchingProcessor.py index a6d700045..8ba9d2725 100644 --- a/src/core/src/core_logic/ConfigurePatchingProcessor.py +++ b/src/core/src/core_logic/ConfigurePatchingProcessor.py @@ -19,7 +19,7 @@ class ConfigurePatchingProcessor(object): - def __init__(self, env_layer, execution_config, composite_logger, telemetry_writer, status_handler, package_manager, auto_assess_service_manager, auto_assess_timer_manager, lifecycle_manager): + def __init__(self, env_layer, execution_config, composite_logger, telemetry_writer, status_handler, package_manager, auto_assess_service_manager, auto_assess_timer_manager, lifecycle_manager, aasm_legacy=None, aatm_legacy=None): self.env_layer = env_layer self.execution_config = execution_config @@ -73,6 +73,7 @@ def set_configure_patching_final_overall_status(self): def __try_set_patch_mode(self): """ Set the patch mode for the VM """ try: + self.composite_logger.log_verbose("[CPP] Processing patch mode configuration...") self.status_handler.set_current_operation(Constants.CONFIGURE_PATCHING) self.current_auto_os_patch_state = self.package_manager.get_current_auto_os_patch_state() @@ -85,9 +86,9 @@ def __try_set_patch_mode(self): if self.execution_config.patch_mode == Constants.PatchModes.AUTOMATIC_BY_PLATFORM and self.current_auto_os_patch_state == Constants.AutomaticOSPatchStates.UNKNOWN: # NOTE: only sending details in error objects for customer visibility on why patch state is unknown, overall configurepatching status will remain successful - self.configure_patching_exception_error = "Could not disable one or more automatic OS update services. Please check if they are configured correctly" + self.configure_patching_exception_error = "Could not disable one or more automatic OS update services. Please check if they are configured correctly." - self.composite_logger.log_debug("Completed processing patch mode configuration.") + self.composite_logger.log_debug("[CPP] Completed processing patch mode configuration.") except Exception as error: self.composite_logger.log_error("Error while processing patch mode configuration. [Error={0}]".format(repr(error))) self.configure_patching_exception_error = error @@ -96,6 +97,7 @@ def __try_set_patch_mode(self): def __try_set_auto_assessment_mode(self): """ Sets the preferred auto-assessment mode for the VM """ try: + self.composite_logger.log_verbose("[CPP] Processing assessment mode configuration...") self.status_handler.set_current_operation(Constants.CONFIGURE_PATCHING_AUTO_ASSESSMENT) self.composite_logger.log_debug("Systemd information: {0}".format(str(self.auto_assess_service_manager.get_version()))) # proactive support telemetry @@ -117,7 +119,7 @@ def __try_set_auto_assessment_mode(self): raise Exception("Unknown assessment mode specified. [AssessmentMode={0}]".format(self.execution_config.assessment_mode)) self.__report_consolidated_configure_patch_status() - self.composite_logger.log_debug("Completed processing automatic assessment mode configuration.") + self.composite_logger.log_debug("[CPP] Completed processing automatic assessment mode configuration.") except Exception as error: # deliberately not setting self.configure_patching_exception_error here as it does not feed into the parent object. Not a bug, if you're thinking about it. self.composite_logger.log_error("Error while processing automatic assessment mode configuration. [Error={0}]".format(repr(error))) @@ -128,6 +130,17 @@ def __try_set_auto_assessment_mode(self): self.composite_logger.log_debug("Restoring status handler operation to {0}.".format(Constants.CONFIGURE_PATCHING)) self.status_handler.set_current_operation(Constants.CONFIGURE_PATCHING) + def __custom_clean_legacy_auto_assess_service(self): + """ Cleans up the legacy auto-assess service """ + self.composite_logger.log_debug("Cleaning up legacy auto-assess service.") + try: + self.auto_assess_service_manager.remove_service() + self.auto_assess_timer_manager.remove_timer() + self.current_auto_assessment_state = Constants.AutoAssessmentStates.DISABLED + except Exception as error: + self.composite_logger.log_error("Error while cleaning up legacy auto-assess service. [Error={0}]".format(repr(error))) + self.configure_patching_successful &= False + def __report_consolidated_configure_patch_status(self, status=Constants.STATUS_TRANSITIONING, error=Constants.DEFAULT_UNSPECIFIED_VALUE): """ Reports """ self.composite_logger.log_debug("Reporting consolidated current configure patch status. [OSPatchState={0}][AssessmentState={1}]".format(self.current_auto_os_patch_state, self.current_auto_assessment_state)) diff --git a/src/core/src/core_logic/ServiceManager.py b/src/core/src/core_logic/ServiceManager.py index 00f87e15b..277d21d2e 100644 --- a/src/core/src/core_logic/ServiceManager.py +++ b/src/core/src/core_logic/ServiceManager.py @@ -58,7 +58,7 @@ def start_service(self): code, out = self.invoke_systemctl(self.service_start_cmd.format(self.service_name), "Starting the service.") return code == 0 - def stop_service(self): + def stop_service(self, service_name=str()): code, out = self.invoke_systemctl(self.service_stop_cmd.format(self.service_name), "Stopping the service.") return code == 0 @@ -74,7 +74,7 @@ def enable_service(self): code, out = self.invoke_systemctl(self.service_enable_cmd.format(self.service_name), "Enabling the service.") return code == 0 - def disable_service(self): + def disable_service(self, service_name=str()): code, out = self.invoke_systemctl(self.service_disable_cmd.format(self.service_name), "Disabling the service.") return code == 0 diff --git a/src/extension/src/Constants.py b/src/extension/src/Constants.py index e68df20ba..d7e71c4b4 100644 --- a/src/extension/src/Constants.py +++ b/src/extension/src/Constants.py @@ -44,8 +44,8 @@ def __iter__(self): HANDLER_ENVIRONMENT_FILE_PATH = os.getcwd() CONFIG_SETTINGS_FILE_EXTENSION = '.settings' STATUS_FILE_EXTENSION = '.status' - CORE_CODE_FILE_NAME = 'MsftLinuxPatchCore.py' - CORE_AUTO_ASSESS_SH_FILE_NAME = "MsftLinuxPatchAutoAssess.sh" + CORE_CODE_FILE_NAME = 'AzGPSLinuxPatchCore.py' + CORE_AUTO_ASSESS_SH_FILE_NAME = "AzGPSLinuxPatchAutoAssess.sh" LOG_FILE_EXTENSION = '.log' LOG_FILES_TO_RETAIN = 15 MAX_LOG_FILES_ALLOWED = 40 diff --git a/src/extension/tests/Test_HandlerManifest.py b/src/extension/tests/Test_HandlerManifest.py index 28c6c7b35..a0f1e4b13 100644 --- a/src/extension/tests/Test_HandlerManifest.py +++ b/src/extension/tests/Test_HandlerManifest.py @@ -38,11 +38,11 @@ def test_handler_manifest_json(self): handler_json = json.loads(file_contents) self.assertEqual(len(handler_json), 1) self.assertEqual(handler_json[0]['version'], 1.0) - self.assertEqual(handler_json[0]['handlerManifest']['disableCommand'], "MsftLinuxPatchExtShim.sh -d") - self.assertEqual(handler_json[0]['handlerManifest']['enableCommand'], "MsftLinuxPatchExtShim.sh -e") - self.assertEqual(handler_json[0]['handlerManifest']['uninstallCommand'], "MsftLinuxPatchExtShim.sh -u") - self.assertEqual(handler_json[0]['handlerManifest']['installCommand'], "MsftLinuxPatchExtShim.sh -i") - self.assertEqual(handler_json[0]['handlerManifest']['updateCommand'], "MsftLinuxPatchExtShim.sh -p") + self.assertEqual(handler_json[0]['handlerManifest']['disableCommand'], "AzGPSLinuxPatchExtShim.sh -d") + self.assertEqual(handler_json[0]['handlerManifest']['enableCommand'], "AzGPSLinuxPatchExtShim.sh -e") + self.assertEqual(handler_json[0]['handlerManifest']['uninstallCommand'], "AzGPSLinuxPatchExtShim.sh -u") + self.assertEqual(handler_json[0]['handlerManifest']['installCommand'], "AzGPSLinuxPatchExtShim.sh -i") + self.assertEqual(handler_json[0]['handlerManifest']['updateCommand'], "AzGPSLinuxPatchExtShim.sh -p") self.assertEqual(handler_json[0]['handlerManifest']['rebootAfterInstall'], False) self.assertEqual(handler_json[0]['handlerManifest']['reportHeartbeat'], False) self.handler_manifest_file_handle.close() diff --git a/src/tools/diagnostics/updatecenter_troubleshooter.py b/src/tools/diagnostics/updatecenter_troubleshooter.py index aeca1783b..3d4f7d026 100644 --- a/src/tools/diagnostics/updatecenter_troubleshooter.py +++ b/src/tools/diagnostics/updatecenter_troubleshooter.py @@ -1257,20 +1257,20 @@ def check_azure_extension(): def check_autoassessment_service(): rule_id = "Linux.AutoAssessment" rule_group_id = "Extensions" - command = "sudo systemctl is-active MsftLinuxPatchAutoAssess.timer" + command = "sudo systemctl is-active AzGPSLinuxPatchAutoAssess.timer" grep_output = os.popen(command).read() if "active" not in str(grep_output): - write_log_output(rule_id, rule_group_id, status_failed, empty_failure_reason," MsftLinuxPatchAutoAssess.timer is not active") + write_log_output(rule_id, rule_group_id, status_failed, empty_failure_reason," AzGPSLinuxPatchAutoAssess.timer is not active") else: - write_log_output(rule_id, rule_group_id, status_passed , empty_failure_reason, " MsftLinuxPatchAutoAssess.timer is active") + write_log_output(rule_id, rule_group_id, status_passed , empty_failure_reason, " AzGPSLinuxPatchAutoAssess.timer is active") - command = "sudo systemctl is-enabled MsftLinuxPatchAutoAssess.service" + command = "sudo systemctl is-enabled AzGPSLinuxPatchAutoAssess.service" grep_output = os.popen(command).read() if "enabled" not in str(grep_output): - write_log_output(rule_id, rule_group_id, status_failed, empty_failure_reason," MsftLinuxPatchAutoAssess.service is not enabled") + write_log_output(rule_id, rule_group_id, status_failed, empty_failure_reason," AzGPSLinuxPatchAutoAssess.service is not enabled") else: - write_log_output(rule_id, rule_group_id, status_passed , empty_failure_reason, " MsftLinuxPatchAutoAssess.service is enabled") + write_log_output(rule_id, rule_group_id, status_passed , empty_failure_reason, " AzGPSLinuxPatchAutoAssess.service is enabled") return 0 From bc7137bc217063fd4d525373d19babb6cb69b861 Mon Sep 17 00:00:00 2001 From: Koshy John Date: Mon, 21 Apr 2025 09:14:41 -0700 Subject: [PATCH 04/10] Gitignore update --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 7de90baaa..075a87dcb 100644 --- a/.gitignore +++ b/.gitignore @@ -135,4 +135,4 @@ dmypy.json # Linux Patch Extension specific /src/core/tests/scratch/ -/src/extension/tests/MsftLinuxPatchAutoAssess.sh +/src/extension/tests/AzGPSLinuxPatchAutoAssess.sh From 5e3f8f421f6626237d314e8ffb13ef76c44b5d9d Mon Sep 17 00:00:00 2001 From: Koshy John Date: Tue, 22 Apr 2025 13:14:51 -0700 Subject: [PATCH 05/10] Additional changes to tests --- .../core_logic/ConfigurePatchingProcessor.py | 39 ++++-- src/core/src/core_logic/ServiceManager.py | 7 ++ src/core/src/core_logic/TimerManager.py | 6 + .../tests/Test_ConfigurePatchingProcessor.py | 38 +++--- src/core/tests/library/ExtStatusAsserter.py | 119 ++++++++++++++++++ src/tools/packager/Package-All.py | 2 +- src/tools/packager/Package-Core.py | 2 +- 7 files changed, 174 insertions(+), 39 deletions(-) create mode 100644 src/core/tests/library/ExtStatusAsserter.py diff --git a/src/core/src/core_logic/ConfigurePatchingProcessor.py b/src/core/src/core_logic/ConfigurePatchingProcessor.py index 8ba9d2725..acd10383e 100644 --- a/src/core/src/core_logic/ConfigurePatchingProcessor.py +++ b/src/core/src/core_logic/ConfigurePatchingProcessor.py @@ -19,7 +19,8 @@ class ConfigurePatchingProcessor(object): - def __init__(self, env_layer, execution_config, composite_logger, telemetry_writer, status_handler, package_manager, auto_assess_service_manager, auto_assess_timer_manager, lifecycle_manager, aasm_legacy=None, aatm_legacy=None): + def __init__(self, env_layer, execution_config, composite_logger, telemetry_writer, status_handler, package_manager, auto_assess_service_manager, auto_assess_timer_manager, lifecycle_manager, + auto_assess_service_manager_legacy=None, auto_assess_timer_manager_legacy=None): self.env_layer = env_layer self.execution_config = execution_config @@ -30,6 +31,8 @@ def __init__(self, env_layer, execution_config, composite_logger, telemetry_writ self.package_manager = package_manager self.auto_assess_service_manager = auto_assess_service_manager self.auto_assess_timer_manager = auto_assess_timer_manager + self.auto_assess_service_manager_legacy = auto_assess_service_manager_legacy + self.auto_assess_timer_manager_legacy = auto_assess_timer_manager_legacy self.lifecycle_manager = lifecycle_manager self.current_auto_os_patch_state = Constants.AutomaticOSPatchStates.UNKNOWN @@ -56,7 +59,7 @@ def start_configure_patching(self): self.configure_patching_exception_error = error # If the tracked operation is Configure patching, we cannot write a final status until assessment has also written a final status (mitigation for a CRP bug) if self.execution_config.operation != Constants.CONFIGURE_PATCHING.lower(): - self.__report_consolidated_configure_patch_status(status=Constants.STATUS_ERROR, error=self.configure_patching_exception_error) + self.__report_consolidated_configure_patch_status(status=Constants.STATUS_ERROR, error=repr(self.configure_patching_exception_error)) self.configure_patching_successful &= False self.composite_logger.log("\nConfigure patching completed.\n") @@ -105,15 +108,21 @@ def __try_set_auto_assessment_mode(self): self.composite_logger.log_debug("No assessment mode config was present. No configuration changes will occur.") elif self.execution_config.assessment_mode == Constants.AssessmentModes.AUTOMATIC_BY_PLATFORM: self.composite_logger.log_debug("Enabling platform-based automatic assessment.") + if not self.auto_assess_service_manager.systemd_exists(): raise Exception("Systemd is not available on this system, and platform-based auto-assessment cannot be configured.") + + self.__erase_auto_assess_config_if_any("legacy", self.auto_assess_service_manager_legacy, self.auto_assess_timer_manager_legacy) self.auto_assess_service_manager.create_and_set_service_idem() self.auto_assess_timer_manager.create_and_set_timer_idem() + self.current_auto_assessment_state = Constants.AutoAssessmentStates.ENABLED elif self.execution_config.assessment_mode == Constants.AssessmentModes.IMAGE_DEFAULT: self.composite_logger.log_debug("Disabling platform-based automatic assessment.") - self.auto_assess_timer_manager.remove_timer() - self.auto_assess_service_manager.remove_service() + + self.__erase_auto_assess_config_if_any("AzGPS", self.auto_assess_service_manager, self.auto_assess_timer_manager) + self.__erase_auto_assess_config_if_any("legacy", self.auto_assess_service_manager_legacy, self.auto_assess_timer_manager_legacy) + self.current_auto_assessment_state = Constants.AutoAssessmentStates.DISABLED else: raise Exception("Unknown assessment mode specified. [AssessmentMode={0}]".format(self.execution_config.assessment_mode)) @@ -123,27 +132,31 @@ def __try_set_auto_assessment_mode(self): except Exception as error: # deliberately not setting self.configure_patching_exception_error here as it does not feed into the parent object. Not a bug, if you're thinking about it. self.composite_logger.log_error("Error while processing automatic assessment mode configuration. [Error={0}]".format(repr(error))) - self.__report_consolidated_configure_patch_status(status=Constants.STATUS_TRANSITIONING, error=error) + self.__report_consolidated_configure_patch_status(status=Constants.STATUS_TRANSITIONING, error=repr(error)) self.configure_patching_successful &= False # revert operation back to parent self.composite_logger.log_debug("Restoring status handler operation to {0}.".format(Constants.CONFIGURE_PATCHING)) self.status_handler.set_current_operation(Constants.CONFIGURE_PATCHING) - def __custom_clean_legacy_auto_assess_service(self): + def __erase_auto_assess_config_if_any(self, service_name, service_manager, timer_manager): """ Cleans up the legacy auto-assess service """ - self.composite_logger.log_debug("Cleaning up legacy auto-assess service.") try: - self.auto_assess_service_manager.remove_service() - self.auto_assess_timer_manager.remove_timer() - self.current_auto_assessment_state = Constants.AutoAssessmentStates.DISABLED + if service_manager is not None and not service_manager.service_exists(): + self.composite_logger.log_debug("[CPP] Cleaning up the {0} auto-assess service.".format(service_name)) + service_manager.remove_service() + + if timer_manager is not None and not timer_manager.timer_exists(): + self.composite_logger.log_debug("[CPP] Cleaning up the {0} auto-assess timer.".format(service_name)) + timer_manager.remove_timer() except Exception as error: - self.composite_logger.log_error("Error while cleaning up legacy auto-assess service. [Error={0}]".format(repr(error))) + self.composite_logger.log_warning("[CPP] Retriable error while cleaning up auto-assess service. [Error={0}]".format(repr(error))) self.configure_patching_successful &= False def __report_consolidated_configure_patch_status(self, status=Constants.STATUS_TRANSITIONING, error=Constants.DEFAULT_UNSPECIFIED_VALUE): - """ Reports """ - self.composite_logger.log_debug("Reporting consolidated current configure patch status. [OSPatchState={0}][AssessmentState={1}]".format(self.current_auto_os_patch_state, self.current_auto_assessment_state)) + # type: (str, str) -> None + """ Reports the consolidated configure patching status """ + self.composite_logger.log_debug("[CPP] Reporting consolidated current configure patch status. [OSPatchState={0}][AssessmentState={1}]".format(self.current_auto_os_patch_state, self.current_auto_assessment_state)) # report error if specified if error != Constants.DEFAULT_UNSPECIFIED_VALUE: diff --git a/src/core/src/core_logic/ServiceManager.py b/src/core/src/core_logic/ServiceManager.py index 277d21d2e..e5d630f4a 100644 --- a/src/core/src/core_logic/ServiceManager.py +++ b/src/core/src/core_logic/ServiceManager.py @@ -35,7 +35,14 @@ def __init__(self, env_layer, execution_config, composite_logger, telemetry_writ self.service_is_active_cmd = "sudo systemctl is-active {0}.service" # region - Service Creation / Removal + def service_exists(self): + # type: () -> bool + """ Check if the service exists """ + service_path = self.__systemd_service_unit_path.format(self.service_name) + return os.path.exists(service_path) + def remove_service(self): + """ Remove the service if it exists """ service_path = self.__systemd_service_unit_path.format(self.service_name) if os.path.exists(service_path): self.stop_service() diff --git a/src/core/src/core_logic/TimerManager.py b/src/core/src/core_logic/TimerManager.py index df9556b54..baa0238ff 100644 --- a/src/core/src/core_logic/TimerManager.py +++ b/src/core/src/core_logic/TimerManager.py @@ -35,6 +35,12 @@ def __init__(self, env_layer, execution_config, composite_logger, telemetry_writ self.timer_is_active_cmd = "sudo systemctl is-active {0}.timer" # region - Time Creation / Removal + def timer_exists(self): + # type: () -> bool + """ Check if the timer exists """ + timer_path = self.__systemd_timer_unit_path.format(self.service_name) + return os.path.exists(timer_path) + def remove_timer(self): timer_path = self.__systemd_timer_unit_path.format(self.service_name) if os.path.exists(timer_path): diff --git a/src/core/tests/Test_ConfigurePatchingProcessor.py b/src/core/tests/Test_ConfigurePatchingProcessor.py index 11f1d38c1..c781255b3 100644 --- a/src/core/tests/Test_ConfigurePatchingProcessor.py +++ b/src/core/tests/Test_ConfigurePatchingProcessor.py @@ -18,6 +18,9 @@ import re import unittest import sys + +from library.ExtStatusAsserter import ExtStatusAsserter + # Conditional import for StringIO try: from StringIO import StringIO # Python 2 @@ -42,16 +45,17 @@ def tearDown(self): # self.runtime.stop() pass - #region Mocks + # region Mocks def mock_package_manager_get_current_auto_os_patch_state_returns_unknown(self): if self.mock_package_manager_get_current_auto_os_patch_state_returns_unknown_call_count == 0: self.mock_package_manager_get_current_auto_os_patch_state_returns_unknown_call_count = 1 return Constants.AutomaticOSPatchStates.DISABLED else: return Constants.AutomaticOSPatchStates.UNKNOWN + def mock_get_current_auto_os_patch_state(self): raise Exception("Mocked Exception") - #endregion Mocks + # endregion Mocks def test_operation_success_for_configure_patching_request_for_apt_with_default_updates_config(self): # create and adjust arguments @@ -138,9 +142,7 @@ def test_operation_success_for_installation_request_with_configure_patching(self # check telemetry events self.__check_telemetry_events(runtime) - # check status file - with runtime.env_layer.file_system.open(runtime.execution_config.status_file_path, 'r') as file_handle: - substatus_file_data = json.load(file_handle)[0]["status"]["substatus"] + # assert self.assertTrue(runtime.package_manager.image_default_patch_configuration_backup_exists()) image_default_patch_configuration_backup = json.loads(runtime.env_layer.file_system.read_with_retry(runtime.package_manager.image_default_patch_configuration_backup_path)) self.assertTrue(image_default_patch_configuration_backup is not None) @@ -150,26 +152,14 @@ def test_operation_success_for_installation_request_with_configure_patching(self self.assertTrue(os_patch_configuration_settings is not None) self.assertTrue('APT::Periodic::Update-Package-Lists "0"' in os_patch_configuration_settings) self.assertTrue('APT::Periodic::Unattended-Upgrade "0"' in os_patch_configuration_settings) - self.assertTrue(substatus_file_data[3]["name"] == Constants.CONFIGURE_PATCHING_SUMMARY) - self.assertTrue(substatus_file_data[3]["status"].lower() == Constants.STATUS_SUCCESS.lower()) - self.assertEqual(len(substatus_file_data), 4) - self.assertTrue(substatus_file_data[0]["name"] == Constants.PATCH_ASSESSMENT_SUMMARY) - self.assertTrue(substatus_file_data[0]["status"].lower() == Constants.STATUS_SUCCESS.lower()) - self.assertTrue(substatus_file_data[1]["name"] == Constants.PATCH_INSTALLATION_SUMMARY) - self.assertTrue(substatus_file_data[1]["status"].lower() == Constants.STATUS_SUCCESS.lower()) - self.assertEqual(json.loads(substatus_file_data[1]["formattedMessage"]["message"])["patches"][0]["name"], "python-samba") - self.assertTrue("Security" in str(json.loads(substatus_file_data[1]["formattedMessage"]["message"])["patches"][0]["classifications"])) - self.assertEqual(json.loads(substatus_file_data[1]["formattedMessage"]["message"])["patches"][1]["name"], "samba-common-bin") - self.assertTrue("Security" in str(json.loads(substatus_file_data[1]["formattedMessage"]["message"])["patches"][1]["classifications"])) - self.assertEqual(json.loads(substatus_file_data[1]["formattedMessage"]["message"])["patches"][2]["name"], "samba-libs") - self.assertTrue("python-samba_2:4.4.5+dfsg-2ubuntu5.4" in str(json.loads(substatus_file_data[1]["formattedMessage"]["message"])["patches"][0]["patchId"])) - self.assertTrue("Security" in str(json.loads(substatus_file_data[1]["formattedMessage"]["message"])["patches"][2]["classifications"])) - self.assertTrue(substatus_file_data[2]["name"] == Constants.PATCH_METADATA_FOR_HEALTHSTORE) - self.assertTrue(substatus_file_data[2]["status"].lower() == Constants.STATUS_SUCCESS.lower()) - substatus_file_data_patch_metadata_summary = json.loads(substatus_file_data[2]["formattedMessage"]["message"]) - self.assertEqual(substatus_file_data_patch_metadata_summary["patchVersion"], "pub_off_sku_2020.09.23") - self.assertTrue(substatus_file_data_patch_metadata_summary["shouldReportToHealthStore"]) + # check status file + ext_status_asserter = ExtStatusAsserter(runtime.execution_config.status_file_path, runtime.env_layer) + ext_status_asserter.assert_status_file_substatuses() + ext_status_asserter.assert_installation_summary_has_patch("python-samba", Constants.PackageClassification.SECURITY, "python-samba_2:4.4.5+dfsg-2ubuntu5.4") + ext_status_asserter.assert_installation_summary_has_patch("samba-common-bin", Constants.PackageClassification.SECURITY) + ext_status_asserter.assert_installation_summary_has_patch("samba-libs", Constants.PackageClassification.SECURITY) + ext_status_asserter.assert_healthstore_status_info(patch_version="pub_off_sku_2020.09.23", should_report=True) runtime.stop() def test_operation_fail_for_configure_patching_telemetry_not_supported(self): diff --git a/src/core/tests/library/ExtStatusAsserter.py b/src/core/tests/library/ExtStatusAsserter.py new file mode 100644 index 000000000..84640384b --- /dev/null +++ b/src/core/tests/library/ExtStatusAsserter.py @@ -0,0 +1,119 @@ +# Copyright 2025 Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Requires Python 2.7+ +import json + +from core.src.bootstrap.Constants import Constants + + +class ExtStatusAsserter(object): + def __init__(self, status_file_path, env_layer): + self.__status_file_path = status_file_path + self.__env_layer = env_layer + + self.__substatus_file_data = self.__read_status_file(self.__status_file_path) + + self.substatus_index_count = 0 + self.configure_patching_substatus_index = self.patch_assessment_substatus_index = self.patch_installation_substatus_index = self.healthstore_substatus_index = None + self.__populate_substatus_indices() + + def __read_status_file(self, status_file_path): + with self.__env_layer.file_system.open(status_file_path, 'r') as file_handle: + return json.load(file_handle)[0]["status"]["substatus"] + + def __populate_substatus_indices(self): + for index, substatus in enumerate(self.__substatus_file_data): + if substatus["name"] == Constants.CONFIGURE_PATCHING_SUMMARY: + self.configure_patching_substatus_index = index + elif substatus["name"] == Constants.PATCH_ASSESSMENT_SUMMARY: + self.patch_assessment_substatus_index = index + elif substatus["name"] == Constants.PATCH_INSTALLATION_SUMMARY: + self.patch_installation_substatus_index = index + elif substatus["name"] == Constants.PATCH_METADATA_FOR_HEALTHSTORE: + self.healthstore_substatus_index = index + + @staticmethod + def get_default_substatus_expectations(): + """Get the default substatus expectations""" + return { + Constants.CONFIGURE_PATCHING_SUMMARY: Constants.STATUS_SUCCESS, + Constants.PATCH_ASSESSMENT_SUMMARY: Constants.STATUS_SUCCESS, + Constants.PATCH_INSTALLATION_SUMMARY: Constants.STATUS_SUCCESS, + Constants.PATCH_METADATA_FOR_HEALTHSTORE: Constants.STATUS_SUCCESS, + } + + def assert_status_file_substatuses(self, substatus_expectations=None): + """Check if the status file has a specific substatus""" + if substatus_expectations is None: + substatus_expectations = self.get_default_substatus_expectations() + + for key, value in substatus_expectations.items(): + if key == Constants.CONFIGURE_PATCHING_SUMMARY and self.__substatus_file_data[self.configure_patching_substatus_index]['status'].lower() != value.lower(): + raise AssertionError("Substatus expectations do not match for {0}.".format(key)) + elif key == Constants.PATCH_ASSESSMENT_SUMMARY and self.__substatus_file_data[self.patch_assessment_substatus_index]['status'].lower() != value.lower(): + raise AssertionError("Substatus expectations do not match for {0}.".format(key)) + elif key == Constants.PATCH_INSTALLATION_SUMMARY and self.__substatus_file_data[self.patch_installation_substatus_index]['status'].lower() != value.lower(): + raise AssertionError("Substatus expectations do not match for {0}.".format(key)) + elif key == Constants.PATCH_METADATA_FOR_HEALTHSTORE and self.__substatus_file_data[self.healthstore_substatus_index]['status'].lower() != value.lower(): + raise AssertionError("Substatus expectations do not match for {0}.".format(key)) + + def assert_assessment_summary_has_patch(self, patch_name, classification=None, patchId=None): + """Check if the assessment summary has a specific patch""" + if self.patch_assessment_substatus_index is None: + raise AssertionError("Patch assessment substatus index not found.") + + assessment_summary = self.__substatus_file_data[self.patch_assessment_substatus_index] + assessment_summary_patches = json.loads(assessment_summary["formattedMessage"]["message"])["patches"] + + for patch in assessment_summary_patches: + if patch["name"] == patch_name: + if classification and classification not in patch["classifications"]: + raise AssertionError("Classification '{classification}' does not match expected value.") + if patchId and patch["patchId"] != patchId: + raise AssertionError("Patch ID '{patchId}' does not match expected value.") + return True + + raise AssertionError("Patch '{patch_name}' not found in assessment summary.") + + def assert_installation_summary_has_patch(self, patch_name, classification=None, patchId=None): + """Check if the installation summary has a specific patch""" + if self.patch_assessment_substatus_index is None: + raise AssertionError("Patch assessment substatus index not found.") + + installation_summary = self.__substatus_file_data[self.patch_assessment_substatus_index] + installation_summary_patches = json.loads(installation_summary["formattedMessage"]["message"])["patches"] + + for patch in installation_summary_patches: + if patch["name"] == patch_name: + if classification and classification not in patch["classifications"]: + raise AssertionError("Classification '{classification}' does not match expected value.") + if patchId and patchId not in str(patch["patchId"]): + raise AssertionError("Patch ID '{patchId}' does not match expected value.") + return True + + raise AssertionError("Patch '{patch_name}' not found in assessment summary.") + + def assert_healthstore_status_info(self, patch_version, should_report=True): + """Check if the healthstore patch version is as expected""" + if self.healthstore_substatus_index is None: + raise AssertionError("Healthstore substatus index not found.") + + healthstore_summary = json.loads(self.__substatus_file_data[self.healthstore_substatus_index]["formattedMessage"]["message"]) + + if should_report and healthstore_summary["shouldReportToHealthStore"] != True: + raise AssertionError("Healthstore summary should report to healthstore.") + + if patch_version != healthstore_summary["patchVersion"]: + raise AssertionError("Healthstore summary patch version does not match expected value.") diff --git a/src/tools/packager/Package-All.py b/src/tools/packager/Package-All.py index f99951321..56ac141c1 100644 --- a/src/tools/packager/Package-All.py +++ b/src/tools/packager/Package-All.py @@ -147,7 +147,7 @@ def generate_compiled_script(source_code_path, merged_file_full_path, merged_fil print('------------- Set Copyright, Version and Environment. Also enforce UNIX-style line endings.') insert_copyright_notice(merged_file_full_path, merged_file_name) - timestamp = datetime.datetime.now(datetime.UTC).strftime("%y%m%d-%H%M") + timestamp = datetime.datetime.now(datetime.timezone.UTC).strftime("%y%m%d-%H%M") replace_text_in_file(merged_file_full_path, '[%exec_name%]', merged_file_name) replace_text_in_file(merged_file_full_path, '[%exec_ver%]', str(new_version)) replace_text_in_file(merged_file_full_path, '[%exec_build_timestamp%]', timestamp) diff --git a/src/tools/packager/Package-Core.py b/src/tools/packager/Package-Core.py index 180a2581c..7f6e34d63 100644 --- a/src/tools/packager/Package-Core.py +++ b/src/tools/packager/Package-Core.py @@ -150,7 +150,7 @@ def generate_compiled_script(source_code_path, merged_file_full_path, merged_fil print('------------- Set Copyright, Version and Environment. Also enforce UNIX-style line endings.') insert_copyright_notice(merged_file_full_path, merged_file_name) - timestamp = datetime.datetime.now(datetime.UTC).strftime("%y%m%d-%H%M") + timestamp = datetime.datetime.now(datetime.timezone.utc).strftime("%y%m%d-%H%M") replace_text_in_file(merged_file_full_path, '[%exec_name%]', merged_file_name) replace_text_in_file(merged_file_full_path, '[%exec_ver%]', str(new_version)) replace_text_in_file(merged_file_full_path, '[%exec_build_timestamp%]', timestamp) From 2be611903e1c33c6a12e59b2275adc904bbc096f Mon Sep 17 00:00:00 2001 From: Koshy John Date: Mon, 28 Apr 2025 13:40:13 -0700 Subject: [PATCH 06/10] Parking ExtStatusAsserter changes --- src/core/tests/library/ExtStatusAsserter.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/core/tests/library/ExtStatusAsserter.py b/src/core/tests/library/ExtStatusAsserter.py index 84640384b..eb074ef3a 100644 --- a/src/core/tests/library/ExtStatusAsserter.py +++ b/src/core/tests/library/ExtStatusAsserter.py @@ -25,6 +25,8 @@ def __init__(self, status_file_path, env_layer): self.__substatus_file_data = self.__read_status_file(self.__status_file_path) + self.__substatus_index_mapping = [[]] + self.substatus_index_count = 0 self.configure_patching_substatus_index = self.patch_assessment_substatus_index = self.patch_installation_substatus_index = self.healthstore_substatus_index = None self.__populate_substatus_indices() From 4c37bff50d7e33285a14e634b0fe2f1ec6e621e8 Mon Sep 17 00:00:00 2001 From: Koshy John Date: Wed, 30 Apr 2025 10:52:06 -0700 Subject: [PATCH 07/10] Additional changes --- src/core/src/bootstrap/Bootstrapper.py | 2 +- src/tools/packager/Package-All.py | 8 ++++++-- src/tools/packager/Package-Core.py | 4 ++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/core/src/bootstrap/Bootstrapper.py b/src/core/src/bootstrap/Bootstrapper.py index 1d1490d58..bb5bce0af 100644 --- a/src/core/src/bootstrap/Bootstrapper.py +++ b/src/core/src/bootstrap/Bootstrapper.py @@ -135,7 +135,7 @@ def build_core_components(self, container): return lifecycle_manager, status_handler def bootstrap_splash_text(self): - self.composite_logger.log("\n\n[%exec_name%] \t -- \t Copyright (c) Microsoft Corporation. All rights reserved. \nApplication version: 3.0.[%exec_build_timestamp%]\n\n") + self.composite_logger.log("\n\n[%exec_name%] \t -- \t Copyright (c) Microsoft Corporation. All rights reserved. \nBuild Date: [%exec_build_date%]\n\n") def basic_environment_health_check(self): self.composite_logger.log("Python version: " + " ".join(sys.version.splitlines())) diff --git a/src/tools/packager/Package-All.py b/src/tools/packager/Package-All.py index 56ac141c1..f7cbbee08 100644 --- a/src/tools/packager/Package-All.py +++ b/src/tools/packager/Package-All.py @@ -20,6 +20,8 @@ Note: Package-All.py internally invokes Package-Core.py to generate AzGPSLinuxPatchCore.py """ from __future__ import print_function + +import shutil import sys import os import errno @@ -147,10 +149,10 @@ def generate_compiled_script(source_code_path, merged_file_full_path, merged_fil print('------------- Set Copyright, Version and Environment. Also enforce UNIX-style line endings.') insert_copyright_notice(merged_file_full_path, merged_file_name) - timestamp = datetime.datetime.now(datetime.timezone.UTC).strftime("%y%m%d-%H%M") + date = datetime.datetime.now(datetime.timezone.utc).strftime("%y.%m.%d") replace_text_in_file(merged_file_full_path, '[%exec_name%]', merged_file_name) replace_text_in_file(merged_file_full_path, '[%exec_ver%]', str(new_version)) - replace_text_in_file(merged_file_full_path, '[%exec_build_timestamp%]', timestamp) + replace_text_in_file(merged_file_full_path, '[%exec_build_date%]', date) replace_text_in_file(merged_file_full_path, 'Constants.UNKNOWN_ENV', environment) replace_text_in_file(merged_file_full_path, '\r\n', '\n') @@ -185,6 +187,8 @@ def main(argv): working_directory = os.path.abspath(os.path.join(source_code_path, os.pardir, os.pardir)) merge_file_directory = os.path.join(working_directory, 'out') try: + if os.path.exists(merge_file_directory): + shutil.rmtree(merge_file_directory) os.makedirs(merge_file_directory) except OSError as e: if e.errno != errno.EEXIST: diff --git a/src/tools/packager/Package-Core.py b/src/tools/packager/Package-Core.py index 7f6e34d63..0a6042f2f 100644 --- a/src/tools/packager/Package-Core.py +++ b/src/tools/packager/Package-Core.py @@ -150,10 +150,10 @@ def generate_compiled_script(source_code_path, merged_file_full_path, merged_fil print('------------- Set Copyright, Version and Environment. Also enforce UNIX-style line endings.') insert_copyright_notice(merged_file_full_path, merged_file_name) - timestamp = datetime.datetime.now(datetime.timezone.utc).strftime("%y%m%d-%H%M") + date = datetime.datetime.now(datetime.timezone.utc).strftime("%y.%m.%d") replace_text_in_file(merged_file_full_path, '[%exec_name%]', merged_file_name) replace_text_in_file(merged_file_full_path, '[%exec_ver%]', str(new_version)) - replace_text_in_file(merged_file_full_path, '[%exec_build_timestamp%]', timestamp) + replace_text_in_file(merged_file_full_path, '[%exec_build_date%]', date) replace_text_in_file(merged_file_full_path, '\r\n', '\n') print('------------- Merged core code was saved to:\n{0}\n'.format(merged_file_full_path)) From e02fd8d8a5ecede6ea1566df1bf09719bc47556e Mon Sep 17 00:00:00 2001 From: Koshy John Date: Thu, 1 May 2025 08:48:10 -0700 Subject: [PATCH 08/10] More test helper changes --- .../core_logic/ConfigurePatchingProcessor.py | 2 +- .../tests/Test_ConfigurePatchingProcessor.py | 17 ++- src/core/tests/library/ExtStatusAsserter.py | 107 +++++++++--------- 3 files changed, 68 insertions(+), 58 deletions(-) diff --git a/src/core/src/core_logic/ConfigurePatchingProcessor.py b/src/core/src/core_logic/ConfigurePatchingProcessor.py index acd10383e..d034e8c29 100644 --- a/src/core/src/core_logic/ConfigurePatchingProcessor.py +++ b/src/core/src/core_logic/ConfigurePatchingProcessor.py @@ -150,7 +150,7 @@ def __erase_auto_assess_config_if_any(self, service_name, service_manager, timer self.composite_logger.log_debug("[CPP] Cleaning up the {0} auto-assess timer.".format(service_name)) timer_manager.remove_timer() except Exception as error: - self.composite_logger.log_warning("[CPP] Retriable error while cleaning up auto-assess service. [Error={0}]".format(repr(error))) + self.composite_logger.log_warning("[CPP] Retriable error while cleaning up auto-assess config. [Error={0}]".format(repr(error))) self.configure_patching_successful &= False def __report_consolidated_configure_patch_status(self, status=Constants.STATUS_TRANSITIONING, error=Constants.DEFAULT_UNSPECIFIED_VALUE): diff --git a/src/core/tests/Test_ConfigurePatchingProcessor.py b/src/core/tests/Test_ConfigurePatchingProcessor.py index c781255b3..63a82e2cd 100644 --- a/src/core/tests/Test_ConfigurePatchingProcessor.py +++ b/src/core/tests/Test_ConfigurePatchingProcessor.py @@ -172,10 +172,16 @@ def test_operation_fail_for_configure_patching_telemetry_not_supported(self): runtime.configure_patching_processor.start_configure_patching() # check status file + ext_status_asserter = ExtStatusAsserter(runtime.execution_config.status_file_path, runtime.env_layer) + ext_status_asserter.assert_status_file_substatuses(substatus_expectations={ + Constants.CONFIGURE_PATCHING_SUMMARY: Constants.STATUS_SUCCESS + }) + with runtime.env_layer.file_system.open(runtime.execution_config.status_file_path, 'r') as file_handle: substatus_file_data = json.load(file_handle)[0]["status"]["substatus"] self.assertEqual(len(substatus_file_data), 1) self.assertTrue(substatus_file_data[0]["name"] == Constants.CONFIGURE_PATCHING_SUMMARY) + if runtime.vm_cloud_type == Constants.VMCloudType.AZURE: self.assertTrue(substatus_file_data[0]["status"].lower() == Constants.STATUS_ERROR.lower()) self.assertTrue(len(json.loads(substatus_file_data[0]["formattedMessage"]["message"])["errors"]["details"]), 1) @@ -355,15 +361,18 @@ def test_configure_patching_raise_exception_auto_assessment_systemd(self): runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True, Constants.APT) runtime.set_legacy_test_type('HappyPath') - # mock swap + # mock swap service manager back_up_auto_assess_service_manager = runtime.configure_patching_processor.auto_assess_service_manager.systemd_exists runtime.configure_patching_processor.auto_assess_service_manager.systemd_exists = lambda: False - self.assertRaises(Exception, runtime.configure_patching_processor.start_configure_patching()) - - # restore runtime.configure_patching_processor.auto_assess_service_manager.systemd_exists = back_up_auto_assess_service_manager + # mock swap legacy timer manager + back_up_auto_assess_timer_manager_legacy = runtime.configure_patching_processor.auto_assess_timer_manager_legacy + runtime.configure_patching_processor.auto_assess_timer_manager_legacy = object() + self.assertRaises(Exception, runtime.configure_patching_processor.start_configure_patching()) + runtime.configure_patching_processor.auto_assess_timer_manager = back_up_auto_assess_timer_manager_legacy + runtime.stop() def __check_telemetry_events(self, runtime): diff --git a/src/core/tests/library/ExtStatusAsserter.py b/src/core/tests/library/ExtStatusAsserter.py index eb074ef3a..b3504d430 100644 --- a/src/core/tests/library/ExtStatusAsserter.py +++ b/src/core/tests/library/ExtStatusAsserter.py @@ -24,27 +24,49 @@ def __init__(self, status_file_path, env_layer): self.__env_layer = env_layer self.__substatus_file_data = self.__read_status_file(self.__status_file_path) - - self.__substatus_index_mapping = [[]] - - self.substatus_index_count = 0 - self.configure_patching_substatus_index = self.patch_assessment_substatus_index = self.patch_installation_substatus_index = self.healthstore_substatus_index = None - self.__populate_substatus_indices() + self.__substatus_high_level_summary = None + self.__load_substatus_high_level_summary(self.__substatus_file_data) def __read_status_file(self, status_file_path): + # type: (str) -> dict with self.__env_layer.file_system.open(status_file_path, 'r') as file_handle: return json.load(file_handle)[0]["status"]["substatus"] - def __populate_substatus_indices(self): - for index, substatus in enumerate(self.__substatus_file_data): - if substatus["name"] == Constants.CONFIGURE_PATCHING_SUMMARY: + @staticmethod + def __get_high_level_summary_template(): + # type: () -> dict + return { + Constants.CONFIGURE_PATCHING_SUMMARY: {"index": -1, "status": None}, + Constants.PATCH_ASSESSMENT_SUMMARY: {"index": -1, "status": None}, + Constants.PATCH_INSTALLATION_SUMMARY: {"index": -1, "status": None}, + Constants.PATCH_METADATA_FOR_HEALTHSTORE: {"index": -1, "status": None}, + } + + def __load_substatus_high_level_summary(self, substatus_file_data): + # type: (dict) -> None + self.__substatus_high_level_summary = self.__get_high_level_summary_template() + + for index, substatus in enumerate(substatus_file_data): + summary_name = substatus["name"] + if summary_name in self.__substatus_high_level_summary: self.configure_patching_substatus_index = index - elif substatus["name"] == Constants.PATCH_ASSESSMENT_SUMMARY: - self.patch_assessment_substatus_index = index - elif substatus["name"] == Constants.PATCH_INSTALLATION_SUMMARY: - self.patch_installation_substatus_index = index - elif substatus["name"] == Constants.PATCH_METADATA_FOR_HEALTHSTORE: - self.healthstore_substatus_index = index + self.__substatus_high_level_summary[summary_name]["index"] = index + self.__substatus_high_level_summary[summary_name]["status"] = substatus["status"] + else: + raise KeyError("Unknown substatus: {0}".format(substatus["name"])) + + def assert_status_file_substatuses(self, operation, expected_status): + """Check if the status file has a specific substatus""" + if operation not in self.__substatus_high_level_summary: + raise KeyError("Unknown operation: {0}".format(operation)) + + substatus_index = self.__substatus_high_level_summary[operation]["index"] + if substatus_index == -1: + raise AssertionError("Substatus index not found for operation: {0}".format(operation)) + + actual_status = self.__substatus_file_data[substatus_index]["status"].lower() + if actual_status != expected_status.lower(): + raise AssertionError("Substatus expectations do not match for {0}. Expected: {1}, Actual: {2}".format(operation, expected_status, actual_status)) @staticmethod def get_default_substatus_expectations(): @@ -62,50 +84,29 @@ def assert_status_file_substatuses(self, substatus_expectations=None): substatus_expectations = self.get_default_substatus_expectations() for key, value in substatus_expectations.items(): - if key == Constants.CONFIGURE_PATCHING_SUMMARY and self.__substatus_file_data[self.configure_patching_substatus_index]['status'].lower() != value.lower(): - raise AssertionError("Substatus expectations do not match for {0}.".format(key)) - elif key == Constants.PATCH_ASSESSMENT_SUMMARY and self.__substatus_file_data[self.patch_assessment_substatus_index]['status'].lower() != value.lower(): - raise AssertionError("Substatus expectations do not match for {0}.".format(key)) - elif key == Constants.PATCH_INSTALLATION_SUMMARY and self.__substatus_file_data[self.patch_installation_substatus_index]['status'].lower() != value.lower(): - raise AssertionError("Substatus expectations do not match for {0}.".format(key)) - elif key == Constants.PATCH_METADATA_FOR_HEALTHSTORE and self.__substatus_file_data[self.healthstore_substatus_index]['status'].lower() != value.lower(): - raise AssertionError("Substatus expectations do not match for {0}.".format(key)) - - def assert_assessment_summary_has_patch(self, patch_name, classification=None, patchId=None): - """Check if the assessment summary has a specific patch""" - if self.patch_assessment_substatus_index is None: - raise AssertionError("Patch assessment substatus index not found.") - - assessment_summary = self.__substatus_file_data[self.patch_assessment_substatus_index] - assessment_summary_patches = json.loads(assessment_summary["formattedMessage"]["message"])["patches"] - - for patch in assessment_summary_patches: - if patch["name"] == patch_name: - if classification and classification not in patch["classifications"]: - raise AssertionError("Classification '{classification}' does not match expected value.") - if patchId and patch["patchId"] != patchId: - raise AssertionError("Patch ID '{patchId}' does not match expected value.") - return True - - raise AssertionError("Patch '{patch_name}' not found in assessment summary.") + if self.__substatus_high_level_summary[key]["index"] == -1: + raise AssertionError("Expected substatus not found: {0}".format(key)) + if self.__substatus_high_level_summary[key]["status"] != value: + raise AssertionError("Substatus expectations do not match for {0}. Expected: {1}, Actual: {2}".format(key, value, self.__substatus_high_level_summary[key]["status"])) - def assert_installation_summary_has_patch(self, patch_name, classification=None, patchId=None): - """Check if the installation summary has a specific patch""" - if self.patch_assessment_substatus_index is None: - raise AssertionError("Patch assessment substatus index not found.") + def assert_summary_has_patch(self, operation, patch_name, classification=None, patchId=None): + """ Check if the defined operation summary has a specific patch """ + substatus_index = self.__substatus_high_level_summary[operation]["index"] + if substatus_index == -1: + raise AssertionError("Substatus index not found for operation: {0}".format(operation)) - installation_summary = self.__substatus_file_data[self.patch_assessment_substatus_index] - installation_summary_patches = json.loads(installation_summary["formattedMessage"]["message"])["patches"] + summary = self.__substatus_file_data[substatus_index] + summary_patches = json.loads(summary["formattedMessage"]["message"])["patches"] - for patch in installation_summary_patches: + for patch in summary_patches: if patch["name"] == patch_name: if classification and classification not in patch["classifications"]: - raise AssertionError("Classification '{classification}' does not match expected value.") - if patchId and patchId not in str(patch["patchId"]): - raise AssertionError("Patch ID '{patchId}' does not match expected value.") + raise AssertionError("Classification '{0}' does not match expected value '{1}' for patch '{2}'.".format(classification, str(patch["classifications"]), patch_name)) + if patchId and patchId != str(patch["patchId"]): + raise AssertionError("Patch ID '{0}' does not match expected value '{1}' for patch '{2}'.".format(patchId, str(patch["patchId"]), patch_name)) return True - raise AssertionError("Patch '{patch_name}' not found in assessment summary.") + raise AssertionError("Patch '{0}' not found in '{1}' summary.".format(patch_name, operation)) def assert_healthstore_status_info(self, patch_version, should_report=True): """Check if the healthstore patch version is as expected""" @@ -118,4 +119,4 @@ def assert_healthstore_status_info(self, patch_version, should_report=True): raise AssertionError("Healthstore summary should report to healthstore.") if patch_version != healthstore_summary["patchVersion"]: - raise AssertionError("Healthstore summary patch version does not match expected value.") + raise AssertionError("Healthstore summary patch version '{0}' does not match expected value {1}.".format(str(healthstore_summary["patchVersion"]), patch_version)) From 0317dd986ee872e169222b2e1a1100a813a3169b Mon Sep 17 00:00:00 2001 From: Koshy John Date: Fri, 2 May 2025 11:58:24 -0700 Subject: [PATCH 09/10] Improved ExtStatusAsserter --- .../core_logic/ConfigurePatchingProcessor.py | 18 ++- .../tests/Test_ConfigurePatchingProcessor.py | 137 +++++++----------- src/core/tests/library/ExtStatusAsserter.py | 137 +++++++++++++----- 3 files changed, 160 insertions(+), 132 deletions(-) diff --git a/src/core/src/core_logic/ConfigurePatchingProcessor.py b/src/core/src/core_logic/ConfigurePatchingProcessor.py index d034e8c29..f01277e22 100644 --- a/src/core/src/core_logic/ConfigurePatchingProcessor.py +++ b/src/core/src/core_logic/ConfigurePatchingProcessor.py @@ -17,10 +17,18 @@ """ Configure Patching """ from core.src.bootstrap.Constants import Constants +from core.src.bootstrap.EnvLayer import EnvLayer +from core.src.core_logic.ExecutionConfig import ExecutionConfig +from core.src.local_loggers.CompositeLogger import CompositeLogger +from core.src.service_interfaces.StatusHandler import StatusHandler +from core.src.package_managers.PackageManager import PackageManager +from core.src.core_logic.ServiceManager import ServiceManager +from core.src.core_logic.TimerManager import TimerManager class ConfigurePatchingProcessor(object): def __init__(self, env_layer, execution_config, composite_logger, telemetry_writer, status_handler, package_manager, auto_assess_service_manager, auto_assess_timer_manager, lifecycle_manager, auto_assess_service_manager_legacy=None, auto_assess_timer_manager_legacy=None): + # type: (EnvLayer, ExecutionConfig, CompositeLogger, TelemetryWriter, StatusHandler, PackageManager, ServiceManager, TimerManager, ServiceManager, TimerManager) -> None self.env_layer = env_layer self.execution_config = execution_config @@ -43,7 +51,7 @@ def __init__(self, env_layer, execution_config, composite_logger, telemetry_writ def start_configure_patching(self): """ Start configure patching """ try: - self.composite_logger.log("\nStarting configure patching... [MachineId: " + self.env_layer.platform.node() +"][ActivityId: " + self.execution_config.activity_id +"][StartTime: " + self.execution_config.start_time +"]") + self.composite_logger.log("[CP] Starting configure patching... [MachineId: " + self.env_layer.platform.node() + "][ActivityId: " + self.execution_config.activity_id + "][StartTime: " + self.execution_config.start_time +"]") self.status_handler.set_current_operation(Constants.CONFIGURE_PATCHING) self.__raise_if_telemetry_unsupported() @@ -120,8 +128,8 @@ def __try_set_auto_assessment_mode(self): elif self.execution_config.assessment_mode == Constants.AssessmentModes.IMAGE_DEFAULT: self.composite_logger.log_debug("Disabling platform-based automatic assessment.") - self.__erase_auto_assess_config_if_any("AzGPS", self.auto_assess_service_manager, self.auto_assess_timer_manager) - self.__erase_auto_assess_config_if_any("legacy", self.auto_assess_service_manager_legacy, self.auto_assess_timer_manager_legacy) + self.__erase_auto_assess_config_if_any(Constants.AUTO_ASSESSMENT_SERVICE_NAME, self.auto_assess_service_manager, self.auto_assess_timer_manager) + self.__erase_auto_assess_config_if_any(Constants.AUTO_ASSESSMENT_SERVICE_NAME_LEGACY, self.auto_assess_service_manager_legacy, self.auto_assess_timer_manager_legacy) self.current_auto_assessment_state = Constants.AutoAssessmentStates.DISABLED else: @@ -143,11 +151,11 @@ def __erase_auto_assess_config_if_any(self, service_name, service_manager, timer """ Cleans up the legacy auto-assess service """ try: if service_manager is not None and not service_manager.service_exists(): - self.composite_logger.log_debug("[CPP] Cleaning up the {0} auto-assess service.".format(service_name)) + self.composite_logger.log_debug("[CPP] Cleaning up the {0} service.".format(service_name)) service_manager.remove_service() if timer_manager is not None and not timer_manager.timer_exists(): - self.composite_logger.log_debug("[CPP] Cleaning up the {0} auto-assess timer.".format(service_name)) + self.composite_logger.log_debug("[CPP] Cleaning up the {0} timer.".format(service_name)) timer_manager.remove_timer() except Exception as error: self.composite_logger.log_warning("[CPP] Retriable error while cleaning up auto-assess config. [Error={0}]".format(repr(error))) diff --git a/src/core/tests/Test_ConfigurePatchingProcessor.py b/src/core/tests/Test_ConfigurePatchingProcessor.py index 63a82e2cd..b417a77ff 100644 --- a/src/core/tests/Test_ConfigurePatchingProcessor.py +++ b/src/core/tests/Test_ConfigurePatchingProcessor.py @@ -80,24 +80,15 @@ def test_operation_success_for_configure_patching_request_for_apt_with_default_u # check telemetry events self.__check_telemetry_events(runtime) - # check status file for configure patching patch mode - with runtime.env_layer.file_system.open(runtime.execution_config.status_file_path, 'r') as file_handle: - substatus_file_data = json.load(file_handle)[0]["status"]["substatus"] - - # check status file for configure patching patch state (and including for 'Platform' initiated assessment data) + # assertions self.assertTrue(runtime.package_manager.image_default_patch_configuration_backup_exists()) - self.assertEqual(len(substatus_file_data), 2) - self.assertTrue(substatus_file_data[0]["name"] == Constants.PATCH_ASSESSMENT_SUMMARY) - message = json.loads(substatus_file_data[0]["formattedMessage"]["message"]) - self.assertTrue(message["startedBy"], Constants.PatchAssessmentSummaryStartedBy.PLATFORM) - self.assertTrue(substatus_file_data[1]["name"] == Constants.CONFIGURE_PATCHING_SUMMARY) - self.assertTrue(substatus_file_data[1]["status"].lower() == Constants.STATUS_SUCCESS.lower()) - message = json.loads(substatus_file_data[1]["formattedMessage"]["message"]) - self.assertEqual(message["automaticOSPatchState"], Constants.AutomaticOSPatchStates.DISABLED) - - # check status file for configure patching assessment state - message = json.loads(substatus_file_data[1]["formattedMessage"]["message"]) - self.assertEqual(message["autoAssessmentStatus"]["autoAssessmentState"], Constants.AutoAssessmentStates.DISABLED) + ext_status_asserter = ExtStatusAsserter(runtime.execution_config.status_file_path, runtime.env_layer) + ext_status_asserter.assert_status_file_substatuses(substatus_expectations={ + Constants.PATCH_ASSESSMENT_SUMMARY: Constants.STATUS_SUCCESS, + Constants.CONFIGURE_PATCHING_SUMMARY: Constants.STATUS_SUCCESS + }) + ext_status_asserter.assert_configure_patching_patch_mode_state(Constants.AutomaticOSPatchStates.DISABLED) + ext_status_asserter.assert_configure_patching_auto_assessment_state(Constants.AutomaticOSPatchStates.DISABLED) # stop test runtime runtime.stop() @@ -116,13 +107,11 @@ def test_operation_success_for_configure_patching_request_for_apt_without_defaul self.__check_telemetry_events(runtime) # check status file - with runtime.env_layer.file_system.open(runtime.execution_config.status_file_path, 'r') as file_handle: - substatus_file_data = json.load(file_handle)[0]["status"]["substatus"] - self.assertEqual(len(substatus_file_data), 2) - self.assertTrue(substatus_file_data[0]["name"] == Constants.PATCH_ASSESSMENT_SUMMARY) # assessment is now part of the CP flow - self.assertTrue(substatus_file_data[0]["status"].lower() == Constants.STATUS_SUCCESS.lower()) - self.assertTrue(substatus_file_data[1]["name"] == Constants.CONFIGURE_PATCHING_SUMMARY) - self.assertTrue(substatus_file_data[1]["status"].lower() == Constants.STATUS_SUCCESS.lower()) + ext_status_asserter = ExtStatusAsserter(runtime.execution_config.status_file_path, runtime.env_layer) + ext_status_asserter.assert_status_file_substatuses(substatus_expectations={ + Constants.PATCH_ASSESSMENT_SUMMARY: Constants.STATUS_SUCCESS, + Constants.CONFIGURE_PATCHING_SUMMARY: Constants.STATUS_SUCCESS + }) runtime.stop() def test_operation_success_for_installation_request_with_configure_patching(self): @@ -156,9 +145,9 @@ def test_operation_success_for_installation_request_with_configure_patching(self # check status file ext_status_asserter = ExtStatusAsserter(runtime.execution_config.status_file_path, runtime.env_layer) ext_status_asserter.assert_status_file_substatuses() - ext_status_asserter.assert_installation_summary_has_patch("python-samba", Constants.PackageClassification.SECURITY, "python-samba_2:4.4.5+dfsg-2ubuntu5.4") - ext_status_asserter.assert_installation_summary_has_patch("samba-common-bin", Constants.PackageClassification.SECURITY) - ext_status_asserter.assert_installation_summary_has_patch("samba-libs", Constants.PackageClassification.SECURITY) + ext_status_asserter.assert_operation_summary_has_patch(Constants.PATCH_INSTALLATION_SUMMARY, "python-samba", Constants.PackageClassification.SECURITY, "python-samba_2:4.4.5+dfsg-2ubuntu5.4") + ext_status_asserter.assert_operation_summary_has_patch(Constants.PATCH_INSTALLATION_SUMMARY,"samba-common-bin", Constants.PackageClassification.SECURITY) + ext_status_asserter.assert_operation_summary_has_patch(Constants.PATCH_INSTALLATION_SUMMARY,"samba-libs", Constants.PackageClassification.SECURITY) ext_status_asserter.assert_healthstore_status_info(patch_version="pub_off_sku_2020.09.23", should_report=True) runtime.stop() @@ -174,22 +163,16 @@ def test_operation_fail_for_configure_patching_telemetry_not_supported(self): # check status file ext_status_asserter = ExtStatusAsserter(runtime.execution_config.status_file_path, runtime.env_layer) ext_status_asserter.assert_status_file_substatuses(substatus_expectations={ - Constants.CONFIGURE_PATCHING_SUMMARY: Constants.STATUS_SUCCESS + Constants.CONFIGURE_PATCHING_SUMMARY: Constants.STATUS_ERROR }) - with runtime.env_layer.file_system.open(runtime.execution_config.status_file_path, 'r') as file_handle: - substatus_file_data = json.load(file_handle)[0]["status"]["substatus"] - self.assertEqual(len(substatus_file_data), 1) - self.assertTrue(substatus_file_data[0]["name"] == Constants.CONFIGURE_PATCHING_SUMMARY) - if runtime.vm_cloud_type == Constants.VMCloudType.AZURE: - self.assertTrue(substatus_file_data[0]["status"].lower() == Constants.STATUS_ERROR.lower()) - self.assertTrue(len(json.loads(substatus_file_data[0]["formattedMessage"]["message"])["errors"]["details"]), 1) - self.assertTrue(Constants.TELEMETRY_NOT_COMPATIBLE_ERROR_MSG in json.loads(substatus_file_data[0]["formattedMessage"]["message"])["errors"]["details"][0]["message"]) - self.assertTrue(Constants.TELEMETRY_NOT_COMPATIBLE_ERROR_MSG in json.loads(substatus_file_data[0]["formattedMessage"]["message"])["autoAssessmentStatus"]["errors"]["details"][0]["message"]) - self.assertTrue(Constants.STATUS_ERROR in json.loads(substatus_file_data[0]["formattedMessage"]["message"])["autoAssessmentStatus"]["autoAssessmentState"]) + ext_status_asserter.assert_status_file_substatus(Constants.CONFIGURE_PATCHING_SUMMARY, Constants.STATUS_ERROR) + ext_status_asserter.assert_operation_summary_has_error(Constants.CONFIGURE_PATCHING_SUMMARY, Constants.TELEMETRY_NOT_COMPATIBLE_ERROR_MSG) + ext_status_asserter.assert_operation_summary_has_error(Constants.CONFIGURE_PATCHING_SUMMARY, Constants.TELEMETRY_NOT_COMPATIBLE_ERROR_MSG, 'autoAssessmentStatus') + ext_status_asserter.assert_configure_patching_auto_assessment_state(Constants.STATUS_ERROR) else: - self.assertTrue(substatus_file_data[0]["status"].lower() == Constants.STATUS_SUCCESS.lower()) + ext_status_asserter.assert_status_file_substatus(Constants.CONFIGURE_PATCHING_SUMMARY, Constants.STATUS_SUCCESS) runtime.stop() def test_patch_mode_set_failure_for_configure_patching(self): @@ -212,21 +195,18 @@ def test_patch_mode_set_failure_for_configure_patching(self): self.__check_telemetry_events(runtime) # check status file - with runtime.env_layer.file_system.open(runtime.execution_config.status_file_path, 'r') as file_handle: - substatus_file_data = json.load(file_handle)[0]["status"]["substatus"] - self.assertEqual(len(substatus_file_data), 2) - self.assertTrue(substatus_file_data[0]["name"] == Constants.PATCH_ASSESSMENT_SUMMARY) # assessment is now part of the CP flow - self.assertTrue(substatus_file_data[0]["status"].lower() == Constants.STATUS_SUCCESS.lower()) - self.assertTrue(substatus_file_data[1]["name"] == Constants.CONFIGURE_PATCHING_SUMMARY) - self.assertTrue(substatus_file_data[1]["status"].lower() == Constants.STATUS_ERROR.lower()) - - #restore + ext_status_asserter = ExtStatusAsserter(runtime.execution_config.status_file_path, runtime.env_layer) + ext_status_asserter.assert_status_file_substatuses(substatus_expectations={ + Constants.PATCH_ASSESSMENT_SUMMARY: Constants.STATUS_SUCCESS, + Constants.CONFIGURE_PATCHING_SUMMARY: Constants.STATUS_ERROR + }) + + # restore runtime.package_manager.get_current_auto_os_patch_state = backup_package_manager_get_current_auto_os_patch_state runtime.stop() def test_configure_patching_with_assessment_mode_by_platform(self): - # create and adjust arguments argument_composer = ArgumentComposer() argument_composer.operation = Constants.CONFIGURE_PATCHING @@ -249,22 +229,14 @@ def test_configure_patching_with_assessment_mode_by_platform(self): # check telemetry events self.__check_telemetry_events(runtime) - # check status file for configure patching patch mode - with runtime.env_layer.file_system.open(runtime.execution_config.status_file_path, 'r') as file_handle: - substatus_file_data = json.load(file_handle)[0]["status"]["substatus"] - - # check status file for configure patching patch state - self.assertEqual(len(substatus_file_data), 2) - self.assertTrue(substatus_file_data[1]["name"] == Constants.CONFIGURE_PATCHING_SUMMARY) - self.assertTrue(substatus_file_data[1]["status"].lower() == Constants.STATUS_SUCCESS.lower()) - message = json.loads(substatus_file_data[1]["formattedMessage"]["message"]) - self.assertEqual(message["automaticOSPatchState"], Constants.AutomaticOSPatchStates.ENABLED) # no change is made on Auto OS updates for patch mode 'ImageDefault' - self.assertTrue(substatus_file_data[0]["name"] == Constants.PATCH_ASSESSMENT_SUMMARY) - self.assertTrue(substatus_file_data[0]["status"].lower() == Constants.STATUS_SUCCESS.lower()) - - # check status file for configure patching assessment state - message = json.loads(substatus_file_data[1]["formattedMessage"]["message"]) - self.assertEqual(message["autoAssessmentStatus"]["autoAssessmentState"], Constants.AutoAssessmentStates.ENABLED) # auto assessment is enabled + # assertions + ext_status_asserter = ExtStatusAsserter(runtime.execution_config.status_file_path, runtime.env_layer) + ext_status_asserter.assert_status_file_substatuses(substatus_expectations={ + Constants.PATCH_ASSESSMENT_SUMMARY: Constants.STATUS_SUCCESS, + Constants.CONFIGURE_PATCHING_SUMMARY: Constants.STATUS_SUCCESS + }) + ext_status_asserter.assert_configure_patching_patch_mode_state(Constants.AutomaticOSPatchStates.ENABLED) # no change is made on Auto OS updates for patch mode 'ImageDefault' + ext_status_asserter.assert_configure_patching_auto_assessment_state(Constants.AutomaticOSPatchStates.ENABLED) # auto assessment is enabled # stop test runtime runtime.stop() @@ -293,23 +265,15 @@ def test_configure_patching_with_patch_mode_and_assessment_mode_by_platform(self # check telemetry events self.__check_telemetry_events(runtime) - # check status file for configure patching patch mode - with runtime.env_layer.file_system.open(runtime.execution_config.status_file_path, 'r') as file_handle: - substatus_file_data = json.load(file_handle)[0]["status"]["substatus"] - - # check status file for configure patching patch state + # assertions self.assertTrue(runtime.package_manager.image_default_patch_configuration_backup_exists()) - self.assertEqual(len(substatus_file_data), 2) - self.assertTrue(substatus_file_data[1]["name"] == Constants.CONFIGURE_PATCHING_SUMMARY) - self.assertTrue(substatus_file_data[1]["status"].lower() == Constants.STATUS_SUCCESS.lower()) - message = json.loads(substatus_file_data[1]["formattedMessage"]["message"]) - self.assertEqual(message["automaticOSPatchState"], Constants.AutomaticOSPatchStates.DISABLED) # auto OS updates are disabled on patch mode 'AutomaticByPlatform' - self.assertTrue(substatus_file_data[0]["name"] == Constants.PATCH_ASSESSMENT_SUMMARY) - self.assertTrue(substatus_file_data[0]["status"].lower() == Constants.STATUS_SUCCESS.lower()) - - # check status file for configure patching assessment state - message = json.loads(substatus_file_data[1]["formattedMessage"]["message"]) - self.assertEqual(message["autoAssessmentStatus"]["autoAssessmentState"], Constants.AutoAssessmentStates.ENABLED) # auto assessment is enabled + ext_status_asserter = ExtStatusAsserter(runtime.execution_config.status_file_path, runtime.env_layer) + ext_status_asserter.assert_status_file_substatuses(substatus_expectations={ + Constants.PATCH_ASSESSMENT_SUMMARY: Constants.STATUS_SUCCESS, + Constants.CONFIGURE_PATCHING_SUMMARY: Constants.STATUS_SUCCESS + }) + ext_status_asserter.assert_configure_patching_patch_mode_state(Constants.AutomaticOSPatchStates.DISABLED) # auto OS updates are disabled on patch mode 'AutomaticByPlatform' + ext_status_asserter.assert_configure_patching_auto_assessment_state(Constants.AutomaticOSPatchStates.ENABLED) # auto assessment is enabled # stop test runtime runtime.stop() @@ -334,7 +298,7 @@ def test_configure_patching_raise_exception_auto_os_patch_state(self): runtime.configure_patching_processor.start_configure_patching() - # restore sdt.out ouptput + # restore sdt.out output sys.stdout = original_stdout # assert @@ -342,11 +306,10 @@ def test_configure_patching_raise_exception_auto_os_patch_state(self): self.assertIn("Error while processing patch mode configuration", output) # check status file - with runtime.env_layer.file_system.open(runtime.execution_config.status_file_path, 'r') as file_handle: - substatus_file_data = json.load(file_handle)[0]["status"]["substatus"] - self.assertEqual(len(substatus_file_data), 1) - self.assertTrue(substatus_file_data[0]["name"] == Constants.CONFIGURE_PATCHING_SUMMARY) - self.assertTrue(substatus_file_data[0]["status"].lower() == Constants.STATUS_TRANSITIONING.lower()) + ext_status_asserter = ExtStatusAsserter(runtime.execution_config.status_file_path, runtime.env_layer) + ext_status_asserter.assert_status_file_substatuses(substatus_expectations={ + Constants.CONFIGURE_PATCHING_SUMMARY: Constants.STATUS_TRANSITIONING + }) # restore runtime.package_manager.get_current_auto_os_patch_state = backup_package_manager_get_current_auto_os_patch_state diff --git a/src/core/tests/library/ExtStatusAsserter.py b/src/core/tests/library/ExtStatusAsserter.py index b3504d430..ab60c6706 100644 --- a/src/core/tests/library/ExtStatusAsserter.py +++ b/src/core/tests/library/ExtStatusAsserter.py @@ -27,14 +27,11 @@ def __init__(self, status_file_path, env_layer): self.__substatus_high_level_summary = None self.__load_substatus_high_level_summary(self.__substatus_file_data) - def __read_status_file(self, status_file_path): - # type: (str) -> dict - with self.__env_layer.file_system.open(status_file_path, 'r') as file_handle: - return json.load(file_handle)[0]["status"]["substatus"] - + # region Data structure helpers @staticmethod def __get_high_level_summary_template(): # type: () -> dict + """ Internal template for in-memory representation of substatus elements """ return { Constants.CONFIGURE_PATCHING_SUMMARY: {"index": -1, "status": None}, Constants.PATCH_ASSESSMENT_SUMMARY: {"index": -1, "status": None}, @@ -42,8 +39,39 @@ def __get_high_level_summary_template(): Constants.PATCH_METADATA_FOR_HEALTHSTORE: {"index": -1, "status": None}, } + def __get_substatus_index_with_assert(self, operation): + # type: (str) -> int + """ Get the index of the substatus """ + if operation not in self.__substatus_high_level_summary: + raise KeyError("Unknown operation: {0}".format(operation)) + + substatus_index = self.__substatus_high_level_summary[operation]["index"] + if substatus_index == -1: + raise AssertionError("Substatus index not found for operation: {0}".format(operation)) + + return substatus_index + + @staticmethod + def get_default_substatus_expectations(): + # type: () -> dict + """ Get the default substatus expectations """ + return { + Constants.CONFIGURE_PATCHING_SUMMARY: Constants.STATUS_SUCCESS, + Constants.PATCH_ASSESSMENT_SUMMARY: Constants.STATUS_SUCCESS, + Constants.PATCH_INSTALLATION_SUMMARY: Constants.STATUS_SUCCESS, + Constants.PATCH_METADATA_FOR_HEALTHSTORE: Constants.STATUS_SUCCESS, + } + # endregion + + # region Data loaders + def __read_status_file(self, status_file_path): + # type: (str) -> dict + with self.__env_layer.file_system.open(status_file_path, 'r') as file_handle: + return json.load(file_handle)[0]["status"]["substatus"] + def __load_substatus_high_level_summary(self, substatus_file_data): # type: (dict) -> None + """ Makes one-time inferences about the structure of the status file """ self.__substatus_high_level_summary = self.__get_high_level_summary_template() for index, substatus in enumerate(substatus_file_data): @@ -54,69 +82,98 @@ def __load_substatus_high_level_summary(self, substatus_file_data): self.__substatus_high_level_summary[summary_name]["status"] = substatus["status"] else: raise KeyError("Unknown substatus: {0}".format(substatus["name"])) + # endregion - def assert_status_file_substatuses(self, operation, expected_status): - """Check if the status file has a specific substatus""" - if operation not in self.__substatus_high_level_summary: - raise KeyError("Unknown operation: {0}".format(operation)) + # region Data Navigators + def __get_substatus_message_as_dict(self, operation): + # type: (str) -> dict + """ Get the substatus message as a dictionary """ + substatus_index = self.__get_substatus_index_with_assert(operation) + return json.loads(self.__substatus_file_data[substatus_index]["formattedMessage"]["message"]) + # endregion - substatus_index = self.__substatus_high_level_summary[operation]["index"] - if substatus_index == -1: - raise AssertionError("Substatus index not found for operation: {0}".format(operation)) + # region Public Assertion methods + def assert_status_file_substatus(self, operation, expected_status): + # type: (str, str) -> None + """ Check if the status file has a specific substatus """ + substatus_index = self.__get_substatus_index_with_assert(operation) actual_status = self.__substatus_file_data[substatus_index]["status"].lower() if actual_status != expected_status.lower(): raise AssertionError("Substatus expectations do not match for {0}. Expected: {1}, Actual: {2}".format(operation, expected_status, actual_status)) - @staticmethod - def get_default_substatus_expectations(): - """Get the default substatus expectations""" - return { - Constants.CONFIGURE_PATCHING_SUMMARY: Constants.STATUS_SUCCESS, - Constants.PATCH_ASSESSMENT_SUMMARY: Constants.STATUS_SUCCESS, - Constants.PATCH_INSTALLATION_SUMMARY: Constants.STATUS_SUCCESS, - Constants.PATCH_METADATA_FOR_HEALTHSTORE: Constants.STATUS_SUCCESS, - } - def assert_status_file_substatuses(self, substatus_expectations=None): - """Check if the status file has a specific substatus""" + # type: (dict) -> None + """ Batch check the status file for substatus expectations """ if substatus_expectations is None: substatus_expectations = self.get_default_substatus_expectations() for key, value in substatus_expectations.items(): - if self.__substatus_high_level_summary[key]["index"] == -1: - raise AssertionError("Expected substatus not found: {0}".format(key)) - if self.__substatus_high_level_summary[key]["status"] != value: - raise AssertionError("Substatus expectations do not match for {0}. Expected: {1}, Actual: {2}".format(key, value, self.__substatus_high_level_summary[key]["status"])) + self.assert_status_file_substatus(key, value) - def assert_summary_has_patch(self, operation, patch_name, classification=None, patchId=None): + def assert_operation_summary_has_patch(self, operation, patch_name, classification=None, patch_id=None): + # type: (str, str, str, str) -> bool """ Check if the defined operation summary has a specific patch """ - substatus_index = self.__substatus_high_level_summary[operation]["index"] - if substatus_index == -1: - raise AssertionError("Substatus index not found for operation: {0}".format(operation)) - - summary = self.__substatus_file_data[substatus_index] - summary_patches = json.loads(summary["formattedMessage"]["message"])["patches"] + substatus_message = self.__get_substatus_message_as_dict(operation) + summary_patches = substatus_message["patches"] for patch in summary_patches: if patch["name"] == patch_name: if classification and classification not in patch["classifications"]: raise AssertionError("Classification '{0}' does not match expected value '{1}' for patch '{2}'.".format(classification, str(patch["classifications"]), patch_name)) - if patchId and patchId != str(patch["patchId"]): - raise AssertionError("Patch ID '{0}' does not match expected value '{1}' for patch '{2}'.".format(patchId, str(patch["patchId"]), patch_name)) + if patch_id and patch_id not in str(patch["patchId"]): + raise AssertionError("Patch ID '{0}' does not match expected value '{1}' for patch '{2}'.".format(patch_id, str(patch["patch_id"]), patch_name)) return True raise AssertionError("Patch '{0}' not found in '{1}' summary.".format(patch_name, operation)) + def assert_operation_summary_has_error(self, operation, error_message, sub_level_for_configure_patching_only=None): + # type: (str, str, str) -> bool + """ Check if the defined operation summary has a specific error """ + substatus_message = self.__get_substatus_message_as_dict(operation) + + if sub_level_for_configure_patching_only not in [None, "autoAssessmentStatus", "patchModeStatus"]: + raise ValueError("sub_level_for_configure_patching_only must be None, 'autoAssessmentStatus', or 'patchModeStatus'.") + + if operation == Constants.CONFIGURE_PATCHING_SUMMARY and sub_level_for_configure_patching_only: + error_detail_list = substatus_message[sub_level_for_configure_patching_only]["errors"]["details"] + else: + error_detail_list = substatus_message["errors"]["details"] + + for error in error_detail_list: + if error_message in error["message"]: + return True + raise AssertionError("Error '{0}' not found in '{1}' summary.".format(error_message, operation)) + + def assert_operation_summary_has_started_by(self, operation, started_by): + # type: (str, str) -> None + """ Check if the defined operation summary has a specific started by """ + substatus_message = self.__get_substatus_message_as_dict(operation) + if substatus_message["startedBy"] != started_by: + raise AssertionError("Started by '{0}' does not match expected value '{1}' for operation '{2}.".format(substatus_message["startedBy"], started_by, operation)) + + def assert_configure_patching_patch_mode_state(self, expected_state): + # type: (str) -> None + """ Check if the patch mode state is as expected """ + substatus_message = self.__get_substatus_message_as_dict(Constants.CONFIGURE_PATCHING_SUMMARY) + if substatus_message["automaticOSPatchState"] != expected_state: + raise AssertionError("Patch mode state '{0}' does not match expected value '{1}'.".format(substatus_message["automaticOSPatchState"], expected_state)) + + def assert_configure_patching_auto_assessment_state(self, expected_state): + # type: (str) -> None + """ Check if the auto-assessment state is as expected """ + substatus_message = self.__get_substatus_message_as_dict(Constants.CONFIGURE_PATCHING_SUMMARY) + if substatus_message["autoAssessmentStatus"]["autoAssessmentState"] != expected_state: + raise AssertionError("Auto-assessment state '{0}' does not match expected value '{1}'.".format(substatus_message["autoAssessmentStatus"]["state"], expected_state)) + def assert_healthstore_status_info(self, patch_version, should_report=True): + # type: (str, bool) -> None """Check if the healthstore patch version is as expected""" - if self.healthstore_substatus_index is None: - raise AssertionError("Healthstore substatus index not found.") - - healthstore_summary = json.loads(self.__substatus_file_data[self.healthstore_substatus_index]["formattedMessage"]["message"]) + healthstore_summary = self.__get_substatus_message_as_dict(Constants.PATCH_METADATA_FOR_HEALTHSTORE) if should_report and healthstore_summary["shouldReportToHealthStore"] != True: raise AssertionError("Healthstore summary should report to healthstore.") if patch_version != healthstore_summary["patchVersion"]: raise AssertionError("Healthstore summary patch version '{0}' does not match expected value {1}.".format(str(healthstore_summary["patchVersion"]), patch_version)) + # endregion From 3c580250711611049e1a71c7dd2c3281b2baf32f Mon Sep 17 00:00:00 2001 From: Koshy John Date: Fri, 9 May 2025 07:42:10 -0700 Subject: [PATCH 10/10] Park changes --- src/core/src/core_logic/ConfigurePatchingProcessor.py | 3 ++- src/core/tests/Test_ConfigurePatchingProcessor.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/core/src/core_logic/ConfigurePatchingProcessor.py b/src/core/src/core_logic/ConfigurePatchingProcessor.py index f01277e22..cd65efcf9 100644 --- a/src/core/src/core_logic/ConfigurePatchingProcessor.py +++ b/src/core/src/core_logic/ConfigurePatchingProcessor.py @@ -25,6 +25,7 @@ from core.src.core_logic.ServiceManager import ServiceManager from core.src.core_logic.TimerManager import TimerManager + class ConfigurePatchingProcessor(object): def __init__(self, env_layer, execution_config, composite_logger, telemetry_writer, status_handler, package_manager, auto_assess_service_manager, auto_assess_timer_manager, lifecycle_manager, auto_assess_service_manager_legacy=None, auto_assess_timer_manager_legacy=None): @@ -51,7 +52,7 @@ def __init__(self, env_layer, execution_config, composite_logger, telemetry_writ def start_configure_patching(self): """ Start configure patching """ try: - self.composite_logger.log("[CP] Starting configure patching... [MachineId: " + self.env_layer.platform.node() + "][ActivityId: " + self.execution_config.activity_id + "][StartTime: " + self.execution_config.start_time +"]") + self.composite_logger.log("[CPP] Starting configure patching... [MachineId: " + self.env_layer.platform.node() + "][ActivityId: " + self.execution_config.activity_id + "][StartTime: " + self.execution_config.start_time +"]") self.status_handler.set_current_operation(Constants.CONFIGURE_PATCHING) self.__raise_if_telemetry_unsupported() diff --git a/src/core/tests/Test_ConfigurePatchingProcessor.py b/src/core/tests/Test_ConfigurePatchingProcessor.py index b417a77ff..d7b48fd60 100644 --- a/src/core/tests/Test_ConfigurePatchingProcessor.py +++ b/src/core/tests/Test_ConfigurePatchingProcessor.py @@ -151,12 +151,13 @@ def test_operation_success_for_installation_request_with_configure_patching(self ext_status_asserter.assert_healthstore_status_info(patch_version="pub_off_sku_2020.09.23", should_report=True) runtime.stop() - def test_operation_fail_for_configure_patching_telemetry_not_supported(self): + def test_operation_fail_for_configure_patching_telemetry_not_supported(self, vm_cloud_type=Constants.VMCloudType.AZURE): argument_composer = ArgumentComposer() argument_composer.operation = Constants.CONFIGURE_PATCHING argument_composer.patch_mode = Constants.PatchModes.AUTOMATIC_BY_PLATFORM argument_composer.events_folder = None runtime = RuntimeCompositor(argument_composer.get_composed_arguments(env_settings=dict(telemetrySupported=False)), True, Constants.APT) + runtime.vm_cloud_type = Constants.VMCloudType.ARC runtime.set_legacy_test_type('HappyPath') runtime.configure_patching_processor.start_configure_patching()