From ae5e9f7af2a3e100f72cc00f89e01fd02a11e3be Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Tue, 22 Oct 2024 16:36:26 +0200 Subject: [PATCH 1/2] refactor: Move parsing of overlay ppa config to functions. The overlay ppa config has multiple uses, move parsing from the function that builds model-level cloud-init user data to separate functions. Signed-off-by: Frode Nordahl --- zaza/utilities/deployment_env.py | 88 +++++++++++++++++++++++--------- 1 file changed, 65 insertions(+), 23 deletions(-) diff --git a/zaza/utilities/deployment_env.py b/zaza/utilities/deployment_env.py index 1631627d..d8b5234d 100644 --- a/zaza/utilities/deployment_env.py +++ b/zaza/utilities/deployment_env.py @@ -92,6 +92,61 @@ def get_overlay_ppas(model_alias='default_alias'): may be used to specify one or more PPAs that will be enabled for all units in the model. + Please refer to docstring of parse_overlay_ppa function for more + information. + + :param model_alias: Name of model alias, defaults to 'default_alias'. + :type model_alias: Option[str] + :returns: List of overlay PPAs. + :rtype: Option[List[Any]] + """ + config = zaza.global_options.get_options() + try: + return config[model_alias].overlay_ppas + except KeyError: + try: + return config.overlay_ppas + except KeyError: + pass + return None + + +def _parse_overlay_ppa_v1(overlay_ppa): + """Parse simple ppa:xxx/yyy formatted overlay_ppa config. + + Example YAML excerpt: + overlay_ppas: + - ppa:ubuntu-security-proposed/ppa + + :param overlay_ppa: Element of overlay_ppas configuration. + :type overlay_ppa: str + """ + return {'source': overlay_ppa} + + +def _parse_overlay_ppa_v2(overlay_ppa): + """Parse advanced format overlay_ppa config. + + Example YAML excerpt: + overlay_ppas: + - source: "deb https://user:pass@private-ppa.launchpad" + key: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + .... + -----END PGP PUBLIC KEY BLOCK----- + :param overlay_ppa: Element of overlay_ppas configuration. + :type overlay_ppa: Dict[str,str] + :returns: Parsed PPA configuration. + :rtype: Dict[str,str] + :raises: TypeError + """ + return {'source': overlay_ppa['source'], + 'key': overlay_ppa['key']} + + +def parse_overlay_ppa(overlay_ppa): + """Parse multiple versions of overlay_ppas elements. + The tests_options section needs to look like: tests_options: @@ -117,20 +172,15 @@ def get_overlay_ppas(model_alias='default_alias'): .... -----END PGP PUBLIC KEY BLOCK----- - :param model: Name of model alias - :type bundle: str - :returns: List of overlay PPAs - :rtype: list[str] + :param overlay_ppa: Element of overlay_ppas configuration. + :type overlay_ppa: Any + :returns: Parsed overlay configuration. + :rtype: Dict[str,str] """ - config = zaza.global_options.get_options() try: - return config[model_alias].overlay_ppas - except KeyError: - try: - return config.overlay_ppas - except KeyError: - pass - return None + return _parse_overlay_ppa_v2(overlay_ppa) + except (TypeError): + return _parse_overlay_ppa_v1(overlay_ppa) def get_cloudinit_userdata(model_alias='default_alias'): @@ -157,17 +207,9 @@ def get_cloudinit_userdata(model_alias='default_alias'): overlay_ppas = get_overlay_ppas(model_alias) if overlay_ppas: for index, overlay_ppa in enumerate(overlay_ppas): - try: - # NOTE: support private PPAs with source and key keys. - cloud_config['apt']['sources']["overlay-ppa-{}".format(index)] = { # noqa - 'source': overlay_ppa['source'], - 'key': overlay_ppa['key'], - } - except (KeyError, TypeError): - # NOTE: simple ppa:xxx/yyy format for backwards compatibility - cloud_config['apt']['sources']["overlay-ppa-{}".format(index)] = { # noqa - 'source': overlay_ppa - } + cloud_config["apt"]["sources"][ + "overlay-ppa-{}".format(index) + ] = parse_overlay_ppa(overlay_ppa) cloudinit_userdata = "#cloud-config\n{}".format( yaml.safe_dump(cloud_config)) From e2140aae1d41ed2a71814f835e2b52d183e861de Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Tue, 22 Oct 2024 18:10:11 +0200 Subject: [PATCH 2/2] Add test config grammar for per app overlay ppa. In some circumstances we want to install packages from overlay PPA on specific applications and not across the whole model. Add configuration grammar, actual implementation of generic setup step will follow in subsequent patch. Signed-off-by: Frode Nordahl --- unit_tests/utilities/test_deployment_env.py | 25 +++++++++++++++ zaza/utilities/deployment_env.py | 35 ++++++++++++++++++--- 2 files changed, 55 insertions(+), 5 deletions(-) diff --git a/unit_tests/utilities/test_deployment_env.py b/unit_tests/utilities/test_deployment_env.py index bfb310c3..5913a7df 100644 --- a/unit_tests/utilities/test_deployment_env.py +++ b/unit_tests/utilities/test_deployment_env.py @@ -90,6 +90,31 @@ def test_get_overlay_ppas(self): ]) ) + config = collections.OrderedDict( + { + 'overlay_ppas': { + 'application': [ + 'ppa:app-ppav1', + { + 'source': 'app-ppav2', + 'key': 'app-bar', + }, + ] + }, + } + ) + get_options_mock.return_value = ro_types.resolve_immutable(config) + self.assertIsNone( + deployment_env.get_overlay_ppas() + ) + self.assertEqual( + deployment_env.get_overlay_ppas(application='application'), + ro_types.ReadOnlyList([ + 'ppa:app-ppav1', + {'source': 'app-ppav2', 'key': 'app-bar'}, + ]) + ) + def test_get_cloudinit_userdata(self): with mock.patch.object(deployment_env, 'get_overlay_ppas', return_value=None): diff --git a/zaza/utilities/deployment_env.py b/zaza/utilities/deployment_env.py index d8b5234d..a6dc3b1a 100644 --- a/zaza/utilities/deployment_env.py +++ b/zaza/utilities/deployment_env.py @@ -85,7 +85,7 @@ def parse_option_list_string(option_list, delimiter=None): return settings -def get_overlay_ppas(model_alias='default_alias'): +def get_overlay_ppas(model_alias='default_alias', application=None): """Get overlay_ppas from global_config. In the config file for the tests, the tests_options.overlay_ppa option @@ -95,20 +95,45 @@ def get_overlay_ppas(model_alias='default_alias'): Please refer to docstring of parse_overlay_ppa function for more information. + When application is provided, additional grammar to constrain the + overlay PPA to specific applications is supported, for example: + + overlay_ppas: + myapp: + - ppa:mylpuser/myppa + myotherapp: + - source: fakesource + key: fakekey + :param model_alias: Name of model alias, defaults to 'default_alias'. :type model_alias: Option[str] + :param application: Name of application to retrieve config for. + :type application: Option[str] :returns: List of overlay PPAs. :rtype: Option[List[Any]] """ config = zaza.global_options.get_options() try: - return config[model_alias].overlay_ppas + config_overlay_ppas = config[model_alias].overlay_ppas except KeyError: try: - return config.overlay_ppas + config_overlay_ppas = config.overlay_ppas except KeyError: - pass - return None + return None + + if application: + try: + return config_overlay_ppas[application] + except (KeyError, TypeError): + return None + + # Avoid returning application dictionary when overlay_ppas config is + # Dict-like object and application is not requested. + try: + config_overlay_ppas.get(None) + return None + except AttributeError: + return config_overlay_ppas def _parse_overlay_ppa_v1(overlay_ppa):