diff --git a/deployer/.gitignore b/deployer/.gitignore deleted file mode 100644 index c9b568f..0000000 --- a/deployer/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*.pyc -*.swp diff --git a/deployer/deployer.py b/deployer/deployer.py deleted file mode 100755 index 04efe61..0000000 --- a/deployer/deployer.py +++ /dev/null @@ -1,352 +0,0 @@ -#!/usr/bin/env python - -import os -import sys - -PYTHON_PATH = os.path.dirname(os.path.abspath(os.path.normpath(sys.argv[0]))) -sys.path.append(PYTHON_PATH) - -from gevent import monkey -monkey.patch_all() - -import gevent -import sys -import argparse -import yaml -import tempfile -import os -import logging - -LOG = logging.getLogger() -LOG.setLevel(logging.DEBUG) -ch = logging.StreamHandler(sys.stdout) -ch.setLevel(logging.DEBUG) -formatter = logging.Formatter( - '%(asctime)s - %(name)s - %(levelname)s - %(message)s') -ch.setFormatter(formatter) -LOG.addHandler(ch) - - -import helpers.utils as utils -import helpers.maasclient as maasclient -import jujuclient -import netaddr -import socket - -from gevent import subprocess -from gevent.queue import Queue, Empty - -from jujuclient import Environment - - -parser = argparse.ArgumentParser(prog="Deployer") -subparsers = parser.add_subparsers(dest="action") - -deploy_parser = subparsers.add_parser('deploy') -teardown_parser = subparsers.add_parser('teardown') - -teardown_parser.add_argument("--search-string", dest="search_string", - type=str, required=True, help="Deploy uuid") -teardown_parser.add_argument("--template", dest="template", - type=str, required=False, help="Juju deployer template") - -deploy_parser.add_argument("--search-string", dest="search_string", - type=str, required=False, help="Deploy uuid") -deploy_parser.add_argument("--template", dest="template", - type=str, required=True, help="Juju deployer template") - - -def exception_handler(green): - LOG.error("Greenlet %r failed with an exception" % green) - sys.exit(1) - - -class MaaSInstanceWatcher(maasclient.Nodes): - - def __init__(self, maas_url, maas_token, queue): - super(MaaSInstanceWatcher, self).__init__(maas_url, maas_token) - self.queue = queue - self.watchers = [] - - def _watch(self, node): - node_state = None - if isinstance(node, maasclient.Node) is False: - raise ValueError("Function got invalid type: %r" % type(node)) - while True: - status = node.substatus() - if node_state != status: - LOG.debug("Node %s changed status to: %s" % (node.data["hostname"], status)) - node_state = status - payload = {"status": status, "instance": node.data["resource_uri"]} - self.queue.put(payload) - if status == maasclient.FAILED_DEPLOYMENT: - return - gevent.sleep(5) - - def start_watcher(self, node): - LOG.debug("Starting watcher for node: %s" % node) - n = self.get(node) - e = gevent.spawn(self._watch, n) - e.link_exception(exception_handler) - self.watchers.append(e) - - -class Deployer(object): - - def __init__(self, options): - self.options = options - self.juju = Environment.connect('maas') - self.search_string = options.search_string - self.bundle = self.options.template - #self.bundle_generator = utils.BundleGenerator(self.options) - self.home = os.environ.get("HOME", "/tmp") - self.workdir = os.path.join(self.home, ".deployer") - self.channel = Queue() - self.eventlets = [] - env_config = self.juju.get_env_config() - self.maas_watcher = MaaSInstanceWatcher( - env_config["Config"]["maas-server"], - env_config["Config"]["maas-oauth"], - self.channel) - - def _ensure_dependencies(self): - pkgs = [] - if utils.which("juju-deployer") is None: - utils.add_apt_ppa("ppa:juju/stable") - utils.apt_update() - pkgs.append("juju-deployer") - if len(pkgs) > 0: - utils.install_apt_packages(pkgs) - - def _ensure_workdir(self): - if os.path.isdir(self.workdir) is False: - os.makedirs(self.workdir, 0o700) - - def _run_deployer(self, bundle): - if os.path.isfile(bundle) is False: - raise Exception("No such bundle file: %s" % bundle) - args = [ - "juju-deployer", "--local-mods", "-S", "-c", bundle - ] - subprocess.check_call(args) - - #def _render_yaml(self, project): - # proj = project.split("/")[-1] - # func = getattr(self.bundle_generator, "%s_bundle" % proj) - # if not func: - # raise ValueError( - # "Project %s is not supported by bundler" % project) - # bundle = func() - # bundle_file = os.path.join(self.workdir, self.search_string) - # with open(bundle_file, "w") as fd: - # yaml.dump(bundle, stream=fd, default_flow_style=False, - # allow_unicode=True, encoding=None) - # return bundle_file - - def _start_maas_watcher(self, machine): - """ - poll MaaS API to monitor machine status. If it switches to - Failed Deployment, then raise an exception - """ - e = gevent.spawn(self.maas_watcher.start_watcher, machine) - e.link_exception(exception_handler) - self.eventlets.append(e) - - def _consume_events(self): - LOG.debug("Starting Consumer") - while True: - try: - event = self.channel.get_nowait() - if event.get("status") == maasclient.FAILED_DEPLOYMENT: - raise Exception("Node %s entered failed deployment state" % - event.get("instance")) - except Empty: - gevent.sleep(1) - continue - - @utils.exec_retry(retry=5) - def _juju_status(self, *args, **kw): - return self.juju.status(*args, **kw) - - def _get_machines(self, status): - machines = [] - m = status.get("Machines") - if m is None: - return machines - for i in m.keys(): - instanceId = m[i].get("InstanceId") - if instanceId == "pending": - continue - machines.append(m[i].get("InstanceId")) - return machines - - def _get_machine_ids(self, status): - m = status.get("Machines") - if m is None: - return [] - return m.keys() - - def _get_service_names(self, status): - m = status.get("Services") - if m is None: - return [] - return m.keys() - - def _analize_units(self, units, debug=False): - all_active = True - for i in units.keys(): - unit = units[i] - if debug: - LOG.debug( - "Unit %s has status: %r" % (i, unit["Workload"]["Status"])) - if unit["UnitAgent"]["Status"] == "error": - raise Exception("Unit %s is in error state: %s" % - (i, unit["UnitAgent"]["Err"])) - if unit["Workload"]["Status"] == "error": - raise Exception("Unit %s workload is in error state: %s" % - (i, unit["Workload"]["Info"])) - if unit["Err"] is not None: - raise Exception("Unit %s is in error state: %s" % - (i, unit["Err"])) - if unit["Workload"]["Status"] != "active": - all_active = False - return all_active - - def _analize_machines(self, machines): - for i in machines.keys(): - machine = machines.get(i) - if machine["Err"]: - raise Exception("MaaS returned error when allocating %s: %s" % - (i, machine["Err"])) - agent = machine.get("Agent") - if agent: - status = agent.get("Status") - info = agent.get("Info") - err = agent.get("Err") - if status == "error" or err: - raise Exception( - "Machine agent is in error state: %r" % info) - - def _resolve_address(self, addr): - try: - netaddr.IPAddress(addr) - return addr - except netaddr.core.AddrFormatError: - return socket.gethostbyname(addr) - - def _write_unit_ips(self, units): - unit_ips = {} - for i in units: - name = i.split("/")[0][:-len("-%s" % self.search_string)].replace('-', "_") - addr = self.juju.get_private_address(i)["PrivateAddress"] - ip = self._resolve_address(addr) - if name in unit_ips: - unit_ips[name] += ",%s" % ip - else: - unit_ips[name] = ip - nodes = os.path.join(os.getcwd(), "nodes") - with open(nodes, "w") as fd: - for i in unit_ips.keys(): - fd.write("%s=%s\n" % (i.upper(), unit_ips[i])) - - def _analize(self, status, debug=False): - """ - Return True if charms have reached active workload state, False if not - raise error any charm reaches error state. - """ - services = status.get("Services") - if services is None: - return False - all_units = {} - for i in services.keys(): - svc = services.get(i) - units = svc.get("Units") - all_units.update(units) - # TODO: only do this if there are changes, not on every iteration. - try: - self._write_unit_ips(all_units) - except jujuclient.EnvError: - LOG.debug("Cound not write unit ips") - all_active = self._analize_units(all_units, debug) - if all_active: - return True - # Juju retains the error returned by the MaaS API in case MaaS - # errored out while the acquire API call was made. In this scenario, - # MaaS will not return a usable node. - machines = status.get("Machines") - if machines is None: - return False - self._analize_machines(machines) - - def _poll_services(self): - """ - This poller works under the assumption that the charms being deployed - have implemented status-set calls that tell us when workload status - changed to active. Poll services, units and instances until all units - have workload status set to active, or untill any of them error out. - """ - LOG.debug("Starting poller") - watched_machines = [] - iteration = 0 - while True: - status = self._juju_status(filters=("*%s*" % self.search_string)) - #LOG.debug("%r" % status) - debug = False - if iteration % 1 == 0: - debug = True - all_active = self._analize(status, debug=debug) - if all_active: - break - machines = self._get_machines(status) - diff = set(machines).difference(set(watched_machines)) - new_machines = list(diff) - for i in new_machines: - self._start_maas_watcher(i) - watched_machines.append(i) - iteration += 1 - gevent.sleep(3) - - def _wait_for_teardown(self, machines=[]): - while True: - has_machines = False - status = self._juju_status() - state_machines = status.get("Machines", {}) - for i in machines: - if state_machines.get(i): - has_machines = True - if has_machines is False: - break - gevent.sleep(3) - - def deploy(self): - self._ensure_workdir() - self._ensure_dependencies() - #bundle = self._render_yaml(self.options.zuul_project) - self._run_deployer(self.bundle) - - e = gevent.spawn(self._consume_events) - e.link_exception(exception_handler) - self.eventlets.append(e) - - self._poll_services() - gevent.killall(self.eventlets) - gevent.killall(self.maas_watcher.watchers) - - def teardown(self): - status = self._juju_status(filters=("*%s*" % self.search_string)) - machines = self._get_machine_ids(status) - service_names = self._get_service_names(status) - for i in service_names: - self.juju.destroy_service(i) - self.juju.destroy_machines(machines, force=True) - self._wait_for_teardown(machines) - - -if __name__ == '__main__': - opt = parser.parse_args() - deployer = Deployer(opt) - if opt.action == "deploy": - deployer.deploy() - if opt.action == "teardown": - deployer.teardown() - diff --git a/deployer/helpers/maasclient.py b/deployer/helpers/maasclient.py deleted file mode 100644 index 8d8bdbb..0000000 --- a/deployer/helpers/maasclient.py +++ /dev/null @@ -1,205 +0,0 @@ -import oauth.oauth as oauth -import httplib2 -import uuid -import urlparse -import json -import datetime -import logging - -LOG = logging.getLogger() - -DEFAULT = 0 -#: The node has been created and has a system ID assigned to i -NEW = 0 -#: Testing and other commissioning steps are taking place. -COMMISSIONING = 1 -#: The commissioning step failed. -FAILED_COMMISSIONING = 2 -#: The node can't be contacted. -MISSING = 3 -#: The node is in the general pool ready to be deployed. -READY = 4 -#: The node is ready for named deployment. -RESERVED = 5 -#: The node has booted into the operating system of its owner' -#: and is ready for use. -DEPLOYED = 6 -#: The node has been removed from service manually until an ad -#: overrides the retirement. -RETIRED = 7 -#: The node is broken: a step in the node lifecyle failed. -#: More details can be found in the node's event log. -BROKEN = 8 -#: The node is being installed. -DEPLOYING = 9 -#: The node has been allocated to a user and is ready for depl -ALLOCATED = 10 -#: The deployment of the node failed. -FAILED_DEPLOYMENT = 11 -#: The node is powering down after a release request. -RELEASING = 12 -#: The releasing of the node failed. -FAILED_RELEASING = 13 -#: The node is erasing its disks. -DISK_ERASING = 14 -#: The node failed to erase its disks. -FAILED_DISK_ERASING = 15 - - - -class MaaSBaseClass(object): - - VERBS = ( - "GET", - "PUT", - "POST", - "DELETE", - ) - - def __init__(self, maas_url, token): - self.maas_url = maas_url - self.url = urlparse.urlparse(self.maas_url) - self.token = token - self._parse_token(token) - - def _parse_token(self, token): - t = token.split(":") - if len(t) != 3: - raise ValueError("Invalid MaaS token") - self.consumer_key = t[0] - self.key = t[1] - self.secret = t[2] - - def _validate_verb(self, verb, body): - if verb not in self.VERBS: - raise ValueError("%s is not supported" % verb) - if verb == "DELETE": - # DELETE requests must have body None - return None - return body - - def _check_response(self, response): - status = response.get("status") - if int(status) > 299: - raise Exception("Request returned status %s" % status) - - @property - def _api_path(self): - uri = "api/1.0/" - return "%s/%s" % (self.url.path.rstrip("/"), uri) - - def _get_resource_uri(self, subresource=None, op=None, params=None): - resource = self.RESOURCE - #LOG.debug("%r --> %r" %(resource, self._api_path)) - if self.RESOURCE.startswith(self._api_path): - resource = self.RESOURCE[len(self._api_path):] - - uri = "api/1.0/%s/" % resource.strip("/") - if subresource: - uri += "%s/" % subresource.strip("/") - if op: - uri = "%s?op=%s" % (uri, op) - if params: - for i in params.keys(): - uri += "&%s=%s" % (i, params[i]) - return uri - - def _dispatch(self, uri, method, body=None): - body = self._validate_verb(method, body) - - resource_tok_string = "oauth_token_secret=%s&oauth_token=%s" % (self.secret, self.key) - resource_token = oauth.OAuthToken.from_string(resource_tok_string) - consumer_token = oauth.OAuthConsumer(self.consumer_key, "") - - oauth_request = oauth.OAuthRequest.from_consumer_and_token( - consumer_token, token=resource_token, http_url=self.maas_url, - parameters={'oauth_nonce': uuid.uuid4().get_hex()}) - - oauth_request.sign_request( - oauth.OAuthSignatureMethod_PLAINTEXT(), consumer_token, - resource_token) - - headers = oauth_request.to_header() - url = "%s/%s" % (self.maas_url, uri) - http = httplib2.Http() - LOG.debug("Sending request to: %s" % url) - response, content = http.request(url, method, body=body, headers=headers) - self._check_response(response) - body = json.loads(content) - return body - - -class ResourceMixin(object): - - def _refresh_data(self): - if self._requested is None: - self._data = self._get() - self._requested = datetime.datetime.utcnow() - - delta = datetime.datetime.utcnow() - self._requested - if delta > datetime.timedelta(seconds=30): - self._data = self._get() - self._requested = datetime.datetime.utcnow() - - if self._data is None: - self._data = self._get() - self._requested = datetime.datetime.utcnow() - return - - @property - def data(self): - self._refresh_data() - return self._data - - -class Node(MaaSBaseClass, ResourceMixin): - - def __init__(self, maas_url, maas_token, resource): - super(Node, self).__init__(maas_url, maas_token) - self.RESOURCE = resource - self._data = None - self._requested = None - - def status(self): - self._refresh_data() - status = self.data.get("status") - if status: - return int(status) - - def substatus(self): - self._refresh_data() - status = self.data.get("substatus") - if status: - return int(status) - - def _get(self): - nodes = self._get_resource_uri() - return self._dispatch(nodes, "GET") - - -class Events(MaaSBaseClass, ResourceMixin): - - RESOURCE = "events" - - def get(self, node=None): - params = None - if node: - params = {"id": node} - return self._get(params=params) - - def _get(self, params=None): - events = self._get_resource_uri(op="query", params=params) - print events - return self._dispatch(events, "GET") - - -class Nodes(MaaSBaseClass): - - RESOURCE = "nodes" - - def list(self): - nodes = self._get_resource_uri(op="list") - return self._dispatch(nodes, "GET") - - def get(self, resource): - return Node(self.maas_url, self.token, resource) diff --git a/deployer/helpers/utils.py b/deployer/helpers/utils.py deleted file mode 100644 index 23cf175..0000000 --- a/deployer/helpers/utils.py +++ /dev/null @@ -1,186 +0,0 @@ -import os -import platform -import gevent - -from gevent import subprocess - -SYS = platform.system() - - -def exec_retry(retry=5): - def wrap(f): - def wrap_f(*args, **kw): - count = 0 - err = "" - while count < retry: - try: - return f(*args, **kw) - break - except Exception as err: - gevent.sleep(3) - err = err - count += 1 - return f(*args, **kw) - return wrap_f - return wrap - - -def is_exe(path): - if os.path.isfile(path) is False: - return False - if SYS == "Windows": - pathext = os.environ.get("PATHEXT", ".COM;.EXE;.BAT;.CMD;.VBS;.VBE;.JS;.JSE;.WSF;.WSH;.MSC") - for i in pathext.split(os.pathsep): - if path.endswith(i): - return True - else: - if os.access(path, os.X_OK): - return True - return False - - -def which(program): - fpath, fname = os.path.split(program) - if fpath: - if is_exe(program): - return program - else: - for path in os.environ["PATH"].split(os.pathsep): - exe_file = os.path.join(path, program) - if is_exe(exe_file): - return exe_file - return None - - -def add_apt_ppa(ppa): - subprocess.check_call([ - "sudo", "-n", "apt-add-repository", "-y", ppa, - ]) - - -def install_apt_packages(pkgs): - apt = ["sudo", "-n", "apt-get", "-y", "--option=Dpkg::Options::=--force-confold", "install"] - apt.extend(pkgs) - subprocess.check_call(apt) - - -def apt_update(): - subprocess.check_call(["sudo", "-n", "apt-get", "update"]) - - -class BundleGenerator(object): - _AD_GIT_URL = 'https://github.com/cloudbase/active-directory.git' - _DEVSTACK_GIT_URL = 'https://github.com/cloudbase/devstack-charm.git' - _HYPER_V_GIT_URL = 'https://github.com/cloudbase/hyperv-charm' - - def __init__(self, options): - self.options = options - - def _get_non_null_values(self, dictionary): - return dict((key, value) for key, value in dictionary.iteritems() - if value is not None) - - def _get_service(self, git_url, charm, nr_units, options): - return {'branch': git_url, - 'charm': charm, - 'num_units': nr_units, - 'options': self._get_non_null_values(options)} - - def _get_ad_service(self, nr_units, domain_name, admin_password, - admin_username=None): - ad_options = {'domain-name': domain_name, - 'administrator': admin_username, - 'password': admin_password} - return self._get_service(self._AD_GIT_URL, - 'local:win2012r2/active-directory', - nr_units, ad_options) - - def _get_hyper_v_service(self, nr_units, download_mirror, extra_python_packages=None, - git_user_email=None, git_user_name=None, wheel_mirror=None, - ppy_mirror=None, vmswitch_name=None, vmswitch_management=None, - ad_user_name=None, enable_freerdp_console=None): - hyper_v_options = {'download-mirror': download_mirror, - 'extra-python-packages': extra_python_packages, - 'git-user-email': git_user_email, - 'git-user-name': git_user_name, - 'wheel-mirror': wheel_mirror, - 'ppy-mirror': ppy_mirror, - 'vmswitch-name': vmswitch_name, - 'vmswitch-management': vmswitch_management, - 'ad-user-name': ad_user_name, - 'enable-freerdp-console': enable_freerdp_console} - return self._get_service(self._HYPER_V_GIT_URL, - 'local:win2012hvr2/hyper-v-ci', - nr_units, hyper_v_options) - - def _get_devstack_service(self, nr_units, vlan_range, heat_image_url, test_image_url, - disabled_services=None, enable_plugins=None, - enabled_services=None, extra_packages=None, - extra_python_packages=None): - devstack_options = {'disabled-services': disabled_services, - 'enable-plugin': enable_plugins, - 'enabled-services': enabled_services, - 'extra-packages': extra_packages, - 'extra-python-packages': extra_python_packages, - 'heat-image-url': heat_image_url, - 'test-image-url': test_image_url, - 'vlan-range': vlan_range} - return self._get_service(self._DEVSTACK_GIT_URL, 'local:trusty/devstack', - nr_units, devstack_options) - - def _get_overrides_options(self, data_ports, external_ports, zuul_branch, zuul_change, - zuul_project, zuul_ref, zuul_url): - return {'data-port': data_ports, - 'external-port': external_ports, - 'zuul-branch': zuul_branch, - 'zuul-change': zuul_change, - 'zuul-project': zuul_project, - 'zuul-ref': zuul_ref, - 'zuul-url': zuul_url} - - def nova_bundle(self): - overrides_options = self._get_overrides_options(self.options.data_ports, - self.options.external_ports, self.options.zuul_branch, - self.options.zuul_change, self.options.zuul_project, - self.options.zuul_ref, self.options.zuul_url) - - hyper_v_service = self._get_hyper_v_service( - nr_units=self.options.nr_hyper_v_units, - download_mirror='http://64.119.130.115/bin', - extra_python_packages=self.options.hyper_v_extra_python_packages, - git_user_email='hyper-v_ci@microsoft.com', - git_user_name='Hyper-V CI', - wheel_mirror='http://64.119.130.115/wheels') - - devstack_service = self._get_devstack_service( - nr_units=self.options.nr_devstack_units, - disabled_services=self.options.devstack_disabled_services, - enable_plugins=self.options.devstack_enabled_plugins, - enabled_services=self.options.devstack_enabled_services, - extra_packages=self.options.devstack_extra_packages, - extra_python_packages=self.options.devstack_extra_python_packages, - heat_image_url='http://10.255.251.230/Fedora.vhdx', - test_image_url='http://10.255.251.230/cirros.vhdx', - vlan_range=self.options.vlan_range) - - hyper_v_service_name = 'hyper-v-ci-%s' % self.options.zuul_uuid - devstack_service_name = 'devstack-%s' % self.options.zuul_uuid - bundle_content = { - 'nova': {'overrides': overrides_options, - 'relations': [[devstack_service_name, hyper_v_service_name]], - 'services': {devstack_service_name: devstack_service, - hyper_v_service_name: hyper_v_service} } - } - - if self.options.nr_ad_units > 0: - ad_service_name = "active-directory" - ad_charm = self._get_ad_service( - nr_units=self.options.nr_ad_units, - domain_name=self.options.ad_domain_name, - admin_password=self.options.ad_admin_password) - ad_charm_dict = {ad_service_name: ad_charm} - bundle_content['nova']['relations'].append([hyper_v_service_name, - ad_service_name]) - bundle_content['nova']['services'].update(ad_charm_dict) - - return bundle_content diff --git a/deployer/requirements.txt b/deployer/requirements.txt deleted file mode 100644 index 4c0e710..0000000 --- a/deployer/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -gevent==1.0 -gzr+lp:~gabriel-samfira/python-jujuclient/python-jujuclient -pyyaml diff --git a/devstack/tests/cinder/included_tests.txt b/devstack/tests/cinder/included_tests.txt deleted file mode 100644 index 4833bbb..0000000 --- a/devstack/tests/cinder/included_tests.txt +++ /dev/null @@ -1 +0,0 @@ -volume diff --git a/devstack/tests/neutron/excluded_tests.txt b/devstack/tests/neutron/excluded_tests.txt deleted file mode 100644 index e69de29..0000000 diff --git a/devstack/tests/neutron/included_tests.txt b/devstack/tests/neutron/included_tests.txt deleted file mode 100644 index 60e7f84..0000000 --- a/devstack/tests/neutron/included_tests.txt +++ /dev/null @@ -1 +0,0 @@ -tempest.api.network diff --git a/devstack/tests/nova/excluded_tests.txt b/devstack/tests/nova/excluded_tests.txt deleted file mode 100644 index bd268d1..0000000 --- a/devstack/tests/nova/excluded_tests.txt +++ /dev/null @@ -1,30 +0,0 @@ -# Hyper-V does not support attaching vNics to a running instance before Threshold -# On Threshold it is supported, requiring Generation 2 -tempest.api.compute.servers.test_attach_interfaces.AttachInterfacesTestJSON.test_create_list_show_delete_interfaces -tempest.api.compute.servers.test_attach_interfaces.AttachInterfacesTestXML.test_create_list_show_delete_interfaces -tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_hotplug_nic - -# Unsupported consoles (Hyper-V uses RDP, not VNC or SPICE) -tempest.api.compute.v3.servers.test_server_actions.ServerActionsV3Test.test_get_spice_console -tempest.api.compute.v3.servers.test_server_actions.ServerActionsV3Test.test_get_vnc_console - -# See Neutron bug https://bugs.launchpad.net/neutron/+bug/1277285 -# Note that corresponding JSON tests pass -tempest.api.network.admin.test_dhcp_agent_scheduler.DHCPAgentSchedulersTestXML.test_add_remove_network_from_dhcp_agent -tempest.api.network.admin.test_l3_agent_scheduler.L3AgentSchedulerTestXML. -tempest.api.network.admin.test_agent_management.AgentManagementTestXML. - -# See Tempest bug: https://bugs.launchpad.net/tempest/+bug/1363986 -tempest.scenario.test_security_groups_basic_ops.TestSecurityGroupsBasicOps.test_cross_tenant_traffic -tempest.scenario.test_security_groups_basic_ops.TestSecurityGroupsBasicOps.test_multiple_security_groups - -# Fails on DevStack. Not related to Hyper-V -tempest.scenario.test_load_balancer_basic.TestLoadBalancerBasic.test_load_balancer_basic - -# Fails on DevStack. requires investigation. -tempest.scenario.test_network_advanced_server_ops.TestNetworkAdvancedServerOps.test_server_connectivity_rebuild - -# Fails on DevStack. requires investigation. -# Note that corresponding XML tests pass -tempest.api.compute.admin.test_simple_tenant_usage.TenantUsagesTestJSON.test_get_usage_tenant -tempest.api.compute.admin.test_simple_tenant_usage.TenantUsagesTestJSON.test_get_usage_tenant_with_non_admin_user diff --git a/devstack/tests/nova/included_tests.txt b/devstack/tests/nova/included_tests.txt deleted file mode 100644 index df80fb2..0000000 --- a/devstack/tests/nova/included_tests.txt +++ /dev/null @@ -1 +0,0 @@ -tempest. diff --git a/devstack/tests/nova/isolated_tests.txt b/devstack/tests/nova/isolated_tests.txt deleted file mode 100644 index 3d77ec2..0000000 --- a/devstack/tests/nova/isolated_tests.txt +++ /dev/null @@ -1,38 +0,0 @@ -tempest.api.compute.admin.test_migrations.MigrationsAdminTest.test_list_migrations_in_flavor_resize_situation -tempest.api.compute.admin.test_servers_negative.ServersAdminNegativeTestJSON.test_resize_server_using_overlimit_ram -tempest.api.compute.admin.test_servers_negative.ServersAdminNegativeTestJSON.test_resize_server_using_overlimit_vcpus -tempest.api.compute.admin.test_servers_negative.ServersAdminNegativeTestXML.test_resize_server_using_overlimit_ram -tempest.api.compute.admin.test_servers_negative.ServersAdminNegativeTestXML.test_resize_server_using_overlimit_vcpus -tempest.api.compute.servers.test_delete_server.DeleteServersTestJSON.test_delete_server_while_in_verify_resize_state -tempest.api.compute.servers.test_delete_server.DeleteServersTestXML.test_delete_server_while_in_verify_resize_state -tempest.api.compute.servers.test_disk_config.ServerDiskConfigTestJSON.test_resize_server_from_auto_to_manual -tempest.api.compute.servers.test_disk_config.ServerDiskConfigTestJSON.test_resize_server_from_manual_to_auto -tempest.api.compute.servers.test_disk_config.ServerDiskConfigTestXML.test_resize_server_from_auto_to_manual -tempest.api.compute.servers.test_disk_config.ServerDiskConfigTestXML.test_resize_server_from_manual_to_auto -tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_resize_server_confirm -tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_resize_server_confirm_from_stopped -tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_resize_server_revert -tempest.api.compute.servers.test_server_actions.ServerActionsTestXML.test_resize_server_confirm -tempest.api.compute.servers.test_server_actions.ServerActionsTestXML.test_resize_server_confirm_from_stopped -tempest.api.compute.servers.test_server_actions.ServerActionsTestXML.test_resize_server_revert -tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_resize_nonexistent_server -tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_resize_server_with_non_existent_flavor -tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_resize_server_with_null_flavor -tempest.api.compute.servers.test_servers_negative.ServersNegativeTestXML.test_resize_nonexistent_server -tempest.api.compute.servers.test_servers_negative.ServersNegativeTestXML.test_resize_server_with_non_existent_flavor -tempest.api.compute.servers.test_servers_negative.ServersNegativeTestXML.test_resize_server_with_null_flavor -tempest.api.compute.test_authorization.AuthorizationTestJSON.test_resize_server_for_alt_account_fails -tempest.api.compute.test_authorization.AuthorizationTestXML.test_resize_server_for_alt_account_fails -tempest.api.compute.v3.admin.test_migrations.MigrationsAdminV3Test.test_list_migrations_in_flavor_resize_situation -tempest.api.compute.v3.admin.test_servers_negative.ServersAdminNegativeV3Test.test_resize_server_using_overlimit_ram -tempest.api.compute.v3.admin.test_servers_negative.ServersAdminNegativeV3Test.test_resize_server_using_overlimit_vcpus -tempest.api.compute.v3.servers.test_delete_server.DeleteServersV3Test.test_delete_server_while_in_verify_resize_state -tempest.api.compute.v3.servers.test_server_actions.ServerActionsV3Test.test_resize_server_confirm -tempest.api.compute.v3.servers.test_server_actions.ServerActionsV3Test.test_resize_server_confirm_from_stopped -tempest.api.compute.v3.servers.test_server_actions.ServerActionsV3Test.test_resize_server_revert -tempest.api.compute.v3.servers.test_servers_negative.ServersNegativeV3Test.test_resize_nonexistent_server -tempest.api.compute.v3.servers.test_servers_negative.ServersNegativeV3Test.test_resize_server_with_non_existent_flavor -tempest.api.compute.v3.servers.test_servers_negative.ServersNegativeV3Test.test_resize_server_with_null_flavor -tempest.scenario.test_network_advanced_server_ops.TestNetworkAdvancedServerOps.test_server_connectivity_resize -tempest.scenario.test_server_advanced_ops.TestServerAdvancedOps.test_resize_server_confirm - diff --git a/infra/deployer/BundleGenerator.py b/infra/deployer/BundleGenerator.py deleted file mode 100644 index 109c2b6..0000000 --- a/infra/deployer/BundleGenerator.py +++ /dev/null @@ -1,116 +0,0 @@ - -class BundleGenerator(object): - _AD_GIT_URL = 'https://github.com/cloudbase/active-directory.git' - _DEVSTACK_GIT_URL = 'https://github.com/cloudbase/devstack-charm.git' - _HYPER_V_GIT_URL = 'https://github.com/cloudbase/hyperv-charm' - - def __init__(self, options): - self.options = options - - def _get_non_null_values(self, dictionary): - return dict((key, value) for key, value in dictionary.iteritems() - if value is not None) - - def _get_service(self, git_url, charm, nr_units, options): - return {'branch': git_url, - 'charm': charm, - 'num_units': nr_units, - 'options': self._get_non_null_values(options)} - - def _get_ad_service(self, nr_units, domain_name, admin_password, - admin_username=None): - ad_options = {'domain-name': domain_name, - 'administrator': admin_username, - 'password': admin_password} - return self._get_service(self._AD_GIT_URL, - 'local:win2012r2/active-directory', - nr_units, ad_options) - - def _get_hyper_v_service(self, nr_units, download_mirror, extra_python_packages=None, - git_user_email=None, git_user_name=None, wheel_mirror=None, - ppy_mirror=None, vmswitch_name=None, vmswitch_management=None, - ad_user_name=None, enable_freerdp_console=None): - hyper_v_options = {'download-mirror': download_mirror, - 'extra-python-packages': extra_python_packages, - 'git-user-email': git_user_email, - 'git-user-name': git_user_name, - 'wheel-mirror': wheel_mirror, - 'ppy-mirror': ppy_mirror, - 'vmswitch-name': vmswitch_name, - 'vmswitch-management': vmswitch_management, - 'ad-user-name': ad_user_name, - 'enable-freerdp-console': enable_freerdp_console} - return self._get_service(self._HYPER_V_GIT_URL, - 'local:win2012hvr2/hyper-v-ci', - nr_units, hyper_v_options) - - def _get_devstack_service(self, nr_units, vlan_range, heat_image_url, test_image_url, - disabled_services=None, enable_plugins=None, - enabled_services=None, extra_packages=None, - extra_python_packages=None): - devstack_options = {'disabled-services': disabled_services, - 'enable-plugin': enable_plugins, - 'enabled-services': enabled_services, - 'extra-packages': extra_packages, - 'extra-python-packages': extra_python_packages, - 'heat-image-url': heat_image_url, - 'test-image-url': test_image_url, - 'vlan-range': vlan_range} - return self._get_service(self._DEVSTACK_GIT_URL, 'local:trusty/devstack', - nr_units, devstack_options) - - def _get_overrides_options(self, data_ports, external_ports, zuul_branch, zuul_change, - zuul_project, zuul_ref, zuul_url): - return {'data-port': data_ports, - 'external-port': external_ports, - 'zuul-branch': zuul_branch, - 'zuul-change': zuul_change, - 'zuul-project': zuul_project, - 'zuul-ref': zuul_ref, - 'zuul-url': zuul_url} - - def nova_bundle(self): - overrides_options = self._get_overrides_options(self.options.data_ports, - self.options.external_ports, self.options.zuul_branch, - self.options.zuul_change, self.options.zuul_project, - self.options.zuul_ref, self.options.zuul_url) - - hyper_v_service = self._get_hyper_v_service( - nr_units=self.options.nr_hyper_v_units, - download_mirror='http://64.119.130.115/bin', - extra_python_packages=self.options.hyper_v_extra_python_packages, - git_user_email='hyper-v_ci@microsoft.com', - git_user_name='Hyper-V CI', - wheel_mirror='http://64.119.130.115/wheels') - - devstack_service = self._get_devstack_service( - nr_units=self.options.nr_devstack_units, - disabled_services=self.options.devstack_disabled_services, - enable_plugins=self.options.devstack_enabled_plugins, - enabled_services=self.options.devstack_enabled_services, - extra_packages=self.options.devstack_extra_packages, - extra_python_packages=self.options.devstack_extra_python_packages, - heat_image_url='http://10.255.251.230/Fedora.vhdx', - test_image_url='http://10.255.251.230/cirros.vhdx', - vlan_range=self.options.vlan_range) - - hyper_v_service_name = 'hyper-v-ci-%s' % self.options.zuul_uuid - devstack_service_name = 'devstack-%s' % self.options.zuul_uuid - bundle_content = { - 'nova': {'overrides': overrides_options, - 'relations': [[devstack_service_name, hyper_v_service_name]], - 'services': {devstack_service_name: devstack_service, - hyper_v_service_name: hyper_v_service} } - } - - if self.options.nr_ad_units > 0: - ad_charm = self._get_ad_service( - nr_units=self.options.nr_ad_units, - domain_name=self.options.ad_domain_name, - admin_password=self.options.ad_admin_password) - ad_charm_dict = {'active-directory': ad_charm} - bundle_content['nova']['relations'].append([hyper_v_service_name, - 'active-directory']) - bundle_content['nova']['services'].update(ad_charm_dict) - - return bundle_content diff --git a/infra/logs/collect_logs.sh b/infra/logs/collect_logs.sh deleted file mode 100644 index e69de29..0000000 diff --git a/infra/logs/utils.sh b/infra/logs/utils.sh deleted file mode 100644 index 613756f..0000000 --- a/infra/logs/utils.sh +++ /dev/null @@ -1,86 +0,0 @@ -function run_wsman_cmd() { - local host=$1 - local cmd=$2 - $BASEDIR/wsmancmd.py -u $win_user -p $win_password -U https://$1:5986/wsman $cmd -} - -function get_win_files() { - local host=$1 - local remote_dir=$2 - local local_dir=$3 - smbclient "//$host/C\$" -c "lcd $local_dir; cd $remote_dir; prompt; mget *" -U "$win_user%$win_password" -} - -function run_wsman_ps() { - local host=$1 - local cmd=$2 - run_wsman_cmd $host "powershell -NonInteractive -ExecutionPolicy RemoteSigned -Command $cmd" -} - -function get_win_hotfixes() { - local host=$1 - run_wsman_cmd $host "wmic qfe list" -} - -function get_win_system_info() { - local host=$1 - run_wsman_cmd $host "systeminfo" -} - -function get_win_time() { - local host=$1 - # Seconds since EPOCH - host_time=`run_wsman_ps $host "[Math]::Truncate([double]::Parse((Get-Date (get-date).ToUniversalTime() -UFormat %s)))" 2>&1` - # Skip the newline - echo ${host_time::-1} -} - -function get_win_hotfixes_log() { - local win_host=$1 - local log_file=$2 - echo "Getting hotfixes details for host: $win_host" - get_win_hotfixes $win_host > $log_file -} - -function get_win_system_info_log() { - local win_host=$1 - local log_file=$2 - echo "Getting system info for host: $win_host" - get_win_system_info $win_host > $log_file -} - -function get_win_host_log_files() { - local host_name=$1 - local local_dir=$2 - get_win_files $host_name "$host_logs_dir" $local_dir -} - -function get_win_host_config_files() { - local host_name=$1 - local local_dir=$2 - mkdir -p $local_dir - - get_win_files $host_name $host_config_dir $local_dir -} - -function check_host_time() { - local host1=$1 - local host2=$2 - host1_time=`get_win_time $host1` - host2_time=`get_win_time $host2` - local_time=`date +%s` - - local delta1=$((local_time - host1_time)) - local delta2=$((local_time - host2_time)) - if [ ${delta1#-} -gt 120 ]; - then - echo "Host $host1 time offset compared to this host is too high: $delta" - return 1 - fi - if [ ${delta2#-} -gt 120 ]; - then - echo "Host $host2 time offset compared to this host is too high: $delta" - return 1 - fi - return 0 -} diff --git a/scripts/bin/get-isolated-tests.sh b/scripts/bin/get-isolated-tests.sh new file mode 100755 index 0000000..a65a6c7 --- /dev/null +++ b/scripts/bin/get-isolated-tests.sh @@ -0,0 +1,40 @@ +#!/bin/bash +set -e + +array_to_regex() +{ + local ar=(${@}) + local regex="" + + for s in "${ar[@]}" + do + if [ "$regex" ]; then + regex+="\\|" + fi + regex+="^"$(echo $s | sed -e 's/[]\/$*.^|[]/\\&/g') + done + echo $regex +} + +tests_dir=$1 + +isolated_tests_file=$2 +exclude_tests_file=$3 + +if [ -f "$exclude_tests_file" ]; then + exclude_tests=(`awk 'NF && $1!~/^#/' $exclude_tests_file`) +fi + +if [ -f "$isolated_tests_file" ]; then + isolated_tests=(`awk 'NF && $1!~/^#/' $isolated_tests_file`) +fi + +exclude_regex=$(array_to_regex ${exclude_tests[@]}) +include_regex=$(array_to_regex ${isolated_tests[@]}) + +if [ ! "$exclude_regex" ]; then + exclude_regex='^$' +fi + +cd $tests_dir +testr list-tests | grep $include_regex | grep -v $exclude_regex diff --git a/devstack/bin/get-results-html.sh b/scripts/bin/get-results-html.sh similarity index 100% rename from devstack/bin/get-results-html.sh rename to scripts/bin/get-results-html.sh diff --git a/devstack/bin/get-tests.sh b/scripts/bin/get-tests.sh similarity index 100% rename from devstack/bin/get-tests.sh rename to scripts/bin/get-tests.sh diff --git a/devstack/bin/parallel-test-runner.sh b/scripts/bin/parallel-test-runner.sh similarity index 100% rename from devstack/bin/parallel-test-runner.sh rename to scripts/bin/parallel-test-runner.sh diff --git a/devstack/bin/run-all-tests.sh b/scripts/bin/run-all-tests.sh similarity index 85% rename from devstack/bin/run-all-tests.sh rename to scripts/bin/run-all-tests.sh index f5edd72..9575266 100755 --- a/devstack/bin/run-all-tests.sh +++ b/scripts/bin/run-all-tests.sh @@ -56,6 +56,8 @@ if [ -z "$LOG_FILE" ]; then LOG_FILE="/home/ubuntu/tempest/subunit-output.log"; if [ -z "$RESULTS_HTML_FILE" ]; then RESULTS_HTML_FILE="/home/ubuntu/tempest/results.html"; fi BASEDIR=$(dirname $0) +SUBUNIT_STATS="/home/ubuntu/tempest/subunit_stats.log" +TEMPEST_OUTPUT="/home/ubuntu/tempest/tempest-output.log" pushd $BASEDIR @@ -82,11 +84,14 @@ $BASEDIR/parallel-test-runner.sh $TESTS_FILE $TESTS_DIR $LOG_FILE \ if [ -f "$ISOLATED_FILE" ]; then echo "Running isolated tests from: $ISOLATED_FILE" + isolated_tests_file=$(tempfile) + $BASEDIR/get-isolated-tests.sh $TESTS_DIR $ISOLATED_FILE $EXCLUDE_FILE > $isolated_tests_file log_tmp=$(tempfile) - $BASEDIR/parallel-test-runner.sh $ISOLATED_FILE $TESTS_DIR $log_tmp \ + $BASEDIR/parallel-test-runner.sh $isolated_tests_file $TESTS_DIR $log_tmp \ $PARALLEL_TESTS $MAX_ATTEMPTS 1 || true cat $log_tmp >> $LOG_FILE + rm $isolated_tests_file rm $log_tmp fi @@ -97,7 +102,9 @@ deactivate echo "Generating HTML report..." $BASEDIR/get-results-html.sh $LOG_FILE $RESULTS_HTML_FILE -subunit-stats $LOG_FILE > /dev/null +cat $LOG_FILE | subunit-trace -n -f > $TEMPEST_OUTPUT 2>&1 || true + +subunit-stats $LOG_FILE > $SUBUNIT_STATS #/dev/null exit_code=$? echo "Total execution time: $SECONDS seconds." diff --git a/devstack/bin/run.sh b/scripts/bin/run.sh similarity index 100% rename from devstack/bin/run.sh rename to scripts/bin/run.sh diff --git a/devstack/bin/subunit2html.py b/scripts/bin/subunit2html.py similarity index 100% rename from devstack/bin/subunit2html.py rename to scripts/bin/subunit2html.py diff --git a/devstack/bin/utils.sh b/scripts/bin/utils.sh similarity index 100% rename from devstack/bin/utils.sh rename to scripts/bin/utils.sh diff --git a/scripts/jobs/common-job-script.sh b/scripts/jobs/common-job-script.sh new file mode 100755 index 0000000..9803b69 --- /dev/null +++ b/scripts/jobs/common-job-script.sh @@ -0,0 +1,182 @@ +#!/bin/bash + +function exec_with_retry () { + local max_retries=$1 + local interval=${2} + local cmd=${@:3} + + local counter=0 + while [ $counter -lt $max_retries ]; do + local exit_code=0 + eval $cmd || exit_code=$? + if [ $exit_code -eq 0 ]; then + return 0 + fi + let counter=counter+1 + + if [ -n "$interval" ]; then + sleep $interval + fi + done + return $exit_code +} + +set -x +set +e + +source ${WORKSPACE}/common-ci/scripts/jobs/${project}-config.sh + +DEPLOYER_PATH="/home/ubuntu/deployer" +JUJU_SSH_KEY="/home/ubuntu/.local/share/juju/ssh/juju_id_rsa" +LOGS_SERVER="10.20.1.14" +LOGS_SSH_KEY="/home/ubuntu/.ssh/norman.pem" +BUNDLE_LOCATION=$(mktemp) + +eval "cat <> $BUNDLE_LOCATION + +cat $BUNDLE_LOCATION + +$DEPLOYER_PATH/deployer.py --clouds-and-credentials $DEPLOYER_PATH/$CI_CREDS deploy --template $BUNDLE_LOCATION --max-unit-retries 10 --timeout 7200 --search-string $UUID +build_exit_code=$? + +source $WORKSPACE/nodes + +exec_with_retry 5 2 ssh -tt -o 'PasswordAuthentication=no' -o 'StrictHostKeyChecking=no' -o 'UserKnownHostsFile=/dev/null' -i $JUJU_SSH_KEY ubuntu@$DEVSTACK \ + "git clone https://github.com/capsali/common-ci.git /home/ubuntu/common-ci" +clone_exit_code=$? + +exec_with_retry 5 2 ssh -tt -o 'PasswordAuthentication=no' -o 'StrictHostKeyChecking=no' -o 'UserKnownHostsFile=/dev/null' -i $JUJU_SSH_KEY ubuntu@$DEVSTACK \ + "git -C /home/ubuntu/common-ci checkout charms" +checkout_exit_code=$? + + +if [[ $build_exit_code -eq 0 ]]; then + #run tempest + + exec_with_retry 5 2 ssh -tt -o 'PasswordAuthentication=no' -o 'StrictHostKeyChecking=no' -o 'UserKnownHostsFile=/dev/null' -i $JUJU_SSH_KEY ubuntu@$DEVSTACK \ + "mkdir -p /home/ubuntu/tempest" + ssh -tt -o 'PasswordAuthentication=no' -o 'StrictHostKeyChecking=no' -o 'UserKnownHostsFile=/dev/null' -i $JUJU_SSH_KEY ubuntu@$DEVSTACK \ + "/home/ubuntu/common-ci/scripts/bin/run-all-tests.sh --include-file /home/ubuntu/common-ci/tests/$project/included-tests.txt \ + --exclude-file /home/ubuntu/common-ci/tests/$project/excluded-tests.txt --isolated-file /home/ubuntu/common-ci/tests/$project/isolated-tests.txt \ + --tests-dir /opt/stack/tempest --parallel-tests 10 --max-attempts 2" + tests_exit_code=$? +fi + +######################### Collect logs ######################### +LOG_DIR="logs/${UUID}" +if [ $LOG_DIR ]; then + rm -rf $LOG_DIR +fi +mkdir -p "$LOG_DIR" + +ssh -tt -o 'PasswordAuthentication=no' -o 'StrictHostKeyChecking=no' -o 'UserKnownHostsFile=/dev/null' -i $JUJU_SSH_KEY ubuntu@$DEVSTACK \ + "sudo /home/ubuntu/common-ci/scripts/logs/collect-logs.sh" + +scp -o 'PasswordAuthentication=no' -o 'StrictHostKeyChecking=no' -o 'UserKnownHostsFile=/dev/null' -i $JUJU_SSH_KEY \ +ubuntu@$DEVSTACK:/home/ubuntu/aggregate.tar.gz $LOG_DIR/aggregate.tar.gz + +tar -zxf $LOG_DIR/aggregate.tar.gz -C $LOG_DIR/ +rm $LOG_DIR/aggregate.tar.gz + +source $WORKSPACE/common-ci/scripts/logs/utils.sh + +for hv in $(echo $HYPERV | tr "," "\n"); do + HV_LOGS=$LOG_DIR/hyperv-logs/$hv + HV_CONFS=$LOG_DIR/hyperv-config/$hv + mkdir -p $HV_LOGS + mkdir -p $HV_CONFS + + run_wsman_ps $hv "New-Item -Force -Type directory -Path C:\Openstack\Eventlog" + set_win_files $hv "\Openstack\Eventlog" "${WORKSPACE}/common-ci/scripts/logs/" "export-eventlog.ps1" + set_win_files $hv "\Openstack\Eventlog" "${WORKSPACE}/common-ci/templates/" "eventlog_css.txt" + set_win_files $hv "\Openstack\Eventlog" "${WORKSPACE}/common-ci/templates/" "eventlog_js.txt" + run_wsman_ps $hv "C:\Openstack\Eventlog\export-eventlog.ps1" + + get_win_files $hv "\openstack\log" $HV_LOGS + get_win_files $hv "\openstack\etc" $HV_CONFS + get_win_files $hv "\juju\log" $HV_LOGS + + run_wsman_cmd $hv 'systeminfo' > $HV_LOGS/systeminfo.log + run_wsman_cmd $hv 'wmic qfe list' > $HV_LOGS/windows-hotfixes.log + run_wsman_cmd $hv 'c:\python27\scripts\pip freeze' > $HV_LOGS/pip-freeze.log + run_wsman_cmd $hv 'ipconfig /all' > $HV_LOGS/ipconfig.log + run_wsman_cmd $hv 'sc qc nova-compute' > $HV_LOGS/nova-compute-service.log + run_wsman_cmd $hv 'sc qc neutron-openvswitch-agent' > $HV_LOGS/neutron-openvswitch-agent-service.log + + run_wsman_ps $hv 'get-netadapter ^| Select-object *' > $HV_LOGS/get-netadapter.log + run_wsman_ps $hv 'get-vmswitch ^| Select-object *' > $HV_LOGS/get-vmswitch.log + run_wsman_ps $hv 'get-WmiObject win32_logicaldisk ^| Select-object *' > $HV_LOGS/disk-free.log + run_wsman_ps $hv 'get-netfirewallprofile ^| Select-Object *' > $HV_LOGS/firewall.log + + run_wsman_ps $hv 'get-process ^| Select-Object *' > $HV_LOGS/get-process.log + run_wsman_ps $hv 'get-service ^| Select-Object *' > $HV_LOGS/get-service.log +done + +wget http://10.20.1.3:8080/job/$JOB_NAME/$BUILD_ID/consoleText -O $LOG_DIR/console.log + +find $LOG_DIR -name "*.log" -exec gzip {} \; + +pushd $LOG_DIR +tar -zcf aggregate.tar.gz . +popd +#tar -zcf $LOG_DIR/aggregate.tar.gz $LOG_DIR + +if [ $project == "ovs" ]; then + if [ ! $UUID ]; then + exit 1 + fi + REMOTE_LOG_PATH="/srv/logs/ovs/tempest-run/$UUID" +else + if [ ! $project ] || [ ! $ZUUL_CHANGE ] || [ ! $ZUUL_PATCHSET ]; then + exit 1 + fi + REMOTE_LOG_PATH="/srv/logs/$project/$ZUUL_CHANGE/$ZUUL_PATCHSET" +fi + +# Copy logs to remote log server +echo "Creating logs destination folder" +ssh -tt -o 'PasswordAuthentication=no' -o 'StrictHostKeyChecking=no' -o 'UserKnownHostsFile=/dev/null' -i $LOGS_SSH_KEY logs@$LOGS_SERVER \ + "rm -r $REMOTE_LOG_PATH" +ssh -tt -o 'PasswordAuthentication=no' -o 'StrictHostKeyChecking=no' -o 'UserKnownHostsFile=/dev/null' -i $LOGS_SSH_KEY logs@$LOGS_SERVER \ + "mkdir -p $REMOTE_LOG_PATH" + #"if [ ! -d $REMOTE_LOG_PATH ]; then mkdir -p $REMOTE_LOG_PATH; else rm -r $REMOTE_LOG_PATH/*; fi" + +echo "Uploading logs" +scp -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking no" -i $LOGS_SSH_KEY $LOG_DIR/aggregate.tar.gz logs@$LOGS_SERVER:$REMOTE_LOG_PATH/aggregate.tar.gz + +echo "Extracting logs" +ssh -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking no" -i $LOGS_SSH_KEY logs@$LOGS_SERVER "tar -xvf $REMOTE_LOG_PATH/aggregate.tar.gz -C $REMOTE_LOG_PATH/ --strip 1" + +# Remove local logs +rm -rf $LOG_DIR +############################################## + +if [ "$DEBUG" != "YES" ]; then + #destroy charms, services and used nodes. + $DEPLOYER_PATH/deployer.py --clouds-and-credentials $DEPLOYER_PATH/$CI_CREDS teardown --search-string $UUID +fi + +if [[ $build_exit_code -ne 0 ]]; then + echo "CI Error while deploying environment" + exit 1 +fi + +if [[ $clone_exit_code -ne 0 ]]; then + echo "CI Error while cloning the scripts repository" + exit 1 +fi + +if [[ $checkout_exit_code -ne 0 ]]; then + echo "CI Error while checking out the scripts repository" + exit 1 +fi + +if [[ $tests_exit_code -ne 0 ]]; then + echo "Tempest tests execution finished with a failure status" + exit 1 +fi + +exit 0 diff --git a/scripts/jobs/neutron-ovs-config.sh b/scripts/jobs/neutron-ovs-config.sh new file mode 100644 index 0000000..76a58a2 --- /dev/null +++ b/scripts/jobs/neutron-ovs-config.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +CI_CREDS="neutron-ovs-creds.yaml" +domain_name="neutronovs.local" +domain_user="neutron-ovs" +test_signing="false" +data_port="00:07:43:13:97:c8 00:07:43:13:96:b8 00:07:43:13:a6:08 00:07:43:14:d2:e8 00:07:43:13:f1:48 00:07:43:13:f1:88 00:07:43:13:b3:88 00:07:43:13:b5:18 00:07:43:13:ea:78 00:07:43:13:f1:68 00:07:43:13:9b:f8 00:07:43:14:12:c8 00:07:43:14:12:78 00:07:43:13:f1:58 00:07:43:14:12:88 00:07:43:14:12:98 00:07:43:13:a0:f8 00:07:43:13:9a:78 00:07:43:14:18:18 00:07:43:13:a1:48 00:07:43:14:1f:38 00:07:43:14:1b:48 00:07:43:14:18:38 00:07:43:13:f4:b8 00:07:43:13:98:48 00:07:43:13:f4:f8 00:07:43:14:18:98 00:07:43:13:f1:28 00:07:43:14:1a:18" +external_port="00:07:43:13:97:c0 00:07:43:13:96:b0 00:07:43:13:a6:00 00:07:43:14:d2:e0 00:07:43:13:f1:40 00:07:43:13:f1:80 00:07:43:13:b3:80 00:07:43:13:b5:10 00:07:43:13:ea:70 00:07:43:13:f1:60 00:07:43:13:9b:f0 00:07:43:14:12:c0 00:07:43:14:12:70 00:07:43:13:f1:50 00:07:43:14:12:80 00:07:43:14:12:90 00:07:43:13:a0:f0 00:07:43:13:9a:70 00:07:43:14:18:10 00:07:43:13:a1:40 00:07:43:14:1f:30 00:07:43:14:1b:40 00:07:43:14:18:30 00:07:43:13:f4:b0 00:07:43:13:98:40 00:07:43:13:f4:f0 00:07:43:14:18:90 00:07:43:13:f1:20 00:07:43:14:1a:10" +prep_project="True" +os_data_network="10.31.4.0/23" +disable_ipv6="false" +win_user="Administrator" +win_password=$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c${1:-24};echo;) +ovs_installer="http://10.20.1.14:8080/openvswitch-hyperv-2.6.1-certified.msi" +heat_image_url="http://10.20.1.14:8080/cirros-latest.vhdx" +test_image_url="http://10.20.1.14:8080/cirros-latest.vhdx" +scenario_img="cirros-latest.vhdx" +vmswitch_management="false" + +if [ "$ZUUL_BRANCH" == "master" ]; then + hyperv_cherry_picks="https://review.openstack.org/openstack/neutron|refs/changes/41/417141/2|master" + devstack_cherry_picks="https://git.openstack.org/openstack/tempest|refs/changes/49/383049/13|master,https://git.openstack.org/openstack/tempest|refs/changes/28/384528/9|master" + post_python_packages="kombu==4.0.1 amqp==2.1.3 SQLAlchemy==1.0.17" +fi + + diff --git a/scripts/jobs/ovs-config.sh b/scripts/jobs/ovs-config.sh new file mode 100644 index 0000000..f881e01 --- /dev/null +++ b/scripts/jobs/ovs-config.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +CI_CREDS="ovs-creds.yaml" +domain_name="openvswitch.local" +domain_user="openvswitch" +test_signing="true" +data_port="E4:1D:2D:22:A0:30 E4:1D:2D:22:A6:30 E4:1D:2D:22:A1:E0 24:8A:07:77:3D:00" +external_port="18:A9:05:58:F7:76 00:23:7D:D2:CF:02 00:23:7D:D2:D8:D2 00:23:7D:D2:D8:72" +ZUUL_BRANCH="master" +prep_project="False" +os_data_network="10.12.3.0/24" +hyperv_cherry_picks="https://review.openstack.org/openstack/neutron|refs/changes/41/417141/2|master" +devstack_cherry_picks="https://git.openstack.org/openstack/tempest|refs/changes/49/383049/13|master,https://git.openstack.org/openstack/tempest|refs/changes/28/384528/9|master" +disable_ipv6="false" +win_user="Administrator" +win_password=$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c${1:-24};echo;) +ovs_installer="http://10.20.1.14:8080/ovs/$UUID/OpenvSwitch.msi" +ovs_certificate="http://10.20.1.14:8080/ovs/$UUID/package.cer" +heat_image_url="http://10.20.1.14:8080/cirros-latest.vhdx" +test_image_url="http://10.20.1.14:8080/cirros-latest.vhdx" +scenario_img="cirros-latest.vhdx" +vmswitch_management="false" +hv_extra_python_packages="setuptools SQLAlchemy==0.9.8 wmi oslo.i18n==1.7.0 pbr==1.2.0 oslo.messaging==4.5.1 lxml==3.6.4" +post_python_packages="kombu==4.0.1 amqp==2.1.3 SQLAlchemy==1.0.17" + +if [ "$ZUUL_BRANCH" == "stable/mitaka" ]; then + post_python_packages="oslo.messaging==5.20.0 psutil==1.2.1 kombu==4.0.1 amqp==2.1.3 SQLAlchemy==1.0.17" +fi diff --git a/scripts/logs/collect-logs.sh b/scripts/logs/collect-logs.sh new file mode 100755 index 0000000..a620da6 --- /dev/null +++ b/scripts/logs/collect-logs.sh @@ -0,0 +1,27 @@ +#!/bin/bash +set +e + +BASEDIR=$(dirname $0) +DEVSTACK_LOGS="/opt/stack/logs/screen" +DEVSTACK_LOG_DIR="/opt/stack/logs" + +TEMPEST_LOGS="/home/ubuntu/tempest" + +LOG_DST="/home/ubuntu/aggregate" +LOG_DST_DEVSTACK="$LOG_DST/devstack-logs" +CONFIG_DST_DEVSTACK="$LOG_DST/devstack-config" + +TAR="tar" +GZIP="gzip -f" + +source $BASEDIR/utils.sh + +emit_info "Collecting devstack logs" +archive_devstack_logs +emit_info "Collecting devstack configs" +archive_devstack_configs +emit_info "Collecting tempest files" +archive_tempest_files + +# Archive everything +pushd $LOG_DST; tar -zcf "$LOG_DST.tar.gz" .; popd diff --git a/scripts/logs/export-eventlog.ps1 b/scripts/logs/export-eventlog.ps1 new file mode 100644 index 0000000..e254758 --- /dev/null +++ b/scripts/logs/export-eventlog.ps1 @@ -0,0 +1,53 @@ +# Loading config and utils + +function dumpeventlog($path){ + + foreach ($i in (get-winevent -ListLog * | ? {$_.RecordCount -gt 0 })) { + $logName = "eventlog_" + $i.LogName + ".evtx" + $logName = $logName.replace(" ","-").replace("/", "-").replace("\", "-") + Write-Host "exporting "$i.LogName" as "$logName + $bkup = Join-Path $path $logName + wevtutil epl $i.LogName $bkup + } +} + +function exporthtmleventlog($path){ + $css = Get-Content $eventlogcsspath -Raw + $js = Get-Content $eventlogjspath -Raw + $HTMLHeader = @" + + + +"@ + + foreach ($i in (get-winevent -ListLog * | ? {$_.RecordCount -gt 0 })) { + $Report = (get-winevent -LogName $i.LogName) + $logName = "eventlog_" + $i.LogName + ".html" + $logName = $logName.replace(" ","-").replace("/", "-").replace("\", "-") + Write-Host "exporting "$i.LogName" as "$logName + $Report = $Report | ConvertTo-Html -Title "${i}" -Head $HTMLHeader -As Table + $Report = $Report | ForEach-Object {$_ -replace "", ''} + $Report = $Report | ForEach-Object {$_ -replace "", '
'} + $bkup = Join-Path $path $logName + $Report = $Report | Set-Content $bkup + } +} + +function cleareventlog(){ + foreach ($i in (get-winevent -ListLog * | ? {$_.RecordCount -gt 0 })) { + wevtutil cl $i.LogName + } +} + +$eventlogPath = "C:\OpenStack\Log\Eventlog" +$eventlogcsspath = "C:\Openstack\Eventlog\eventlog_css.txt" +$eventlogjspath = "C:\Openstack\Eventlog\eventlog_js.txt" + +if (Test-Path $eventlogPath){ + Remove-Item $eventlogPath -recurse -force +} + +New-Item -ItemType Directory -Force -Path $eventlogPath + +dumpeventlog $eventlogPath +exporthtmleventlog $eventlogPath diff --git a/scripts/logs/utils.sh b/scripts/logs/utils.sh new file mode 100644 index 0000000..9f8219e --- /dev/null +++ b/scripts/logs/utils.sh @@ -0,0 +1,207 @@ +#!/bin/bash + +BASEDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +LOG_DST="/home/ubuntu/aggregate" + +TAR="tar" +GZIP="gzip -f" + +function emit_error() { + echo "ERROR: $1" + exit 1 +} + +function emit_warning() { + echo "WARNING: $1" + return 0 +} + +function emit_info() { + echo "INFO: $1" + return 0 +} + +function run_wsman_cmd() { + local host=$1 + local cmd=$2 + $BASEDIR/wsmancmd.py -s -H $host -a certificate -c /home/ubuntu/.ssl/winrm_client_cert.pem -k /home/ubuntu/.ssl/winrm_client_cert.key "$cmd" +} + +function get_win_files() { + local host=$1 + local remote_dir=$2 + local local_dir=$3 + if [ ! -d "$local_dir" ];then + mkdir "$local_dir" + fi + smbclient "//$host/C\$" -c "prompt OFF; cd $remote_dir" -U "$win_user%$win_password" + if [ $? -ne 0 ];then + echo "Folder $remote_dir does not exists" + return 0 + fi + smbclient "//$host/C\$" -c "prompt OFF; recurse ON; lcd $local_dir; cd $remote_dir; mget *" -U "$win_user%$win_password" +} + +function set_win_files() { + local host=$1 + local remote_dir=$2 + local local_dir=$3 + local local_file=$4 + smbclient "//$host/C\$" -c "prompt OFF; cd $remote_dir" -U "$win_user%$win_password" + if [ $? -ne 0 ];then + echo "Folder $remote_dir does not exists" + return 0 + fi + smbclient "//$host/C\$" --directory $remote_dir -c "prompt OFF; recurse ON; lcd $local_dir; put $local_file" -U "$win_user%$win_password" +} + +function run_wsman_ps() { + local host=$1 + local cmd=$2 + run_wsman_cmd $host "powershell -NonInteractive -ExecutionPolicy RemoteSigned -Command $cmd" +} + +function get_win_hotfixes() { + local host=$1 + run_wsman_cmd $host "wmic qfe list" +} + +function get_win_system_info() { + local host=$1 + run_wsman_cmd $host "systeminfo" +} + +function get_win_time() { + local host=$1 + # Seconds since EPOCH + host_time=`run_wsman_ps $host "[Math]::Truncate([double]::Parse((Get-Date (get-date).ToUniversalTime() -UFormat %s)))" 2>&1` + # Skip the newline + echo ${host_time::-1} +} + +function get_win_hotfixes_log() { + local win_host=$1 + local log_file=$2 + emit_info "Getting hotfixes details for host: $win_host" + get_win_hotfixes $win_host > $log_file +} + +function get_win_system_info_log() { + local win_host=$1 + local log_file=$2 + emit_info "Getting system info for host: $win_host" + get_win_system_info $win_host > $log_file +} + +function get_win_host_log_files() { + local host_name=$1 + local local_dir=$2 + get_win_files $host_name "$host_logs_dir" $local_dir +} + +function get_win_host_config_files() { + local host_name=$1 + local local_dir=$2 + mkdir -p $local_dir + + get_win_files $host_name $host_config_dir $local_dir +} + +function check_host_time() { + local host1=$1 + local host2=$2 + host1_time=`get_win_time $host1` + host2_time=`get_win_time $host2` + local_time=`date +%s` + + local delta1=$((local_time - host1_time)) + local delta2=$((local_time - host2_time)) + if [ ${delta1#-} -gt 120 ]; + then + emit_info "Host $host1 time offset compared to this host is too high: $delta" + return 1 + fi + if [ ${delta2#-} -gt 120 ]; + then + emit_info "Host $host2 time offset compared to this host is too high: $delta" + return 1 + fi + return 0 +} + +function archive_devstack_logs() { + local LOG_DST_DEVSTACK=${1:-$LOG_DST/devstack-logs} + local DEVSTACK_LOGS="/opt/stack/logs/screen" + + if [ ! -d "$LOG_DST_DEVSTACK" ] + then + mkdir -p "$LOG_DST_DEVSTACK" || emit_error "L30: Failed to create $LOG_DST_DEVSTACK" + fi + + for i in `ls -A $DEVSTACK_LOGS` + do + if [ -h "$DEVSTACK_LOGS/$i" ] + then + REAL=$(readlink "$DEVSTACK_LOGS/$i") + $GZIP -c "$REAL" > "$LOG_DST_DEVSTACK/$i.gz" || emit_warning "L38: Failed to archive devstack logs: $i" + fi + done + $GZIP -c /var/log/mysql/error.log > "$LOG_DST_DEVSTACK/mysql_error.log.gz" + $GZIP -c /var/log/cloud-init.log > "$LOG_DST_DEVSTACK/cloud-init.log.gz" + $GZIP -c /var/log/cloud-init-output.log > "$LOG_DST_DEVSTACK/cloud-init-output.log.gz" + $GZIP -c /var/log/dmesg > "$LOG_DST_DEVSTACK/dmesg.log.gz" + $GZIP -c /var/log/kern.log > "$LOG_DST_DEVSTACK/kern.log.gz" + $GZIP -c /var/log/syslog > "$LOG_DST_DEVSTACK/syslog.log.gz" + + mkdir -p "$LOG_DST_DEVSTACK/rabbitmq" + cp /var/log/rabbitmq/* "$LOG_DST_DEVSTACK/rabbitmq" + sudo rabbitmqctl status > "$LOG_DST_DEVSTACK/rabbitmq/status.txt" 2>&1 + $GZIP $LOG_DST_DEVSTACK/rabbitmq/* + mkdir -p "$LOG_DST_DEVSTACK/openvswitch" + cp /var/log/openvswitch/* "$LOG_DST_DEVSTACK/openvswitch" + $GZIP $LOG_DST_DEVSTACK/openvswitch/* + for j in `ls -A /var/log/juju`; do + $GZIP -c /var/log/juju/$j > "$LOG_DST_DEVSTACK/$j.gz" + done +} + +function archive_devstack_configs() { + local CONFIG_DST_DEVSTACK=${1:-$LOG_DST/devstack-config} + + if [ ! -d "$CONFIG_DST_DEVSTACK" ] + then + mkdir -p "$CONFIG_DST_DEVSTACK" || emit_warning "L38: Failed to archive devstack configs" + fi + + for i in cinder glance keystone neutron nova openvswitch + do + cp -r -L "/etc/$i" "$CONFIG_DST_DEVSTACK/$i" || continue + done + for file in `find "$CONFIG_DST_DEVSTACK/$i" -type f` + do + $GZIP $file + done + + $GZIP -c /home/ubuntu/devstack/local.conf > "$CONFIG_DST_DEVSTACK/local.conf.gz" + $GZIP -c /opt/stack/tempest/etc/tempest.conf > "$CONFIG_DST_DEVSTACK/tempest.conf.gz" + df -h > "$CONFIG_DST_DEVSTACK/df.txt" 2>&1 && $GZIP "$CONFIG_DST_DEVSTACK/df.txt" + iptables-save > "$CONFIG_DST_DEVSTACK/iptables.txt" 2>&1 && $GZIP "$CONFIG_DST_DEVSTACK/iptables.txt" + dpkg-query -l > "$CONFIG_DST_DEVSTACK/dpkg-l.txt" 2>&1 && $GZIP "$CONFIG_DST_DEVSTACK/dpkg-l.txt" + pip freeze > "$CONFIG_DST_DEVSTACK/pip-freeze.txt" 2>&1 && $GZIP "$CONFIG_DST_DEVSTACK/pip-freeze.txt" + ps axwu > "$CONFIG_DST_DEVSTACK/pidstat.txt" 2>&1 && $GZIP "$CONFIG_DST_DEVSTACK/pidstat.txt" + ifconfig -a -v > "$CONFIG_DST_DEVSTACK/ifconfig.txt" 2>&1 && $GZIP "$CONFIG_DST_DEVSTACK/ifconfig.txt" + sudo ovs-vsctl -v show > "$CONFIG_DST_DEVSTACK/ovs_bridges.txt" 2>&1 && $GZIP "$CONFIG_DST_DEVSTACK/ovs_bridges.txt" +} + +function archive_tempest_files() { + local TEMPEST_LOGS="/home/ubuntu/tempest" + local LOG_DST_TEMPEST=${1:-$LOG_DST/tempest} + if [ ! -d "$LOG_DST_TEMPEST" ]; then + mkdir -p "$LOG_DST_TEMPEST" + fi + for i in `ls -A $TEMPEST_LOGS` + do + $GZIP "$TEMPEST_LOGS/$i" -c > "$LOG_DST_TEMPEST/$i.gz" || emit_error "L133: Failed to archive tempest logs" + done +} + diff --git a/scripts/logs/wsmancmd.py b/scripts/logs/wsmancmd.py new file mode 100755 index 0000000..32ca789 --- /dev/null +++ b/scripts/logs/wsmancmd.py @@ -0,0 +1,171 @@ +#!/usr/bin/python + +# Copyright 2013 Cloudbase Solutions Srl +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import getopt +import sys + +from winrm import protocol + +AUTH_BASIC = "basic" +AUTH_KERBEROS = "kerberos" +AUTH_CERTIFICATE = "certificate" + +DEFAULT_PORT_HTTP = 5985 +DEFAULT_PORT_HTTPS = 5986 + +CODEPAGE_UTF8 = 65001 + +def print_usage(): + print ("%s [-U ] [-H ] [-P ] [-s] " + "[-a ] " + "[-u ] [-p ] " + "[-c -k ] " + " [cmd_args]" % sys.argv[0]) + + +def parse_args(): + args_ok = False + auth = AUTH_BASIC + username = None + password = None + url = None + host = None + port = None + use_ssl = False + cmd = None + cert_pem = None + cert_key_pem = None + is_powershell_cmd = False + + try: + show_usage = False + opts, args = getopt.getopt(sys.argv[1:], "hsU:H:P:u:p:c:k:a:", + "powershell") + for opt, arg in opts: + if opt == "-h": + show_usage = True + if opt == "-s": + use_ssl = True + if opt == "-H": + host = arg + if opt == "-P": + port = arg + if opt == "-U": + url = arg + elif opt == "-a": + auth = arg + elif opt == "-u": + username = arg + elif opt == "-p": + password = arg + elif opt == "-c": + cert_pem = arg + elif opt == "-k": + cert_key_pem = arg + elif opt == "--powershell": + is_powershell_cmd = True + + cmd = args + + if (show_usage or not + (cmd and + (url and not host and not port and not use_ssl) or + host and ((bool(port) ^ bool(use_ssl) or + not port and not use_ssl)) and + (auth == AUTH_BASIC and username and password or + auth == AUTH_CERTIFICATE and cert_pem and cert_key_pem or + auth == AUTH_KERBEROS))): + print_usage() + else: + args_ok = True + + except getopt.GetoptError: + print_usage() + + return (args_ok, url, host, use_ssl, port, auth, username, password, + cert_pem, cert_key_pem, cmd, is_powershell_cmd) + + +def run_wsman_cmd(url, auth, username, password, cert_pem, cert_key_pem, cmd): + protocol.Protocol.DEFAULT_TIMEOUT = 3600 + + if not auth: + auth = AUTH_BASIC + + auth_transport_map = {AUTH_BASIC: 'plaintext', + AUTH_KERBEROS: 'kerberos', + AUTH_CERTIFICATE: 'ssl'} + + p = protocol.Protocol(endpoint=url, + transport=auth_transport_map[auth], + username=username, + password=password, + cert_pem=cert_pem, + cert_key_pem=cert_key_pem) + + shell_id = p.open_shell(codepage=CODEPAGE_UTF8) + + command_id = p.run_command(shell_id, cmd[0], cmd[1:]) + std_out, std_err, status_code = p.get_command_output(shell_id, command_id) + + p.cleanup_command(shell_id, command_id) + p.close_shell(shell_id) + + return (std_out, std_err, status_code) + + +def get_url(url, host, use_ssl, port): + if url: + return url + else: + if not port: + if use_ssl: + port = DEFAULT_PORT_HTTPS + else: + port = DEFAULT_PORT_HTTP + + if use_ssl: + protocol = "https" + else: + protocol = "http" + + return ("%(protocol)s://%(host)s:%(port)s/wsman" % locals()) + + +def main(): + exit_code = 1 + + (args_ok, url, host, use_ssl, port, auth, username, password, + cert_pem, cert_key_pem, cmd, is_powershell_cmd) = parse_args() + if args_ok: + url = get_url(url, host, use_ssl, port) + + if is_powershell_cmd: + cmd = ["powershell.exe", "-ExecutionPolicy", "RemoteSigned", + "-NonInteractive", "-Command"] + cmd + + std_out, std_err, exit_code = run_wsman_cmd(url, auth, username, + password, cert_pem, + cert_key_pem, cmd) + sys.stdout.write(std_out) + sys.stderr.write(std_err) + + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/templates/bundle.template b/templates/bundle.template new file mode 100644 index 0000000..bd052d1 --- /dev/null +++ b/templates/bundle.template @@ -0,0 +1,77 @@ +relations: + - ["devstack-${UUID}", "hyperv-${UUID}"] + - ["active-directory", "hyperv-${UUID}"] + +services: + active-directory: + charm: cs:~cloudbaseit/active-directory-5 + num_units: 1 + series: win2016 + constraints: "tags=$ADTAGS" + options: + administrator-password: ${ad_password} + safe-mode-password: ${ad_password} + domain-name: ${domain_name} + domain-user: ${domain_user} + domain-user-password: ${ad_password} + devstack-${UUID}: + charm: /home/ubuntu/charms/ubuntu/devstack + num_units: 1 + constraints: "tags=$TAGS" + series: xenial + options: + disabled-services: horizon n-novnc n-net n-cpu ceilometer-acompute s-proxy s-object s-container s-account + enabled-services: rabbit mysql key n-api n-crt n-obj n-cond n-sch n-cauth + neutron q-svc q-agt q-dhcp q-l3 q-meta q-lbaas q-fwaas q-metering q-vpn + g-api g-reg cinder c-api c-vol c-sch heat h-api h-api-cfn h-api-cw h-eng tempest + extra-packages: build-essential libpython-all-dev python-all python-dev python3-all + python3-dev g++ g++-4.8 pkg-config libvirt-dev smbclient libxml2-dev libxslt1-dev zlib1g-dev + extra-python-packages: "git+https://github.com/petrutlucian94/pywinrm.git lxml==3.6.4" + heat-image-url: ${heat_image_url} + test-image-url: ${test_image_url} + ml2-mechanism: openvswitch + tenant-network-type: vxlan + enable-tunneling: True + enable-live-migration: True + ntp-server: pool.ntp.org + vlan-range: 2500:2550 + nameservers: 10.20.1.37 8.8.8.8 + enable-vlans: False + scenario-img: ${scenario_img} + cherry-picks: ${devstack_cherry_picks} + pypi-mirror: http://10.20.1.8:8080/cloudbase/CI/+simple/ + data-port: ${data_port} + external-port: ${external_port} + zuul-branch: ${ZUUL_BRANCH} + zuul-change: "${ZUUL_CHANGE}" + zuul-project: ${ZUUL_PROJECT} + zuul-ref: ${ZUUL_REF} + zuul-url: ${ZUUL_URL} + prep-project: ${prep_project} + pip-version: "pip==8.1.1" + hyperv-${UUID}: + charm: /home/ubuntu/charms/windows/hyper-v-ci + num_units: 2 + series: win2016 + constraints: "tags=$TAGS" + options: + administrator-password: ${win_password} + vmswitch-management: ${vmswitch_management} + ovs-installer-url: ${ovs_installer} + ovs-certificate-url: ${ovs_certificate} + test-signing: ${test_signing} + network-type: ovs + os-data-network: ${os_data_network} + extra-python-packages: ${hv_extra_python_packages} + post-python-packages: ${post_python_packages} + git-user-email: "mcapsali@gmail.com" + git-user-name: "capsali" + cherry-picks: "${hyperv_cherry_picks}" + pypi-mirror: http://10.20.1.8:8080/cloudbase/CI/+simple/ + data-port: ${data_port} + zuul-branch: ${ZUUL_BRANCH} + zuul-change: "${ZUUL_CHANGE}" + zuul-project: ${ZUUL_PROJECT} + zuul-ref: ${ZUUL_REF} + zuul-url: ${ZUUL_URL} + pip-version: "pip==8.1.1" diff --git a/templates/eventlog_css.txt b/templates/eventlog_css.txt new file mode 100644 index 0000000..e05d6ac --- /dev/null +++ b/templates/eventlog_css.txt @@ -0,0 +1,44 @@ + +body { +color: black; +} +#table{ +-moz-user-select:none; +cursor:default; +font-family:Arial,sans-serif; +font-size:11px; +width:100%; +} +#table th#hoverTH { +background:#555555 none repeat scroll 0%; +} +#table.sortable th { +background:#666666 none repeat scroll 0%; +border-bottom:1px solid #444444; +border-left:1px solid #555555; +border-top:1px solid #444444; +color:#FFFFFF; +cursor:pointer; +padding:4px 0pt 4px 9px; +text-align:left; +} + +table tr:nth-child(2n) td { +background:#EDF3FE none repeat scroll 0%; +border-bottom:1px solid #E8F0FF; +border-right:1px solid #FFFFFF; +border-top:1px solid #E8F0FF; +} +table tr:nth-child(2n+1) td { +background:#FFFFFF none repeat scroll 0%; +border-bottom:1px solid #FFFFFF; +border-right:1px solid #FFFFFF; +border-top:1px solid #FFFFFF; +} + +#table tr td { +border-bottom:1px solid #FFFFFF; +border-left:1px solid #D9D9D9; +border-top:1px solid #FFFFFF; +padding:3px 8px; +} diff --git a/templates/eventlog_js.txt b/templates/eventlog_js.txt new file mode 100644 index 0000000..8f07716 --- /dev/null +++ b/templates/eventlog_js.txt @@ -0,0 +1,477 @@ + +var stIsIE = /*@cc_on!@*/false; + +sorttable = { + init: function() { + // quit if this function has already been called + if (arguments.callee.done) return; + // flag this function so we don't do the same thing twice + arguments.callee.done = true; + // kill the timer + if (_timer) clearInterval(_timer); + + if (!document.createElement || !document.getElementsByTagName) return; + + sorttable.DATE_RE = /^(\d\d?)[\/\.-](\d\d?)[\/\.-]((\d\d)?\d\d)$/; + + forEach(document.getElementsByTagName('table'), function(table) { + if (table.className.search(/\bsortable\b/) != -1) { + sorttable.makeSortable(table); + } + }); + + }, + + makeSortable: function(table) { + if (table.getElementsByTagName('thead').length == 0) { + // table doesn't have a tHead. Since it should have, create one and + // put the first table row in it. + the = document.createElement('thead'); + the.appendChild(table.rows[0]); + table.insertBefore(the,table.firstChild); + } + // Safari doesn't support table.tHead, sigh + if (table.tHead == null) table.tHead = table.getElementsByTagName('thead')[0]; + + if (table.tHead.rows.length != 1) return; // can't cope with two header rows + + // Sorttable v1 put rows with a class of "sortbottom" at the bottom (as + // "total" rows, for example). This is B&R, since what you're supposed + // to do is put them in a tfoot. So, if there are sortbottom rows, + // for backwards compatibility, move them to tfoot (creating it if needed). + sortbottomrows = []; + for (var i=0; i5' : ' ▴'; + this.appendChild(sortrevind); + return; + } + if (this.className.search(/\bsorttable_sorted_reverse\b/) != -1) { + // if we're already sorted by this column in reverse, just + // re-reverse the table, which is quicker + sorttable.reverse(this.sorttable_tbody); + this.className = this.className.replace('sorttable_sorted_reverse', + 'sorttable_sorted'); + this.removeChild(document.getElementById('sorttable_sortrevind')); + sortfwdind = document.createElement('span'); + sortfwdind.id = "sorttable_sortfwdind"; + sortfwdind.innerHTML = stIsIE ? ' 6' : ' ▾'; + this.appendChild(sortfwdind); + return; + } + + // remove sorttable_sorted classes + theadrow = this.parentNode; + forEach(theadrow.childNodes, function(cell) { + if (cell.nodeType == 1) { // an element + cell.className = cell.className.replace('sorttable_sorted_reverse',''); + cell.className = cell.className.replace('sorttable_sorted',''); + } + }); + sortfwdind = document.getElementById('sorttable_sortfwdind'); + if (sortfwdind) { sortfwdind.parentNode.removeChild(sortfwdind); } + sortrevind = document.getElementById('sorttable_sortrevind'); + if (sortrevind) { sortrevind.parentNode.removeChild(sortrevind); } + + this.className += ' sorttable_sorted'; + sortfwdind = document.createElement('span'); + sortfwdind.id = "sorttable_sortfwdind"; + sortfwdind.innerHTML = stIsIE ? ' 6' : ' ▾'; + this.appendChild(sortfwdind); + + // build an array to sort. This is a Schwartzian transform thing, + // i.e., we "decorate" each row with the actual sort key, + // sort based on the sort keys, and then put the rows back in order + // which is a lot faster because you only do getInnerText once per row + row_array = []; + col = this.sorttable_columnindex; + rows = this.sorttable_tbody.rows; + for (var j=0; j$]?[\d,.]+%?$/)) { + return sorttable.sort_numeric; + } + // check for a date: dd/mm/yyyy or dd/mm/yy + // can have / or . or - as separator + // can be mm/dd as well + possdate = text.match(sorttable.DATE_RE) + if (possdate) { + // looks like a date + first = parseInt(possdate[1]); + second = parseInt(possdate[2]); + if (first > 12) { + // definitely dd/mm + return sorttable.sort_ddmm; + } else if (second > 12) { + return sorttable.sort_mmdd; + } else { + // looks like a date, but we can't tell which, so assume + // that it's dd/mm (English imperialism!) and keep looking + sortfn = sorttable.sort_ddmm; + } + } + } + } + return sortfn; + }, + + getInnerText: function(node) { + // gets the text we want to use for sorting for a cell. + // strips leading and trailing whitespace. + // this is *not* a generic getInnerText function; it's special to sorttable. + // for example, you can override the cell text with a customkey attribute. + // it also gets .value for fields. + + if (!node) return ""; + + hasInputs = (typeof node.getElementsByTagName == 'function') && + node.getElementsByTagName('input').length; + + if (node.getAttribute("sorttable_customkey") != null) { + return node.getAttribute("sorttable_customkey"); + } + else if (typeof node.textContent != 'undefined' && !hasInputs) { + return node.textContent.replace(/^\s+|\s+$/g, ''); + } + else if (typeof node.innerText != 'undefined' && !hasInputs) { + return node.innerText.replace(/^\s+|\s+$/g, ''); + } + else if (typeof node.text != 'undefined' && !hasInputs) { + return node.text.replace(/^\s+|\s+$/g, ''); + } + else { + switch (node.nodeType) { + case 3: + if (node.nodeName.toLowerCase() == 'input') { + return node.value.replace(/^\s+|\s+$/g, ''); + } + case 4: + return node.nodeValue.replace(/^\s+|\s+$/g, ''); + break; + case 1: + case 11: + var innerText = ''; + for (var i = 0; i < node.childNodes.length; i++) { + innerText += sorttable.getInnerText(node.childNodes[i]); + } + return innerText.replace(/^\s+|\s+$/g, ''); + break; + default: + return ''; + } + } + }, + + reverse: function(tbody) { + // reverse the rows in a tbody + newrows = []; + for (var i=0; i=0; i--) { + tbody.appendChild(newrows[i]); + } + delete newrows; + }, + + /* sort functions + each sort function takes two parameters, a and b + you are comparing a[0] and b[0] */ + sort_numeric: function(a,b) { + aa = parseFloat(a[0].replace(/[^0-9.-]/g,'')); + if (isNaN(aa)) aa = 0; + bb = parseFloat(b[0].replace(/[^0-9.-]/g,'')); + if (isNaN(bb)) bb = 0; + return aa-bb; + }, + sort_alpha: function(a,b) { + if (a[0]==b[0]) return 0; + if (a[0] 0 ) { + var q = list[i]; list[i] = list[i+1]; list[i+1] = q; + swap = true; + } + } // for + t--; + + if (!swap) break; + + for(var i = t; i > b; --i) { + if ( comp_func(list[i], list[i-1]) < 0 ) { + var q = list[i]; list[i] = list[i-1]; list[i-1] = q; + swap = true; + } + } // for + b++; + + } // while(swap) + } +} + +/* ****************************************************************** + Supporting functions: bundled here to avoid depending on a library + ****************************************************************** */ + +// Dean Edwards/Matthias Miller/John Resig + +/* for Mozilla/Opera9 */ +if (document.addEventListener) { + document.addEventListener("DOMContentLoaded", sorttable.init, false); +} + +/* for Internet Explorer */ +/*@cc_on @*/ +/*@if (@_win32) + document.write("