add 3rd party integration tests

Author: OriHoch

README.md

@@ -93,21 +93,69 @@ Log-in to Rancher and create a new workload:
   * `CLOUDCLI_API_SERVER` = `https://api.example.com`
   * `CLOUDCLI_PROVIDER` = `proxy`
 
-## Running libcloud tests
+## Running 3rd party integration tests
 
 Create a Python virtualenv
 
 ```
 mkdir tests/venv
 python3 -m virtualenv -p python3 tests/venv
-tests/venv/bin/pip install https://github.com/OriHoch/libcloud/archive/kamatera-compute.zip
 ```
 
-Run the cloudcli-server on localhost and then run the tests:
+Start cloudcli-server localy
+
+#### Running Libcloud tests
+
+Install Libcloud development version
+
+```
+tests/venv/bin/python3 -m pip install -e git+https://git-wip-us.apache.org/repos/asf/libcloud.git@trunk#egg=apache-libcloud
+```
+
+Run the tests:
 
 ```
 export API_CLIENT_ID=XXX
 export API_SECRET=YYY
 export API_SERVER=localhost
-tests/venv/bin/python tests/libcloud_create_node.py && tests/venv/bin/python tests/libcloud_node_operations.py 
+tests/venv/bin/python3 tests/libcloud_create_node.py && tests/venv/bin/python3 tests/libcloud_node_operations.py 
+```
+
+#### Running Ansible tests
+
+[Install latest Ansible](https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html)
+
+Verify version: `ansible-galaxy --version | grep "ansible-galaxy 2.9."`
+
+Install the Kamatera collection:
+
+```
+ansible-galaxy collection install --force https://github.com/OriHoch/ansible-collection-kamatera/releases/download/v0.0.1/kamatera-kamatera-0.0.0.tar.gz
+```
+
+Run the tests:
+
+```
+export KAMATERA_API_CLIENT_ID=
+export KAMATERA_API_SECRET=
+export KAMATERA_API_URL=http://localhost:8000
+tests/venv/bin/python3 tests/ansible.py
+```
+
+#### Running Salt tests
+
+Install Salt with Kamatera
+
+```
+tests/venv/bin/python3 -m pip install https://github.com/OriHoch/salt/archive/kamatera-cloud.zip
+```
+
+Run the tests:
+
+```
+export KAMATERA_API_CLIENT_ID=
+export KAMATERA_API_SECRET=
+export KAMATERA_API_URL=http://localhost:8000
+export SALT_BIN_DIR=`pwd`/tests/venv/bin
+tests/venv/bin/python3 tests/salt.py
 ```

tests/ansible.py

@@ -0,0 +1,51 @@
+import subprocess
+import tempfile
+import json
+import os
+
+
+testsdir = os.path.dirname(__file__)
+outputdir = os.path.abspath(os.path.join(testsdir, "output"))
+os.makedirs(outputdir, exist_ok=True)
+
+subprocess.check_call(["ansible-playbook", "tests/ansible_playbooks/compute_datacenters_playbook.yml", "-e", "output_dir=%s" % outputdir])
+
+with open(os.path.join(outputdir, 'kamatera_datacenters.json')) as f:
+  datacenters = json.load(f)
+assert set(datacenters[0].keys()) == set(['category', 'subCategory', 'name', 'id']), set(datacenters[0].keys())
+for k,v in datacenters[0].items():
+  assert len(v) > 1, '%s=%s' % (k, v)
+
+subprocess.check_call(["ansible-playbook", "tests/ansible_playbooks/compute_options_playbook.yml", "-e", "output_dir=%s" % outputdir, "-e", "datacenter=EU-LO"])
+
+with open(os.path.join(outputdir, 'kamatera_datacenter_capabilities.json')) as f:
+  capabilities = json.load(f)
+assert set(capabilities.keys()) == set(['cpuTypes', 'defaultMonthlyTrafficPackage', 'diskSizeGB', 'monthlyTrafficPackage']), set(capabilities.keys())
+for k,v in capabilities.items():
+  assert len(v) > 0, '%s=%s' % (k,v)
+
+with open(os.path.join(outputdir, 'kamatera_datacenter_images.json')) as f:
+  images = json.load(f)
+assert set(images[0].keys()) == set(['datacenter', 'code', 'name', 'osDiskSizeGB', 'ramMBMin', 'os', 'id']), set(images[0].keys())
+for k,v in images[0].items():
+  if k == 'osDiskSizeGB':
+    assert v > 0, '%s=%s' % (k,v)
+  else:
+    assert len(v) > 0, '%s=%s' % (k,v)
+for image in images:
+  if image['os'] == 'Ubuntu' and image['code'] == '18.04 64bit':
+    image_id = image['id']
+
+create_args = [
+    "-e", "datacenter=EU-LO",
+    "-e", "image=%s" % image_id,
+    "-e", "cpu_type=%s" % capabilities['cpuTypes'][0]['id'],
+    "-e", "cpu_cores=%s" % capabilities['cpuTypes'][0]['cpuCores'][2],
+    "-e", "ram_mb=%s" % capabilities['cpuTypes'][0]['ramMB'][4],
+    "-e", "disk_size_gb=%s" % capabilities['diskSizeGB'][4],
+    "-e", "disk_size_2_gb=%s" % capabilities['diskSizeGB'][2]
+]
+
+subprocess.check_call(["ansible-playbook", "tests/ansible_playbooks/compute_playbook.yml", "-e", "output_dir=%s" % outputdir, "-e", "server_name=ansibletest", *create_args])
+
+print("Great Success!")

tests/ansible_playbooks/compute_datacenters_playbook.yml

@@ -0,0 +1,8 @@
+- name: Get capabilities
+  hosts: localhost
+  tasks:
+    - kamatera.kamatera.kamatera_compute_options: {}
+      register: datacenters
+    - copy:
+        content: "{{ datacenters.datacenters | to_nice_json }}"
+        dest: "{{ output_dir | default('/tmp') }}/kamatera_datacenters.json"

tests/ansible_playbooks/compute_options_playbook.yml

@@ -0,0 +1,12 @@
+- name: Get capabilities
+  hosts: localhost
+  tasks:
+    - kamatera.kamatera.kamatera_compute_options:
+        datacenter: "{{ datacenter }}"
+      register: capabilities
+    - copy:
+        content: "{{ capabilities.images | to_nice_json }}"
+        dest: "{{ output_dir | default('/tmp') }}/kamatera_datacenter_images.json"
+    - copy:
+        content: "{{ capabilities.capabilities | to_nice_json }}"
+        dest: "{{ output_dir | default('/tmp') }}/kamatera_datacenter_capabilities.json"

tests/ansible_playbooks/compute_playbook.yml

@@ -0,0 +1,46 @@
+- name: Provisioning and operations example
+  hosts: localhost
+  tasks:
+    - kamatera.kamatera.kamatera_compute:
+        server_names: ['{{ server_name }}_1','{{ server_name }}_2']
+        state: present
+        datacenter: "{{ datacenter }}"
+        image: "{{ image }}"
+        cpu_type: "{{ cpu_type }}"
+        cpu_cores: "{{ cpu_cores }}"
+        ram_mb: "{{ ram_mb }}"
+        disk_size_gb: "{{ disk_size_gb }}"
+        extra_disk_sizes_gb:
+          - "{{ disk_size_2_gb }}"
+      register: res
+    - copy:
+        content: "{{ res | to_nice_json }}"
+        dest: "{{ output_dir | default('/tmp') }}/kamatera_create_servers.json"
+    - kamatera.kamatera.kamatera_compute:
+        server_names: '{{ res.server_names }}'
+        state: restarted
+      register: res
+    - copy:
+        content: "{{ res | to_nice_json }}"
+        dest: "{{ output_dir | default('/tmp') }}/kamatera_restart_servers.json"
+    - kamatera.kamatera.kamatera_compute:
+        server_names: '{{ res.server_names }}'
+        state: stopped
+      register: res
+    - copy:
+        content: "{{ res | to_nice_json }}"
+        dest: "{{ output_dir | default('/tmp') }}/kamatera_stop_servers.json"
+    - kamatera.kamatera.kamatera_compute:
+        server_names: '{{ res.server_names }}'
+        state: running
+      register: res
+    - copy:
+        content: "{{ res | to_nice_json }}"
+        dest: "{{ output_dir | default('/tmp') }}/kamatera_start_servers.json"
+    - kamatera.kamatera.kamatera_compute:
+        server_names: '{{ res.server_names }}'
+        state: absent
+      register: res
+    - copy:
+        content: "{{ res | to_nice_json }}"
+        dest: "{{ output_dir | default('/tmp') }}/kamatera_delete_servers.json"

tests/salt.py

@@ -0,0 +1,121 @@
+import json
+import os
+import subprocess
+
+testsdir = os.path.dirname(__file__)
+outputdir = os.path.abspath(os.path.join(testsdir, "output"))
+os.makedirs(outputdir, exist_ok=True)
+
+os.makedirs("/etc/salt/cloud.providers.d", exist_ok=True)
+if os.path.exists("/etc/salt/cloud.providers.d/cloudcli-server-test-kamatera.conf"):
+    os.unlink("/etc/salt/cloud.providers.d/cloudcli-server-test-kamatera.conf")
+with open("/etc/salt/cloud.providers.d/cloudcli-server-test-kamatera.conf", "w") as f:
+    f.write("""
+cloudcli-server-test-kamatera:
+    driver: kamatera
+    api_client_id: {KAMATERA_API_CLIENT_ID}
+    api_secret: {KAMATERA_API_SECRET}
+    api_url: {KAMATERA_API_URL}
+""".format(**os.environ))
+
+print("listing locations...")
+with open(os.path.join(outputdir, "salt-locations.json"), "wb") as f:
+    f.write(subprocess.check_output(["%s/salt-cloud" % os.environ["SALT_BIN_DIR"], "--out=json", "--list-locations", "cloudcli-server-test-kamatera"]))
+
+with open(os.path.join(outputdir, "salt-locations.json")) as f:
+    locations = json.load(f)["cloudcli-server-test-kamatera"]["kamatera"]
+assert locations["IL"] == "Rosh Haayin, Israel (Middle East)"
+assert len(locations) > 10
+print("OK")
+
+print("listing sizes...")
+with open(os.path.join(outputdir, "salt-sizes.json"), "wb") as f:
+    f.write(subprocess.check_output(["%s/salt-cloud" % os.environ["SALT_BIN_DIR"], "--out=json", "--location=IL", "--list-sizes", "cloudcli-server-test-kamatera"]))
+
+with open(os.path.join(outputdir, "salt-sizes.json")) as f:
+    sizes = json.load(f)["cloudcli-server-test-kamatera"]["kamatera"]
+assert len(sizes) > 3
+assert sizes["B"]["name"] == "Type B - General Purpose"
+print("OK")
+
+print("listing available server options...")
+with open(os.path.join(outputdir, "salt-server-options.json"), "wb") as f:
+    f.write(subprocess.check_output(["%s/salt-cloud" % os.environ["SALT_BIN_DIR"], "--out=json", "--location=IL", "-f", "avail_server_options", "cloudcli-server-test-kamatera"]))
+
+with open(os.path.join(outputdir, "salt-server-options.json")) as f:
+    server_options = json.load(f)["cloudcli-server-test-kamatera"]["kamatera"]
+assert len(server_options) > 1
+assert server_options["monthlyTrafficPackage"]["t5000"] == "5000GB/month on 10Gbit/sec port"
+print("OK")
+
+print("listing images...")
+with open(os.path.join(outputdir, "salt-images.json"), "wb") as f:
+    f.write(subprocess.check_output(["%s/salt-cloud" % os.environ["SALT_BIN_DIR"], "--out=json", "--location=IL", "--list-images", "cloudcli-server-test-kamatera"]))
+
+with open(os.path.join(outputdir, "salt-images.json")) as f:
+    images = json.load(f)["cloudcli-server-test-kamatera"]["kamatera"]
+assert len(images) > 10
+assert images["IL:6000C29a5a7220dcf84716e7bba74215"] == "Ubuntu Server version 18.04 LTS (bionic) 64-bit"
+print("OK")
+
+os.makedirs("/etc/salt/cloud.profiles.d", exist_ok=True)
+if os.path.exists("/etc/salt/cloud.profiles.d/cloudcli-server-test.conf"):
+    os.unlink("/etc/salt/cloud.profiles.d/cloudcli-server-test.conf")
+with open("/etc/salt/cloud.profiles.d/cloudcli-server-test.conf", "w") as f:
+    f.write("""
+cloudcli-server-test:
+    provider: cloudcli-server-test-kamatera
+    location: IL
+    cpu_type: B
+    cpu_cores: 2
+    ram_mb: 2048
+    disk_size_gb: 20
+    extra_disk_sizes_gb:
+    - 15
+    billing_cycle: hourly
+    monthly_traffic_package: t5000
+    image: IL:6000C29a5a7220dcf84716e7bba74215
+    networks:
+    - name: wan
+      ip: auto
+    daily_backup: false
+    managed: false
+""")
+
+print("creating server...")
+with open(os.path.join(outputdir, "salt-create-server.json"), "wb") as f:
+    f.write(subprocess.check_output(["%s/salt-cloud" % os.environ["SALT_BIN_DIR"], "--out=json", "-p", "cloudcli-server-test", "salttest"]))
+
+with open(os.path.join(outputdir, "salt-create-server.json")) as f:
+    jsonlines = None
+    for line in f.readlines():
+        if line.strip() == "{":
+            jsonlines = ["{"]
+        elif jsonlines is not None:
+            jsonlines.append(line.strip())
+    created_server = json.loads("\n".join(jsonlines))["salttest"]
+assert created_server["deployed"] == True
+created_server_name = created_server["name"]
+print("created_server_name = " + created_server_name)
+print("OK")
+
+print("listing servers..")
+with open(os.path.join(outputdir, "salt-query.json"), "wb") as f:
+    f.write(subprocess.check_output(["%s/salt-cloud" % os.environ["SALT_BIN_DIR"], "--out=json", "-Q"]))
+
+with open(os.path.join(outputdir, "salt-query.json")) as f:
+    servers = json.load(f)["cloudcli-server-test-kamatera"]["kamatera"]
+assert len(servers) > 0
+assert servers[created_server_name]["state"] == "running"
+print("OK")
+
+print("deleting server...")
+with open(os.path.join(outputdir, "salt-delete.json"), "wb") as f:
+    f.write(subprocess.check_output(["%s/salt-cloud" % os.environ["SALT_BIN_DIR"], "--out=json", "-y", "-d", created_server_name]))
+
+with open(os.path.join(outputdir, "salt-delete.json")) as f:
+    servers = json.load(f)["cloudcli-server-test-kamatera"]["kamatera"]
+assert len(servers) > 0
+assert servers[created_server_name]["state"] == "destroyed"
+assert servers[created_server_name]["success"] == True
+print("OK")