chore: Handle Data notifications example

examples/create-data-object.py

@@ -6,7 +6,9 @@
 
 from nuvla.api import Api as Nuvla
 from nuvla.api.resources.data import DataObjectS3
+from nuvla.api.resources.data import DataRecord
 from nuvla.api.resources.user import User
+from nuvla.api.api import NuvlaError
 from utils import nuvla_conf_user_pass
 
 
@@ -24,86 +26,244 @@
 # Login to Nuvla.
 #
 user_api = User(nuvla)
-user_api.login_password(username, password)
+# user_api.login_password(username, password)
+user_api.login(username, password)
+
+# REQUIRED: ID of S3 credential in Nuvla.
+# either we already know this and it is hardcoded below
+# s3_cred_id = 'credential/362e6088-394d-4de9-9667-8e1a40136aa6'
+# s3_cred_id = 'credential/0f70f0fb-c226-4fc2-a98b-61f0646813d7'
+
+# OR we can get the credentials from a login if we want...
+# as in below
+creds = nuvla.get('credential')
+s3_cred_id = None
+for cred in creds.data['resources']:
+    try:
+        parent = nuvla.get(cred['parent'])
+        if "s3" in str(parent.data['subtype']) and "Exoscale" in str(parent.data['name']):
+            s3_cred_id = str(cred['id'])
+            print(f"Found Exoscale S3 credential: {s3_cred_id}")
+    except (NuvlaError, KeyError):
+        pass
+# if we didn't find the credential, we should exit
+if not s3_cred_id:
+    print("No Exoscale S3 credential found. Exiting.")
+    exit(1)
 
 #
 # Set correct values for variable below.
 #
-
-# REQUIRED: ID of S3 credential in Nuvla.
-s3_cred_id = 'credential/362e6088-394d-4de9-9667-8e1a40136aa6'
-
 # Bucket to store the test objects.
-bucket = 'cloud.animals'
-
+# bucket will be created if it doesn't exist?
+#
+bucket = 'bucket-for-extract-project-demo-purposes-002'
+# can we get the bucket name from the credential?
 
 #
 # data-object API object.
 #
 data_obj_api = DataObjectS3(nuvla)
 
+def generate_event(local_event_record: dict) -> str:
+    '''
+    Generate an event in Nuvla
+    :param local_event_record: dict - event record
+    :return: str - event id
+    '''
+    try:
+        cimi_event = nuvla.add('event', local_event_record)
+    except NuvlaError as e:
+        print(f'Failed to create event: {e}')
+        return None
+    event_id = cimi_event.data['resource-id']
+    print('created event:', event_id, "with data:")
+    pp(cimi_event.data)
+    return event_id
+
+# should generate an event here as a data record was create
+# maybe this should be combined?
+
+def create_data_record(local_data_record: dict) -> str:
+    '''
+    Create a data record in Nuvla
+    :param local_data_record: dict - data record
+    :return: str - data record id
+    if the data record already exists, it will be deleted and recreated
+    possibly should be a recursive function
+    '''
+    try:
+        dr_id_created = dr_api.add(local_data_record)
+    except NuvlaError as e:
+        print(f'Failed to create data record: {e}')
+        for arg in e.args:
+            if 'data-record' in arg:
+                dr_id_created = arg.split(' ')[-1]
+                print(f'Data record already exists, data record id: {dr_id_created}. Deleting and recreating.')
+                deleted_dr_id = dr_api.delete(dr_id_created)
+                if deleted_dr_id:
+                    print('Deleted data record id:', deleted_dr_id)
+                    dr_id_created = create_data_record(local_data_record)
+                    break
+
+        return dr_id_created
+
+    return dr_id_created
+
+def create_data_object(local_content, local_bucket: str, local_object_path: str, local_s3_cred_id: str, local_content_type: str, local_tags) -> str:
+    try:
+        object_id = data_obj_api.create(local_content, local_bucket, 
+                                        local_object_path, local_s3_cred_id,
+                                        content_type=local_content_type,
+                                        tags=local_tags)
+    except NuvlaError as e:
+        print(f'Failed to create data object: {e}') 
+        for arg in e.args:
+            if 'data-object' in arg:
+                object_id = arg.split(' ')[-1]
+                print(f'\n\nObject already exists, object id: {object_id}. Deleting and recreating.')
+                deleted_object_id = data_obj_api.delete(object_id)
+                if deleted_object_id:
+                    print(f'Deleted object id: {deleted_object_id}')
+                    object_id = create_data_object(local_content, local_bucket, local_object_path, local_s3_cred_id, local_content_type, local_tags)
+                    print(f'Created object id: {object_id}\n\n')
+                    break
+        return object_id
+    return object_id
 
 #
 # Create binary object on S3 and register it in Nuvla.
 #
+print("\nCreating binary object\n")
+# here is the file that actually gets uploaded to S3
 content = open('data/african-lion.jpg', 'rb').read()
+# here is the path where the file will be stored in S3
+object_path = 'africa/african-lion3.jpg'
+# here is the content type of the file
+content_type = 'image/scary-lions'
 
-object_path = 'africa/african-lion.jpg'
-content_type = 'image/jpg'
-
-# Add object.
-# Bucket will be created if it doesn't exit.
-bin_object_id = data_obj_api.create(content, bucket, object_path, s3_cred_id,
-                                    content_type=content_type,
-                                    tags=["zoo", "africa", "lion"])
-print('.png object id:', bin_object_id)
 
+# Add binary object.
+# Bucket will be created if it doesn't exist.
+# tags can be added to the object
+tags = ["zoo", "africa", "lion"]
+bin_object_id = create_data_object(content, bucket, \
+                              object_path, s3_cred_id, \
+                              content_type, tags)
 # Get object document.
+# print out the object document
 obj_doc = data_obj_api.get(bin_object_id)
 pp(obj_doc)
 
+# create a data record for the object
+dr_api = DataRecord(nuvla)
+# find the S3 infrastructure service
+# either it's known and hardcoded or we can search for it using the credential
+# s3_infra_service_id = 'infrastructure-service/09b08b49-2408-4b80-a7cc-73f420903fd5'
+cred_doc = nuvla.get(s3_cred_id) 
+s3_infra_service_id=cred_doc.data['parent']
+if not s3_infra_service_id:
+    print('No infrastructure service found for the credential. Exiting.')
+    exit(1)
+
+data_record = {
+    "infrastructure-service": s3_infra_service_id,
+
+    "description": "Lions in Africa",
+    "name": "African Lion",
+    "object": object_path,
+    "bucket": bucket,
+    "content-type": "animals/lion",
+    "bytes": len(content),
+    "platform": "S3",
+    "tags": ["zoo", "africa", "lion", "whatevs"],
+    "another-field": "another-value",
+    "yet-another-field": "yet-another-value",
+    "and-another-field": "and-another-value",   
+}
+
+# dr_id = dr_api.add(data_record)
+print('\n\ncreating data record\n\n')
+dr_id = create_data_record(data_record)
+print('created data record:', dr_id)
+pp(dr_api.get(dr_id))
+
+event_record = {
+    "category": "user",
+    "content": {
+      "resource": {
+        "href": dr_id, # this must be the data record id
+        "content": {
+          "content-type": content_type,
+        },
+      },
+      "state": "created",
+    },
+    "severity": "medium",
+    "tags": data_record['tags'],
+}
+
+event = generate_event(event_record)
+event_structure = nuvla.get(event).data
+pp(event_structure)
+
 # Download object and store it locally to a file.
-local_fn = './data/local-african-lion.jpg'
-data_obj_api.get_to_file(bin_object_id, local_fn)
+local_filename = './data/local-african-lion.jpg'
+data_obj_api.get_to_file(bin_object_id, local_filename)
 # Verify checksum.
 assert hashlib.md5(content).hexdigest() \
-       == hashlib.md5(open(local_fn, 'rb').read()).hexdigest()
-os.unlink(local_fn)
+       == hashlib.md5(open(local_filename, 'rb').read()).hexdigest()
+os.unlink(local_filename)
 
+# get a list of data records
+records = dr_api.id_by_name('African Lion', filter=f"tags='lion'")
+# pp(records)
+for record in records:
+    # pp(dr_api.get(record))
+    try:
+        dr_api.get(record)['content']['resource']['href']
+        print(f'New style data {record} record, probably should NOT delete it.')
+    except KeyError:
+        # print(f'Old style data {record} record, could delete it.') 
+        print(f'Deleted old style data record:', dr_api.delete(record))      
+# exit(1)
 
-#
-# Create text object on S3 and register it in Nuvla.
+# Create a text object on S3 and register it in Nuvla.
 #
 content = open('./data/african-lion.txt', 'r').read()
 
-object_path = 'africa/african-lion.txt'
+object_path = 'africa/african-lion-zebra.txt'
 content_type = 'plain/text'
 
-# Add object.
-# Bucket will be created if doesn't exit.
-str_object_id = data_obj_api.create(content, bucket, object_path, s3_cred_id,
-                                    content_type=content_type,
-                                    tags=["zoo", "africa", "lion"])
-print('.txt object id:', str_object_id)
+# Add text object.
+# Bucket will be created if doesn't exist.
+
+print("\nCreating text object\n")
+str_object_id = create_data_object(content, bucket, \
+                              object_path, s3_cred_id, \
+                              content_type, tags)
 
 # Get object document.
 obj_doc = data_obj_api.get(str_object_id)
 pp(obj_doc)
 
 # Download object and store it locally to a file.
-local_fn = './data/local-african-lion.txt'
-data_obj_api.get_to_file(str_object_id, local_fn)
+local_filename = './data/local-african-lion.txt'
+data_obj_api.get_to_file(str_object_id, local_filename)
 # Verify checksum.
 assert hashlib.md5(content.encode()).hexdigest() \
-       == hashlib.md5(open(local_fn, 'r').read().encode()).hexdigest()
-os.unlink(local_fn)
+       == hashlib.md5(open(local_filename, 'r').read().encode()).hexdigest()
+os.unlink(local_filename)
 
+# here we could do the same data_record and event creation as above
+# but we won't for now
 
 #
 # Delete object from Nuvla and S3. Bucket gets deleted as well if it's empty.
+# these are commented out to avoid deleting the objects.
 #
-assert bin_object_id == data_obj_api.delete(bin_object_id)
-assert str_object_id == data_obj_api.delete(str_object_id)
+# assert bin_object_id == data_obj_api.delete(bin_object_id)
+# assert str_object_id == data_obj_api.delete(str_object_id)
 
 # Logs out the user.
 user_api.logout()

examples/create-data-record.py

@@ -1,5 +1,7 @@
 #!/usr/bin/env python3
 
+import random
+from pprint import pprint as pp
 from nuvla.api import Api as Nuvla
 from nuvla.api.resources.data import DataRecord
 from nuvla.api.resources.user import User
@@ -20,7 +22,7 @@
 # Login to Nuvla.
 #
 user_api = User(nuvla)
-user_api.login_password(username, password)
+user_api.login(username, password)
 
 #
 # Data record API object.
@@ -41,28 +43,38 @@
 #
 # Add data record that describes an object on S3.
 #
-s3_infra_service_id = 'infrastructure-service/771fa0f1-a38d-400b-bad4-3f7600f069af'
+# s3_infra_service_id = 'infrastructure-service/09b08b49-2408-4b80-a7cc-73f420903fd5'
+cred_doc = nuvla.get(s3_cred_id) 
+print('credential doc:', cred_doc.data['parent'])
+s3_infra_service_id=cred_doc.data['parent']
+if not s3_infra_service_id:
+    print('No infrastructure service found for the credential. Exiting.')
+    exit(1)
+
+bin_object_id = 'data-object/1-2-3-4-5'
+content_type = 'animals/african-lion'
+
 data_record = {
     "infrastructure-service": s3_infra_service_id,
 
     "description": "Lions in Africa",
     "name": "African Lion",
-
     "object": "african-lion.jpg",
     "bucket": "cloud.animals",
-    "content-type": "animals/lion",
-
-    "bytes": 12499950,
+    "content-type": "animals/lion", # here is where the content type is defined and the "application/taska" can be added.
+    "bytes": random.randint(1000, 100000),
     "platform": "S3",
-
-    "tags": ["zoo", "africa", "lion"]
+    "another-field": "another-value",  # this field will be ignored
+    "tags": ["zoo", "africa", "lion", "whatevs"],
 }
 dr_id = dr_api.add(data_record)
 print('created data record:', dr_id)
 
+pp(dr_api.get(dr_id))
 
 #
 # Delete data record.
+# OR this can be commented out to keep the data record.
 #
 assert dr_id == dr_api.delete(dr_id)
 print('deleted data record:', dr_id)

examples/create-infra-service-cred.py

@@ -1,5 +1,8 @@
 #!/usr/bin/env python3
 
+import yaml
+from pprint import pprint
+
 from nuvla.api import Api as Nuvla
 from nuvla.api.resources.credential import (CredentialK8s,
                                             CredentialS3,
@@ -13,7 +16,12 @@
 nuvla = Nuvla()
 # nuvla = Nuvla(endpoint='https://nuvla.io', insecure=True)
 
-nuvla.login_password(username, password)
+
+userpass = {"href" : "session-template/password",
+         "username" : username,
+         "password" : password
+         }
+nuvla.login(userpass)
 
 # Fake infra service ID to which all credentials will be attached.
 infra_service_id = "infrastructure-service/1-2-3-4-5"
@@ -34,8 +42,18 @@
 #
 #k8s_cred_id = cred_k8s.create(ca, cert, key, infra_service_id, "My K8s creds")
 
+def read_yaml_file(filepath):
+    with open(filepath, 'r') as file:
+        try:
+            return yaml.safe_load(file)
+        except yaml.YAMLError as exc:
+            print(exc)
+
+config = read_yaml_file('data/example-k8s-config.yaml')
+print(config)
+
 k8s_cred_id = cred_k8s \
-    .create_from_config('data/example-k8s-config.yaml', infra_service_id,
+    .create_from_config(config, infra_service_id,
                         context='first-k8s-cluster-user', name='My K8s creds')
 print('Kubernetes creds: ', k8s_cred_id)