Docker-ized

.gitignore

@@ -1,3 +1,5 @@
 __pycache__/circles.cpython-36.pyc
 __pycache__/teams_helper.cpython-36.pyc
 __pycache__/circle_predictor.cpython-36.pyc
+secrets/secret.json
+bash/fix_access.sh

Dockerfile

@@ -0,0 +1,32 @@
+FROM python:3.6
+# setting up the pip environment
+RUN pip install azure.storage azure.keyvault azure.common matplotlib numpy pandas opencv-python requests
+
+# setting up the user and the logging folder
+RUN groupadd -r drillbit && useradd -r -g drillbit drillbit
+RUN mkdir /var/log/drillbit-logs && chown drillbit /var/log/drillbit-logs
+RUN mkdir /opt/drillbit/ && chown drillbit /opt/drillbit/
+RUN mkdir /opt/drillbit/cache && chown drillbit /opt/drillbit/cache
+
+# getting gosu
+RUN set -eux; \
+        apt-get update; \
+        apt-get install -y gosu; \
+        rm -rf /var/lib/apt/lists/*; \
+# verify that the binary works
+        gosu drillbit true
+
+# copying scripts over
+COPY circle_predictor.py /opt/drillbit/
+COPY circle_service.py /opt/drillbit/
+COPY circles.py /opt/drillbit/
+COPY teams_helper.py /opt/drillbit/
+COPY secret_helper.py /opt/drillbit/
+COPY status.json /opt/drillbit/
+COPY bash/drillbit.sh /opt/drillbit/
+RUN chown drillbit /opt/drillbit/status.json
+RUN chmod +xxx /opt/drillbit/drillbit.sh
+
+ENTRYPOINT ["/opt/drillbit/drillbit.sh"]
+
+CMD [ "hotdog" ]

README.md

@@ -0,0 +1,18 @@
+# Award winning NOT a HOTDOG ripoff app for Oil and Gas Applications
+
+Now better with Docker, Key Vault, and Azure Cointainer Services.
+Prereqs:
+- storage account with following blobs:
+    - circles
+    - photos
+    - $web (enable the web portion of the blob account)
+- azure key vault
+- (optional) azure container registry
+- (optional) azure kubernetes service
+
+To create and run a docker container locally, fill out the secrets/env file with the correct values for the app and key vault
+
+```bash
+docker built -t drillbit .
+docker run --env-file ./secrets/env drillbit
+```

bash/drillbit.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -ex
+if [ "$1" = 'hotdog' ]; then
+    while true; do
+        gosu drillbit python3 /opt/drillbit/circle_service.py 
+    done
+else
+    exec "$@"
+fi

circle_predictor.py

@@ -6,10 +6,7 @@ def upload_circle(api_url, api_key, path):
     """uploads the circle imager to the vision API"""
     headers = {"Prediction-Key": api_key, "Content-Type": "multipart/form-data"}
     img_data = open(path,'rb')
-    print(api_url)
     resp = requests.post(url=api_url, headers=headers, data=img_data)
-    print(resp)
-
     if resp.status_code == 200:
         return resp.json()
     else:

circle_service.py

@@ -2,9 +2,12 @@
 import os
 import requests
 import json
+import logging
+import datetime
 from circles import get_circles
 from circle_predictor import predict_circles, interpret_results
 from teams_helper import teams_message
+from secret_helper import SecretHelper
 
 
 
@@ -18,55 +21,59 @@ def upload_circles(folder, bbService):
         if filename.startswith("original."):
             bbService.create_blob_from_path("$web", blob_path_to_file, full_path_to_file)
 
+if __name__ == "__main__":
+    logfile = "/var/log/drillbit-logs/log-%s.log" % datetime.datetime.now().strftime("%Y-%m-%d")
+    logging.basicConfig(filename=logfile,level=logging.INFO)
+    if "APPID" in os.environ:
+        appid = os.environ["APPID"]
+    if "TENANT" in os.environ:
+        tenant = os.environ["TENANT"]
+    if "APPKEY" in os.environ:
+        key = os.environ["APPKEY"]
+    if "RESOURCE" in os.environ:
+        resource = os.environ["RESOURCE"]
+    if "VAULT" in os.environ:
+        vbu = "https://%s.vault.azure.net" % os.environ["VAULT"]
+    if not(appid and tenant and key and resource):
+        logging.critical("[---] you need to have the right evironment variables")
+        exit()
+
+    sh = SecretHelper(client_id=appid, secret=key, tenant=tenant, resource=resource)
+    block_blob_service = BlockBlobService(account_name=sh.get_secret(vbu, "blobaccount").value, account_key=sh.get_secret(vbu, "blobkey").value)
+    container_name='photos'
+    status = json.load(open('/opt/drillbit/status.json'))
+    while True:
+        generator = block_blob_service.list_blobs(container_name)
+        for blob in generator:
+            if blob.name.lower().endswith('png') or blob.name.lower().endswith('jpg') or blob.name.lower().endswith('jpeg'):
+                if not blob.name in status:
+                    stripped_name = blob.name.split("/")[-1]
+                    filen = stripped_name.replace(".","_")
+                    foldername = "/opt/drillbit/cache/%s" % filen
+                    try:
+                        if not os.path.isdir(foldername):
+                            os.mkdir(foldername)
+                    except:
+                        logging.error("[-] unable to create folder: %s" % foldername)
+                    extent = blob.name.split(".")[-1]
+                    full_path_to_file2 = "%s/original.%s" % (foldername , extent)
+                    block_blob_service.get_blob_to_path(container_name, blob.name, full_path_to_file2)
+                    try:
+                        if get_circles(full_path_to_file2, foldername):
+                            upload_circles(foldername, block_blob_service)
+                            circle_predictions = predict_circles(sh.get_secret(vbu, "apiurl").value, sh.get_secret(vbu, "apikey").value, foldername)
+                            irm = interpret_results(blob.name, circle_predictions)
+                            json.dump(circle_predictions, open(foldername+"/predictions.json","w"))
+                            teams_message(irm['message'], sh.get_secret(vbu, "teamshook").value, irm['colour'], "%s/%s/original.%s" % (sh.get_secret(vbu, "webblob").value,foldername.split("/")[-1], extent))
+                        else:
+                            teams_message("NOT DRILLBIT. probable data quality issues with %s" % blob.name, sh.get_secret(vbu, "teamshook").value,"800080", "%s/%s/original.%s" % (sh.get_secret(vbu, "webblob").value,foldername.split("/")[-1], extent))         
+                        status[blob.name]=foldername
+                        json.dump(status,open('/opt/drillbit/status.json','w'))
+                        logging.info("[+] processed: %s" % blob.name)
+                    except:
+                        logging.error("[-] erroring when trying to find circles: %s" % blob.name)
+                        teams_message("processing issues with %s" % blob.name, sh.get_secret(vbu, "teamshook").value,"800080")
+                        raise
 
-def send_flow(flowurl, message):
-    """sends a request to get a flow going"""
-    try:
-        resp = requests.post(url=flowurl, json=message)
-        if resp.status_code == 202:
-            print("[+] successfully sent flow")
-    except:
-        print("[-] issues with flow")
-
-creds = json.load(open('/etc/hackathon/creds.json'))
-
-block_blob_service = BlockBlobService(account_name=creds['blobacct'], account_key=creds['blobkey'])
-container_name='photos'
-status = json.load(open('/etc/hackathon/status.json'))
-while True:
-    generator = block_blob_service.list_blobs(container_name)
-    for blob in generator:
-        if blob.name.lower().endswith('png') or blob.name.lower().endswith('jpg') or blob.name.lower().endswith('jpeg'):
-            if not blob.name in status:
-                stripped_name = blob.name.split("/")[-1]
-                filen = stripped_name.replace(".","_")
-                foldername = "/tmp/%s" % filen
-                try:
-                    if not os.path.isdir(foldername):
-                        os.mkdir(foldername)
-                except:
-                    print("[-] unable to create folder: %s" % foldername)
-                extent = blob.name.split(".")[-1]
-                full_path_to_file2 = "%s/original.%s" % (foldername , extent)
-                block_blob_service.get_blob_to_path(container_name, blob.name, full_path_to_file2)
-                try:
-                    if get_circles(full_path_to_file2, foldername):
-                        upload_circles(foldername, block_blob_service)
-                        circle_predictions = predict_circles(creds['api_url'], creds['api_key'], foldername)
-                        irm = interpret_results(blob.name, circle_predictions)
-                        json.dump(circle_predictions, open(foldername+"/predictions.json","w"))
-                        teams_message(irm['message'],creds['teamshook'],irm['colour'], "%s/%s/original.%s" % (creds['webblob'],foldername.split("/")[-1], extent))
-                        #send_flow(creds['flowurl'],{"message":"DRILLBIT - found circles in: %s" % blob.name})           
-                    else:
-                        #send_flow(creds['flowurl'],{"message":"NOT DRILLBIT - unable to find circles in: %s" % blob.name})
-                        teams_message("NOT DRILLBIT. probable data quality issues with %s" % blob.name,creds['teamshook'],"800080", "%s/%s/original.%s" % (creds['webblob'],foldername.split("/")[-1], extent))         
-                    status[blob.name]=foldername
-                    json.dump(status,open('/etc/hackathon/status.json','w'))
-                    print("[+] processed: %s" % blob.name)
-                except:
-                    print("[-] erroring when trying to find circles: %s" % blob.name)
-                    teams_message("processing issues with %s" % blob.name, creds['teamshook'],"800080")
-                    raise
-
-            else:
-                print("[*] already processed: %s" % blob.name)
+                else:
+                    logging.error("[*] already processed: %s" % blob.name)

drillbit.yaml

@@ -0,0 +1,36 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: drillbit
+spec:
+  containers:
+  - name: drillbit
+    image: drillbit.azurecr.io/drillbit
+    env:
+      - name: APPID
+        valueFrom:
+          secretKeyRef:
+            name: drillbit
+            key: appid
+      - name: APPKEY
+        valueFrom:
+          secretKeyRef:
+            name: drillbit
+            key: key
+      - name: RESOURCE
+        valueFrom:
+          secretKeyRef:
+            name: drillbit
+            key: resource
+      - name: TENANT
+        valueFrom:
+          secretKeyRef:
+            name: drillbit
+            key: tenant
+      - name: VAULT
+        valueFrom:
+          secretKeyRef:
+            name: drillbit
+            key: vault
+  restartPolicy: Never
+

secret_helper.py

@@ -0,0 +1,19 @@
+from azure.keyvault import KeyVaultClient, KeyVaultAuthentication
+from azure.common.credentials import ServicePrincipalCredentials
+
+class SecretHelper():
+    """helper class to retrieve credentials from Azure Key Valut"""
+    def __init__(self, client_id, secret, tenant, resource):
+        """Initialize the connection to Key Vault"""
+        credentials = ServicePrincipalCredentials(
+            client_id = client_id,
+            secret = secret,
+            tenant = tenant,
+            resource = resource
+        )
+        self.client = KeyVaultClient(credentials)
+
+    def get_secret(self, vault_base_url, secret_name, secret_version=""):
+        """get a secret from Azure Key Vault"""
+        secret_bundle = self.client.get_secret(vault_base_url=vault_base_url, secret_name=secret_name, secret_version=secret_version)
+        return secret_bundle

secrets/env

@@ -0,0 +1,5 @@
+APPID=
+APPKEY=
+TENANT=
+RESOURCE=https://vault.azure.net
+VAULT=

status.json

@@ -0,0 +1 @@
+{}