diff --git a/.gitignore b/.gitignore index aa435cf..b0180bb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ __pycache__/circles.cpython-36.pyc __pycache__/teams_helper.cpython-36.pyc __pycache__/circle_predictor.cpython-36.pyc +secrets/secret.json +bash/fix_access.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..1f53f4f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,32 @@ +FROM python:3.6 +# setting up the pip environment +RUN pip install azure.storage azure.keyvault azure.common matplotlib numpy pandas opencv-python requests + +# setting up the user and the logging folder +RUN groupadd -r drillbit && useradd -r -g drillbit drillbit +RUN mkdir /var/log/drillbit-logs && chown drillbit /var/log/drillbit-logs +RUN mkdir /opt/drillbit/ && chown drillbit /opt/drillbit/ +RUN mkdir /opt/drillbit/cache && chown drillbit /opt/drillbit/cache + +# getting gosu +RUN set -eux; \ + apt-get update; \ + apt-get install -y gosu; \ + rm -rf /var/lib/apt/lists/*; \ +# verify that the binary works + gosu drillbit true + +# copying scripts over +COPY circle_predictor.py /opt/drillbit/ +COPY circle_service.py /opt/drillbit/ +COPY circles.py /opt/drillbit/ +COPY teams_helper.py /opt/drillbit/ +COPY secret_helper.py /opt/drillbit/ +COPY status.json /opt/drillbit/ +COPY bash/drillbit.sh /opt/drillbit/ +RUN chown drillbit /opt/drillbit/status.json +RUN chmod +xxx /opt/drillbit/drillbit.sh + +ENTRYPOINT ["/opt/drillbit/drillbit.sh"] + +CMD [ "hotdog" ] \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..0d0e204 --- /dev/null +++ b/README.md @@ -0,0 +1,18 @@ +# Award winning NOT a HOTDOG ripoff app for Oil and Gas Applications + +Now better with Docker, Key Vault, and Azure Cointainer Services. +Prereqs: +- storage account with following blobs: + - circles + - photos + - $web (enable the web portion of the blob account) +- azure key vault +- (optional) azure container registry +- (optional) azure kubernetes service + +To create and run a docker container locally, fill out the secrets/env file with the correct values for the app and key vault + +```bash +docker built -t drillbit . +docker run --env-file ./secrets/env drillbit +``` diff --git a/bash/drillbit.sh b/bash/drillbit.sh new file mode 100644 index 0000000..afcd281 --- /dev/null +++ b/bash/drillbit.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -ex +if [ "$1" = 'hotdog' ]; then + while true; do + gosu drillbit python3 /opt/drillbit/circle_service.py + done +else + exec "$@" +fi diff --git a/circle_predictor.py b/circle_predictor.py index a3ce375..6c9d3dd 100644 --- a/circle_predictor.py +++ b/circle_predictor.py @@ -6,10 +6,7 @@ def upload_circle(api_url, api_key, path): """uploads the circle imager to the vision API""" headers = {"Prediction-Key": api_key, "Content-Type": "multipart/form-data"} img_data = open(path,'rb') - print(api_url) resp = requests.post(url=api_url, headers=headers, data=img_data) - print(resp) - if resp.status_code == 200: return resp.json() else: diff --git a/circle_service.py b/circle_service.py index c7d82de..4794e78 100644 --- a/circle_service.py +++ b/circle_service.py @@ -2,9 +2,12 @@ import os import requests import json +import logging +import datetime from circles import get_circles from circle_predictor import predict_circles, interpret_results from teams_helper import teams_message +from secret_helper import SecretHelper @@ -18,55 +21,59 @@ def upload_circles(folder, bbService): if filename.startswith("original."): bbService.create_blob_from_path("$web", blob_path_to_file, full_path_to_file) +if __name__ == "__main__": + logfile = "/var/log/drillbit-logs/log-%s.log" % datetime.datetime.now().strftime("%Y-%m-%d") + logging.basicConfig(filename=logfile,level=logging.INFO) + if "APPID" in os.environ: + appid = os.environ["APPID"] + if "TENANT" in os.environ: + tenant = os.environ["TENANT"] + if "APPKEY" in os.environ: + key = os.environ["APPKEY"] + if "RESOURCE" in os.environ: + resource = os.environ["RESOURCE"] + if "VAULT" in os.environ: + vbu = "https://%s.vault.azure.net" % os.environ["VAULT"] + if not(appid and tenant and key and resource): + logging.critical("[---] you need to have the right evironment variables") + exit() + + sh = SecretHelper(client_id=appid, secret=key, tenant=tenant, resource=resource) + block_blob_service = BlockBlobService(account_name=sh.get_secret(vbu, "blobaccount").value, account_key=sh.get_secret(vbu, "blobkey").value) + container_name='photos' + status = json.load(open('/opt/drillbit/status.json')) + while True: + generator = block_blob_service.list_blobs(container_name) + for blob in generator: + if blob.name.lower().endswith('png') or blob.name.lower().endswith('jpg') or blob.name.lower().endswith('jpeg'): + if not blob.name in status: + stripped_name = blob.name.split("/")[-1] + filen = stripped_name.replace(".","_") + foldername = "/opt/drillbit/cache/%s" % filen + try: + if not os.path.isdir(foldername): + os.mkdir(foldername) + except: + logging.error("[-] unable to create folder: %s" % foldername) + extent = blob.name.split(".")[-1] + full_path_to_file2 = "%s/original.%s" % (foldername , extent) + block_blob_service.get_blob_to_path(container_name, blob.name, full_path_to_file2) + try: + if get_circles(full_path_to_file2, foldername): + upload_circles(foldername, block_blob_service) + circle_predictions = predict_circles(sh.get_secret(vbu, "apiurl").value, sh.get_secret(vbu, "apikey").value, foldername) + irm = interpret_results(blob.name, circle_predictions) + json.dump(circle_predictions, open(foldername+"/predictions.json","w")) + teams_message(irm['message'], sh.get_secret(vbu, "teamshook").value, irm['colour'], "%s/%s/original.%s" % (sh.get_secret(vbu, "webblob").value,foldername.split("/")[-1], extent)) + else: + teams_message("NOT DRILLBIT. probable data quality issues with %s" % blob.name, sh.get_secret(vbu, "teamshook").value,"800080", "%s/%s/original.%s" % (sh.get_secret(vbu, "webblob").value,foldername.split("/")[-1], extent)) + status[blob.name]=foldername + json.dump(status,open('/opt/drillbit/status.json','w')) + logging.info("[+] processed: %s" % blob.name) + except: + logging.error("[-] erroring when trying to find circles: %s" % blob.name) + teams_message("processing issues with %s" % blob.name, sh.get_secret(vbu, "teamshook").value,"800080") + raise -def send_flow(flowurl, message): - """sends a request to get a flow going""" - try: - resp = requests.post(url=flowurl, json=message) - if resp.status_code == 202: - print("[+] successfully sent flow") - except: - print("[-] issues with flow") - -creds = json.load(open('/etc/hackathon/creds.json')) - -block_blob_service = BlockBlobService(account_name=creds['blobacct'], account_key=creds['blobkey']) -container_name='photos' -status = json.load(open('/etc/hackathon/status.json')) -while True: - generator = block_blob_service.list_blobs(container_name) - for blob in generator: - if blob.name.lower().endswith('png') or blob.name.lower().endswith('jpg') or blob.name.lower().endswith('jpeg'): - if not blob.name in status: - stripped_name = blob.name.split("/")[-1] - filen = stripped_name.replace(".","_") - foldername = "/tmp/%s" % filen - try: - if not os.path.isdir(foldername): - os.mkdir(foldername) - except: - print("[-] unable to create folder: %s" % foldername) - extent = blob.name.split(".")[-1] - full_path_to_file2 = "%s/original.%s" % (foldername , extent) - block_blob_service.get_blob_to_path(container_name, blob.name, full_path_to_file2) - try: - if get_circles(full_path_to_file2, foldername): - upload_circles(foldername, block_blob_service) - circle_predictions = predict_circles(creds['api_url'], creds['api_key'], foldername) - irm = interpret_results(blob.name, circle_predictions) - json.dump(circle_predictions, open(foldername+"/predictions.json","w")) - teams_message(irm['message'],creds['teamshook'],irm['colour'], "%s/%s/original.%s" % (creds['webblob'],foldername.split("/")[-1], extent)) - #send_flow(creds['flowurl'],{"message":"DRILLBIT - found circles in: %s" % blob.name}) - else: - #send_flow(creds['flowurl'],{"message":"NOT DRILLBIT - unable to find circles in: %s" % blob.name}) - teams_message("NOT DRILLBIT. probable data quality issues with %s" % blob.name,creds['teamshook'],"800080", "%s/%s/original.%s" % (creds['webblob'],foldername.split("/")[-1], extent)) - status[blob.name]=foldername - json.dump(status,open('/etc/hackathon/status.json','w')) - print("[+] processed: %s" % blob.name) - except: - print("[-] erroring when trying to find circles: %s" % blob.name) - teams_message("processing issues with %s" % blob.name, creds['teamshook'],"800080") - raise - - else: - print("[*] already processed: %s" % blob.name) \ No newline at end of file + else: + logging.error("[*] already processed: %s" % blob.name) \ No newline at end of file diff --git a/drillbit.yaml b/drillbit.yaml new file mode 100644 index 0000000..5062061 --- /dev/null +++ b/drillbit.yaml @@ -0,0 +1,36 @@ +apiVersion: v1 +kind: Pod +metadata: + name: drillbit +spec: + containers: + - name: drillbit + image: drillbit.azurecr.io/drillbit + env: + - name: APPID + valueFrom: + secretKeyRef: + name: drillbit + key: appid + - name: APPKEY + valueFrom: + secretKeyRef: + name: drillbit + key: key + - name: RESOURCE + valueFrom: + secretKeyRef: + name: drillbit + key: resource + - name: TENANT + valueFrom: + secretKeyRef: + name: drillbit + key: tenant + - name: VAULT + valueFrom: + secretKeyRef: + name: drillbit + key: vault + restartPolicy: Never + diff --git a/secret_helper.py b/secret_helper.py new file mode 100644 index 0000000..5fa33da --- /dev/null +++ b/secret_helper.py @@ -0,0 +1,19 @@ +from azure.keyvault import KeyVaultClient, KeyVaultAuthentication +from azure.common.credentials import ServicePrincipalCredentials + +class SecretHelper(): + """helper class to retrieve credentials from Azure Key Valut""" + def __init__(self, client_id, secret, tenant, resource): + """Initialize the connection to Key Vault""" + credentials = ServicePrincipalCredentials( + client_id = client_id, + secret = secret, + tenant = tenant, + resource = resource + ) + self.client = KeyVaultClient(credentials) + + def get_secret(self, vault_base_url, secret_name, secret_version=""): + """get a secret from Azure Key Vault""" + secret_bundle = self.client.get_secret(vault_base_url=vault_base_url, secret_name=secret_name, secret_version=secret_version) + return secret_bundle diff --git a/secrets/env b/secrets/env new file mode 100644 index 0000000..5ebcca9 --- /dev/null +++ b/secrets/env @@ -0,0 +1,5 @@ +APPID= +APPKEY= +TENANT= +RESOURCE=https://vault.azure.net +VAULT= \ No newline at end of file diff --git a/status.json b/status.json new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/status.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/test6.jpg b/test6.jpg deleted file mode 100644 index 1a32559..0000000 Binary files a/test6.jpg and /dev/null differ diff --git a/test7.png b/test7.png deleted file mode 100644 index fe4848e..0000000 Binary files a/test7.png and /dev/null differ