From cf5a90ee53982e802b960034cb9f4b80e450d1e7 Mon Sep 17 00:00:00 2001 From: almahmoud Date: Tue, 1 Feb 2022 16:25:36 -0500 Subject: [PATCH 1/2] Add docker building --- .github/workflows/build_containers.yaml | 28 +++++++++++++++++++ Dockerfile | 7 +++++ Dockerfile.dev | 11 ++++++++ scripts/bucket_upload.py | 37 +++++++++++++++++++++++++ 4 files changed, 83 insertions(+) create mode 100644 .github/workflows/build_containers.yaml create mode 100644 Dockerfile create mode 100644 Dockerfile.dev create mode 100644 scripts/bucket_upload.py diff --git a/.github/workflows/build_containers.yaml b/.github/workflows/build_containers.yaml new file mode 100644 index 00000000..0c9ac363 --- /dev/null +++ b/.github/workflows/build_containers.yaml @@ -0,0 +1,28 @@ +name: Build Image +on: [push, workflow_dispatch] +jobs: + test: + name: Build image + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python: ['3.9'] + extra: [full, openstack, aws, gcp, azure] + steps: + - uses: actions/checkout@v2 + - run: docker build . -t almahmoud/cloudbridge:${{matrix.python}}-${{matrix.extra}} --build-arg EXTRA=${{matrix.extra}} --build-arg PYTHON=${{matrix.python}} + - name: Login to docker hub + uses: actions-hub/docker/login@master + env: + DOCKER_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Push to docker hub + uses: actions-hub/docker@master + with: + args: push almahmoud/cloudbridge:${{matrix.python}}-${{matrix.extra}} + - run: docker build . -t almahmoud/cloudbridge:dev-${{matrix.python}}-${{matrix.extra}} -f Dockerfile.dev --build-arg EXTRA=${{matrix.extra}} --build-arg PYTHON=${{matrix.python}} + - name: Push to docker hub + uses: actions-hub/docker@master + with: + args: push almahmoud/cloudbridge:dev-${{matrix.python}}-${{matrix.extra}} diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..f4026960 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,7 @@ +ARG EXTRA=full +ARG PYTHON=3.9 + +FROM python:$PYTHON-slim +RUN pip install --no-cache-dir cloudbridge[$EXTRA] + +COPY scripts/ cloudbridge_scripts/ diff --git a/Dockerfile.dev b/Dockerfile.dev new file mode 100644 index 00000000..973a6dc7 --- /dev/null +++ b/Dockerfile.dev @@ -0,0 +1,11 @@ +ARG EXTRA=full +ARG PYTHON=latest +FROM python:$PYTHON + +RUN apt-get update \ + && apt-get --force-yes install -y curl vim git \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* +RUN pip install --no-cache-dir pep8 flake8 pyflakes pyyaml ipython cloudbridge[$EXTRA] + +COPY scripts/ cloudbridge_scripts/ diff --git a/scripts/bucket_upload.py b/scripts/bucket_upload.py new file mode 100644 index 00000000..02e1c5a5 --- /dev/null +++ b/scripts/bucket_upload.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 + +import sys +import argparse + +from cloudbridge.factory import CloudProviderFactory +from cloudbridge.factory import ProviderList +from cloudbridge.interfaces.exceptions import DuplicateResourceException + +def main(arguments): + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('filepath', help="File to upload", type=str) + parser.add_argument('destination', help="Destination in bucket", type=str) + parser.add_argument('-b', '--bucket', help="Name of destination bucket", type=str) + parser.add_argument('-p', '--provider', + choices=[ProviderList.__dict__[p] for p in ProviderList.__dict__.keys() if all(x not in p for x in ['_', 'MOCK'])], + help='Name of provider to use') + parser.add_argument('-r', '--retries', help="Maximum number of retries. Default = 5", type=int, default=5) + args = parser.parse_args(arguments) + provider = CloudProviderFactory().create_provider(args.provider, {}) + try: + bucket = provider.storage.buckets.create(args.bucket) + except DuplicateResourceException: + bucket = provider.storage.buckets.get(args.bucket) + ob = bucket.objects.create(args.destination) + uploaded = False + count = 0 + while not uploaded: + print(f'Trying for the {count}th time') + uploaded = ob.upload_from_file(args.filepath) + count += 1 + print(f'Uploaded: {args.filepath} to bucket {args.bucket} at location {args.destination}') + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) \ No newline at end of file From d6e5897fdb38650b9e41f8a7d94bf45f7bc99e19 Mon Sep 17 00:00:00 2001 From: almahmoud Date: Tue, 1 Feb 2022 16:32:31 -0500 Subject: [PATCH 2/2] Change docker repo --- .github/workflows/build_containers.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_containers.yaml b/.github/workflows/build_containers.yaml index 0c9ac363..689ddd17 100644 --- a/.github/workflows/build_containers.yaml +++ b/.github/workflows/build_containers.yaml @@ -11,7 +11,7 @@ jobs: extra: [full, openstack, aws, gcp, azure] steps: - uses: actions/checkout@v2 - - run: docker build . -t almahmoud/cloudbridge:${{matrix.python}}-${{matrix.extra}} --build-arg EXTRA=${{matrix.extra}} --build-arg PYTHON=${{matrix.python}} + - run: docker build . -t cloudve/cloudbridge:${{matrix.python}}-${{matrix.extra}} --build-arg EXTRA=${{matrix.extra}} --build-arg PYTHON=${{matrix.python}} - name: Login to docker hub uses: actions-hub/docker/login@master env: @@ -20,9 +20,9 @@ jobs: - name: Push to docker hub uses: actions-hub/docker@master with: - args: push almahmoud/cloudbridge:${{matrix.python}}-${{matrix.extra}} - - run: docker build . -t almahmoud/cloudbridge:dev-${{matrix.python}}-${{matrix.extra}} -f Dockerfile.dev --build-arg EXTRA=${{matrix.extra}} --build-arg PYTHON=${{matrix.python}} + args: push cloudve/cloudbridge:${{matrix.python}}-${{matrix.extra}} + - run: docker build . -t cloudve/cloudbridge:dev-${{matrix.python}}-${{matrix.extra}} -f Dockerfile.dev --build-arg EXTRA=${{matrix.extra}} --build-arg PYTHON=${{matrix.python}} - name: Push to docker hub uses: actions-hub/docker@master with: - args: push almahmoud/cloudbridge:dev-${{matrix.python}}-${{matrix.extra}} + args: push cloudve/cloudbridge:dev-${{matrix.python}}-${{matrix.extra}}