From 8abadb9d6b8319dbc0501534e3584c48bb151369 Mon Sep 17 00:00:00 2001 From: "Brock A. Martin" Date: Mon, 1 Jul 2019 13:21:41 -0500 Subject: [PATCH 1/5] Start minio server in /data The docker volume in docker-compose is being mounted at /data rather than /export. Without changing this, any data stored by minio at runtime is lost since it is not persistently stored in a volume. --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 0caa4b8..86b5176 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -46,7 +46,7 @@ services: minio: image: minio/minio:RELEASE.2018-11-06T01-01-02Z - command: server /export + command: server /data environment: *env ports: - 80:9000 @@ -54,4 +54,4 @@ services: - minio:/data volumes: - minio: \ No newline at end of file + minio: From 99559500a075d74a242902b30a66f9b401972fcb Mon Sep 17 00:00:00 2001 From: "Brock A. Martin" Date: Mon, 1 Jul 2019 13:49:05 -0500 Subject: [PATCH 2/5] Remove unnecessary /app volume mounts The /app directory has already been copied into the docker image. There is no reason to mount it again at runtime via a volume. --- docker-compose.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 0caa4b8..745635b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,8 +15,6 @@ services: - beat - rabbitmq restart: 'no' - volumes: - - ./app:/app worker-minio: build: . @@ -27,8 +25,6 @@ services: - beat - rabbitmq restart: 'no' - volumes: - - ./app:/app beat: build: . @@ -38,8 +34,6 @@ services: depends_on: - rabbitmq restart: 'no' - volumes: - - ./app:/app rabbitmq: image: rabbitmq:3.7.8 @@ -54,4 +48,4 @@ services: - minio:/data volumes: - minio: \ No newline at end of file + minio: From 55f0c21e28fd70afcbfce52c3cc630263a278f0a Mon Sep 17 00:00:00 2001 From: "Brock A. Martin" Date: Mon, 1 Jul 2019 14:00:31 -0500 Subject: [PATCH 3/5] Add worker-minio dependency on minio The worker-minio containers depend on the minio service starting first. --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index e9b1e97..42eb0a2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,6 +24,7 @@ services: depends_on: - beat - rabbitmq + - minio restart: 'no' beat: From 93bf86e1f36e65c4289a568f392744dc13c24ec5 Mon Sep 17 00:00:00 2001 From: "Brock A. Martin" Date: Mon, 1 Jul 2019 14:32:26 -0500 Subject: [PATCH 4/5] Add documentation for docker-compose/docker-stack The added text in the README.md file helps to differentiate the uses of the docker-compose.yml and docker-stack.yml files as well as explains a little on how to manage the example service. --- README.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/README.md b/README.md index 15a6be7..538d935 100644 --- a/README.md +++ b/README.md @@ -1 +1,25 @@ # celery-docker + +## using `docker-compose` + +### Up +`$ docker-compose up -d` + +### Scale +`$ docker-compose up -d --scale worker=2` +`$ docker-compose up -d --scale worker=1` + +### Down +`$ docker-compose down` + +## using `docker-stack` + +### Up +`$ docker stack deploy -c docker-stack.yml celery-docker-example` + +### Scale +`$ docker service scale celery-docker-example_worker=2` +`$ docker service scale celery-docker-example_worker=1` + +### Down +`$ docker stack rm celery-docker-example` From ea89c60afd192919c129497443a766334ba4efdb Mon Sep 17 00:00:00 2001 From: "Brock A. Martin" Date: Mon, 1 Jul 2019 14:34:39 -0500 Subject: [PATCH 5/5] Add docker-stack.yml file This new file allows the example service to be started on a docker swarm. --- docker-stack.yml | 58 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 docker-stack.yml diff --git a/docker-stack.yml b/docker-stack.yml new file mode 100644 index 0000000..1d8e9ef --- /dev/null +++ b/docker-stack.yml @@ -0,0 +1,58 @@ +version: '3.4' +services: + worker: + image: &img worker + command: [celery, worker, --app=worker.app, --pool=gevent, --concurrency=20, --loglevel=INFO] + environment: &env + - CELERY_BROKER_URL=amqp://guest:guest@rabbitmq:5672 + - MINIO_HOST=minio:9000 + - MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE + - MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY + - NEWSPAPER_URLS=https://www.theguardian.com,https://www.nytimes.com + - NEWSPAPER_SCHEDULE=300 + depends_on: + - beat + - rabbitmq + deploy: + restart_policy: + condition: 'none' + + worker-minio: + image: *img + command: [celery, worker, --app=worker.app, --pool=gevent, --concurrency=20, --queues=minio, --loglevel=INFO] + environment: *env + depends_on: + - beat + - rabbitmq + - minio + deploy: + restart_policy: + condition: 'none' + + beat: + image: *img + command: [celery, beat, --app=worker.app, --loglevel=INFO] + environment: *env + depends_on: + - rabbitmq + deploy: + restart_policy: + condition: 'none' + + rabbitmq: + image: rabbitmq:3.7.8 + + minio: + image: minio/minio:RELEASE.2018-11-06T01-01-02Z + command: server /data + environment: *env + ports: + - 80:9000 + volumes: + - minio:/data + deploy: + placement: + constraints: [node.role == manager] + +volumes: + minio: