#15 Add flysystem-process-bundle demo

.docker/.gitignore

@@ -1,2 +1,3 @@
 .env
 mysql/data
+sftp/data

.docker/compose.yaml

@@ -72,5 +72,13 @@ services:
       retries: 100
     ports:
       - "${MYSQL_PORT:-3306}:3306"
+  sftp:
+    image: atmoz/sftp
+    volumes:
+      - ./sftp:/home/sftp/
+    ports:
+      - "22:22"
+    command: sftp:password:1000
+
 volumes:
   process_bundle_demo_data:

.env

@@ -35,3 +35,11 @@ DATABASE_URL="mysql://app:app@mysql:3306/app?serverVersion=9.1.0&charset=utf8mb4
 # DATABASE_URL="mysql://app:!ChangeMe!@database:3306/app?serverVersion=10.11.2-MariaDB&charset=utf8mb4"
 #DATABASE_URL="postgresql://app:!ChangeMe!@database:5432/app?serverVersion=15&charset=utf8"
 ###< doctrine/doctrine-bundle ###
+
+###> docker/atmoz-sftp ###
+SFTP_HOST=sftp
+SFTP_PORT=22
+SFTP_USERNAME=sftp
+SFTP_PASSWORD=password
+SFTP_ROOT=data
+###< docker/atmoz-sftp ###

composer.json

@@ -11,8 +11,9 @@
         "cleverage/doctrine-process-bundle": "dev-prepare-release",
         "cleverage/process-bundle": "dev-prepare-release",
         "cleverage/soap-process-bundle": "dev-prepare-release",
-        "cleverage/rest-process-bundle": "dev-prepare-release",
         "cleverage/ui-process-bundle": "dev-prepare-release",
+        "cleverage/flysystem-process-bundle": "dev-prepare-release",
+        "league/flysystem-sftp-v3": "^3.0",
         "symfony/flex": "^v2.3"
     },
     "require-dev": {

config/bundles.php

@@ -17,4 +17,6 @@
     Doctrine\Bundle\FixturesBundle\DoctrineFixturesBundle::class => ['dev' => true, 'test' => true],
     CleverAge\SoapProcessBundle\CleverAgeSoapProcessBundle::class => ['all' => true],
     CleverAge\RestProcessBundle\CleverAgeRestProcessBundle::class => ['all' => true],
+    CleverAge\FlysystemProcessBundle\CleverAgeFlysystemProcessBundle::class => ['all' => true],
+    \League\FlysystemBundle\FlysystemBundle::class => ['all' => true]
 ];

config/packages/flysystem.yaml

@@ -0,0 +1,17 @@
+parameters:
+  local.storage.dir: '%kernel.project_dir%/var/storage/local'
+flysystem:
+  storages:
+    local.storage:
+      adapter: 'local'
+      options:
+        directory: '%local.storage.dir%'
+
+    remote.storage:
+      adapter: 'sftp'
+      options:
+        host: '%env(string:SFTP_HOST)%'
+        port: '%env(int:SFTP_PORT)%'
+        username: '%env(string:SFTP_USERNAME)%'
+        password: '%env(string:SFTP_PASSWORD)%'
+        root: '%env(string:SFTP_ROOT)%'

config/packages/process/demo.sftp_copy.yaml

@@ -0,0 +1,45 @@
+clever_age_process:
+    configurations:
+        demo.sftp_copy:
+            description: >
+                A sample process to copy files between flysystem storages.
+                source_filesystem context variable is used to define source filesystem.
+                destination_filesystem context variable is used to define destination filesystem.
+                file_pattern context variable is used to define file_pattern to match.
+                Following help show us how to delete all .csv files from filesystem remote.storage to local.storage
+                See config/packages/flysystem.yaml to see configured flysystem/storages.
+            help: >
+                bin/console cleverage:process:execute demo.sftp_copy -c source_filesystem:remote.storage -c destination_filesystem:local.storage -c file_pattern:/.csv$/ -vv
+            options:
+                ui:
+                    source: Bar
+                    target: Foo
+            tasks:
+                copy:
+                    service: '@CleverAge\FlysystemProcessBundle\Task\FileFetchTask'
+                    description: >
+                        Download .csv files from remote.storage to local.storage.
+                    outputs: aggregate
+                    options:
+                        source_filesystem: '{{ source_filesystem }}'
+                        destination_filesystem: '{{ destination_filesystem }}'
+                        file_pattern: '{{ file_pattern }}'
+                        remove_source: true
+                aggregate:
+                    service: '@CleverAge\ProcessBundle\Task\AggregateIterableTask'
+                    description: >
+                        Aggregate files previously downloaded. Then when all files are downloaded pass to read_local_dir
+                        task to read each files one by one.
+                    outputs: read_local_dir
+                read_local_dir:
+                    service: '@CleverAge\ProcessBundle\Task\File\FolderBrowserTask'
+                    outputs: log_local_file
+                    options:
+                        folder_path: '%local.storage.dir%'
+                log_local_file:
+                    service: '@CleverAge\ProcessBundle\Task\Reporting\LoggerTask'
+                    options:
+                        message: Read file.
+                        level: info
+
+

config/packages/process/demo.sftp_purge.yaml

@@ -0,0 +1,36 @@
+clever_age_process:
+    configurations:
+        demo.sftp_purge:
+            description: >
+                A sample process to remove files from flysystem storage.
+                filesystem context variable is used to define source filesystem.
+                file_pattern context variable is used to define file_pattern to match.
+                Following help show us how to delete all .csv files from filesystem remote.storage
+                See config/packages/flysystem.yaml to see configured flysystem/storages.
+            help: >
+                bin/console cleverage:process:execute demo.sftp_purge -c filesystem:remote.storage -c file_pattern:/.csv$/ -vv
+            options:
+                ui:
+                    source: Bar
+                    target: Foo
+            tasks:
+                get_files:
+                    service: '@CleverAge\FlysystemProcessBundle\Task\ListContentTask'
+                    description: >
+                        List .csv files from remote.storage.
+                        See config/packages/flysystem.yaml to see configured flysystem/storages.
+                    outputs: get_file_path
+                    options:
+                        filesystem: '{{ filesystem }}'
+                        file_pattern: '{{ file_pattern }}'
+                get_file_path:
+                    service: '@CleverAge\ProcessBundle\Task\PropertyGetterTask'
+                    description: >
+                        Get filepath within flysystem/storage.
+                    outputs: remove
+                    options:
+                        property: 'path'
+                remove:
+                    service: '@CleverAge\FlysystemProcessBundle\Task\RemoveFileTask'
+                    options:
+                        filesystem: 'remote.storage'