diff --git a/.docker/.gitignore b/.docker/.gitignore index 41a4e0e..cdeab93 100644 --- a/.docker/.gitignore +++ b/.docker/.gitignore @@ -1,2 +1,3 @@ .env mysql/data +sftp/data \ No newline at end of file diff --git a/.docker/compose.yaml b/.docker/compose.yaml index 279235a..85937fb 100644 --- a/.docker/compose.yaml +++ b/.docker/compose.yaml @@ -72,5 +72,13 @@ services: retries: 100 ports: - "${MYSQL_PORT:-3306}:3306" + sftp: + image: atmoz/sftp + volumes: + - ./sftp:/home/sftp/ + ports: + - "22:22" + command: sftp:password:1000 + volumes: process_bundle_demo_data: \ No newline at end of file diff --git a/.env b/.env index 333324b..a09a7a3 100644 --- a/.env +++ b/.env @@ -35,3 +35,11 @@ DATABASE_URL="mysql://app:app@mysql:3306/app?serverVersion=9.1.0&charset=utf8mb4 # DATABASE_URL="mysql://app:!ChangeMe!@database:3306/app?serverVersion=10.11.2-MariaDB&charset=utf8mb4" #DATABASE_URL="postgresql://app:!ChangeMe!@database:5432/app?serverVersion=15&charset=utf8" ###< doctrine/doctrine-bundle ### + +###> docker/atmoz-sftp ### +SFTP_HOST=sftp +SFTP_PORT=22 +SFTP_USERNAME=sftp +SFTP_PASSWORD=password +SFTP_ROOT=data +###< docker/atmoz-sftp ### \ No newline at end of file diff --git a/composer.json b/composer.json index e4b6116..e1c6a51 100644 --- a/composer.json +++ b/composer.json @@ -11,8 +11,9 @@ "cleverage/doctrine-process-bundle": "dev-prepare-release", "cleverage/process-bundle": "dev-prepare-release", "cleverage/soap-process-bundle": "dev-prepare-release", - "cleverage/rest-process-bundle": "dev-prepare-release", "cleverage/ui-process-bundle": "dev-prepare-release", + "cleverage/flysystem-process-bundle": "dev-prepare-release", + "league/flysystem-sftp-v3": "^3.0", "symfony/flex": "^v2.3" }, "require-dev": { diff --git a/config/bundles.php b/config/bundles.php index b9ae9e9..bc64251 100644 --- a/config/bundles.php +++ b/config/bundles.php @@ -17,4 +17,6 @@ Doctrine\Bundle\FixturesBundle\DoctrineFixturesBundle::class => ['dev' => true, 'test' => true], CleverAge\SoapProcessBundle\CleverAgeSoapProcessBundle::class => ['all' => true], CleverAge\RestProcessBundle\CleverAgeRestProcessBundle::class => ['all' => true], + CleverAge\FlysystemProcessBundle\CleverAgeFlysystemProcessBundle::class => ['all' => true], + \League\FlysystemBundle\FlysystemBundle::class => ['all' => true] ]; diff --git a/config/packages/flysystem.yaml b/config/packages/flysystem.yaml new file mode 100644 index 0000000..f70675c --- /dev/null +++ b/config/packages/flysystem.yaml @@ -0,0 +1,17 @@ +parameters: + local.storage.dir: '%kernel.project_dir%/var/storage/local' +flysystem: + storages: + local.storage: + adapter: 'local' + options: + directory: '%local.storage.dir%' + + remote.storage: + adapter: 'sftp' + options: + host: '%env(string:SFTP_HOST)%' + port: '%env(int:SFTP_PORT)%' + username: '%env(string:SFTP_USERNAME)%' + password: '%env(string:SFTP_PASSWORD)%' + root: '%env(string:SFTP_ROOT)%' \ No newline at end of file diff --git a/config/packages/process/demo.sftp_copy.yaml b/config/packages/process/demo.sftp_copy.yaml new file mode 100644 index 0000000..7b90f36 --- /dev/null +++ b/config/packages/process/demo.sftp_copy.yaml @@ -0,0 +1,45 @@ +clever_age_process: + configurations: + demo.sftp_copy: + description: > + A sample process to copy files between flysystem storages. + source_filesystem context variable is used to define source filesystem. + destination_filesystem context variable is used to define destination filesystem. + file_pattern context variable is used to define file_pattern to match. + Following help show us how to delete all .csv files from filesystem remote.storage to local.storage + See config/packages/flysystem.yaml to see configured flysystem/storages. + help: > + bin/console cleverage:process:execute demo.sftp_copy -c source_filesystem:remote.storage -c destination_filesystem:local.storage -c file_pattern:/.csv$/ -vv + options: + ui: + source: Bar + target: Foo + tasks: + copy: + service: '@CleverAge\FlysystemProcessBundle\Task\FileFetchTask' + description: > + Download .csv files from remote.storage to local.storage. + outputs: aggregate + options: + source_filesystem: '{{ source_filesystem }}' + destination_filesystem: '{{ destination_filesystem }}' + file_pattern: '{{ file_pattern }}' + remove_source: true + aggregate: + service: '@CleverAge\ProcessBundle\Task\AggregateIterableTask' + description: > + Aggregate files previously downloaded. Then when all files are downloaded pass to read_local_dir + task to read each files one by one. + outputs: read_local_dir + read_local_dir: + service: '@CleverAge\ProcessBundle\Task\File\FolderBrowserTask' + outputs: log_local_file + options: + folder_path: '%local.storage.dir%' + log_local_file: + service: '@CleverAge\ProcessBundle\Task\Reporting\LoggerTask' + options: + message: Read file. + level: info + + diff --git a/config/packages/process/demo.sftp_purge.yaml b/config/packages/process/demo.sftp_purge.yaml new file mode 100644 index 0000000..1ac24d8 --- /dev/null +++ b/config/packages/process/demo.sftp_purge.yaml @@ -0,0 +1,36 @@ +clever_age_process: + configurations: + demo.sftp_purge: + description: > + A sample process to remove files from flysystem storage. + filesystem context variable is used to define source filesystem. + file_pattern context variable is used to define file_pattern to match. + Following help show us how to delete all .csv files from filesystem remote.storage + See config/packages/flysystem.yaml to see configured flysystem/storages. + help: > + bin/console cleverage:process:execute demo.sftp_purge -c filesystem:remote.storage -c file_pattern:/.csv$/ -vv + options: + ui: + source: Bar + target: Foo + tasks: + get_files: + service: '@CleverAge\FlysystemProcessBundle\Task\ListContentTask' + description: > + List .csv files from remote.storage. + See config/packages/flysystem.yaml to see configured flysystem/storages. + outputs: get_file_path + options: + filesystem: '{{ filesystem }}' + file_pattern: '{{ file_pattern }}' + get_file_path: + service: '@CleverAge\ProcessBundle\Task\PropertyGetterTask' + description: > + Get filepath within flysystem/storage. + outputs: remove + options: + property: 'path' + remove: + service: '@CleverAge\FlysystemProcessBundle\Task\RemoveFileTask' + options: + filesystem: 'remote.storage'