diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index 6b730a18..12173cbb 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -46,6 +46,7 @@ use Utopia\Migration\Resources\Functions\Deployment; use Utopia\Migration\Resources\Functions\EnvVar; use Utopia\Migration\Resources\Functions\Func; +use Utopia\Migration\Resources\Integrations\Platform; use Utopia\Migration\Resources\Sites\Deployment as SiteDeployment; use Utopia\Migration\Resources\Sites\EnvVar as SiteEnvVar; use Utopia\Migration\Resources\Sites\Site; @@ -83,7 +84,9 @@ public function __construct( string $endpoint, string $key, protected UtopiaDatabase $database, - protected array $collectionStructure + protected array $collectionStructure, + protected ?UtopiaDatabase $dbForPlatform = null, + protected string $projectInternalId = '', ) { $this->project = $project; $this->endpoint = $endpoint; @@ -142,6 +145,9 @@ public static function getSupportedResources(): array Resource::TYPE_SITE, Resource::TYPE_SITE_DEPLOYMENT, Resource::TYPE_SITE_VARIABLE, + + // Integrations + Resource::TYPE_PLATFORM, ]; } @@ -221,7 +227,6 @@ public function report(array $resources = [], array $resourceIds = []): array $scope = 'sites.write'; $this->sites->create('', '', Framework::OTHER(), BuildRuntime::STATIC1()); } - } catch (AppwriteException $e) { if ($e->getCode() === 403) { throw new \Exception('Missing scope: ' . $scope, previous: $e); @@ -253,6 +258,7 @@ protected function import(array $resources, callable $callback): void try { $this->database->setPreserveDates(true); + $this->dbForPlatform?->setPreserveDates(true); $responseResource = match ($resource->getGroup()) { Transfer::GROUP_DATABASES => $this->importDatabaseResource($resource, $isLast), @@ -260,6 +266,7 @@ protected function import(array $resources, callable $callback): void Transfer::GROUP_AUTH => $this->importAuthResource($resource), Transfer::GROUP_FUNCTIONS => $this->importFunctionResource($resource), Transfer::GROUP_SITES => $this->importSiteResource($resource), + Transfer::GROUP_INTEGRATIONS => $this->importIntegrationsResource($resource), default => throw new \Exception('Invalid resource group'), }; } catch (\Throwable $e) { @@ -277,6 +284,7 @@ protected function import(array $resources, callable $callback): void $responseResource = $resource; } finally { $this->database->setPreserveDates(false); + $this->dbForPlatform?->setPreserveDates(false); } $this->cache->update($responseResource); @@ -1059,7 +1067,6 @@ protected function createRow(Row $resource, bool $isLast): bool 'database_' . $databaseInternalId . '_collection_' . $tableInternalId, $this->rowBuffer )); - } finally { $this->rowBuffer = []; } @@ -1730,4 +1737,70 @@ private function importSiteDeployment(SiteDeployment $deployment): Resource return $deployment; } + + /** + * @throws \Exception + */ + public function importIntegrationsResource(Resource $resource): Resource + { + if ($this->dbForPlatform === null || $this->projectInternalId === '') { + throw new \Exception('Platform database not available for integrations migration'); + } + + switch ($resource->getName()) { + case Resource::TYPE_PLATFORM: + /** @var Platform $resource */ + $this->createPlatform($resource); + break; + } + + if ($resource->getStatus() !== Resource::STATUS_SKIPPED) { + $resource->setStatus(Resource::STATUS_SUCCESS); + } + + return $resource; + } + + /** + * @throws \Throwable + */ + protected function createPlatform(Platform $resource): bool + { + $existing = $this->dbForPlatform->findOne('platforms', [ + Query::equal('projectId', [$this->project]), + Query::equal('type', [$resource->getType()]), + Query::equal('name', [$resource->getPlatformName()]), + ]); + + if ($existing !== false && !$existing->isEmpty()) { + $resource->setStatus(Resource::STATUS_SKIPPED, 'Platform already exists'); + return false; + } + + $createdAt = $this->normalizeDateTime($resource->getCreatedAt()); + $updatedAt = $this->normalizeDateTime($resource->getUpdatedAt(), $createdAt); + + try { + $this->dbForPlatform->createDocument('platforms', new UtopiaDocument([ + '$id' => ID::unique(), + '$permissions' => $resource->getPermissions(), + 'projectInternalId' => $this->projectInternalId, + 'projectId' => $this->project, + 'type' => $resource->getType(), + 'name' => $resource->getPlatformName(), + 'key' => $resource->getKey(), + 'store' => $resource->getStore(), + 'hostname' => $resource->getHostname(), + '$createdAt' => $createdAt, + '$updatedAt' => $updatedAt, + ])); + } catch (DuplicateException) { + $resource->setStatus(Resource::STATUS_SKIPPED, 'Platform already exists'); + return false; + } + + $this->dbForPlatform->purgeCachedDocument('projects', $this->project); + + return true; + } } diff --git a/src/Migration/Resource.php b/src/Migration/Resource.php index e338385c..f0d275af 100644 --- a/src/Migration/Resource.php +++ b/src/Migration/Resource.php @@ -60,6 +60,9 @@ abstract class Resource implements \JsonSerializable public const TYPE_ENVIRONMENT_VARIABLE = 'environment-variable'; + // Integrations + public const TYPE_PLATFORM = 'platform'; + // legacy terminologies public const TYPE_DOCUMENT = 'document'; public const TYPE_ATTRIBUTE = 'attribute'; @@ -89,6 +92,7 @@ abstract class Resource implements \JsonSerializable self::TYPE_ENVIRONMENT_VARIABLE, self::TYPE_TEAM, self::TYPE_MEMBERSHIP, + self::TYPE_PLATFORM, // legacy self::TYPE_DOCUMENT, diff --git a/src/Migration/Resources/Integrations/Platform.php b/src/Migration/Resources/Integrations/Platform.php new file mode 100644 index 00000000..c81b6f04 --- /dev/null +++ b/src/Migration/Resources/Integrations/Platform.php @@ -0,0 +1,104 @@ +id = $id; + $this->createdAt = $createdAt; + $this->updatedAt = $updatedAt; + } + + /** + * @param array $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['id'], + $array['type'], + $array['name'], + $array['key'] ?? '', + $array['store'] ?? '', + $array['hostname'] ?? '', + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + /** + * @return array + */ + public function jsonSerialize(): array + { + return [ + 'id' => $this->id, + 'type' => $this->type, + 'name' => $this->name, + 'key' => $this->key, + 'store' => $this->store, + 'hostname' => $this->hostname, + 'createdAt' => $this->createdAt, + 'updatedAt' => $this->updatedAt, + ]; + } + + public static function getName(): string + { + return Resource::TYPE_PLATFORM; + } + + public function getGroup(): string + { + return Transfer::GROUP_INTEGRATIONS; + } + + public function getType(): string + { + return $this->type; + } + + public function getPlatformName(): string + { + return $this->name; + } + + public function getKey(): string + { + return $this->key; + } + + public function getStore(): string + { + return $this->store; + } + + public function getHostname(): string + { + return $this->hostname; + } +} diff --git a/src/Migration/Source.php b/src/Migration/Source.php index a6b154a1..81f61c16 100644 --- a/src/Migration/Source.php +++ b/src/Migration/Source.php @@ -41,6 +41,11 @@ public function getSitesBatchSize(): int return static::$defaultBatchSize; } + public function getIntegrationsBatchSize(): int + { + return static::$defaultBatchSize; + } + /** * @param array $resources * @return void @@ -95,6 +100,7 @@ public function exportResources(array $resources): void Transfer::GROUP_STORAGE => Transfer::GROUP_STORAGE_RESOURCES, Transfer::GROUP_FUNCTIONS => Transfer::GROUP_FUNCTIONS_RESOURCES, Transfer::GROUP_SITES => Transfer::GROUP_SITES_RESOURCES, + Transfer::GROUP_INTEGRATIONS => Transfer::GROUP_INTEGRATIONS_RESOURCES, ]; foreach ($mapping as $group => $resources) { @@ -126,6 +132,9 @@ public function exportResources(array $resources): void case Transfer::GROUP_SITES: $this->exportGroupSites($this->getSitesBatchSize(), $resources); break; + case Transfer::GROUP_INTEGRATIONS: + $this->exportGroupIntegrations($this->getIntegrationsBatchSize(), $resources); + break; } } } @@ -169,4 +178,12 @@ abstract protected function exportGroupFunctions(int $batchSize, array $resource * @param array $resources Resources to export */ abstract protected function exportGroupSites(int $batchSize, array $resources): void; + + /** + * Export Integrations Group + * + * @param int $batchSize + * @param array $resources Resources to export + */ + abstract protected function exportGroupIntegrations(int $batchSize, array $resources): void; } diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index 47a03461..aef0ab21 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -44,6 +44,7 @@ use Utopia\Migration\Resources\Functions\Deployment; use Utopia\Migration\Resources\Functions\EnvVar; use Utopia\Migration\Resources\Functions\Func; +use Utopia\Migration\Resources\Integrations\Platform; use Utopia\Migration\Resources\Sites\Deployment as SiteDeployment; use Utopia\Migration\Resources\Sites\EnvVar as SiteEnvVar; use Utopia\Migration\Resources\Sites\Site; @@ -76,6 +77,10 @@ class Appwrite extends Source private Reader $database; + private bool $consoleKeyFetched = false; + + private ?string $consoleKey = null; + /** * @throws \Exception */ @@ -116,6 +121,38 @@ public function __construct( } } + public function setConsoleKey(string $key): void + { + $this->consoleKey = $key; + $this->consoleKeyFetched = true; + } + + /** + * @return array|null + */ + protected function getConsoleHeaders(): ?array + { + if (!$this->consoleKeyFetched) { + $this->consoleKeyFetched = true; + + try { + $response = $this->call('GET', '/migrations/appwrite/console-key'); + $this->consoleKey = $response['key'] ?? null; + } catch (\Throwable) { + $this->consoleKey = null; + } + } + + if ($this->consoleKey === null) { + return null; + } + + return [ + 'x-appwrite-project' => 'console', + 'x-appwrite-key' => $this->consoleKey, + ]; + } + public static function getName(): string { return 'Appwrite'; @@ -158,7 +195,8 @@ public static function getSupportedResources(): array Resource::TYPE_SITE_DEPLOYMENT, Resource::TYPE_SITE_VARIABLE, - // Settings + // Integrations + Resource::TYPE_PLATFORM, ]; } @@ -196,6 +234,7 @@ public function report(array $resources = [], array $resourceIds = []): array $this->reportStorage($resources, $report, $resourceIds); $this->reportFunctions($resources, $report, $resourceIds); $this->reportSites($resources, $report, $resourceIds); + $this->reportIntegrations($resources, $report, $resourceIds); $report['version'] = $this->call( 'GET', @@ -1996,6 +2035,99 @@ private function exportSiteDeploymentData(Site $site, array $deployment): void } } + /** + * @param array $resources + * @param array $report + * @param array> $resourceIds + */ + private function reportIntegrations(array $resources, array &$report, array $resourceIds = []): void + { + if (\in_array(Resource::TYPE_PLATFORM, $resources)) { + $consoleHeaders = $this->getConsoleHeaders(); + + if ($consoleHeaders === null) { + return; + } + + try { + $response = $this->call('GET', '/projects/' . $this->project . '/platforms', $consoleHeaders); + $report[Resource::TYPE_PLATFORM] = $response['total'] ?? 0; + } catch (\Throwable) { + $report[Resource::TYPE_PLATFORM] = 0; + } + } + } + + /** + * @param int $batchSize + * @param array $resources + */ + protected function exportGroupIntegrations(int $batchSize, array $resources): void + { + if (\in_array(Resource::TYPE_PLATFORM, $resources)) { + $this->exportWithConsoleHeaders( + Resource::TYPE_PLATFORM, + Transfer::GROUP_INTEGRATIONS, + $this->exportPlatforms(...) + ); + } + } + + protected function exportWithConsoleHeaders(string $resourceType, string $group, callable $callback): void + { + $consoleHeaders = $this->getConsoleHeaders(); + + if ($consoleHeaders === null) { + $this->addError(new Exception( + $resourceType, + $group, + message: 'Console key unavailable for source instance', + )); + return; + } + + try { + $callback($consoleHeaders); + } catch (\Throwable $e) { + $this->addError(new Exception( + $resourceType, + $group, + message: $e->getMessage(), + code: $e->getCode(), + previous: $e + )); + } + } + + /** + * @throws AppwriteException + */ + private function exportPlatforms(array $consoleHeaders): void + { + $response = $this->call('GET', '/projects/' . $this->project . '/platforms', $consoleHeaders); + + if (empty($response['platforms'])) { + return; + } + + $platforms = []; + + foreach ($response['platforms'] as $platform) { + $platforms[] = new Platform( + $platform['$id'] ?? '', + $platform['type'] ?? '', + $platform['name'] ?? '', + $platform['key'] ?? '', + $platform['store'] ?? '', + $platform['hostname'] ?? '', + createdAt: $platform['$createdAt'] ?? '', + updatedAt: $platform['$updatedAt'] ?? '', + ); + } + + $this->callback($platforms); + } + /** * Build queries with optional filtering by resource IDs */ diff --git a/src/Migration/Sources/CSV.php b/src/Migration/Sources/CSV.php index 3cd227c5..7b8e479f 100644 --- a/src/Migration/Sources/CSV.php +++ b/src/Migration/Sources/CSV.php @@ -380,6 +380,11 @@ protected function exportGroupSites(int $batchSize, array $resources): void throw new \Exception('Not Implemented'); } + protected function exportGroupIntegrations(int $batchSize, array $resources): void + { + throw new \Exception('Not Implemented'); + } + /** * @param callable(resource $stream, string $delimiter): void $callback * @return void diff --git a/src/Migration/Sources/Firebase.php b/src/Migration/Sources/Firebase.php index b1ef5b72..efe7f006 100644 --- a/src/Migration/Sources/Firebase.php +++ b/src/Migration/Sources/Firebase.php @@ -667,7 +667,6 @@ protected function exportGroupStorage(int $batchSize, array $resources): void previous: $e )); } - } private function exportBuckets(int $batchsize): void @@ -813,4 +812,9 @@ protected function exportGroupSites(int $batchSize, array $resources): void { throw new \Exception('Not implemented'); } + + protected function exportGroupIntegrations(int $batchSize, array $resources): void + { + throw new \Exception('Not implemented'); + } } diff --git a/src/Migration/Sources/JSON.php b/src/Migration/Sources/JSON.php index 8abf63d7..944824dc 100644 --- a/src/Migration/Sources/JSON.php +++ b/src/Migration/Sources/JSON.php @@ -209,6 +209,11 @@ protected function exportGroupSites(int $batchSize, array $resources): void throw new \Exception('Not Implemented'); } + protected function exportGroupIntegrations(int $batchSize, array $resources): void + { + throw new \Exception('Not Implemented'); + } + /** * @param callable(Items): void $callback * @throws \Exception|JsonMachineException diff --git a/src/Migration/Sources/NHost.php b/src/Migration/Sources/NHost.php index 5c563248..c0829704 100644 --- a/src/Migration/Sources/NHost.php +++ b/src/Migration/Sources/NHost.php @@ -853,4 +853,9 @@ protected function exportGroupSites(int $batchSize, array $resources): void { throw new \Exception('Not Implemented'); } + + protected function exportGroupIntegrations(int $batchSize, array $resources): void + { + throw new \Exception('Not Implemented'); + } } diff --git a/src/Migration/Transfer.php b/src/Migration/Transfer.php index b4da5007..04dc01fc 100644 --- a/src/Migration/Transfer.php +++ b/src/Migration/Transfer.php @@ -16,7 +16,7 @@ class Transfer public const GROUP_DATABASES = 'databases'; - public const GROUP_SETTINGS = 'settings'; + public const GROUP_INTEGRATIONS = 'integrations'; public const GROUP_AUTH_RESOURCES = [ Resource::TYPE_USER, @@ -50,7 +50,9 @@ class Transfer Resource::TYPE_ROW, ]; - public const GROUP_SETTINGS_RESOURCES = []; + public const GROUP_INTEGRATIONS_RESOURCES = [ + Resource::TYPE_PLATFORM, + ]; public const ALL_PUBLIC_RESOURCES = [ Resource::TYPE_USER, @@ -70,6 +72,9 @@ class Transfer Resource::TYPE_COLUMN, Resource::TYPE_ROW, + // Integrations + Resource::TYPE_PLATFORM, + // legacy Resource::TYPE_DOCUMENT, Resource::TYPE_ATTRIBUTE, @@ -83,6 +88,7 @@ class Transfer Resource::TYPE_SITE, Resource::TYPE_USER, Resource::TYPE_TEAM, + Resource::TYPE_PLATFORM, ]; public const STORAGE_MAX_CHUNK_SIZE = 1024 * 1024 * 5; // 5MB @@ -342,7 +348,7 @@ public static function extractServices(array $services): array self::GROUP_GENERAL => array_merge($resources, []), self::GROUP_AUTH => array_merge($resources, self::GROUP_AUTH_RESOURCES), self::GROUP_DATABASES => array_merge($resources, self::GROUP_DATABASES_RESOURCES), - self::GROUP_SETTINGS => array_merge($resources, self::GROUP_SETTINGS_RESOURCES), + self::GROUP_INTEGRATIONS => array_merge($resources, self::GROUP_INTEGRATIONS_RESOURCES), default => throw new \Exception('No service group found'), }; } diff --git a/tests/Migration/Unit/Adapters/MockDestination.php b/tests/Migration/Unit/Adapters/MockDestination.php index d8258b3e..333a3b70 100644 --- a/tests/Migration/Unit/Adapters/MockDestination.php +++ b/tests/Migration/Unit/Adapters/MockDestination.php @@ -51,6 +51,7 @@ public static function getSupportedResources(): array Resource::TYPE_ENVIRONMENT_VARIABLE, Resource::TYPE_TEAM, Resource::TYPE_MEMBERSHIP, + Resource::TYPE_PLATFORM, ]; } diff --git a/tests/Migration/Unit/Adapters/MockSource.php b/tests/Migration/Unit/Adapters/MockSource.php index 64e5640f..0353219f 100644 --- a/tests/Migration/Unit/Adapters/MockSource.php +++ b/tests/Migration/Unit/Adapters/MockSource.php @@ -80,6 +80,7 @@ public static function getSupportedResources(): array Resource::TYPE_ENVIRONMENT_VARIABLE, Resource::TYPE_TEAM, Resource::TYPE_MEMBERSHIP, + Resource::TYPE_PLATFORM, // legacy Resource::TYPE_DOCUMENT, @@ -177,4 +178,21 @@ protected function exportGroupSites(int $batchSize, array $resources): void $this->handleResourceTransfer(Transfer::GROUP_SITES, $resource); } } + + /** + * Export Integrations Group + * + * @param int $batchSize Max 100 + * @param string[] $resources Resources to export + */ + protected function exportGroupIntegrations(int $batchSize, array $resources): void + { + foreach (Transfer::GROUP_INTEGRATIONS_RESOURCES as $resource) { + if (!\in_array($resource, $resources)) { + continue; + } + + $this->handleResourceTransfer(Transfer::GROUP_INTEGRATIONS, $resource); + } + } }