diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index 05c62470..32de9758 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -50,6 +50,7 @@ use Utopia\Migration\Resources\Functions\Deployment; use Utopia\Migration\Resources\Functions\EnvVar; use Utopia\Migration\Resources\Functions\Func; +use Utopia\Migration\Resources\Integrations\Platform; use Utopia\Migration\Resources\Messaging\Message; use Utopia\Migration\Resources\Messaging\Provider; use Utopia\Migration\Resources\Messaging\Subscriber; @@ -99,7 +100,9 @@ public function __construct( string $key, protected UtopiaDatabase $dbForProject, callable $getDatabasesDB, - protected array $collectionStructure + protected array $collectionStructure, + protected UtopiaDatabase $dbForPlatform, + protected string $projectInternalId, ) { $this->project = $project; $this->endpoint = $endpoint; @@ -169,6 +172,9 @@ public static function getSupportedResources(): array Resource::TYPE_SITE, Resource::TYPE_SITE_DEPLOYMENT, Resource::TYPE_SITE_VARIABLE, + + // Integrations + Resource::TYPE_PLATFORM, ]; } @@ -281,7 +287,6 @@ public function report(array $resources = [], array $resourceIds = []): array $scope = 'sites.write'; $this->sites->create('', '', Framework::OTHER(), BuildRuntime::STATIC1()); } - } catch (AppwriteException $e) { if ($e->getCode() === 403) { throw new \Exception( @@ -317,6 +322,7 @@ protected function import(array $resources, callable $callback): void try { $this->dbForProject->setPreserveDates(true); + $this->dbForPlatform?->setPreserveDates(true); $responseResource = match ($resource->getGroup()) { Transfer::GROUP_DATABASES => $this->importDatabaseResource($resource, $isLast), @@ -325,6 +331,7 @@ protected function import(array $resources, callable $callback): void Transfer::GROUP_FUNCTIONS => $this->importFunctionResource($resource), Transfer::GROUP_MESSAGING => $this->importMessagingResource($resource), Transfer::GROUP_SITES => $this->importSiteResource($resource), + Transfer::GROUP_INTEGRATIONS => $this->importIntegrationsResource($resource), default => throw new \Exception('Invalid resource group', Exception::CODE_VALIDATION), }; } catch (\Throwable $e) { @@ -342,6 +349,7 @@ protected function import(array $resources, callable $callback): void $responseResource = $resource; } finally { $this->dbForProject->setPreserveDates(false); + $this->dbForPlatform?->setPreserveDates(false); } $this->cache->update($responseResource); @@ -1071,7 +1079,6 @@ protected function createRecord(Row $resource, bool $isLast): bool 'database_' . $databaseInternalId . '_collection_' . $tableInternalId, $this->rowBuffer )); - } finally { $this->rowBuffer = []; } @@ -2189,6 +2196,68 @@ private function importSiteDeployment(SiteDeployment $deployment): Resource return $deployment; } + /** + * @throws \Exception + */ + public function importIntegrationsResource(Resource $resource): Resource + { + switch ($resource->getName()) { + case Resource::TYPE_PLATFORM: + /** @var Platform $resource */ + $this->createPlatform($resource); + break; + } + + if ($resource->getStatus() !== Resource::STATUS_SKIPPED) { + $resource->setStatus(Resource::STATUS_SUCCESS); + } + + return $resource; + } + + /** + * @throws \Throwable + */ + protected function createPlatform(Platform $resource): bool + { + $existing = $this->dbForPlatform->findOne('platforms', [ + Query::equal('projectId', [$this->project]), + Query::equal('type', [$resource->getType()]), + Query::equal('name', [$resource->getPlatformName()]), + ]); + + if ($existing !== false && !$existing->isEmpty()) { + $resource->setStatus(Resource::STATUS_SKIPPED, 'Platform already exists'); + return false; + } + + $createdAt = $this->normalizeDateTime($resource->getCreatedAt()); + $updatedAt = $this->normalizeDateTime($resource->getUpdatedAt(), $createdAt); + + try { + $this->dbForPlatform->createDocument('platforms', new UtopiaDocument([ + '$id' => ID::unique(), + '$permissions' => $resource->getPermissions(), + 'projectInternalId' => $this->projectInternalId, + 'projectId' => $this->project, + 'type' => $resource->getType(), + 'name' => $resource->getPlatformName(), + 'key' => $resource->getKey(), + 'store' => $resource->getStore(), + 'hostname' => $resource->getHostname(), + '$createdAt' => $createdAt, + '$updatedAt' => $updatedAt, + ])); + } catch (DuplicateException) { + $resource->setStatus(Resource::STATUS_SKIPPED, 'Platform already exists'); + return false; + } + + $this->dbForPlatform->purgeCachedDocument('projects', $this->project); + + return true; + } + private function validateFieldsForIndexes(Index $resource, UtopiaDocument $table, array &$lengths) { /** diff --git a/src/Migration/Resource.php b/src/Migration/Resource.php index 1dc198c6..e624fe2c 100644 --- a/src/Migration/Resource.php +++ b/src/Migration/Resource.php @@ -71,6 +71,8 @@ abstract class Resource implements \JsonSerializable public const TYPE_ENVIRONMENT_VARIABLE = 'environment-variable'; + // Integrations + public const TYPE_PLATFORM = 'platform'; public const TYPE_SUBSCRIBER = 'subscriber'; public const TYPE_MESSAGE = 'message'; @@ -106,6 +108,7 @@ abstract class Resource implements \JsonSerializable self::TYPE_ENVIRONMENT_VARIABLE, self::TYPE_TEAM, self::TYPE_MEMBERSHIP, + self::TYPE_PLATFORM, self::TYPE_PROVIDER, self::TYPE_TOPIC, self::TYPE_SUBSCRIBER, diff --git a/src/Migration/Resources/Integrations/Platform.php b/src/Migration/Resources/Integrations/Platform.php new file mode 100644 index 00000000..c81b6f04 --- /dev/null +++ b/src/Migration/Resources/Integrations/Platform.php @@ -0,0 +1,104 @@ +id = $id; + $this->createdAt = $createdAt; + $this->updatedAt = $updatedAt; + } + + /** + * @param array $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['id'], + $array['type'], + $array['name'], + $array['key'] ?? '', + $array['store'] ?? '', + $array['hostname'] ?? '', + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + /** + * @return array + */ + public function jsonSerialize(): array + { + return [ + 'id' => $this->id, + 'type' => $this->type, + 'name' => $this->name, + 'key' => $this->key, + 'store' => $this->store, + 'hostname' => $this->hostname, + 'createdAt' => $this->createdAt, + 'updatedAt' => $this->updatedAt, + ]; + } + + public static function getName(): string + { + return Resource::TYPE_PLATFORM; + } + + public function getGroup(): string + { + return Transfer::GROUP_INTEGRATIONS; + } + + public function getType(): string + { + return $this->type; + } + + public function getPlatformName(): string + { + return $this->name; + } + + public function getKey(): string + { + return $this->key; + } + + public function getStore(): string + { + return $this->store; + } + + public function getHostname(): string + { + return $this->hostname; + } +} diff --git a/src/Migration/Source.php b/src/Migration/Source.php index b6eb8c42..2dcf205f 100644 --- a/src/Migration/Source.php +++ b/src/Migration/Source.php @@ -46,6 +46,11 @@ public function getSitesBatchSize(): int return static::$defaultBatchSize; } + public function getIntegrationsBatchSize(): int + { + return static::$defaultBatchSize; + } + /** * @param array $resources * @return void @@ -109,6 +114,7 @@ public function exportResources(array $resources): void Transfer::GROUP_FUNCTIONS => Transfer::GROUP_FUNCTIONS_RESOURCES, Transfer::GROUP_MESSAGING => Transfer::GROUP_MESSAGING_RESOURCES, Transfer::GROUP_SITES => Transfer::GROUP_SITES_RESOURCES, + Transfer::GROUP_INTEGRATIONS => Transfer::GROUP_INTEGRATIONS_RESOURCES, ]; foreach ($mapping as $group => $resources) { @@ -143,6 +149,9 @@ public function exportResources(array $resources): void case Transfer::GROUP_SITES: $this->exportGroupSites($this->getSitesBatchSize(), $resources); break; + case Transfer::GROUP_INTEGRATIONS: + $this->exportGroupIntegrations($this->getIntegrationsBatchSize(), $resources); + break; } } } @@ -194,4 +203,12 @@ abstract protected function exportGroupMessaging(int $batchSize, array $resource * @param array $resources Resources to export */ abstract protected function exportGroupSites(int $batchSize, array $resources): void; + + /** + * Export Integrations Group + * + * @param int $batchSize + * @param array $resources Resources to export + */ + abstract protected function exportGroupIntegrations(int $batchSize, array $resources): void; } diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index 4de9c1bb..1f51fa6b 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -53,6 +53,7 @@ use Utopia\Migration\Resources\Functions\Deployment; use Utopia\Migration\Resources\Functions\EnvVar; use Utopia\Migration\Resources\Functions\Func; +use Utopia\Migration\Resources\Integrations\Platform; use Utopia\Migration\Resources\Messaging\Message; use Utopia\Migration\Resources\Messaging\Provider; use Utopia\Migration\Resources\Messaging\Subscriber; @@ -96,6 +97,10 @@ class Appwrite extends Source */ protected mixed $getDatabasesDB; + private bool $consoleKeyFetched = false; + + private ?string $consoleKey = null; + /** * @throws \Exception */ @@ -147,6 +152,38 @@ public function __construct( } + public function setConsoleKey(string $key): void + { + $this->consoleKey = $key; + $this->consoleKeyFetched = true; + } + + /** + * @return array|null + */ + protected function getConsoleHeaders(): ?array + { + if (!$this->consoleKeyFetched) { + $this->consoleKeyFetched = true; + + try { + $response = $this->call('POST', '/migrations/appwrite/console-key'); + $this->consoleKey = $response['key'] ?? null; + } catch (\Throwable) { + $this->consoleKey = null; + } + } + + if ($this->consoleKey === null) { + return null; + } + + return [ + 'x-appwrite-project' => 'console', + 'x-appwrite-key' => $this->consoleKey, + ]; + } + public static function getName(): string { return 'Appwrite'; @@ -200,7 +237,8 @@ public static function getSupportedResources(): array Resource::TYPE_SITE_DEPLOYMENT, Resource::TYPE_SITE_VARIABLE, - // Settings + // Integrations + Resource::TYPE_PLATFORM, ]; } @@ -239,6 +277,7 @@ public function report(array $resources = [], array $resourceIds = []): array $this->reportFunctions($resources, $report, $resourceIds); $this->reportMessaging($resources, $report, $resourceIds); $this->reportSites($resources, $report, $resourceIds); + $this->reportIntegrations($resources, $report, $resourceIds); $report['version'] = $this->call( 'GET', @@ -2188,6 +2227,29 @@ private function exportSiteDeploymentData(Site $site, array $deployment): void } } + /** + * @param array $resources + * @param array $report + * @param array> $resourceIds + */ + private function reportIntegrations(array $resources, array &$report, array $resourceIds = []): void + { + if (\in_array(Resource::TYPE_PLATFORM, $resources)) { + $consoleHeaders = $this->getConsoleHeaders(); + + if ($consoleHeaders === null) { + return; + } + + try { + $response = $this->call('GET', '/projects/' . $this->project . '/platforms', $consoleHeaders); + $report[Resource::TYPE_PLATFORM] = $response['total'] ?? 0; + } catch (\Throwable) { + $report[Resource::TYPE_PLATFORM] = 0; + } + } + } + /** * @param string $databaseType * @param array $database { @@ -2212,6 +2274,47 @@ public static function getDatabase(string $databaseType, array $database): Resou } } + /** + * @param int $batchSize + * @param array $resources + */ + protected function exportGroupIntegrations(int $batchSize, array $resources): void + { + if (\in_array(Resource::TYPE_PLATFORM, $resources)) { + $this->exportWithConsoleHeaders( + Resource::TYPE_PLATFORM, + Transfer::GROUP_INTEGRATIONS, + $this->exportPlatforms(...) + ); + } + } + + protected function exportWithConsoleHeaders(string $resourceType, string $group, callable $callback): void + { + $consoleHeaders = $this->getConsoleHeaders(); + + if ($consoleHeaders === null) { + $this->addError(new Exception( + $resourceType, + $group, + message: 'Console key unavailable for source instance', + )); + return; + } + + try { + $callback($consoleHeaders); + } catch (\Throwable $e) { + $this->addError(new Exception( + $resourceType, + $group, + message: $e->getMessage(), + code: $e->getCode(), + previous: $e + )); + } + } + /** * eg., tables,collections * @param string $databaseType @@ -2242,6 +2345,35 @@ public static function getEntity(string $databaseType, array $entity): Resource } } + /** + * @throws AppwriteException + */ + private function exportPlatforms(array $consoleHeaders): void + { + $response = $this->call('GET', '/projects/' . $this->project . '/platforms', $consoleHeaders); + + if (empty($response['platforms'])) { + return; + } + + $platforms = []; + + foreach ($response['platforms'] as $platform) { + $platforms[] = new Platform( + $platform['$id'] ?? '', + $platform['type'] ?? '', + $platform['name'] ?? '', + $platform['key'] ?? '', + $platform['store'] ?? '', + $platform['hostname'] ?? '', + createdAt: $platform['$createdAt'] ?? '', + updatedAt: $platform['$updatedAt'] ?? '', + ); + } + + $this->callback($platforms); + } + /** * eg.,documents/attributes * @param string $databaseType diff --git a/src/Migration/Sources/CSV.php b/src/Migration/Sources/CSV.php index 7d02090f..3454f4a1 100644 --- a/src/Migration/Sources/CSV.php +++ b/src/Migration/Sources/CSV.php @@ -429,6 +429,11 @@ protected function exportGroupSites(int $batchSize, array $resources): void throw new \Exception('Not Implemented'); } + protected function exportGroupIntegrations(int $batchSize, array $resources): void + { + throw new \Exception('Not Implemented'); + } + /** * @param callable(resource $stream, string $delimiter): void $callback * @return void diff --git a/src/Migration/Sources/Firebase.php b/src/Migration/Sources/Firebase.php index 4495fb5b..94615746 100644 --- a/src/Migration/Sources/Firebase.php +++ b/src/Migration/Sources/Firebase.php @@ -667,7 +667,6 @@ protected function exportGroupStorage(int $batchSize, array $resources): void previous: $e )); } - } private function exportBuckets(int $batchsize): void @@ -818,4 +817,9 @@ protected function exportGroupSites(int $batchSize, array $resources): void { throw new \Exception('Not implemented'); } + + protected function exportGroupIntegrations(int $batchSize, array $resources): void + { + throw new \Exception('Not implemented'); + } } diff --git a/src/Migration/Sources/JSON.php b/src/Migration/Sources/JSON.php index 15e7b6aa..2a2ff81f 100644 --- a/src/Migration/Sources/JSON.php +++ b/src/Migration/Sources/JSON.php @@ -217,6 +217,11 @@ protected function exportGroupSites(int $batchSize, array $resources): void throw new \Exception('Not Implemented'); } + protected function exportGroupIntegrations(int $batchSize, array $resources): void + { + throw new \Exception('Not Implemented'); + } + /** * @param callable(Items): void $callback * @throws \Exception|JsonMachineException diff --git a/src/Migration/Sources/NHost.php b/src/Migration/Sources/NHost.php index 4aac49f8..c6bfa419 100644 --- a/src/Migration/Sources/NHost.php +++ b/src/Migration/Sources/NHost.php @@ -951,4 +951,9 @@ protected function exportGroupSites(int $batchSize, array $resources): void { throw new \Exception('Not Implemented'); } + + protected function exportGroupIntegrations(int $batchSize, array $resources): void + { + throw new \Exception('Not Implemented'); + } } diff --git a/src/Migration/Transfer.php b/src/Migration/Transfer.php index 561ff283..59ed486c 100644 --- a/src/Migration/Transfer.php +++ b/src/Migration/Transfer.php @@ -22,7 +22,7 @@ class Transfer public const GROUP_DATABASES_VECTOR_DB = 'vectorsdb'; - public const GROUP_SETTINGS = 'settings'; + public const GROUP_INTEGRATIONS = 'integrations'; public const GROUP_MESSAGING = 'messaging'; @@ -58,6 +58,9 @@ class Transfer Resource::TYPE_ROW, ]; + public const GROUP_INTEGRATIONS_RESOURCES = [ + Resource::TYPE_PLATFORM, + ]; public const GROUP_DOCUMENTSDB_RESOURCES = [ Resource::TYPE_DATABASE_DOCUMENTSDB, Resource::TYPE_COLLECTION, @@ -117,6 +120,9 @@ class Transfer Resource::TYPE_SUBSCRIBER, Resource::TYPE_MESSAGE, + // Integrations + Resource::TYPE_PLATFORM, + // legacy Resource::TYPE_DOCUMENT, Resource::TYPE_ATTRIBUTE, @@ -132,6 +138,7 @@ class Transfer Resource::TYPE_SITE, Resource::TYPE_USER, Resource::TYPE_TEAM, + Resource::TYPE_PLATFORM, Resource::TYPE_PROVIDER, Resource::TYPE_TOPIC, Resource::TYPE_MESSAGE, @@ -394,10 +401,10 @@ public static function extractServices(array $services): array self::GROUP_GENERAL => array_merge($resources, []), self::GROUP_AUTH => array_merge($resources, self::GROUP_AUTH_RESOURCES), self::GROUP_DATABASES => array_merge($resources, self::GROUP_DATABASES_RESOURCES), + self::GROUP_INTEGRATIONS => array_merge($resources, self::GROUP_INTEGRATIONS_RESOURCES), self::GROUP_DATABASES_TABLES_DB => array_merge($resources, self::GROUP_TABLESDB_RESOURCES), self::GROUP_DATABASES_DOCUMENTS_DB => array_merge($resources, self::GROUP_DOCUMENTSDB_RESOURCES), self::GROUP_DATABASES_VECTOR_DB => array_merge($resources, self::GROUP_VECTORSDB_RESOURCES), - self::GROUP_SETTINGS => array_merge($resources, self::GROUP_SETTINGS_RESOURCES), self::GROUP_MESSAGING => array_merge($resources, self::GROUP_MESSAGING_RESOURCES), default => throw new \Exception('No service group found'), }; diff --git a/tests/Migration/Unit/Adapters/MockDestination.php b/tests/Migration/Unit/Adapters/MockDestination.php index fe1da710..2aa528a8 100644 --- a/tests/Migration/Unit/Adapters/MockDestination.php +++ b/tests/Migration/Unit/Adapters/MockDestination.php @@ -51,6 +51,7 @@ public static function getSupportedResources(): array Resource::TYPE_ENVIRONMENT_VARIABLE, Resource::TYPE_TEAM, Resource::TYPE_MEMBERSHIP, + Resource::TYPE_PLATFORM, Resource::TYPE_PROVIDER, Resource::TYPE_TOPIC, Resource::TYPE_SUBSCRIBER, diff --git a/tests/Migration/Unit/Adapters/MockSource.php b/tests/Migration/Unit/Adapters/MockSource.php index 889eeb3c..1524dea8 100644 --- a/tests/Migration/Unit/Adapters/MockSource.php +++ b/tests/Migration/Unit/Adapters/MockSource.php @@ -80,6 +80,7 @@ public static function getSupportedResources(): array Resource::TYPE_ENVIRONMENT_VARIABLE, Resource::TYPE_TEAM, Resource::TYPE_MEMBERSHIP, + Resource::TYPE_PLATFORM, Resource::TYPE_PROVIDER, Resource::TYPE_TOPIC, Resource::TYPE_SUBSCRIBER, @@ -198,4 +199,21 @@ protected function exportGroupSites(int $batchSize, array $resources): void $this->handleResourceTransfer(Transfer::GROUP_SITES, $resource); } } + + /** + * Export Integrations Group + * + * @param int $batchSize Max 100 + * @param string[] $resources Resources to export + */ + protected function exportGroupIntegrations(int $batchSize, array $resources): void + { + foreach (Transfer::GROUP_INTEGRATIONS_RESOURCES as $resource) { + if (!\in_array($resource, $resources)) { + continue; + } + + $this->handleResourceTransfer(Transfer::GROUP_INTEGRATIONS, $resource); + } + } }