diff --git a/examples/web/src/pages/ai-demo.tsx b/examples/web/src/pages/ai-demo.tsx index 223bc100..df1918fb 100644 --- a/examples/web/src/pages/ai-demo.tsx +++ b/examples/web/src/pages/ai-demo.tsx @@ -8,6 +8,7 @@ import Elevenlabs from '@imgly/plugin-ai-audio-generation-web/elevenlabs'; import Anthropic from '@imgly/plugin-ai-text-generation-web/anthropic'; import OpenAIText from '@imgly/plugin-ai-text-generation-web/open-ai'; import FalAiSticker from '@imgly/plugin-ai-sticker-generation-web/fal-ai'; +import BackgroundRemovalPlugin, { FalAi } from '@imgly/plugin-background-removal-web'; import { useRef } from 'react'; import { rateLimitMiddleware } from '@imgly/plugin-ai-generation-web'; @@ -154,6 +155,23 @@ function App() { }); }; + + + await instance.addPlugin(BackgroundRemovalPlugin({ + provider: await FalAi.Birefnet2({ + proxyUrl: import.meta.env.VITE_FAL_AI_PROXY_URL, + headers: { + 'x-client-version': '1.0.0', + 'User-Agent': 'CreativeEditor-AI-Demo/1.0' + }, + timeout: 30000, + debug: true + })({ cesdk: instance }), + ui: { + locations: ['canvasMenu', 'dock'] + } + })); + instance.addPlugin( AiApps({ debug: true, diff --git a/packages/plugin-background-removal-web/README.md b/packages/plugin-background-removal-web/README.md index d8f33cf4..2769cc34 100644 --- a/packages/plugin-background-removal-web/README.md +++ b/packages/plugin-background-removal-web/README.md @@ -17,8 +17,11 @@ npm install @imgly/plugin-background-removal-web onnxruntime-web@1.21.0 ## Usage -Adding the plugin to CE.SDK will automatically add a background removal -canvas menu entry for every block with an image fill. +This plugin supports both **client-side** and **server-side** background removal: + +### Client-Side Background Removal (Default) + +For client-side background removal using the `@imgly/background-removal-js` library (no external API required): ```typescript import CreativeEditorSDK from '@cesdk/cesdk-js'; @@ -38,6 +41,8 @@ const config = { const cesdk = await CreativeEditorSDK.create(container, config); await cesdk.addDefaultAssetSources(); await cesdk.addDemoAssetSources({ sceneMode: 'Design' }); + +// Client-side background removal (default behavior) await cesdk.addPlugin(BackgroundRemovalPlugin()); // Add the canvas menu component for background removal @@ -49,6 +54,62 @@ cesdk.ui.setCanvasMenuOrder([ await cesdk.createDesignScene(); ``` +### Server-Side Background Removal (Fal-AI) + +For server-side background removal using Fal-AI or other providers: + +```typescript +import CreativeEditorSDK from '@cesdk/cesdk-js'; +import BackgroundRemovalPlugin, { FalAi } from '@imgly/plugin-background-removal-web'; + +// ... CESDK setup ... + +// Server-side background removal with pre-configured models +await cesdk.addPlugin(BackgroundRemovalPlugin({ + provider: await FalAi.Birefnet2({ + proxyUrl: 'your-proxy-url', + headers: { + 'x-client-version': '1.0.0', + 'User-Agent': 'CreativeEditor-AI-Demo/1.0' + }, + timeout: 30000, + debug: true + })({ cesdk }), + ui: { + locations: ['canvasMenu', 'dock'] + } +})); + +// Available pre-configured models: +// - FalAi.Birefnet2 - High-quality background removal (fal-ai/birefnet/v2) +// - FalAi.Birefnet - Original Birefnet model (fal-ai/birefnet) +// - FalAi.BriaBackgroundRemove - Bria's background removal (fal-ai/bria/background/remove) +// - FalAi.RembgEnhance - Enhanced background removal (smoretalk-ai/rembg-enhance) +// - FalAi.ImageutilsRembg - Image utils background removal (fal-ai/imageutils/rembg) + +// Or use any custom model with createProvider: +await cesdk.addPlugin(BackgroundRemovalPlugin({ + provider: await FalAi.createProvider( + 'your-custom-model-key', + { + proxyUrl: 'your-proxy-url', + headers: { + 'x-client-version': '1.0.0', + 'User-Agent': 'CreativeEditor-AI-Demo/1.0' + }, + timeout: 30000, + debug: true + } + )({ cesdk }), + ui: { + locations: ['canvasMenu', 'dock'] + } +})); +``` + +Adding the plugin to CE.SDK will automatically add a background removal +canvas menu entry for every block with an image fill. + ## Configuration ### Adding Components @@ -207,3 +268,8 @@ async function uploadBlob( return url; } ``` + +## Performance Considerations + +- **Client-side**: Processes images locally, no network requests, but requires more client resources. +- **Server-side**: Offloads processing to servers, requires network requests, but uses less client resources. \ No newline at end of file diff --git a/packages/plugin-background-removal-web/esbuild/config.mjs b/packages/plugin-background-removal-web/esbuild/config.mjs index 04a6719d..51e41283 100644 --- a/packages/plugin-background-removal-web/esbuild/config.mjs +++ b/packages/plugin-background-removal-web/esbuild/config.mjs @@ -15,9 +15,25 @@ export default ({ isDevelopment }) => { `${chalk.yellow('Building version:')} ${chalk.bold(packageJson.version)}` ); - return baseConfig({ + // Base configuration that applies to all builds + const baseOptions = { isDevelopment, external: ['@imgly/background-removal', '@cesdk/cesdk-js'], pluginVersion: packageJson.version - }); + }; + + // Get the base configuration + const config = baseConfig(baseOptions); + + // Set entry points and output configuration + config.entryPoints = [ + './src/index.ts', + './src/fal-ai/index.ts' + ]; + config.outExtension = { '.js': '.mjs' }; + config.outdir = './dist'; + config.outbase = './src'; + config.outfile = undefined; + + return config; }; diff --git a/packages/plugin-background-removal-web/package.json b/packages/plugin-background-removal-web/package.json index dad145b0..8d52670c 100644 --- a/packages/plugin-background-removal-web/package.json +++ b/packages/plugin-background-removal-web/package.json @@ -32,6 +32,10 @@ ".": { "import": "./dist/index.mjs", "types": "./dist/index.d.ts" + }, + "./fal-ai": { + "import": "./dist/fal-ai/index.mjs", + "types": "./dist/fal-ai/index.d.ts" } }, "homepage": "https://img.ly/products/creative-sdk", @@ -77,11 +81,12 @@ "injected": true } }, - "peerDependencies": { - "@cesdk/cesdk-js": "^1.32.0", - "onnxruntime-web": "1.21.0" - }, "dependencies": { "@imgly/background-removal": "1.7.0" + }, + "peerDependencies": { + "@cesdk/cesdk-js": "^1.32.0", + "onnxruntime-web": "1.21.0", + "@fal-ai/client": "^1.3.0" } } diff --git a/packages/plugin-background-removal-web/src/fal-ai/createBackgroundRemovalProvider.ts b/packages/plugin-background-removal-web/src/fal-ai/createBackgroundRemovalProvider.ts new file mode 100644 index 00000000..2fa6d09a --- /dev/null +++ b/packages/plugin-background-removal-web/src/fal-ai/createBackgroundRemovalProvider.ts @@ -0,0 +1,120 @@ +import CreativeEditorSDK from '@cesdk/cesdk-js'; +import { fal } from '@fal-ai/client'; +import { BackgroundRemovalProvider } from '../processBackgroundRemoval'; +import { uploadImageInputToFalIfNeeded } from './utils'; + +type BackgroundRemovalProviderConfiguration = { + proxyUrl: string; + debug?: boolean; + headers?: Record; + timeout?: number; +}; + +export function createBackgroundRemovalProvider( + modelKey: string, + config: BackgroundRemovalProviderConfiguration +): (context: { + cesdk: CreativeEditorSDK; +}) => Promise { + return async ({ cesdk }: { cesdk: CreativeEditorSDK }) => { + // Configure fal client + fal.config({ + proxyUrl: config.proxyUrl, + requestMiddleware: config.headers + ? async (request) => { + return { + ...request, + headers: { + ...request.headers, + ...config.headers + } + }; + } + : undefined + }); + + const provider: BackgroundRemovalProvider = { + type: 'custom', + processImageFileURI: async (imageFileURI: string): Promise => { + try { + if (config.debug) { + // eslint-disable-next-line no-console + console.log('Processing background removal with:', { + imageFileURI, + modelKey + }); + } + + // Upload image if needed (for blob: or buffer: URLs) + const processedImageUrl = await uploadImageInputToFalIfNeeded( + imageFileURI, + cesdk + ); + + const input = { + image_url: processedImageUrl || imageFileURI, + sync_mode: true + }; + + const controller = new AbortController(); + const timeoutId = config.timeout + ? setTimeout(() => controller.abort(), config.timeout) + : undefined; + + try { + const response = await fal.subscribe(modelKey, { + input, + logs: config.debug, + abortSignal: controller.signal + }); + + if (timeoutId) clearTimeout(timeoutId); + + if (config.debug) { + // eslint-disable-next-line no-console + console.log('Background removal response:', response); + } + + const resultUrl = response.data?.image?.url; + if (!resultUrl) { + throw new Error('No processed image URL in response'); + } + + return resultUrl; + } finally { + if (timeoutId) clearTimeout(timeoutId); + } + } catch (error) { + // eslint-disable-next-line no-console + console.error('Background removal error:', error); + throw error; + } + }, + + processSourceSet: async (sourceSet) => { + if (sourceSet.length === 0) { + throw new Error('No sources provided'); + } + + // Process the highest resolution source (first in the sorted array) + const highestResSource = sourceSet[0]; + const processedUrl = await (provider as any).processImageFileURI( + highestResSource.uri + ); + + // Return a new source set with the processed image + return [ + { + uri: processedUrl, + width: highestResSource.width, + height: highestResSource.height + } + ]; + } + }; + + return provider; + }; +} + +export default createBackgroundRemovalProvider; diff --git a/packages/plugin-background-removal-web/src/fal-ai/index.ts b/packages/plugin-background-removal-web/src/fal-ai/index.ts new file mode 100644 index 00000000..38ed5334 --- /dev/null +++ b/packages/plugin-background-removal-web/src/fal-ai/index.ts @@ -0,0 +1,50 @@ +import { createBackgroundRemovalProvider } from './createBackgroundRemovalProvider'; + +const FalAi = { + /** + * Creates a FalAI background removal provider for any model + * @param modelKey - The FalAI model key (e.g., 'fal-ai/birefnet', 'fal-ai/birefnet/v2') + * @param config - Provider configuration + */ + createProvider: createBackgroundRemovalProvider, + + /** + * Pre-configured provider for Birefnet v2 model + * @param config - Provider configuration + */ + Birefnet2: (config: Parameters[1]) => + createBackgroundRemovalProvider('fal-ai/birefnet/v2', config), + + /** + * Pre-configured provider for Birefnet model + * @param config - Provider configuration + */ + Birefnet: (config: Parameters[1]) => + createBackgroundRemovalProvider('fal-ai/birefnet', config), + + /** + * Pre-configured provider for Bria Background Remove model + * @param config - Provider configuration + */ + BriaBackgroundRemove: ( + config: Parameters[1] + ) => createBackgroundRemovalProvider('fal-ai/bria/background/remove', config), + + /** + * Pre-configured provider for Rembg Enhance model + * @param config - Provider configuration + */ + RembgEnhance: ( + config: Parameters[1] + ) => createBackgroundRemovalProvider('smoretalk-ai/rembg-enhance', config), + + /** + * Pre-configured provider for Imageutils Rembg model + * @param config - Provider configuration + */ + ImageutilsRembg: ( + config: Parameters[1] + ) => createBackgroundRemovalProvider('fal-ai/imageutils/rembg', config) +}; + +export default FalAi; diff --git a/packages/plugin-background-removal-web/src/fal-ai/utils.ts b/packages/plugin-background-removal-web/src/fal-ai/utils.ts new file mode 100644 index 00000000..b1113ce2 --- /dev/null +++ b/packages/plugin-background-removal-web/src/fal-ai/utils.ts @@ -0,0 +1,41 @@ +import { fal } from '@fal-ai/client'; +import { mimeTypeToExtension } from '@imgly/plugin-utils'; +import CreativeEditorSDK from '@cesdk/cesdk-js'; + +export async function uploadImageInputToFalIfNeeded( + imageUrl?: string, + cesdk?: CreativeEditorSDK +): Promise { + if (imageUrl == null) return undefined; + if (imageUrl.startsWith('blob:')) { + const mimeType = + cesdk != null + ? await cesdk.engine.editor.getMimeType(imageUrl) + : 'image/png'; + const imageUrlResponse = await fetch(imageUrl); + const imageUrlBlob = await imageUrlResponse.blob(); + const imageUrlFile = new File( + [imageUrlBlob], + `image.${mimeTypeToExtension(mimeType)}`, + { + type: mimeType + } + ); + return fal.storage.upload(imageUrlFile); + } + if (cesdk != null && imageUrl.startsWith('buffer:')) { + const mimeType = await cesdk.engine.editor.getMimeType(imageUrl); + const length = cesdk.engine.editor.getBufferLength(imageUrl); + const data = cesdk.engine.editor.getBufferData(imageUrl, 0, length); + const imageUrlFile = new File( + [data], + `image.${mimeTypeToExtension(mimeType)}`, + { + type: mimeType + } + ); + return fal.storage.upload(imageUrlFile); + } + + return imageUrl; +} diff --git a/packages/plugin-background-removal-web/src/index.ts b/packages/plugin-background-removal-web/src/index.ts index e18e19b0..6758c1a0 100644 --- a/packages/plugin-background-removal-web/src/index.ts +++ b/packages/plugin-background-removal-web/src/index.ts @@ -6,4 +6,15 @@ const Plugin = (pluginConfiguration?: PluginConfiguration) => ({ ...plugin(pluginConfiguration) }); +// Default export supports both client-side and provider-based usage export default Plugin; + +// Export FalAI providers for server-side background removal +export { default as FalAi } from './fal-ai'; + +// Export types +export type { BackgroundRemovalProvider } from './processBackgroundRemoval'; +export type { PluginConfiguration } from './plugin'; + +// Re-export the plugin function for direct usage +export { plugin }; diff --git a/packages/plugin-background-removal-web/src/plugin.ts b/packages/plugin-background-removal-web/src/plugin.ts index b04d15c4..38af1063 100644 --- a/packages/plugin-background-removal-web/src/plugin.ts +++ b/packages/plugin-background-removal-web/src/plugin.ts @@ -33,6 +33,7 @@ export default ( cesdk, blockId, metadata, + // Use default client-side provider if no provider is specified pluginConfiguration.provider ?? { type: '@imgly/background-removal' } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2f95d192..b2158445 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -158,7 +158,7 @@ importers: version: file:packages/plugin-ai-video-generation-web(@cesdk/cesdk-js@1.49.1(react@18.3.1)) '@imgly/plugin-background-removal-web': specifier: workspace:* - version: file:packages/plugin-background-removal-web(@cesdk/cesdk-js@1.49.1(react@18.3.1))(onnxruntime-web@1.21.0) + version: file:packages/plugin-background-removal-web(@cesdk/cesdk-js@1.49.1(react@18.3.1))(@fal-ai/client@1.3.0)(onnxruntime-web@1.21.0) '@imgly/plugin-cutout-library-web': specifier: workspace:* version: file:packages/plugin-cutout-library-web(@cesdk/cesdk-js@1.49.1(react@18.3.1)) @@ -563,6 +563,9 @@ importers: packages/plugin-background-removal-web: dependencies: + '@fal-ai/client': + specifier: ^1.3.0 + version: 1.3.0 '@imgly/background-removal': specifier: 1.7.0 version: 1.7.0(onnxruntime-web@1.21.0) @@ -1361,6 +1364,7 @@ packages: resolution: {directory: packages/plugin-background-removal-web, type: directory} peerDependencies: '@cesdk/cesdk-js': ^1.32.0 + '@fal-ai/client': ^1.3.0 onnxruntime-web: 1.21.0 '@imgly/plugin-cutout-library-web@file:packages/plugin-cutout-library-web': @@ -5058,9 +5062,10 @@ snapshots: dependencies: '@cesdk/cesdk-js': 1.49.1(react@18.3.1) - '@imgly/plugin-background-removal-web@file:packages/plugin-background-removal-web(@cesdk/cesdk-js@1.49.1(react@18.3.1))(onnxruntime-web@1.21.0)': + '@imgly/plugin-background-removal-web@file:packages/plugin-background-removal-web(@cesdk/cesdk-js@1.49.1(react@18.3.1))(@fal-ai/client@1.3.0)(onnxruntime-web@1.21.0)': dependencies: '@cesdk/cesdk-js': 1.49.1(react@18.3.1) + '@fal-ai/client': 1.3.0 '@imgly/background-removal': 1.7.0(onnxruntime-web@1.21.0) onnxruntime-web: 1.21.0