Fireworks: image generation responds differs depending on model

[gr2m]

Our current example for image generation using the @ai-sdk/fireworks provider works when using accounts/fireworks/models/stable-diffusion-xl-1024-v1-0 as provider.

ai/examples/ai-core/src/generate-image/fireworks.ts

Line 9 in fe2831a

'accounts/fireworks/models/stable-diffusion-xl-1024-v1-0',

But it fails when using accounts/fireworks/models/flux-kontext-pro. Unlike with the stable-diffusion model, flux-kontext-pro requests return a JSON object with a request_id which then needs to be sent repeatedly to https://docs.fireworks.ai/api-reference/get-generated-image-from-flux-kontex until it responds with result.status === 'Ready' and and result.result.sample set to an image URL, which then can be downloaded to retrieve the actual image.

[gr2m]

For reference, here is how it works with the flux model

const MODEL_ID = 'accounts/fireworks/models/stable-diffusion-xl-1024-v1-0';

async function editImageNative() {
  const imageBuffer = readFileSync('data/comic-cat.png');
  const base64Image = imageBuffer.toString('base64');

  console.log('Editing image with Fireworks Kontext API...');

  const response = await fetch(
    `https://api.fireworks.ai/inference/v1/workflows/${MODEL_ID}`,
    {
      method: 'POST',
      headers: {
        Authorization: `Bearer ${process.env.FIREWORKS_API_KEY}`,
        'Content-Type': 'application/json',
      },
      body: JSON.stringify({
        prompt: 'Turn the cat into a golden retriever dog',
        input_image: base64Image,
        safety_tolerance: 6,
      }),
    },
  );

  if (!response.ok) {
    const error = await response.text();
    console.error('Request Error for %s:', MODEL_ID, error);
    throw new Error(`Fireworks API error: ${response.status}`);
  }

  // Response is binary image data
  const taskResponse = await response.json();
  console.dir(taskResponse);
  const imageUrl = await getImageUrl(
    String(process.env.FIREWORKS_API_KEY),
    MODEL_ID,
    taskResponse.request_id,
  );

  console.log('OUTPUT IMAGE:');
  console.log('Image URL:', imageUrl);

  // Fetch the image data from the URL if needed  
  const imageResponse = await fetch(imageUrl);
  const arrayBuffer = await imageResponse.arrayBuffer();
  await presentImages([
    {
      base64: '',
      mediaType: 'image/jpeg',
      uint8Array: new Uint8Array(arrayBuffer),
    },
  ]);
}

async function getImageUrl(apiKey: string, modelId: string, requestId: string) {
  const url = `https://api.fireworks.ai/inference/v1/workflows/${modelId}/get_result`;

  console.log('checking for result of task %s at %s', requestId, url);
  const response = await fetch(url, {
    method: 'POST',
    headers: {
      'Content-Type': 'application/json',
      Authorization: `Bearer ${apiKey}`,
    },
    body: JSON.stringify({ id: requestId }),
  });

  const result = await response.json();
  console.dir(result);

  if (result.status !== 'Ready') {
    console.log('no result yet, waiting 3s before retrying...');
    await new Promise(resolve => setTimeout(resolve, 3000));
    return getImageUrl(apiKey, modelId, requestId);
  }

  return result.result.sample
}