1047 lines
38 KiB
TypeScript
1047 lines
38 KiB
TypeScript
import { NextRequest, NextResponse } from 'next/server';
|
||
import sharp from 'sharp';
|
||
import { ImageGenerationClient, Config, HeaderUtils } from 'coze-coding-dev-sdk';
|
||
import { buildCustomApiHeaders, fetchWithRetry, parseCustomApiError, parseCustomApiJsonWithProgress } from '@/lib/custom-api-fetch';
|
||
import { getAspectRatioPromptHint, resolveImageSize } from '@/lib/model-config';
|
||
import { localStorage } from '@/lib/local-storage';
|
||
import { fetchPublicHttpUrl } from '@/lib/remote-fetch';
|
||
import { isTrustedInternalGenerationRequest, isUuid, resolveServerApiConfig } from '@/lib/server-api-config';
|
||
import { updateGenerationJobProgress } from '@/lib/generation-job-estimates';
|
||
import {
|
||
compressImageBufferForUpstream,
|
||
dataUrlToImageBuffer,
|
||
imageBufferToDataUrl,
|
||
} from '@/lib/server-image-compression';
|
||
|
||
interface CustomApiConfig {
|
||
apiUrl: string;
|
||
modelName: string;
|
||
apiKey: string;
|
||
provider: string;
|
||
customApiKeyId?: string;
|
||
systemApiId?: string;
|
||
}
|
||
|
||
const GENERATION_TIMEOUT = 300_000;
|
||
const GENERATION_TIMEOUT_SECONDS = GENERATION_TIMEOUT / 1000;
|
||
const MAX_UPSTREAM_REFERENCE_IMAGE_BYTES = Number(process.env.MAX_UPSTREAM_REFERENCE_IMAGE_BYTES || 700 * 1024);
|
||
|
||
interface TargetImageSize {
|
||
width: number;
|
||
height: number;
|
||
}
|
||
|
||
interface PersistedImageResult {
|
||
url: string;
|
||
width: number;
|
||
height: number;
|
||
bytes: number;
|
||
}
|
||
|
||
export const runtime = 'nodejs';
|
||
|
||
function parseImageSize(size: string | undefined): TargetImageSize | null {
|
||
const match = size?.match(/^(\d{2,5})x(\d{2,5})$/i);
|
||
if (!match) return null;
|
||
const width = Number(match[1]);
|
||
const height = Number(match[2]);
|
||
return width > 0 && height > 0 ? { width, height } : null;
|
||
}
|
||
|
||
function resolveTargetImageSize(
|
||
size: string | undefined,
|
||
aspectRatio: string | undefined,
|
||
resolution: string | undefined,
|
||
quality: string | undefined,
|
||
): TargetImageSize | null {
|
||
const explicit = parseImageSize(size);
|
||
if (explicit) return explicit;
|
||
|
||
if (aspectRatio && aspectRatio !== 'original' && resolution) {
|
||
return parseImageSize(resolveImageSize(aspectRatio, resolution));
|
||
}
|
||
|
||
const squareByQuality: Record<string, string> = {
|
||
'1K': '1024x1024',
|
||
'1080P': '1024x1024',
|
||
'2K': '2048x2048',
|
||
'4K': '4096x4096',
|
||
};
|
||
return parseImageSize(quality ? squareByQuality[quality] : undefined);
|
||
}
|
||
|
||
function formatTargetSize(targetSize: TargetImageSize): string {
|
||
return `${targetSize.width}x${targetSize.height}`;
|
||
}
|
||
|
||
function imageMeetsTargetSize(width: number, height: number, targetSize: TargetImageSize): boolean {
|
||
return width >= targetSize.width && height >= targetSize.height;
|
||
}
|
||
|
||
function getImageExtension(mimeType: string | null | undefined, fallbackUrl?: string): string {
|
||
const normalized = mimeType?.split(';')[0].trim().toLowerCase();
|
||
const mimeExt: Record<string, string> = {
|
||
'image/png': 'png',
|
||
'image/jpeg': 'jpg',
|
||
'image/jpg': 'jpg',
|
||
'image/webp': 'webp',
|
||
};
|
||
if (normalized && mimeExt[normalized]) return mimeExt[normalized];
|
||
const urlExt = fallbackUrl?.split('?')[0].match(/\.([a-z0-9]+)$/i)?.[1];
|
||
return urlExt || 'png';
|
||
}
|
||
|
||
function parseImageDataUrl(dataUrl: string): { buffer: Buffer; mimeType: string; ext: string } | null {
|
||
const match = dataUrl.match(/^data:(image\/[^;]+);base64,(.+)$/);
|
||
if (!match) return null;
|
||
const [, mimeType, base64Data] = match;
|
||
return {
|
||
buffer: Buffer.from(base64Data, 'base64'),
|
||
mimeType,
|
||
ext: getImageExtension(mimeType),
|
||
};
|
||
}
|
||
|
||
async function persistImageWithMetadata(url: string, prefix: string): Promise<PersistedImageResult | null> {
|
||
let buffer: Buffer;
|
||
let mimeType = 'image/png';
|
||
let ext = 'png';
|
||
|
||
if (url.startsWith('data:')) {
|
||
const parsed = parseImageDataUrl(url);
|
||
if (!parsed) return null;
|
||
buffer = parsed.buffer;
|
||
mimeType = parsed.mimeType;
|
||
ext = parsed.ext;
|
||
} else {
|
||
const existingKey = localStorage.getKeyFromPublicUrl(url);
|
||
if (existingKey && localStorage.fileExists(existingKey)) {
|
||
buffer = localStorage.readFile(existingKey);
|
||
ext = existingKey.split('.').pop() || ext;
|
||
} else if (url.startsWith('http')) {
|
||
const response = await withTimeout(fetchPublicHttpUrl(url), 30_000, 'Fetch generated image');
|
||
if (!response.ok) throw new Error(`下载生成图片失败: ${response.status}`);
|
||
mimeType = response.headers.get('content-type')?.split(';')[0] || mimeType;
|
||
buffer = Buffer.from(await response.arrayBuffer());
|
||
ext = getImageExtension(mimeType, url);
|
||
} else {
|
||
return null;
|
||
}
|
||
}
|
||
|
||
const metadata = await sharp(buffer, { failOn: 'none' }).metadata();
|
||
if (!metadata.width || !metadata.height) {
|
||
throw new Error('无法读取生成图片尺寸');
|
||
}
|
||
|
||
const fileName = `${prefix}/${Date.now()}-${Math.random().toString(36).slice(2, 8)}.${ext}`;
|
||
const fileKey = await withTimeout(
|
||
localStorage.uploadFile({
|
||
fileContent: buffer,
|
||
fileName,
|
||
contentType: mimeType,
|
||
}),
|
||
30_000,
|
||
'Local uploadFile',
|
||
);
|
||
const presignedUrl = await withTimeout(
|
||
localStorage.generatePresignedUrl({ key: fileKey, expireTime: 2592000 }),
|
||
10_000,
|
||
'Local generatePresignedUrl',
|
||
);
|
||
|
||
return {
|
||
url: presignedUrl,
|
||
width: metadata.width,
|
||
height: metadata.height,
|
||
bytes: buffer.length,
|
||
};
|
||
}
|
||
|
||
async function persistQualifiedImageUrls(
|
||
urls: string[],
|
||
prefix: string,
|
||
targetSize: TargetImageSize | null,
|
||
context: string,
|
||
): Promise<{ images: string[]; rejected: string[] }> {
|
||
const images: string[] = [];
|
||
const rejected: string[] = [];
|
||
|
||
for (const url of urls) {
|
||
try {
|
||
const persisted = await persistImageWithMetadata(url, prefix);
|
||
if (!persisted) {
|
||
rejected.push('无法读取生成图片');
|
||
continue;
|
||
}
|
||
if (targetSize && !imageMeetsTargetSize(persisted.width, persisted.height, targetSize)) {
|
||
const message = `${persisted.width}x${persisted.height} < ${formatTargetSize(targetSize)}`;
|
||
console.warn(`[${context}] Rejected low-resolution image:`, message);
|
||
rejected.push(message);
|
||
continue;
|
||
}
|
||
console.log(`[${context}] Accepted generated image:`, `${persisted.width}x${persisted.height}`, 'bytes:', persisted.bytes);
|
||
images.push(persisted.url);
|
||
} catch (err) {
|
||
const message = err instanceof Error ? err.message : '图片处理失败';
|
||
console.warn(`[${context}] Failed to persist generated image:`, message);
|
||
rejected.push(message);
|
||
}
|
||
}
|
||
|
||
return { images, rejected };
|
||
}
|
||
|
||
async function fetchCustomImageGeneration(
|
||
endpoint: string,
|
||
apiKey: string,
|
||
requestBody: Record<string, unknown>,
|
||
onProgress?: (progress: Record<string, unknown>) => void | Promise<void>,
|
||
): Promise<{ ok: true; images: string[] } | { ok: false; response: Response; errorText: string }> {
|
||
const response = await fetchWithRetry(
|
||
endpoint,
|
||
{ method: 'POST', headers: buildCustomApiHeaders(apiKey), body: JSON.stringify(requestBody) },
|
||
GENERATION_TIMEOUT,
|
||
1,
|
||
);
|
||
|
||
if (!response.ok) {
|
||
return { ok: false, response, errorText: await response.text() };
|
||
}
|
||
|
||
const data = await parseCustomApiJsonWithProgress(response, onProgress);
|
||
return { ok: true, images: extractImagesFromGenerationsResponse(data as Record<string, unknown>) };
|
||
}
|
||
|
||
async function requestQualifiedCustomImages(
|
||
endpoint: string,
|
||
apiKey: string,
|
||
requestBody: Record<string, unknown>,
|
||
targetCount: number,
|
||
targetSize: TargetImageSize | null,
|
||
onProgress?: (progress: Record<string, unknown>) => void | Promise<void>,
|
||
): Promise<{ images: string[]; rejected: string[]; upstreamError?: { status: number; text: string } }> {
|
||
const accepted: string[] = [];
|
||
const rejected: string[] = [];
|
||
const maxAttempts = Math.max(targetCount * 3, 3);
|
||
|
||
for (let attempt = 1; attempt <= maxAttempts && accepted.length < targetCount; attempt += 1) {
|
||
const remaining = targetCount - accepted.length;
|
||
const requestCount = attempt === 1
|
||
? Math.max(remaining, Number(requestBody.n) || 1)
|
||
: 1;
|
||
const response = await fetchCustomImageGeneration(
|
||
endpoint,
|
||
apiKey,
|
||
{ ...requestBody, n: requestCount },
|
||
onProgress,
|
||
);
|
||
|
||
if (!response.ok) {
|
||
return {
|
||
images: accepted,
|
||
rejected,
|
||
upstreamError: { status: response.response.status, text: response.errorText },
|
||
};
|
||
}
|
||
|
||
if (response.images.length === 0) {
|
||
rejected.push('响应中无图片数据');
|
||
continue;
|
||
}
|
||
|
||
const persisted = await persistQualifiedImageUrls(
|
||
response.images,
|
||
'generated/images',
|
||
targetSize,
|
||
`Custom API Image attempt ${attempt}`,
|
||
);
|
||
accepted.push(...persisted.images);
|
||
rejected.push(...persisted.rejected);
|
||
}
|
||
|
||
return { images: accepted.slice(0, targetCount), rejected };
|
||
}
|
||
|
||
function lowResolutionError(targetSize: TargetImageSize | null, rejected: string[]): string {
|
||
const target = targetSize ? `要求 ${formatTargetSize(targetSize)}` : '要求的分辨率';
|
||
const actual = rejected.length > 0 ? `，实际返回：${rejected.join('；')}` : '';
|
||
return `上游返回图片分辨率不符合${target}${actual}`;
|
||
}
|
||
|
||
/** Helper: wrap a promise with a timeout that rejects with a descriptive message */
|
||
function withTimeout<T>(promise: Promise<T>, ms: number, label: string): Promise<T> {
|
||
return new Promise<T>((resolve, reject) => {
|
||
const timer = setTimeout(() => reject(new Error(`${label} timed out after ${ms}ms`)), ms);
|
||
promise.then(
|
||
(v) => { clearTimeout(timer); resolve(v); },
|
||
(e) => { clearTimeout(timer); reject(e); },
|
||
);
|
||
});
|
||
}
|
||
|
||
/**
|
||
* Upload a base64 data URL to S3 storage and return a presigned URL.
|
||
*/
|
||
async function uploadDataUrlAndGetPublicUrl(dataUrl: string): Promise<string | null> {
|
||
try {
|
||
const match = dataUrl.match(/^data:(image\/[^;]+);base64,(.+)$/);
|
||
if (!match) return null;
|
||
const [, mimeType, base64Data] = match;
|
||
const ext = mimeType.split('/')[1] || 'png';
|
||
const buffer = Buffer.from(base64Data, 'base64');
|
||
const fileName = `img2img-ref/${Date.now()}-${Math.random().toString(36).slice(2, 8)}.${ext}`;
|
||
|
||
const fileKey = await localStorage.uploadFile({
|
||
fileContent: buffer,
|
||
fileName,
|
||
contentType: mimeType,
|
||
});
|
||
|
||
if (!fileKey) {
|
||
console.error('[Upload Ref Image] uploadFile returned empty key');
|
||
return null;
|
||
}
|
||
|
||
const presignedUrl = await localStorage.generatePresignedUrl({
|
||
key: fileKey,
|
||
expireTime: 3600,
|
||
});
|
||
|
||
console.log('[Upload Ref Image] Success, key:', fileKey, 'url length:', presignedUrl?.length);
|
||
return presignedUrl || null;
|
||
} catch (err) {
|
||
console.error('[Upload Ref Image Error]', err instanceof Error ? err.message : err);
|
||
return null;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Derive the chat completions endpoint URL from an images/generations URL.
|
||
*/
|
||
function deriveChatCompletionsUrl(imagesUrl: string): string {
|
||
if (imagesUrl.includes('/chat/completions')) return imagesUrl;
|
||
return imagesUrl
|
||
.replace(/\/images\/(generations|edits).*/i, '/chat/completions')
|
||
.replace(/\/+$/, '');
|
||
}
|
||
|
||
/**
|
||
* Derive the images/edits endpoint URL from an images/generations URL.
|
||
* This is the official OpenAI endpoint for image-to-image.
|
||
*/
|
||
function deriveImagesEditsUrl(imagesUrl: string): string {
|
||
if (imagesUrl.includes('/images/edits')) return imagesUrl;
|
||
return imagesUrl
|
||
.replace(/\/images\/generations.*/i, '/images/edits')
|
||
.replace(/\/+$/, '');
|
||
}
|
||
|
||
/**
|
||
* Extract image URLs/data from a chat completions response.
|
||
*/
|
||
function extractImagesFromChatResponse(data: Record<string, unknown>): string[] {
|
||
const images: string[] = [];
|
||
const choices = data.choices as Array<Record<string, unknown>> | undefined;
|
||
if (Array.isArray(choices)) {
|
||
for (const choice of choices) {
|
||
const message = choice.message as Record<string, unknown> | undefined;
|
||
if (!message) continue;
|
||
const content = message.content;
|
||
|
||
if (typeof content === 'string') {
|
||
if (content.startsWith('data:image/') || content.startsWith('http')) {
|
||
images.push(content);
|
||
}
|
||
const mdMatch = content.match(/!\[.*?\]\((data:image\/[^)]+)\)/);
|
||
if (mdMatch) images.push(mdMatch[1]);
|
||
const urlMatch = content.match(/(https?:\/\/[^\s"']+\.(png|jpg|jpeg|webp)[^\s"']*)/i);
|
||
if (urlMatch) images.push(urlMatch[1]);
|
||
} else if (Array.isArray(content)) {
|
||
for (const item of content as Array<Record<string, unknown>>) {
|
||
if (item.type === 'image_url' && item.image_url) {
|
||
const url = (item.image_url as Record<string, unknown>).url;
|
||
if (typeof url === 'string') images.push(url);
|
||
}
|
||
if (item.type === 'image' && item.image) {
|
||
const imgData = item.image as Record<string, unknown>;
|
||
if (typeof imgData.url === 'string') images.push(imgData.url);
|
||
if (typeof imgData.b64_json === 'string') {
|
||
images.push(`data:image/png;base64,${imgData.b64_json}`);
|
||
}
|
||
}
|
||
if (item.type === 'text' && typeof item.text === 'string') {
|
||
const text = item.text as string;
|
||
if (text.startsWith('data:image/')) images.push(text);
|
||
if (text.startsWith('http') && /\.(png|jpg|jpeg|webp)/i.test(text)) images.push(text);
|
||
const mdMatch = text.match(/!\[.*?\]\((data:image\/[^)]+)\)/);
|
||
if (mdMatch) images.push(mdMatch[1]);
|
||
const urlMatch = text.match(/(https?:\/\/[^\s"']+\.(png|jpg|jpeg|webp)[^\s"']*)/i);
|
||
if (urlMatch) images.push(urlMatch[1]);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
return images;
|
||
}
|
||
|
||
function objectKeysFromUnknown(value: unknown): string[] {
|
||
if (!value || typeof value !== 'object' || Array.isArray(value)) return [];
|
||
return Object.keys(value);
|
||
}
|
||
|
||
/**
|
||
* Extract images from images/generations or images/edits response format.
|
||
*/
|
||
function extractImagesFromGenerationsResponse(data: Record<string, unknown>): string[] {
|
||
const images: string[] = [];
|
||
if (Array.isArray(data.data)) {
|
||
for (const item of data.data as Array<Record<string, unknown>>) {
|
||
if (typeof item === 'string') { images.push(item); continue; }
|
||
if (item.b64_json && typeof item.b64_json === 'string') {
|
||
images.push(`data:image/png;base64,${item.b64_json}`);
|
||
}
|
||
if (item.url && typeof item.url === 'string') images.push(item.url);
|
||
}
|
||
} else if (typeof data.url === 'string') {
|
||
images.push(data.url);
|
||
} else if (typeof data.image_url === 'string') {
|
||
images.push(data.image_url);
|
||
}
|
||
|
||
const streamEvents = data.__streamEvents;
|
||
if (Array.isArray(streamEvents)) {
|
||
for (const event of streamEvents) {
|
||
if (!event || typeof event !== 'object' || Array.isArray(event)) continue;
|
||
images.push(...extractImagesFromGenerationsResponse(event as Record<string, unknown>));
|
||
}
|
||
}
|
||
|
||
return images;
|
||
}
|
||
|
||
/** Track which strategy produced a result */
|
||
interface StrategyResult {
|
||
success: boolean;
|
||
images?: string[];
|
||
error?: string;
|
||
status?: number;
|
||
strategyName: string;
|
||
}
|
||
|
||
/**
|
||
* Try a single API request strategy and return the result.
|
||
*/
|
||
async function tryImageStrategy(
|
||
url: string,
|
||
headers: Record<string, string>,
|
||
body: Record<string, unknown>,
|
||
strategyName: string,
|
||
isChatFormat: boolean,
|
||
onProgress?: (progress: Record<string, unknown>) => void | Promise<void>,
|
||
): Promise<StrategyResult> {
|
||
console.log(`[Custom API img2img → ${strategyName}] URL:`, url,
|
||
'| model:', body.model,
|
||
'| body_keys:', Object.keys(body).join(','));
|
||
|
||
try {
|
||
const response = await fetchWithRetry(
|
||
url,
|
||
{
|
||
method: 'POST',
|
||
headers,
|
||
body: JSON.stringify(body),
|
||
},
|
||
GENERATION_TIMEOUT,
|
||
1,
|
||
);
|
||
|
||
if (response.ok) {
|
||
const data = await parseCustomApiJsonWithProgress(response, onProgress);
|
||
let images = isChatFormat
|
||
? extractImagesFromChatResponse(data as Record<string, unknown>)
|
||
: [];
|
||
if (images.length === 0) {
|
||
images = extractImagesFromGenerationsResponse(data as Record<string, unknown>);
|
||
}
|
||
|
||
if (images.length > 0) {
|
||
console.log(`[Custom API img2img → ${strategyName} SUCCESS] Got`, images.length, 'images');
|
||
return { success: true, images, strategyName };
|
||
}
|
||
|
||
console.warn(`[Custom API img2img → ${strategyName}] OK but no images extracted, keys:`, objectKeysFromUnknown(data));
|
||
return { success: false, error: '响应中无图片数据', strategyName };
|
||
}
|
||
|
||
const errorText = await response.text();
|
||
console.warn(`[Custom API img2img → ${strategyName} FAILED]`, response.status, errorText.slice(0, 200));
|
||
return { success: false, error: parseCustomApiError(response.status, errorText), status: response.status, strategyName };
|
||
} catch (err) {
|
||
const msg = err instanceof Error ? err.message : '请求异常';
|
||
console.warn(`[Custom API img2img → ${strategyName} ERROR]`, msg);
|
||
return { success: false, error: msg, strategyName };
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Try images/edits endpoint with multipart/form-data format.
|
||
*
|
||
* CRITICAL: This is the format Cherry Studio (Electron app) uses for img2img.
|
||
* OpenAI's official /v1/images/edits endpoint uses multipart/form-data, NOT JSON.
|
||
* API proxies like mozhevip.top route based on Content-Type:
|
||
* - multipart/form-data → routed to img2img account pool → WORKS
|
||
* - application/json → routed to wrong pool → 503 "No available compatible accounts"
|
||
*
|
||
* This is why the same API+Key works in Cherry Studio but not from our server.
|
||
*/
|
||
async function tryEditsWithFormData(
|
||
url: string,
|
||
apiKey: string,
|
||
model: string,
|
||
prompt: string,
|
||
imageBuffer: Buffer,
|
||
imageMimeType: string,
|
||
size: string | undefined,
|
||
strength: number | undefined,
|
||
count: number,
|
||
onProgress?: (progress: Record<string, unknown>) => void | Promise<void>,
|
||
): Promise<StrategyResult> {
|
||
const strategyName = '策略2: images/edits (FormData)';
|
||
console.log(`[Custom API img2img → ${strategyName}] URL:`, url, '| model:', model);
|
||
|
||
try {
|
||
// Build multipart/form-data manually (Node.js doesn't have native FormData that works with fetch)
|
||
const boundary = `----FormBoundary${Date.now()}${Math.random().toString(36).slice(2)}`;
|
||
const parts: Buffer[] = [];
|
||
|
||
// Add text fields
|
||
const textFields: Record<string, string> = {
|
||
model,
|
||
prompt,
|
||
stream: 'true',
|
||
};
|
||
if (size) textFields.size = size;
|
||
if (count > 1) textFields.n = String(count);
|
||
if (strength !== undefined) textFields.strength = String(strength);
|
||
|
||
for (const [key, value] of Object.entries(textFields)) {
|
||
parts.push(Buffer.from(
|
||
`--${boundary}\r\nContent-Disposition: form-data; name="${key}"\r\n\r\n${value}\r\n`
|
||
));
|
||
}
|
||
|
||
// Add image file field
|
||
const ext = imageMimeType.split('/')[1] || 'png';
|
||
parts.push(Buffer.from(
|
||
`--${boundary}\r\nContent-Disposition: form-data; name="image"; filename="image.${ext}"\r\nContent-Type: ${imageMimeType}\r\n\r\n`
|
||
));
|
||
parts.push(imageBuffer);
|
||
parts.push(Buffer.from(`\r\n`));
|
||
|
||
// Close boundary
|
||
parts.push(Buffer.from(`--${boundary}--\r\n`));
|
||
|
||
const bodyBuffer = Buffer.concat(parts);
|
||
|
||
const response = await fetchWithRetry(
|
||
url,
|
||
{
|
||
method: 'POST',
|
||
headers: {
|
||
'Authorization': `Bearer ${apiKey}`,
|
||
'Content-Type': `multipart/form-data; boundary=${boundary}`,
|
||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
|
||
},
|
||
body: bodyBuffer,
|
||
},
|
||
GENERATION_TIMEOUT,
|
||
1,
|
||
);
|
||
|
||
if (response.ok) {
|
||
const data = await parseCustomApiJsonWithProgress(response, onProgress);
|
||
const images = extractImagesFromGenerationsResponse(data as Record<string, unknown>);
|
||
if (images.length > 0) {
|
||
console.log(`[Custom API img2img → ${strategyName} SUCCESS] Got`, images.length, 'images');
|
||
return { success: true, images, strategyName };
|
||
}
|
||
console.warn(`[Custom API img2img → ${strategyName}] OK but no images, keys:`, objectKeysFromUnknown(data));
|
||
return { success: false, error: '响应中无图片数据', strategyName };
|
||
}
|
||
|
||
const errorText = await response.text();
|
||
console.warn(`[Custom API img2img → ${strategyName} FAILED]`, response.status, errorText.slice(0, 200));
|
||
return { success: false, error: parseCustomApiError(response.status, errorText), status: response.status, strategyName };
|
||
} catch (err) {
|
||
const msg = err instanceof Error ? err.message : '请求异常';
|
||
console.warn(`[Custom API img2img → ${strategyName} ERROR]`, msg);
|
||
return { success: false, error: msg, strategyName };
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Image-to-image via custom API with multi-strategy approach.
|
||
* Tries 3 different endpoint formats in order:
|
||
* 1. /v1/chat/completions with image_url (Cherry Studio / OpenAI multimodal style)
|
||
* 2. /v1/images/edits with image (Official OpenAI image edit endpoint)
|
||
* 3. /v1/images/generations with init_image (Reference code / Stable Diffusion style)
|
||
*/
|
||
async function customApiImageToImage(
|
||
customApiConfig: CustomApiConfig,
|
||
prompt: string,
|
||
negativePrompt: string | undefined,
|
||
image: string,
|
||
strength: number | undefined,
|
||
size: string | undefined,
|
||
count: number,
|
||
targetSize: TargetImageSize | null,
|
||
aspectRatio?: string,
|
||
onProgress?: (progress: Record<string, unknown>) => void | Promise<void>,
|
||
): Promise<NextResponse> {
|
||
const endpoint = customApiConfig.apiUrl;
|
||
if (!endpoint) {
|
||
return NextResponse.json({ error: '自定义API未配置请求地址' }, { status: 400 });
|
||
}
|
||
if (!customApiConfig.modelName) {
|
||
return NextResponse.json({ error: '自定义API未配置模型名称，请在设置中填写模型名称（如 gpt-image-2）' }, { status: 400 });
|
||
}
|
||
|
||
let normalizedImage = image;
|
||
|
||
// Prepare image buffer for FormData upload
|
||
let imageBuffer: Buffer | null = null;
|
||
let imageMimeType = 'image/png';
|
||
if (image.startsWith('data:')) {
|
||
const parsedImage = dataUrlToImageBuffer(image);
|
||
if (parsedImage) {
|
||
imageMimeType = parsedImage.mimeType;
|
||
imageBuffer = parsedImage.buffer;
|
||
}
|
||
} else {
|
||
// It's a URL - download it first
|
||
try {
|
||
const imgRes = await fetchPublicHttpUrl(image);
|
||
if (imgRes.ok) {
|
||
const contentType = imgRes.headers.get('content-type') || 'image/png';
|
||
imageMimeType = contentType.split(';')[0];
|
||
const arrayBuf = await imgRes.arrayBuffer();
|
||
imageBuffer = Buffer.from(arrayBuf);
|
||
}
|
||
} catch (e) {
|
||
console.warn('[Custom API img2img] Failed to download reference image from URL:', e);
|
||
}
|
||
}
|
||
|
||
if (imageBuffer) {
|
||
try {
|
||
const compressed = await compressImageBufferForUpstream(
|
||
{ buffer: imageBuffer, mimeType: imageMimeType },
|
||
{ maxBytes: MAX_UPSTREAM_REFERENCE_IMAGE_BYTES },
|
||
);
|
||
if (compressed.changed) {
|
||
console.log('[Custom API img2img] Compressed reference image:', compressed.originalBytes, '→', compressed.buffer.length);
|
||
}
|
||
imageBuffer = compressed.buffer;
|
||
imageMimeType = compressed.mimeType;
|
||
normalizedImage = imageBufferToDataUrl({ buffer: imageBuffer, mimeType: imageMimeType });
|
||
} catch (err) {
|
||
console.warn('[Custom API img2img] Reference image compression failed, using original:', err instanceof Error ? err.message : err);
|
||
}
|
||
}
|
||
|
||
// Upload reference image to S3 to get a public URL (for strategies that use URL instead of file upload)
|
||
let imageUrl = normalizedImage;
|
||
if (normalizedImage.startsWith('data:')) {
|
||
console.log('[Custom API img2img] Uploading reference image to S3 to reduce payload...');
|
||
const uploadedUrl = await uploadDataUrlAndGetPublicUrl(normalizedImage);
|
||
if (uploadedUrl) {
|
||
imageUrl = uploadedUrl;
|
||
console.log('[Custom API img2img] Using S3 URL, size reduction:', normalizedImage.length, '→', imageUrl.length);
|
||
} else {
|
||
console.warn('[Custom API img2img] S3 upload failed, falling back to data URL in request body');
|
||
}
|
||
}
|
||
|
||
// Build prompt text with optional negative prompt and strength hints
|
||
let promptText = prompt;
|
||
if (negativePrompt) {
|
||
promptText += `\n\n负面提示词（排除以下元素）: ${negativePrompt}`;
|
||
}
|
||
if (strength !== undefined && strength !== 0.5) {
|
||
promptText += `\n\n[重绘幅度: ${strength.toFixed(2)}，${strength < 0.5 ? '尽量保留参考图特征' : '更贴近提示词描述'}]`;
|
||
}
|
||
// Augment prompt with aspect ratio hint
|
||
if (aspectRatio) {
|
||
const hint = getAspectRatioPromptHint(aspectRatio);
|
||
if (hint) promptText += `\n\n[${hint}]`;
|
||
}
|
||
|
||
const headers = buildCustomApiHeaders(customApiConfig.apiKey);
|
||
const denoisingStrength = strength ?? 0.5;
|
||
|
||
// --- Strategy 1: /v1/images/edits with multipart/form-data ---
|
||
// This is THE format Cherry Studio uses! OpenAI's official endpoint.
|
||
// API proxies route multipart/form-data to the correct img2img account pool.
|
||
let result1: StrategyResult | null = null;
|
||
if (imageBuffer) {
|
||
const editsUrl = deriveImagesEditsUrl(endpoint);
|
||
result1 = await tryEditsWithFormData(
|
||
editsUrl,
|
||
customApiConfig.apiKey,
|
||
customApiConfig.modelName,
|
||
promptText,
|
||
imageBuffer,
|
||
imageMimeType,
|
||
size,
|
||
denoisingStrength,
|
||
count,
|
||
onProgress,
|
||
);
|
||
if (result1.success && result1.images) {
|
||
const persisted = await persistQualifiedImageUrls(result1.images, 'generated/images', targetSize, 'Custom API img2img strategy1');
|
||
if (persisted.images.length > 0) return NextResponse.json({ images: persisted.images });
|
||
result1 = { ...result1, success: false, error: lowResolutionError(targetSize, persisted.rejected) };
|
||
}
|
||
}
|
||
|
||
// --- Strategy 2: chat/completions with image_url (multimodal style) ---
|
||
const chatUrl = deriveChatCompletionsUrl(endpoint);
|
||
const chatBody: Record<string, unknown> = {
|
||
model: customApiConfig.modelName,
|
||
stream: true,
|
||
messages: [
|
||
{
|
||
role: 'user',
|
||
content: [
|
||
{ type: 'image_url', image_url: { url: imageUrl } },
|
||
{ type: 'text', text: promptText },
|
||
],
|
||
},
|
||
],
|
||
size: size || '1024x1024',
|
||
n: count,
|
||
};
|
||
const result2 = await tryImageStrategy(chatUrl, headers, chatBody, '策略2: chat/completions', true, onProgress);
|
||
if (result2.success && result2.images) {
|
||
const persisted = await persistQualifiedImageUrls(result2.images, 'generated/images', targetSize, 'Custom API img2img strategy2');
|
||
if (persisted.images.length > 0) return NextResponse.json({ images: persisted.images });
|
||
result2.success = false;
|
||
result2.error = lowResolutionError(targetSize, persisted.rejected);
|
||
}
|
||
|
||
// --- Strategy 3: /v1/images/generations with init_image (Reference code / SD style) ---
|
||
let rawBase64 = normalizedImage;
|
||
if (normalizedImage.startsWith('data:')) {
|
||
const commaIndex = normalizedImage.indexOf(',');
|
||
if (commaIndex !== -1) rawBase64 = normalizedImage.substring(commaIndex + 1);
|
||
}
|
||
|
||
const imgBody: Record<string, unknown> = {
|
||
model: customApiConfig.modelName,
|
||
prompt: promptText,
|
||
n: count,
|
||
size: size || '1024x1024',
|
||
response_format: 'b64_json',
|
||
stream: true,
|
||
init_image: rawBase64,
|
||
denoising_strength: denoisingStrength,
|
||
};
|
||
const result3 = await tryImageStrategy(endpoint, headers, imgBody, '策略3: images/generations+init_image', false, onProgress);
|
||
if (result3.success && result3.images) {
|
||
const persisted = await persistQualifiedImageUrls(result3.images, 'generated/images', targetSize, 'Custom API img2img strategy3');
|
||
if (persisted.images.length > 0) return NextResponse.json({ images: persisted.images });
|
||
result3.success = false;
|
||
result3.error = lowResolutionError(targetSize, persisted.rejected);
|
||
}
|
||
|
||
const upstreamError = result1?.error || result2.error || result3.error;
|
||
const upstreamStatus = result1?.status || result2.status || result3.status || 502;
|
||
return NextResponse.json(
|
||
{
|
||
error: upstreamError || '图生图失败',
|
||
},
|
||
{ status: upstreamStatus >= 500 ? 502 : upstreamStatus }
|
||
);
|
||
}
|
||
|
||
export async function POST(request: NextRequest) {
|
||
try {
|
||
const body = await request.json();
|
||
const {
|
||
prompt,
|
||
negativePrompt,
|
||
model = 'doubao-seedream-5-0-260128',
|
||
quality = '2K',
|
||
size,
|
||
aspectRatio,
|
||
resolution,
|
||
count = 1,
|
||
guidanceScale = 7,
|
||
image,
|
||
strength,
|
||
customApiConfig,
|
||
} = body as {
|
||
prompt?: string;
|
||
negativePrompt?: string;
|
||
model?: string;
|
||
quality?: string;
|
||
size?: string;
|
||
aspectRatio?: string;
|
||
resolution?: string;
|
||
count?: number;
|
||
guidanceScale?: number;
|
||
image?: string;
|
||
strength?: number;
|
||
customApiConfig?: CustomApiConfig;
|
||
};
|
||
|
||
if (!prompt) {
|
||
return NextResponse.json({ error: '请提供创作描述' }, { status: 400 });
|
||
}
|
||
|
||
if (prompt.length < 2) {
|
||
return NextResponse.json({ error: '创作描述过短，请输入更详细的描述' }, { status: 400 });
|
||
}
|
||
|
||
const trustedInternalRequest = isTrustedInternalGenerationRequest(request);
|
||
const trustedUserId = trustedInternalRequest
|
||
? request.headers.get('x-miaojing-generation-user-id')
|
||
: null;
|
||
const generationJobId = trustedInternalRequest
|
||
? request.headers.get('x-miaojing-generation-job-id')
|
||
: null;
|
||
const resolvedCustomApiConfig = await resolveServerApiConfig(
|
||
request,
|
||
customApiConfig,
|
||
isUuid(trustedUserId) ? trustedUserId : null,
|
||
);
|
||
const handleUpstreamProgress = (progress: Record<string, unknown>) => updateGenerationJobProgress(
|
||
isUuid(generationJobId) ? generationJobId : null,
|
||
progress,
|
||
);
|
||
const targetSize = resolveTargetImageSize(size, aspectRatio, resolution, quality);
|
||
|
||
// Log all incoming parameters for debugging
|
||
console.log('[Image Generation] Params:', JSON.stringify({
|
||
model,
|
||
size,
|
||
aspectRatio,
|
||
resolution,
|
||
count,
|
||
guidanceScale,
|
||
hasCustomApi: !!resolvedCustomApiConfig,
|
||
customApiUrl: resolvedCustomApiConfig?.apiUrl,
|
||
customApiModel: resolvedCustomApiConfig?.modelName,
|
||
hasImage: !!image,
|
||
strength,
|
||
promptLength: prompt.length,
|
||
}));
|
||
|
||
// ---- Custom API mode ----
|
||
if (resolvedCustomApiConfig && resolvedCustomApiConfig.apiKey) {
|
||
const resolvedApiKey = resolvedCustomApiConfig.apiKey;
|
||
try {
|
||
// Image-to-image: use multi-strategy approach
|
||
if (image) {
|
||
return await customApiImageToImage(
|
||
resolvedCustomApiConfig as CustomApiConfig,
|
||
prompt,
|
||
negativePrompt,
|
||
image,
|
||
strength,
|
||
size,
|
||
count,
|
||
targetSize,
|
||
aspectRatio,
|
||
handleUpstreamProgress,
|
||
);
|
||
}
|
||
|
||
// Text-to-image: use images/generations format
|
||
const endpoint = resolvedCustomApiConfig.apiUrl;
|
||
if (!endpoint) {
|
||
return NextResponse.json({ error: '自定义API未配置请求地址' }, { status: 400 });
|
||
}
|
||
if (!resolvedCustomApiConfig.modelName) {
|
||
return NextResponse.json({ error: '自定义API未配置模型名称，请在设置中填写模型名称（如 gpt-image-2）' }, { status: 400 });
|
||
}
|
||
|
||
// Ensure n is at least 1
|
||
const n = Math.max(1, count || 1);
|
||
|
||
// Augment prompt with aspect ratio hint as fallback
|
||
// Many APIs ignore size/aspect_ratio params, so embedding in prompt helps
|
||
const ratioHint = aspectRatio ? getAspectRatioPromptHint(aspectRatio) : '';
|
||
const augmentedPrompt = ratioHint ? `${prompt}\n\n[${ratioHint}]` : prompt;
|
||
|
||
const requestBody: Record<string, unknown> = {
|
||
model: resolvedCustomApiConfig.modelName,
|
||
prompt: augmentedPrompt,
|
||
n,
|
||
size: size || '1024x1024',
|
||
response_format: 'b64_json',
|
||
stream: true,
|
||
};
|
||
if (negativePrompt) {
|
||
requestBody.negative_prompt = negativePrompt;
|
||
}
|
||
// Pass guidance_scale for diffusion models (CFG scale)
|
||
if (guidanceScale && guidanceScale !== 7) {
|
||
requestBody.guidance_scale = guidanceScale;
|
||
}
|
||
// Pass aspect_ratio for APIs that prefer it over pixel size
|
||
if (aspectRatio) {
|
||
requestBody.aspect_ratio = aspectRatio;
|
||
}
|
||
|
||
console.log('[Custom API Image] Text-to-image, sending to:', endpoint,
|
||
'| model:', requestBody.model,
|
||
'| size:', requestBody.size,
|
||
'| n:', requestBody.n,
|
||
'| aspect_ratio:', requestBody.aspect_ratio,
|
||
'| stream:', requestBody.stream,
|
||
'| guidance_scale:', requestBody.guidance_scale,
|
||
'| prompt_length:', prompt.length,
|
||
'| augmented_prompt_length:', augmentedPrompt.length);
|
||
|
||
let customGenerationResult: Awaited<ReturnType<typeof requestQualifiedCustomImages>>;
|
||
try {
|
||
customGenerationResult = await requestQualifiedCustomImages(
|
||
endpoint,
|
||
resolvedApiKey,
|
||
requestBody,
|
||
n,
|
||
targetSize,
|
||
handleUpstreamProgress,
|
||
);
|
||
} catch (fetchError: unknown) {
|
||
if (fetchError instanceof DOMException && fetchError.name === 'AbortError') {
|
||
return NextResponse.json({ error: `自定义API请求超时（${GENERATION_TIMEOUT_SECONDS}秒）` }, { status: 504 });
|
||
}
|
||
const msg = fetchError instanceof Error ? fetchError.message : '请求失败';
|
||
if (msg.includes('ECONNREFUSED') || msg.includes('ENOTFOUND') || msg.includes('fetch failed')) {
|
||
return NextResponse.json({ error: `无法连接到自定义API: ${msg}。请检查 API 地址` }, { status: 502 });
|
||
}
|
||
return NextResponse.json({ error: `自定义API网络错误: ${msg}` }, { status: 502 });
|
||
}
|
||
|
||
if (customGenerationResult.upstreamError) {
|
||
const { status, text } = customGenerationResult.upstreamError;
|
||
console.error('[Custom API Image Error]', status, text.slice(0, 500));
|
||
return NextResponse.json(
|
||
{ error: parseCustomApiError(status, text) },
|
||
{ status: status >= 500 ? 502 : status }
|
||
);
|
||
}
|
||
|
||
if (customGenerationResult.images.length === 0) {
|
||
return NextResponse.json({ error: lowResolutionError(targetSize, customGenerationResult.rejected) }, { status: 502 });
|
||
}
|
||
console.log('[Custom API Image] Persisted', customGenerationResult.images.length, '/', n, 'qualified images',
|
||
'| target:', targetSize ? formatTargetSize(targetSize) : 'none');
|
||
return NextResponse.json({ images: customGenerationResult.images });
|
||
} catch (customError: unknown) {
|
||
const msg = customError instanceof Error ? customError.message : '自定义API请求异常';
|
||
console.error('[Custom API Image Exception]', msg);
|
||
return NextResponse.json({ error: `自定义API异常: ${msg}` }, { status: 502 });
|
||
}
|
||
}
|
||
|
||
// ---- Default mode: use coze-coding-dev-sdk ----
|
||
const customHeaders = HeaderUtils.extractForwardHeaders(request.headers);
|
||
const config = new Config();
|
||
const client = new ImageGenerationClient(config, customHeaders);
|
||
|
||
let sdkSize: string;
|
||
if (size) {
|
||
sdkSize = size;
|
||
} else if (aspectRatio && resolution) {
|
||
// Resolve from aspect ratio + resolution
|
||
const sizeMap: Record<string, Record<string, string>> = {
|
||
'1:1': { '1080P': '1024x1024', '2K': '2048x2048', '4K': '4096x4096' },
|
||
'16:9': { '1080P': '1920x1080', '2K': '2560x1440', '4K': '3840x2160' },
|
||
'9:16': { '1080P': '1080x1920', '2K': '1440x2560', '4K': '2160x3840' },
|
||
'4:3': { '1080P': '1440x1080', '2K': '2560x1920', '4K': '4096x3072' },
|
||
'3:4': { '1080P': '1080x1440', '2K': '1920x2560', '4K': '3072x4096' },
|
||
};
|
||
sdkSize = sizeMap[aspectRatio]?.[resolution] || '1024x1024';
|
||
} else {
|
||
sdkSize = quality === '4K' ? '4K' : quality === '1K' ? '1K' : '2K';
|
||
}
|
||
|
||
const generateRequest: Record<string, unknown> = {
|
||
prompt,
|
||
model,
|
||
size: sdkSize,
|
||
watermark: false,
|
||
};
|
||
|
||
if (negativePrompt) {
|
||
generateRequest.negativePrompt = negativePrompt;
|
||
}
|
||
|
||
if (image) {
|
||
if (image.startsWith('data:')) {
|
||
const uploadedUrl = await uploadDataUrlAndGetPublicUrl(image);
|
||
if (uploadedUrl) {
|
||
generateRequest.image = uploadedUrl;
|
||
} else {
|
||
console.warn('[Image Gen] Failed to upload reference image, skipping');
|
||
}
|
||
} else {
|
||
generateRequest.image = image;
|
||
}
|
||
}
|
||
|
||
let response;
|
||
try {
|
||
const debugRequest = { ...generateRequest };
|
||
if (typeof debugRequest.image === 'string' && debugRequest.image.length > 100) {
|
||
debugRequest.image = `${debugRequest.image.substring(0, 60)}... (${debugRequest.image.length} chars)`;
|
||
}
|
||
console.log('[SDK Image Request]', JSON.stringify(debugRequest));
|
||
response = await client.generate(generateRequest as unknown as Parameters<typeof client.generate>[0]);
|
||
} catch (sdkError: unknown) {
|
||
const sdkMessage = sdkError instanceof Error ? sdkError.message : '图片生成请求失败';
|
||
let detail = '';
|
||
try {
|
||
const errObj = sdkError as { response?: { status?: number; data?: unknown; statusText?: string } };
|
||
if (errObj.response) {
|
||
const dataStr = errObj.response.data ? JSON.stringify(errObj.response.data) : '';
|
||
detail = `status=${errObj.response.status} data=${dataStr.substring(0, 500)}`;
|
||
}
|
||
} catch { /* ignore */ }
|
||
console.error('[Image Generation SDK Error]', sdkMessage, detail);
|
||
if (image) {
|
||
return NextResponse.json({
|
||
error: '图生图生成失败: 内置模型图生图功能暂不可用。建议使用自定义API重试。',
|
||
}, { status: 503 });
|
||
}
|
||
return NextResponse.json({ error: `图片生成服务暂时不可用: ${sdkMessage}` }, { status: 503 });
|
||
}
|
||
|
||
const helper = client.getResponseHelper(response);
|
||
if (!helper.success) {
|
||
const errorMsg = helper.errorMessages.length > 0 ? helper.errorMessages.join('; ') : '图片生成失败';
|
||
return NextResponse.json({ error: errorMsg }, { status: 500 });
|
||
}
|
||
|
||
const images = helper.imageUrls;
|
||
if (images.length === 0) {
|
||
return NextResponse.json({ error: '图片生成失败，请稍后重试' }, { status: 500 });
|
||
}
|
||
|
||
const persistedImages = await persistQualifiedImageUrls(images, 'generated/images', targetSize, 'SDK Image');
|
||
if (persistedImages.images.length === 0) {
|
||
return NextResponse.json({ error: lowResolutionError(targetSize, persistedImages.rejected) }, { status: 502 });
|
||
}
|
||
return NextResponse.json({ images: persistedImages.images });
|
||
} catch (error: unknown) {
|
||
const message = error instanceof Error ? error.message : '图片生成失败';
|
||
console.error('[Image Generation Error]', message, error instanceof Error ? error.stack : '');
|
||
return NextResponse.json({ error: `生成失败: ${message}` }, { status: 500 });
|
||
}
|
||
}
|