Harden data portability and backup restore
This commit is contained in:
FengLee
@@ -51,10 +51,11 @@ Use this table before searching.
|
||||
| User auth/login/register/profile | `src/lib/session-auth.ts`, `src/lib/auth-store.ts` | `src/app/api/auth/*`, `src/app/api/profile/*` |
|
||||
| Admin console | `src/app/console/page.tsx`, `src/app/console/dashboard/page.tsx`, `src/modules/console/pages/*` | `src/components/admin/*`, `src/app/api/admin/*` |
|
||||
| Canvas (legacy, disabled in UI) | `src/app/canvas/page.tsx`, `src/components/canvas/infinite-canvas-workspace.tsx`, `src/components/canvas/react-flow-canvas.tsx` | `/canvas` intentionally returns 404 and navbar must not show `画布`; legacy source/API files remain only for future cleanup or explicit re-enable work. |
|
||||
| Gallery and creation history | `src/app/gallery/page.tsx`, `src/app/profile/page.tsx`, `src/components/profile/creation-history-tab.tsx`, `src/components/image-metadata-badge.tsx` | `src/lib/creation-history-store.ts`, `src/app/api/gallery/*`, `src/app/api/creation-history/route.ts`. Gallery/detail image previews show actual ratio and natural resolution in the upper-right badge. |
|
||||
| Gallery and creation history | `src/app/gallery/page.tsx`, `src/app/profile/page.tsx`, `src/components/profile/creation-history-tab.tsx`, `src/components/image-metadata-badge.tsx` | `src/lib/creation-history-store.ts`, `src/app/api/gallery/*`, `src/app/api/creation-history/route.ts`. Gallery/detail image previews show actual ratio and natural resolution in the upper-right badge. History also refreshes on `miaojing_auth_updated` after login/account switch. |
|
||||
| Local files/downloads | `src/lib/local-storage.ts`, `src/app/api/local-storage/[...path]/route.ts` | `src/app/api/download/route.ts` |
|
||||
| Email and policy pages | `src/lib/email-service.ts`, `src/components/site-policy-page.tsx` | `src/app/api/email/*`, `src/app/about/page.tsx`, `src/app/terms/page.tsx`, `src/app/privacy/page.tsx`, `src/app/help/page.tsx` |
|
||||
| Upgrade/deploy/backup | `scripts/*`, `ecosystem.config.cjs` | `src/app/api/admin/upgrade/route.ts`, `src/components/admin/system-upgrade-tab.tsx` |
|
||||
| Data backup/import/export | `src/components/admin/data-management-tab.tsx` | `src/app/api/admin/data-export/route.ts`, `src/app/api/admin/data-import/route.ts`, `src/lib/local-storage.ts`. Export includes `_media` for local-storage assets; import restores media, remaps custom IDs, runs in a transaction, and dedupes works by URL/source URL/media SHA. |
|
||||
|
||||
## Current Known Source Warning
|
||||
|
||||
@@ -80,7 +81,8 @@ Before treating `ag-psd` or `@xyflow/react` as source bugs, run `corepack pnpm i
|
||||
- Do not write raw API keys to responses or logs. Use encryption helpers in `src/lib/server-crypto.ts` and safe mapping functions.
|
||||
- Do not create ad hoc storage paths. Use `src/lib/local-storage.ts` and preserve path traversal checks.
|
||||
- Do not create a second generation flow without checking `src/lib/generation-job-client.ts`, `src/lib/generation-job-runner.ts`, and `src/lib/generation-job-worker.ts`.
|
||||
- Do not change admin upgrade behavior without checking both `src/app/api/admin/upgrade/route.ts` and `scripts/admin-upgrade-runner.mjs`.
|
||||
- Do not change admin upgrade behavior without checking both `src/app/api/admin/upgrade/route.ts`, `scripts/admin-upgrade-runner.mjs`, `scripts/backup-create.sh`, and `scripts/backup-restore.sh`.
|
||||
- Do not change backup/import/restore behavior without preserving transaction boundaries, media restore behavior, dedupe rules, and pre-restore safety backups.
|
||||
- Do not change public content rendering without checking both backend persistence (`site-config`, `announcements`) and frontend consumers (`site-config-sync`, policy pages, footer, popup).
|
||||
|
||||
## Documentation Maintenance Rule
|
||||
|
||||
@@ -126,8 +126,8 @@ All routes in this section require admin unless noted.
|
||||
| GET/POST/PUT/DELETE | `/api/admin/system-apis` | `src/app/api/admin/system-apis/route.ts` | System API config CRUD with encrypted keys and pricing metadata. |
|
||||
| GET/POST/PUT/DELETE | `/api/admin/model-recommendations` | `src/app/api/admin/model-recommendations/route.ts` | Managed model recommendations. |
|
||||
| GET/DELETE | `/api/admin/generation-jobs` | `src/app/api/admin/generation-jobs/route.ts` | Admin task listing and deletion. |
|
||||
| GET | `/api/admin/data-export` | `src/app/api/admin/data-export/route.ts` | Export business data. |
|
||||
| POST | `/api/admin/data-import` | `src/app/api/admin/data-import/route.ts` | Import business data. |
|
||||
| GET | `/api/admin/data-export` | `src/app/api/admin/data-export/route.ts` | Export business data plus `_media` entries for local-storage assets referenced by works and site config. `_meta` reports media count/bytes/missing/skipped. |
|
||||
| POST | `/api/admin/data-import` | `src/app/api/admin/data-import/route.ts` | Import business data. Accepts optional `_media`; restores media to sha-based keys, remaps users/custom API keys/works, imports in a transaction with per-row savepoints, and dedupes works by URL/source URL/media SHA. |
|
||||
| GET/PUT/POST | `/api/admin/email-settings` | `src/app/api/admin/email-settings/route.ts` | Read/update/test email settings. |
|
||||
| GET | `/api/admin/email-recipients` | `src/app/api/admin/email-recipients/route.ts` | Email recipient list. |
|
||||
| POST | `/api/admin/send-email` | `src/app/api/admin/send-email/route.ts` | Send admin email. |
|
||||
|
||||
@@ -208,7 +208,7 @@ Scripts:
|
||||
- `scripts/start.sh`: production start path.
|
||||
- `scripts/deploy-or-upgrade.sh`: deployment/upgrade automation.
|
||||
- `scripts/admin-upgrade-runner.mjs`: package extraction/build/restart runner.
|
||||
- `scripts/backup-create.sh`, `backup-list.sh`, `backup-restore.sh`: backups.
|
||||
- `scripts/backup-create.sh`, `backup-list.sh`, `backup-restore.sh`: backups. Backup creation validates `pg_dump` output and tar integrity; restore validates archive/dump contents, creates a pre-restore safety dump/copy, uses a single DB transaction, and swaps local storage atomically.
|
||||
- `scripts/apply-database-patch.sh`: DB patch execution.
|
||||
|
||||
Runtime:
|
||||
@@ -219,7 +219,16 @@ Runtime:
|
||||
- `src/app/api/admin/upgrade/route.ts`
|
||||
- `src/components/admin/system-upgrade-tab.tsx`
|
||||
|
||||
When changing deploy/upgrade behavior, validate package limits, disk checks, backup creation, rollback paths, PM2 restart command, and health checks.
|
||||
When changing deploy/upgrade behavior, validate package limits, disk checks, backup creation, rollback paths, restore safety backups, PM2 restart command, and health checks.
|
||||
|
||||
## Data Portability
|
||||
|
||||
Admin data export/import is a portability layer, separate from the full tar backup scripts:
|
||||
|
||||
- `src/app/api/admin/data-export/route.ts` exports database business tables and bundles local-storage media referenced by `works` and `site_config` under `_media`.
|
||||
- `src/app/api/admin/data-import/route.ts` accepts older DB-only exports and newer media-inclusive exports. Newer exports restore media files to stable sha-based local-storage keys before writing work rows.
|
||||
- Import runs one DB transaction with per-row savepoints, remaps user/work/custom API key IDs, preserves old source URL/media SHA markers in `works.params`, and merges repeated imports by URL/source URL/media SHA.
|
||||
- Older exports without `_media` can restore database rows but cannot recreate missing local files by themselves; copy `LOCAL_STORAGE_DIR` or use a newer export for full gallery/history image recovery.
|
||||
|
||||
## Security Boundaries
|
||||
|
||||
|
||||
@@ -68,9 +68,9 @@ Use this guide when the user reports behavior. Start from the symptom row, inspe
|
||||
|
||||
| Symptom | Check Files | What To Verify |
|
||||
| --- | --- | --- |
|
||||
| History missing after generation | `src/lib/creation-history-store.ts`, `src/app/api/creation-history/route.ts`, create panel component | History POST, `works` insert, URL not data URL except reverse prompt placeholder. |
|
||||
| History missing after generation or login/account switch | `src/lib/creation-history-store.ts`, `src/app/api/creation-history/route.ts`, create panel component | History POST, `works` insert, URL not data URL except reverse prompt placeholder, and `miaojing_auth_updated` triggers a fresh server fetch. |
|
||||
| Published work not in gallery | `src/lib/creation-history-store.ts`, `src/app/api/gallery/publish/route.ts`, `src/app/api/gallery/route.ts`, `src/app/gallery/page.tsx` | `is_public = true`, `status = completed`, media copied to gallery folder, filters. |
|
||||
| Imported gallery images do not render after production data import | `src/app/api/admin/data-import/route.ts`, `src/lib/local-storage.ts`, `src/app/api/local-storage/[...path]/route.ts`, DB `works.result_url` | Check whether production media files were copied into `LOCAL_STORAGE_DIR`; DB rows alone are not enough for relative `/api/local-storage/*` URLs. |
|
||||
| Imported gallery images do not render after production data import | `src/app/api/admin/data-export/route.ts`, `src/app/api/admin/data-import/route.ts`, `src/lib/local-storage.ts`, `src/app/api/local-storage/[...path]/route.ts`, DB `works.result_url` | New exports should include `_media`; import should persist media to local storage. If using an older export without `_media`, DB rows alone cannot recreate missing `/api/local-storage/*` files. |
|
||||
| Gallery delete does not remove public item | `src/app/api/gallery/route.ts`, admin UI route using it | DELETE unpublishes by setting `is_public = false`, not hard delete. |
|
||||
| Search/filter/sort wrong | `src/app/api/gallery/route.ts`, `src/app/gallery/page.tsx` | Query params `type`, `limit`, `offset`, `sort`, `q/search`; SQL where/order. |
|
||||
| Gallery search box looks inconsistent with the rest of the UI | `src/app/gallery/page.tsx` | The search field is a custom glass panel with an inner focused input surface; avoid reverting it to a plain transparent input row. |
|
||||
@@ -91,9 +91,9 @@ Use this guide when the user reports behavior. Start from the symptom row, inspe
|
||||
| Dashboard stats wrong | `src/modules/console/pages/console-dashboard-page.tsx`, `src/app/api/admin/dashboard/route.ts`, `src/app/api/admin/stats/route.ts` | SQL source tables, safe query fallbacks, admin auth. |
|
||||
| User management bug | `src/components/admin/user-management-tab.tsx`, `src/app/api/admin/users/route.ts` | Role/tier mapping, active flag, admin auth. |
|
||||
| Order/payment bug | `src/components/admin/order-management-tab.tsx`, `src/components/admin/payment-tab.tsx`, `src/app/api/admin/orders/route.ts`, `src/app/api/admin/payment-methods/route.ts`, `src/lib/server-payment-config.ts` | Payment config encryption, order status, request shape. |
|
||||
| Data import/export bug | `src/components/admin/data-management-tab.tsx`, `src/app/api/admin/data-export/route.ts`, `src/app/api/admin/data-import/route.ts` | JSON format, table coverage, admin auth. |
|
||||
| Data import/export bug | `src/components/admin/data-management-tab.tsx`, `src/app/api/admin/data-export/route.ts`, `src/app/api/admin/data-import/route.ts` | JSON format, table coverage, admin auth, `_media` coverage, import transaction/savepoints, work dedupe by URL/source URL/media SHA. |
|
||||
| Admin email send/settings bug | `src/components/admin/settings-tab.tsx`, `src/app/api/admin/email-settings/route.ts`, `src/app/api/admin/send-email/route.ts`, `src/lib/email-service.ts` | SMTP config, template rendering, MIME/body encoding. |
|
||||
| Upgrade page stuck/fails | `src/components/admin/system-upgrade-tab.tsx`, `src/app/api/admin/upgrade/route.ts`, `scripts/admin-upgrade-runner.mjs` | State dir, package limits, disk checks, backup, PM2 restart command, stale status. |
|
||||
| Upgrade page stuck/fails | `src/components/admin/system-upgrade-tab.tsx`, `src/app/api/admin/upgrade/route.ts`, `scripts/admin-upgrade-runner.mjs`, `scripts/backup-create.sh`, `scripts/backup-restore.sh` | State dir, package limits, disk checks, backup validation, restore safety backup, PM2 restart command, stale status. |
|
||||
| Platform logs missing | `src/components/admin/log-management-tab.tsx`, `src/lib/platform-logs.ts`, routes that call `writePlatformLog` | Runtime schema exists, log retention, API calls actually write logs. |
|
||||
|
||||
## Storage, Download, Files
|
||||
|
||||
@@ -129,7 +129,7 @@ Use this document to jump directly to code before broad searching.
|
||||
| Payment | `src/components/admin/payment-tab.tsx` | `src/app/api/admin/payment-methods/route.ts`, `src/lib/server-payment-config.ts` |
|
||||
| Orders | `src/components/admin/order-management-tab.tsx` | `src/app/api/admin/orders/route.ts` |
|
||||
| Announcements | `src/components/admin/announcement-tab.tsx` | `src/app/api/announcements/route.ts` |
|
||||
| Data import/export | `src/components/admin/data-management-tab.tsx` | `src/app/api/admin/data-export/route.ts`, `src/app/api/admin/data-import/route.ts` |
|
||||
| Data import/export | `src/components/admin/data-management-tab.tsx` | `src/app/api/admin/data-export/route.ts`, `src/app/api/admin/data-import/route.ts`. Export bundles local-storage URLs from works/site config into `_media`; import restores those files, maps old IDs, merges duplicate works, and runs DB writes in a transaction. |
|
||||
| System upgrade | `src/components/admin/system-upgrade-tab.tsx` | `src/app/api/admin/upgrade/route.ts`, `scripts/admin-upgrade-runner.mjs` |
|
||||
| Logs/tasks | `src/components/admin/log-management-tab.tsx`, `src/components/admin/task-management-tab.tsx` | `src/lib/platform-logs.ts`, `src/app/api/admin/generation-jobs/route.ts` |
|
||||
| Settings | `src/components/admin/settings-tab.tsx` | `src/app/api/site-config/route.ts`, `src/app/api/admin/email-settings/route.ts`, `src/app/api/admin/send-email/route.ts` |
|
||||
@@ -165,6 +165,6 @@ Use this document to jump directly to code before broad searching.
|
||||
| Start | `scripts/start.sh` |
|
||||
| Dev | `scripts/dev.sh` |
|
||||
| Deploy/upgrade | `scripts/deploy-or-upgrade.sh` |
|
||||
| Backup | `scripts/backup-create.sh`, `scripts/backup-list.sh`, `scripts/backup-restore.sh` |
|
||||
| Backup | `scripts/backup-create.sh`, `scripts/backup-list.sh`, `scripts/backup-restore.sh`. Restore uses `pg_restore --single-transaction`, validates archive/dump contents, atomically swaps local storage, and keeps a pre-restore safety backup. |
|
||||
| Admin upgrade runner | `scripts/admin-upgrade-runner.mjs` |
|
||||
| Boundary checks | `scripts/check-boundaries.sh` |
|
||||
|
||||
@@ -269,7 +269,10 @@ async function rollbackAfterFailure(message) {
|
||||
run('bash', ['./scripts/backup-restore.sh', state.backupFile], {
|
||||
cwd: projectRoot,
|
||||
label: '恢复数据备份',
|
||||
env: { COZE_WORKSPACE_PATH: projectRoot },
|
||||
env: {
|
||||
COZE_WORKSPACE_PATH: projectRoot,
|
||||
RESTORE_SAFETY_DIR: path.join(stateRoot, 'restore-safety'),
|
||||
},
|
||||
});
|
||||
logStep('数据回滚完成', '数据库、存储目录和环境配置已恢复');
|
||||
}
|
||||
|
||||
@@ -38,8 +38,13 @@ command -v pg_dump >/dev/null 2>&1 || {
|
||||
echo "pg_dump is required to create backups." >&2
|
||||
exit 1
|
||||
}
|
||||
command -v pg_restore >/dev/null 2>&1 || {
|
||||
echo "pg_restore is required to verify backups." >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
pg_dump "${LOCAL_DB_URL}" --format=custom --file "${TMP_DIR}/database.dump"
|
||||
pg_restore --list "${TMP_DIR}/database.dump" >/dev/null
|
||||
|
||||
STORAGE_SOURCE="${LOCAL_STORAGE_DIR:-${COZE_WORKSPACE_PATH}/local-storage}"
|
||||
if [ -d "${STORAGE_SOURCE}" ]; then
|
||||
@@ -57,13 +62,16 @@ fi
|
||||
cat > "${TMP_DIR}/manifest.json" <<EOF
|
||||
{
|
||||
"app": "miaojingAI",
|
||||
"formatVersion": 2,
|
||||
"createdAt": "$(date -Iseconds)",
|
||||
"hostname": "$(hostname)",
|
||||
"storagePath": "${STORAGE_SOURCE}",
|
||||
"includes": ["database.dump", "local-storage", ".env.local", "package.json"]
|
||||
}
|
||||
EOF
|
||||
|
||||
tar -czf "${BACKUP_FILE}" -C "${TMP_DIR}" .
|
||||
tar -tzf "${BACKUP_FILE}" >/dev/null
|
||||
chmod 600 "${BACKUP_FILE}"
|
||||
|
||||
find "${BACKUP_DIR}" -maxdepth 1 -name 'miaojing-backup-*.tar.gz' -type f \
|
||||
|
||||
@@ -3,7 +3,9 @@ set -Eeuo pipefail
|
||||
|
||||
COZE_WORKSPACE_PATH="${COZE_WORKSPACE_PATH:-$(pwd)}"
|
||||
BACKUP_FILE="${1:-}"
|
||||
TIMESTAMP="$(date +%Y%m%d-%H%M%S)"
|
||||
TMP_DIR="$(mktemp -d)"
|
||||
RESTORE_SAFETY_DIR="${RESTORE_SAFETY_DIR:-}"
|
||||
|
||||
cleanup() {
|
||||
rm -rf "${TMP_DIR}"
|
||||
@@ -40,26 +42,72 @@ command -v pg_restore >/dev/null 2>&1 || {
|
||||
echo "pg_restore is required to restore backups." >&2
|
||||
exit 1
|
||||
}
|
||||
command -v pg_dump >/dev/null 2>&1 || {
|
||||
echo "pg_dump is required to create restore safety backups." >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
tar -tzf "${BACKUP_FILE}" >/dev/null
|
||||
tar -xzf "${BACKUP_FILE}" -C "${TMP_DIR}"
|
||||
|
||||
if [ ! -f "${TMP_DIR}/database.dump" ]; then
|
||||
echo "Invalid backup: missing database.dump." >&2
|
||||
exit 2
|
||||
fi
|
||||
pg_restore --list "${TMP_DIR}/database.dump" >/dev/null
|
||||
|
||||
pg_restore --clean --if-exists --no-owner --dbname "${LOCAL_DB_URL}" "${TMP_DIR}/database.dump"
|
||||
SAFETY_ROOT="${RESTORE_SAFETY_DIR:-${COZE_WORKSPACE_PATH}/backups/restore-safety}"
|
||||
SAFETY_DIR="${SAFETY_ROOT}/pre-restore-${TIMESTAMP}"
|
||||
mkdir -p "${SAFETY_DIR}"
|
||||
chmod 700 "${SAFETY_ROOT}" "${SAFETY_DIR}"
|
||||
|
||||
pg_dump "${LOCAL_DB_URL}" --format=custom --file "${SAFETY_DIR}/database-before-restore.dump"
|
||||
pg_restore --list "${SAFETY_DIR}/database-before-restore.dump" >/dev/null
|
||||
|
||||
STORAGE_TARGET="${LOCAL_STORAGE_DIR:-${COZE_WORKSPACE_PATH}/local-storage}"
|
||||
if [ -e "${STORAGE_TARGET}" ]; then
|
||||
mkdir -p "${SAFETY_DIR}/storage-parent"
|
||||
cp -a "${STORAGE_TARGET}" "${SAFETY_DIR}/storage-parent/$(basename "${STORAGE_TARGET}")"
|
||||
fi
|
||||
STORAGE_PARENT="$(dirname "${STORAGE_TARGET}")"
|
||||
STORAGE_NAME="$(basename "${STORAGE_TARGET}")"
|
||||
PREVIOUS_STORAGE="${SAFETY_DIR}/${STORAGE_NAME}.previous"
|
||||
STAGED_STORAGE="${TMP_DIR}/${STORAGE_NAME}.staged"
|
||||
if [ -d "${TMP_DIR}/local-storage" ]; then
|
||||
rm -rf "${STAGED_STORAGE}"
|
||||
cp -a "${TMP_DIR}/local-storage" "${STAGED_STORAGE}"
|
||||
fi
|
||||
|
||||
if [ -f ".env.local" ]; then
|
||||
cp ".env.local" "${SAFETY_DIR}/.env.local.before-restore"
|
||||
chmod 600 "${SAFETY_DIR}/.env.local.before-restore"
|
||||
fi
|
||||
|
||||
pg_restore --clean --if-exists --no-owner --single-transaction --dbname "${LOCAL_DB_URL}" "${TMP_DIR}/database.dump"
|
||||
|
||||
if [ -d "${TMP_DIR}/local-storage" ]; then
|
||||
STORAGE_TARGET="${LOCAL_STORAGE_DIR:-${COZE_WORKSPACE_PATH}/local-storage}"
|
||||
rm -rf "${STORAGE_TARGET}"
|
||||
mkdir -p "$(dirname "${STORAGE_TARGET}")"
|
||||
cp -a "${TMP_DIR}/local-storage" "${STORAGE_TARGET}"
|
||||
mkdir -p "${STORAGE_PARENT}"
|
||||
if [ -e "${STORAGE_TARGET}" ]; then
|
||||
mv "${STORAGE_TARGET}" "${PREVIOUS_STORAGE}"
|
||||
fi
|
||||
if ! mv "${STAGED_STORAGE}" "${STORAGE_PARENT}/${STORAGE_NAME}"; then
|
||||
rm -rf "${STORAGE_PARENT:?}/${STORAGE_NAME}"
|
||||
if [ -e "${PREVIOUS_STORAGE}" ]; then
|
||||
mv "${PREVIOUS_STORAGE}" "${STORAGE_PARENT}/${STORAGE_NAME}"
|
||||
fi
|
||||
echo "Storage restore failed; previous storage was restored." >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -f "${TMP_DIR}/.env.local" ]; then
|
||||
cp "${TMP_DIR}/.env.local" ".env.local"
|
||||
cp "${TMP_DIR}/.env.local" ".env.local.restore-next"
|
||||
mv ".env.local.restore-next" ".env.local"
|
||||
chmod 600 ".env.local"
|
||||
fi
|
||||
|
||||
find "${SAFETY_ROOT}" -maxdepth 1 -type d -name 'pre-restore-*' \
|
||||
-printf '%T@ %p\n' | sort -rn | awk 'NR>10 {print $2}' | xargs -r rm -rf
|
||||
|
||||
echo "Restore completed from ${BACKUP_FILE}"
|
||||
echo "Pre-restore safety backup: ${SAFETY_DIR}"
|
||||
|
||||
@@ -1,6 +1,19 @@
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { requireAdmin } from '@/lib/admin-auth';
|
||||
import { getDbClient } from '@/storage/database/local-db';
|
||||
import { localStorage } from '@/lib/local-storage';
|
||||
import crypto from 'crypto';
|
||||
|
||||
type ExportMediaEntry = {
|
||||
contentType: string;
|
||||
encoding: 'base64';
|
||||
data: string;
|
||||
size: number;
|
||||
sha256: string;
|
||||
};
|
||||
|
||||
const MAX_EXPORT_MEDIA_BYTES = 800 * 1024 * 1024;
|
||||
const MAX_EXPORT_SINGLE_MEDIA_BYTES = 100 * 1024 * 1024;
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
const authError = await requireAdmin(request);
|
||||
@@ -48,15 +61,22 @@ export async function GET(request: NextRequest) {
|
||||
data.auth_users = result.rows || [];
|
||||
} catch { data.auth_users = []; }
|
||||
|
||||
const mediaExport = collectExportMedia(data);
|
||||
|
||||
const exportData = {
|
||||
_meta: {
|
||||
version: '1.0',
|
||||
version: '1.1',
|
||||
platform: 'miaojing',
|
||||
exported_at: new Date().toISOString(),
|
||||
tables: Object.keys(data),
|
||||
counts: Object.fromEntries(Object.entries(data).map(([k, v]) => [k, v.length])),
|
||||
media_files: Object.keys(mediaExport.media).length,
|
||||
media_bytes: mediaExport.bytes,
|
||||
media_missing: mediaExport.missing,
|
||||
media_skipped: mediaExport.skipped,
|
||||
},
|
||||
data,
|
||||
_media: mediaExport.media,
|
||||
};
|
||||
|
||||
return NextResponse.json(exportData);
|
||||
@@ -68,3 +88,75 @@ export async function GET(request: NextRequest) {
|
||||
return NextResponse.json({ error: err instanceof Error ? err.message : '导出失败' }, { status: 500 });
|
||||
}
|
||||
}
|
||||
|
||||
function collectExportMedia(data: Record<string, unknown[]>): {
|
||||
media: Record<string, ExportMediaEntry>;
|
||||
bytes: number;
|
||||
missing: string[];
|
||||
skipped: string[];
|
||||
} {
|
||||
const urls = new Set<string>();
|
||||
for (const row of data.works || []) {
|
||||
collectLocalStorageUrls(row, urls);
|
||||
}
|
||||
for (const row of data.site_config || []) {
|
||||
collectLocalStorageUrls(row, urls);
|
||||
}
|
||||
|
||||
const media: Record<string, ExportMediaEntry> = {};
|
||||
const missing: string[] = [];
|
||||
const skipped: string[] = [];
|
||||
let bytes = 0;
|
||||
|
||||
for (const url of urls) {
|
||||
const key = localStorage.getKeyFromPublicUrl(url);
|
||||
if (!key || !localStorage.fileExists(key)) {
|
||||
missing.push(url);
|
||||
continue;
|
||||
}
|
||||
const buffer = localStorage.readFile(key);
|
||||
if (buffer.byteLength > MAX_EXPORT_SINGLE_MEDIA_BYTES) {
|
||||
skipped.push(url);
|
||||
continue;
|
||||
}
|
||||
if (bytes + buffer.byteLength > MAX_EXPORT_MEDIA_BYTES) {
|
||||
skipped.push(url);
|
||||
continue;
|
||||
}
|
||||
bytes += buffer.byteLength;
|
||||
media[url] = {
|
||||
contentType: getContentTypeFromKey(key),
|
||||
encoding: 'base64',
|
||||
data: buffer.toString('base64'),
|
||||
size: buffer.byteLength,
|
||||
sha256: crypto.createHash('sha256').update(buffer).digest('hex'),
|
||||
};
|
||||
}
|
||||
|
||||
return { media, bytes, missing, skipped };
|
||||
}
|
||||
|
||||
function collectLocalStorageUrls(value: unknown, output: Set<string>): void {
|
||||
if (typeof value === 'string') {
|
||||
if (localStorage.getKeyFromPublicUrl(value)) output.add(value);
|
||||
return;
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
value.forEach(item => collectLocalStorageUrls(item, output));
|
||||
return;
|
||||
}
|
||||
if (value && typeof value === 'object') {
|
||||
Object.values(value as Record<string, unknown>).forEach(item => collectLocalStorageUrls(item, output));
|
||||
}
|
||||
}
|
||||
|
||||
function getContentTypeFromKey(key: string): string {
|
||||
const ext = key.split('.').pop()?.toLowerCase();
|
||||
if (ext === 'jpg' || ext === 'jpeg') return 'image/jpeg';
|
||||
if (ext === 'png') return 'image/png';
|
||||
if (ext === 'webp') return 'image/webp';
|
||||
if (ext === 'gif') return 'image/gif';
|
||||
if (ext === 'mp4') return 'video/mp4';
|
||||
if (ext === 'webm') return 'video/webm';
|
||||
return 'application/octet-stream';
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ import { requireAdmin } from '@/lib/admin-auth';
|
||||
import { localStorage } from '@/lib/local-storage';
|
||||
import { encryptSecret, previewSecret } from '@/lib/server-crypto';
|
||||
import { getDbClient } from '@/storage/database/local-db';
|
||||
import crypto from 'crypto';
|
||||
|
||||
interface ImportMeta {
|
||||
version: string;
|
||||
@@ -15,11 +16,21 @@ interface ImportMeta {
|
||||
interface ImportPayload {
|
||||
_meta: ImportMeta;
|
||||
data: Record<string, unknown[]>;
|
||||
_media?: Record<string, ImportMediaEntry>;
|
||||
options?: {
|
||||
skipAuth?: boolean;
|
||||
};
|
||||
}
|
||||
|
||||
type ImportMediaEntry = {
|
||||
contentType?: string;
|
||||
encoding?: 'base64';
|
||||
data?: string;
|
||||
dataUrl?: string;
|
||||
size?: number;
|
||||
sha256?: string;
|
||||
};
|
||||
|
||||
const MAX_ROWS_PER_TABLE = 5000;
|
||||
const UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
||||
const UUID_ID_TABLES = new Set([
|
||||
@@ -40,7 +51,7 @@ const TABLE_COLUMNS: Record<string, string[]> = {
|
||||
site_config: ['id', 'site_name', 'site_tab_title', 'site_description', 'site_keywords', 'logo_url', 'favicon_url', 'announcement', 'membership_enabled', 'terms_of_service', 'privacy_policy', 'about_us', 'help_center', 'filing_info', 'filing_url', 'public_security_filing_info', 'public_security_filing_url', 'updated_at'],
|
||||
site_stats: ['id', 'total_visits', 'total_users', 'total_generations', 'updated_at'],
|
||||
announcements: ['id', 'title', 'content', 'type', 'is_active', 'starts_at', 'expires_at', 'created_at', 'updated_at'],
|
||||
works: ['id', 'user_id', 'title', 'type', 'prompt', 'negative_prompt', 'params', 'result_url', 'thumbnail_url', 'width', 'height', 'duration', 'status', 'is_public', 'likes_count', 'views_count', 'created_at', 'updated_at'],
|
||||
works: ['id', 'user_id', 'title', 'type', 'prompt', 'negative_prompt', 'params', 'result_url', 'thumbnail_url', 'width', 'height', 'duration', 'status', 'is_public', 'likes_count', 'views_count', 'credits_cost', 'created_at', 'updated_at'],
|
||||
credit_transactions: ['id', 'user_id', 'amount', 'balance_after', 'type', 'description', 'related_work_id', 'created_at'],
|
||||
orders: ['id', 'user_id', 'order_no', 'product_type', 'product_name', 'amount', 'credits_amount', 'status', 'payment_method', 'paid_at', 'created_at', 'updated_at'],
|
||||
user_api_keys: ['id', 'user_id', 'provider', 'supplier_name', 'api_url', 'model_name', 'note', 'api_key_encrypted', 'api_key_preview', 'type', 'is_active', 'created_at', 'updated_at'],
|
||||
@@ -77,6 +88,7 @@ type ImportContext = {
|
||||
emailUserIdMap: Map<string, string>;
|
||||
apiKeyIdMap: Map<string, string>;
|
||||
apiKeyOwnerIdMap: Map<string, string>;
|
||||
media: Record<string, ImportMediaEntry>;
|
||||
columnCache: Map<string, Set<string>>;
|
||||
defaultableColumnCache: Map<string, Set<string>>;
|
||||
};
|
||||
@@ -98,7 +110,8 @@ export async function POST(request: NextRequest) {
|
||||
const result: Record<string, ImportResult> = {};
|
||||
|
||||
try {
|
||||
const context = await buildImportContext(client, data);
|
||||
const context = await buildImportContext(client, data, body._media || {});
|
||||
await client.query('BEGIN');
|
||||
|
||||
if (!skipAuth && Array.isArray(data.auth_users)) {
|
||||
result.auth_users = await importRows(client, 'auth.users', AUTH_USER_COLUMNS, data.auth_users, context);
|
||||
@@ -115,7 +128,13 @@ export async function POST(request: NextRequest) {
|
||||
result[table] = await importRows(client, table, allowedColumns, Array.isArray(rows) ? rows : [], context);
|
||||
}
|
||||
|
||||
result.dedupe_works = await dedupeWorks(client);
|
||||
await client.query('COMMIT');
|
||||
|
||||
return NextResponse.json({ success: true, message: '数据导入完成', details: result, meta: _meta });
|
||||
} catch (error) {
|
||||
await client.query('ROLLBACK').catch(() => undefined);
|
||||
throw error;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
@@ -143,19 +162,23 @@ async function importRows(
|
||||
const defaultableColumns = await getDefaultableColumns(client, table, context);
|
||||
const effectiveAllowedColumns = allowedColumns.filter(col => existingColumns.has(col));
|
||||
|
||||
for (const rawRow of rows) {
|
||||
const row = await normalizeImportRow(table, rawRow as Record<string, unknown>, context);
|
||||
const cols = Object.keys(row).filter(col => (
|
||||
effectiveAllowedColumns.includes(col)
|
||||
&& !(row[col] == null && defaultableColumns.has(col))
|
||||
));
|
||||
if (!cols.includes('id') || cols.length === 0) {
|
||||
skipped++;
|
||||
errors.push(`${table}: 缺少 id 或没有允许导入的字段`);
|
||||
continue;
|
||||
}
|
||||
|
||||
for (let rowIndex = 0; rowIndex < rows.length; rowIndex += 1) {
|
||||
const savepoint = `import_row_${table.replace(/[^a-zA-Z0-9_]/g, '_')}_${rowIndex}`;
|
||||
try {
|
||||
await client.query(`SAVEPOINT ${savepoint}`);
|
||||
const rawRow = rows[rowIndex];
|
||||
const row = await normalizeImportRow(table, rawRow as Record<string, unknown>, context);
|
||||
const cols = Object.keys(row).filter(col => (
|
||||
effectiveAllowedColumns.includes(col)
|
||||
&& !(row[col] == null && defaultableColumns.has(col))
|
||||
));
|
||||
if (!cols.includes('id') || cols.length === 0) {
|
||||
skipped++;
|
||||
errors.push(`${table}: 缺少 id 或没有允许导入的字段`);
|
||||
await client.query(`RELEASE SAVEPOINT ${savepoint}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const vals = cols.map(col => row[col]);
|
||||
const placeholders = cols.map((_, i) => `$${i + 1}`).join(', ');
|
||||
const conflictCols = CONFLICT_COLUMNS[table] || ['id'];
|
||||
@@ -174,7 +197,10 @@ async function importRows(
|
||||
} else {
|
||||
skipped++;
|
||||
}
|
||||
await client.query(`RELEASE SAVEPOINT ${savepoint}`);
|
||||
} catch (e) {
|
||||
await client.query(`ROLLBACK TO SAVEPOINT ${savepoint}`).catch(() => undefined);
|
||||
await client.query(`RELEASE SAVEPOINT ${savepoint}`).catch(() => undefined);
|
||||
skipped++;
|
||||
errors.push(`${table}: ${e instanceof Error ? e.message : 'unknown error'}`);
|
||||
}
|
||||
@@ -186,6 +212,7 @@ async function importRows(
|
||||
async function buildImportContext(
|
||||
client: Awaited<ReturnType<typeof getDbClient>>,
|
||||
data: Record<string, unknown[]>,
|
||||
media: Record<string, ImportMediaEntry>,
|
||||
): Promise<ImportContext> {
|
||||
const userIdMap = new Map<string, string>();
|
||||
const workIdMap = new Map<string, string>();
|
||||
@@ -249,6 +276,7 @@ async function buildImportContext(
|
||||
emailUserIdMap,
|
||||
apiKeyIdMap,
|
||||
apiKeyOwnerIdMap,
|
||||
media,
|
||||
columnCache: new Map(),
|
||||
defaultableColumnCache: new Map(),
|
||||
});
|
||||
@@ -262,23 +290,45 @@ async function buildImportContext(
|
||||
|
||||
const works = Array.isArray(data.works) ? data.works : [];
|
||||
const workUrls = new Map<string, string>();
|
||||
const workMediaShas = new Map<string, string>();
|
||||
for (const raw of works) {
|
||||
const row = raw as Record<string, unknown>;
|
||||
seedUuidMap(workIdMap, row.id);
|
||||
if (typeof row.id === 'string' && typeof row.result_url === 'string' && row.result_url.trim() && !isDataUrl(row.result_url)) {
|
||||
workUrls.set(row.result_url.trim(), row.id);
|
||||
if (typeof row.id === 'string' && typeof row.result_url === 'string' && row.result_url.trim()) {
|
||||
if (!isDataUrl(row.result_url)) {
|
||||
workUrls.set(row.result_url.trim(), row.id);
|
||||
}
|
||||
const mediaSha = getImportMediaSha256(row.result_url, media);
|
||||
if (mediaSha) {
|
||||
workMediaShas.set(mediaSha, row.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (workUrls.size > 0) {
|
||||
if (workUrls.size > 0 || workMediaShas.size > 0) {
|
||||
const existing = await client.query(
|
||||
'SELECT id, result_url FROM works WHERE result_url = ANY($1)',
|
||||
[[...workUrls.keys()]],
|
||||
`SELECT id, result_url, params
|
||||
FROM works
|
||||
WHERE result_url = ANY($1)
|
||||
OR params->>'importSourceUrl' = ANY($1)
|
||||
OR params->>'resultMediaSha256' = ANY($2)`,
|
||||
[[...workUrls.keys()], [...workMediaShas.keys()]],
|
||||
);
|
||||
for (const row of existing.rows) {
|
||||
const importedId = workUrls.get(row.result_url);
|
||||
if (importedId && importedId !== row.id) {
|
||||
workIdMap.set(importedId, row.id);
|
||||
}
|
||||
const params = (row.params || {}) as Record<string, unknown>;
|
||||
const sourceUrl = typeof params.importSourceUrl === 'string' ? params.importSourceUrl : '';
|
||||
const sourceMatchId = sourceUrl ? workUrls.get(sourceUrl) : undefined;
|
||||
if (sourceMatchId && sourceMatchId !== row.id) {
|
||||
workIdMap.set(sourceMatchId, row.id);
|
||||
}
|
||||
const sha = typeof params.resultMediaSha256 === 'string' ? params.resultMediaSha256 : '';
|
||||
const shaMatchId = sha ? workMediaShas.get(sha) : undefined;
|
||||
if (shaMatchId && shaMatchId !== row.id) {
|
||||
workIdMap.set(shaMatchId, row.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -288,6 +338,7 @@ async function buildImportContext(
|
||||
emailUserIdMap,
|
||||
apiKeyIdMap,
|
||||
apiKeyOwnerIdMap,
|
||||
media,
|
||||
columnCache: new Map(),
|
||||
defaultableColumnCache: new Map(),
|
||||
};
|
||||
@@ -339,18 +390,29 @@ async function normalizeImportRow(table: string, row: Record<string, unknown>, c
|
||||
if (importedUserId) {
|
||||
next.user_id = context.userIdMap.get(importedUserId) || importedUserId;
|
||||
}
|
||||
if (typeof next.result_url === 'string') {
|
||||
next.result_url = await persistImportMedia(next.result_url, getWorkMediaFolder(next.type, 'results'));
|
||||
}
|
||||
if (typeof next.thumbnail_url === 'string') {
|
||||
next.thumbnail_url = await persistImportMedia(next.thumbnail_url, 'imported/works/thumbnails');
|
||||
}
|
||||
if (next.params && typeof next.params === 'object') {
|
||||
next.params = await sanitizeImportMedia(next.params, 'imported/works/references');
|
||||
next.params = { ...(next.params as Record<string, unknown>) };
|
||||
remapCustomModelId(next.params as Record<string, unknown>, context);
|
||||
if ((!next.user_id || next.user_id === SYSTEM_USER_ID) && findUserIdByCustomModel(next, context)) {
|
||||
next.user_id = findUserIdByCustomModel(next, context);
|
||||
}
|
||||
} else {
|
||||
next.params = {};
|
||||
}
|
||||
if (typeof next.result_url === 'string') {
|
||||
const originalResultUrl = next.result_url;
|
||||
const mediaSha = getImportMediaSha256(originalResultUrl, context.media) || getDataUrlSha256(originalResultUrl);
|
||||
next.result_url = await persistImportMedia(originalResultUrl, getWorkMediaFolder(next.type, 'results'), context);
|
||||
if (mediaSha && next.params && typeof next.params === 'object') {
|
||||
(next.params as Record<string, unknown>).importSourceUrl = originalResultUrl;
|
||||
(next.params as Record<string, unknown>).resultMediaSha256 = mediaSha;
|
||||
}
|
||||
}
|
||||
if (typeof next.thumbnail_url === 'string') {
|
||||
next.thumbnail_url = await persistImportMedia(next.thumbnail_url, 'imported/works/thumbnails', context);
|
||||
}
|
||||
if (next.params && typeof next.params === 'object') {
|
||||
next.params = await sanitizeImportMedia(next.params, 'imported/works/references', context);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -482,6 +544,7 @@ function getMergeAssignments(table: string, cols: string[]): string[] {
|
||||
if (has('width')) assignments.push(`width = COALESCE(target.width, EXCLUDED.width)`);
|
||||
if (has('height')) assignments.push(`height = COALESCE(target.height, EXCLUDED.height)`);
|
||||
if (has('duration')) assignments.push(`duration = COALESCE(target.duration, EXCLUDED.duration)`);
|
||||
if (has('credits_cost')) assignments.push(`credits_cost = COALESCE(target.credits_cost, EXCLUDED.credits_cost)`);
|
||||
if (has('updated_at')) assignments.push(`updated_at = GREATEST(COALESCE(target.updated_at, EXCLUDED.updated_at), COALESCE(EXCLUDED.updated_at, target.updated_at))`);
|
||||
return assignments;
|
||||
}
|
||||
@@ -603,35 +666,176 @@ function extensionFromMime(mime: string): string {
|
||||
return 'bin';
|
||||
}
|
||||
|
||||
async function persistImportMedia(value: string, folder: string): Promise<string> {
|
||||
if (!isDataUrl(value)) return value;
|
||||
function getImportMediaSha256(value: string, media: Record<string, ImportMediaEntry>): string | null {
|
||||
const entry = media[value];
|
||||
if (!entry) return null;
|
||||
if (typeof entry.sha256 === 'string' && /^[a-f0-9]{64}$/i.test(entry.sha256)) {
|
||||
return entry.sha256.toLowerCase();
|
||||
}
|
||||
const decoded = decodeImportMediaEntry(entry);
|
||||
return decoded ? decoded.sha256 : null;
|
||||
}
|
||||
|
||||
function getDataUrlSha256(value: string): string | null {
|
||||
if (!isDataUrl(value)) return null;
|
||||
const decoded = decodeDataUrl(value);
|
||||
return decoded ? decoded.sha256 : null;
|
||||
}
|
||||
|
||||
function decodeDataUrl(value: string): { buffer: Buffer; mime: string; sha256: string } | null {
|
||||
const match = value.match(/^data:([^;,]+)?(;base64)?,([\s\S]*)$/i);
|
||||
if (!match) return value;
|
||||
if (!match) return null;
|
||||
|
||||
const mime = match[1] || 'application/octet-stream';
|
||||
const isBase64 = Boolean(match[2]);
|
||||
const payload = match[3] || '';
|
||||
const buffer = isBase64 ? Buffer.from(payload, 'base64') : Buffer.from(decodeURIComponent(payload));
|
||||
return {
|
||||
buffer,
|
||||
mime,
|
||||
sha256: crypto.createHash('sha256').update(buffer).digest('hex'),
|
||||
};
|
||||
}
|
||||
|
||||
function decodeImportMediaEntry(entry: ImportMediaEntry): { buffer: Buffer; mime: string; sha256: string } | null {
|
||||
if (typeof entry.dataUrl === 'string' && entry.dataUrl.trim()) {
|
||||
return decodeDataUrl(entry.dataUrl.trim());
|
||||
}
|
||||
if (entry.encoding === 'base64' && typeof entry.data === 'string') {
|
||||
const buffer = Buffer.from(entry.data, 'base64');
|
||||
return {
|
||||
buffer,
|
||||
mime: entry.contentType || 'application/octet-stream',
|
||||
sha256: crypto.createHash('sha256').update(buffer).digest('hex'),
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function persistImportMedia(value: string, folder: string, context?: ImportContext): Promise<string> {
|
||||
const entry = context?.media[value];
|
||||
const decoded = entry ? decodeImportMediaEntry(entry) : decodeDataUrl(value);
|
||||
if (!decoded) return value;
|
||||
|
||||
const { buffer, mime, sha256 } = decoded;
|
||||
const ext = extensionFromMime(mime);
|
||||
const key = `${folder}/${Date.now()}-${crypto.randomUUID()}.${ext}`;
|
||||
const key = `${folder}/${sha256}.${ext}`;
|
||||
const savedKey = await localStorage.uploadFile({ fileContent: buffer, fileName: key, contentType: mime });
|
||||
return localStorage.generatePresignedUrl({ key: savedKey, expireTime: 2592000 });
|
||||
}
|
||||
|
||||
async function sanitizeImportMedia(value: unknown, folder: string): Promise<unknown> {
|
||||
async function sanitizeImportMedia(value: unknown, folder: string, context: ImportContext): Promise<unknown> {
|
||||
if (typeof value === 'string') {
|
||||
return persistImportMedia(value, folder);
|
||||
return persistImportMedia(value, folder, context);
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
return Promise.all(value.map(item => sanitizeImportMedia(item, folder)));
|
||||
return Promise.all(value.map(item => sanitizeImportMedia(item, folder, context)));
|
||||
}
|
||||
if (value && typeof value === 'object') {
|
||||
const output: Record<string, unknown> = {};
|
||||
for (const [key, nested] of Object.entries(value as Record<string, unknown>)) {
|
||||
output[key] = await sanitizeImportMedia(nested, folder);
|
||||
output[key] = await sanitizeImportMedia(nested, folder, context);
|
||||
}
|
||||
return output;
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
async function dedupeWorks(client: Awaited<ReturnType<typeof getDbClient>>): Promise<ImportResult> {
|
||||
const errors: string[] = [];
|
||||
let removed = 0;
|
||||
|
||||
for (const expression of [
|
||||
"NULLIF(result_url, '')",
|
||||
"NULLIF(params->>'importSourceUrl', '')",
|
||||
"NULLIF(params->>'resultMediaSha256', '')",
|
||||
]) {
|
||||
try {
|
||||
await client.query(`
|
||||
WITH ranked AS (
|
||||
SELECT id,
|
||||
FIRST_VALUE(id) OVER (
|
||||
PARTITION BY ${expression}
|
||||
ORDER BY is_public DESC, updated_at DESC NULLS LAST, created_at DESC NULLS LAST, id DESC
|
||||
) AS keep_id,
|
||||
ROW_NUMBER() OVER (
|
||||
PARTITION BY ${expression}
|
||||
ORDER BY is_public DESC, updated_at DESC NULLS LAST, created_at DESC NULLS LAST, id DESC
|
||||
) AS rn
|
||||
FROM works
|
||||
WHERE ${expression} IS NOT NULL
|
||||
),
|
||||
duplicates AS (
|
||||
SELECT id, keep_id FROM ranked WHERE rn > 1
|
||||
)
|
||||
DELETE FROM work_likes wl
|
||||
USING duplicates d, work_likes kept
|
||||
WHERE wl.work_id = d.id
|
||||
AND kept.work_id = d.keep_id
|
||||
AND kept.user_id = wl.user_id
|
||||
`);
|
||||
await client.query(`
|
||||
WITH ranked AS (
|
||||
SELECT id,
|
||||
FIRST_VALUE(id) OVER (
|
||||
PARTITION BY ${expression}
|
||||
ORDER BY is_public DESC, updated_at DESC NULLS LAST, created_at DESC NULLS LAST, id DESC
|
||||
) AS keep_id,
|
||||
ROW_NUMBER() OVER (
|
||||
PARTITION BY ${expression}
|
||||
ORDER BY is_public DESC, updated_at DESC NULLS LAST, created_at DESC NULLS LAST, id DESC
|
||||
) AS rn
|
||||
FROM works
|
||||
WHERE ${expression} IS NOT NULL
|
||||
),
|
||||
duplicates AS (
|
||||
SELECT id, keep_id FROM ranked WHERE rn > 1
|
||||
)
|
||||
UPDATE work_likes wl
|
||||
SET work_id = d.keep_id
|
||||
FROM duplicates d
|
||||
WHERE wl.work_id = d.id
|
||||
`);
|
||||
await client.query(`
|
||||
WITH ranked AS (
|
||||
SELECT id,
|
||||
FIRST_VALUE(id) OVER (
|
||||
PARTITION BY ${expression}
|
||||
ORDER BY is_public DESC, updated_at DESC NULLS LAST, created_at DESC NULLS LAST, id DESC
|
||||
) AS keep_id,
|
||||
ROW_NUMBER() OVER (
|
||||
PARTITION BY ${expression}
|
||||
ORDER BY is_public DESC, updated_at DESC NULLS LAST, created_at DESC NULLS LAST, id DESC
|
||||
) AS rn
|
||||
FROM works
|
||||
WHERE ${expression} IS NOT NULL
|
||||
),
|
||||
duplicates AS (
|
||||
SELECT id, keep_id FROM ranked WHERE rn > 1
|
||||
)
|
||||
UPDATE credit_transactions ct
|
||||
SET related_work_id = d.keep_id
|
||||
FROM duplicates d
|
||||
WHERE ct.related_work_id = d.id
|
||||
`);
|
||||
const result = await client.query(`
|
||||
WITH ranked AS (
|
||||
SELECT id,
|
||||
ROW_NUMBER() OVER (
|
||||
PARTITION BY ${expression}
|
||||
ORDER BY is_public DESC, updated_at DESC NULLS LAST, created_at DESC NULLS LAST, id DESC
|
||||
) AS rn
|
||||
FROM works
|
||||
WHERE ${expression} IS NOT NULL
|
||||
)
|
||||
DELETE FROM works
|
||||
WHERE id IN (SELECT id FROM ranked WHERE rn > 1)
|
||||
`);
|
||||
removed += result.rowCount || 0;
|
||||
} catch (error) {
|
||||
errors.push(error instanceof Error ? error.message : '作品去重失败');
|
||||
}
|
||||
}
|
||||
|
||||
return { imported: 0, skipped: removed, errors };
|
||||
}
|
||||
|
||||
@@ -26,6 +26,9 @@ interface ImportTableResult {
|
||||
errors: string[];
|
||||
}
|
||||
|
||||
const MAX_IMPORT_FILE_BYTES = 1024 * 1024 * 1024;
|
||||
const MAX_IMPORT_FILE_LABEL = '1GB';
|
||||
|
||||
function getAdminAuthHeaders(): HeadersInit {
|
||||
try {
|
||||
const raw = window.localStorage.getItem('miaojing_auth');
|
||||
@@ -85,8 +88,8 @@ export default function DataManagementTab() {
|
||||
}
|
||||
|
||||
const file = fileInput.files[0];
|
||||
if (file.size > 50 * 1024 * 1024) {
|
||||
toast.error('文件大小不能超过 50MB');
|
||||
if (file.size > MAX_IMPORT_FILE_BYTES) {
|
||||
toast.error(`文件大小不能超过 ${MAX_IMPORT_FILE_LABEL}`);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -117,7 +120,8 @@ export default function DataManagementTab() {
|
||||
const totalImported = Object.values(result.details || {}).reduce(
|
||||
(sum: number, r: unknown) => sum + ((r as ImportTableResult).imported || 0), 0
|
||||
);
|
||||
toast.success(`数据导入完成，共导入 ${totalImported} 条记录`);
|
||||
const deduped = (result.details?.dedupe_works as ImportTableResult | undefined)?.skipped || 0;
|
||||
toast.success(`数据导入完成，共导入 ${totalImported} 条记录${deduped > 0 ? `，合并重复作品 ${deduped} 条` : ''}`);
|
||||
} catch (err) {
|
||||
toast.error(err instanceof Error ? err.message : '导入失败');
|
||||
} finally {
|
||||
@@ -137,7 +141,7 @@ export default function DataManagementTab() {
|
||||
数据导出
|
||||
</CardTitle>
|
||||
<CardDescription>
|
||||
一键导出平台所有数据（用户、作品、订单、配置等），生成 JSON 备份文件
|
||||
一键导出平台业务数据和本地媒体文件，生成可迁移恢复的 JSON 备份文件
|
||||
</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent className="space-y-4">
|
||||
@@ -146,6 +150,7 @@ export default function DataManagementTab() {
|
||||
<ul className="text-sm text-muted-foreground space-y-1 ml-4">
|
||||
<li>用户资料 (profiles) + 认证账号 (auth_users)</li>
|
||||
<li>创作作品 (works) + 点赞记录 (work_likes)</li>
|
||||
<li>作品、参考图、Logo、站点图标等本地媒体文件 (_media)</li>
|
||||
<li>积分记录 (credit_transactions) + 订单 (orders)</li>
|
||||
<li>用户 API 密钥 (user_api_keys)</li>
|
||||
<li>公告 (announcements) + 网站配置 (site_config) + 访问统计 (site_stats)</li>
|
||||
@@ -166,7 +171,7 @@ export default function DataManagementTab() {
|
||||
数据导入
|
||||
</CardTitle>
|
||||
<CardDescription>
|
||||
从备份文件恢复数据到当前平台，或迁移到新平台。支持 upsert 模式（已存在的记录更新，不存在则创建）
|
||||
从备份文件恢复数据到当前平台，或迁移到新平台。导入会在事务内合并记录、还原媒体，并按来源 URL/媒体哈希合并重复作品
|
||||
</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent className="space-y-4">
|
||||
@@ -177,6 +182,8 @@ export default function DataManagementTab() {
|
||||
</div>
|
||||
<ul className="text-sm text-muted-foreground space-y-1 ml-6">
|
||||
<li>导入将合并数据，不会删除已有记录（upsert 模式）</li>
|
||||
<li>新版导出文件会包含 _media，可完整恢复画廊图片、创作历史图片和站点图片</li>
|
||||
<li>旧版导出文件不含 _media 时，只能恢复数据库记录；如媒体文件未同步，画廊图片仍可能无法显示</li>
|
||||
<li>新平台的数据库需先运行初始化脚本 (init-database.sql)</li>
|
||||
<li>认证账号会自动创建，密码为随机临时密码，用户需通过管理员重置</li>
|
||||
<li>如新平台已有相同邮箱的用户，将更新其信息而非重复创建</li>
|
||||
@@ -199,7 +206,7 @@ export default function DataManagementTab() {
|
||||
hover:file:bg-primary/20
|
||||
file:cursor-pointer"
|
||||
/>
|
||||
<p className="text-xs text-muted-foreground">仅支持 .json 格式，文件大小不超过 50MB</p>
|
||||
<p className="text-xs text-muted-foreground">仅支持 .json 格式，文件大小不超过 {MAX_IMPORT_FILE_LABEL}</p>
|
||||
</div>
|
||||
|
||||
{/* Options */}
|
||||
@@ -233,7 +240,7 @@ export default function DataManagementTab() {
|
||||
<span className="text-muted-foreground">{table}</span>
|
||||
<span className="font-medium">
|
||||
{r.imported > 0 && <span className="text-emerald-600">{r.imported} 导入</span>}
|
||||
{r.skipped > 0 && <span className="text-amber-600 ml-2">{r.skipped} 跳过</span>}
|
||||
{r.skipped > 0 && <span className="text-amber-600 ml-2">{r.skipped} {table === 'dedupe_works' ? '合并' : '跳过'}</span>}
|
||||
{r.imported === 0 && r.skipped === 0 && <span className="text-muted-foreground">无数据</span>}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
@@ -254,10 +254,12 @@ export function useCreationHistory() {
|
||||
}).catch(() => setRecords(loadRecords()));
|
||||
};
|
||||
window.addEventListener('creation-history-updated', handler);
|
||||
window.addEventListener('miaojing_auth_updated', handler);
|
||||
window.addEventListener('storage', handler);
|
||||
|
||||
return () => {
|
||||
window.removeEventListener('creation-history-updated', handler);
|
||||
window.removeEventListener('miaojing_auth_updated', handler);
|
||||
window.removeEventListener('storage', handler);
|
||||
};
|
||||
}, []);
|
||||
|
||||
Reference in New Issue
Block a user