/** * Async admin user export jobs (PostgreSQL persistence + local filesystem). */ import fs from 'fs'; import os from 'os'; import path from 'path'; import { v4 as uuidv4 } from 'uuid'; import * as XLSX from 'xlsx'; import * as adminService from './adminService'; import * as adminExportJobRepository from '../repositories/adminExportJobRepository'; import type { AdminExportJobRow } from '../repositories/adminExportJobRepository'; import * as countryRepository from '../repositories/countryRepository'; import { sanitizeExportColumns, type AdminUserExportColumn, } from '../utils/adminUserListQuery'; import { buildExportRows } from './adminUserExportFormat'; import type { AdminSortBy } from '../repositories/userRepository'; const EXPORT_DIR = path.join(process.cwd(), 'data', 'exports'); const JOB_TTL_MS = 60 * 60 * 1000; const PAGE_SIZE = 500; const DEFAULT_MAX_ROWS = 100_000; const ABSOLUTE_MAX_ROWS = 500_000; const POLL_CLEANUP_MS = 15 * 60 * 1000; const WORKER_POLL_MS = 1500; const STALE_JOB_MINUTES = 30; export type ExportJobStatus = 'queued' | 'running' | 'completed' | 'failed'; export type AdminUserExportRequest = { search?: string; roleName?: 'user' | 'moderator' | 'admin' | 'super_admin'; emailVerified?: boolean; country?: string; inactiveSinceDays?: number; minMainBalance?: string; maxMainBalance?: string; minBonusReceived?: string; maxBonusReceived?: string; accountStatus?: 'active' | 'disabled' | 'suspended'; hasReceivedBonus?: boolean; createdFrom?: string; createdTo?: string; minAge?: number; maxAge?: number; sortBy?: AdminSortBy; sortDir?: 'asc' | 'desc'; columns?: string[]; maxRows?: number; userIds?: string[]; format?: 'xlsx' | 'csv'; }; export type ExportJobView = { id: string; status: ExportJobStatus; progress: number; rowCount: number; totalEstimate: number | null; error: string | null; fileName: string | null; createdAt: string; completedAt: string | null; jobKind?: 'users' | 'reports'; reportKey?: string | null; }; const WORKER_ID = `${os.hostname()}-${process.pid}-${uuidv4().slice(0, 8)}`; let workerBusy = false; let workerStarted = false; function ensureExportDir(): void { fs.mkdirSync(EXPORT_DIR, { recursive: true }); } function clampMaxRows(maxRows?: number): number { if (maxRows === undefined || maxRows === null || !Number.isFinite(maxRows) || maxRows <= 0) { return DEFAULT_MAX_ROWS; } return Math.min(Math.floor(maxRows), ABSOLUTE_MAX_ROWS); } function rowToView(row: AdminExportJobRow): ExportJobView { return { id: row.id, status: row.status, progress: row.progress, rowCount: row.row_count, totalEstimate: row.total_estimate, error: row.error, fileName: row.file_name, createdAt: row.created_at.toISOString(), completedAt: row.completed_at?.toISOString() ?? null, jobKind: row.job_kind, reportKey: row.report_key, }; } export async function startAdminUserExport( requestedBy: string, request: AdminUserExportRequest ): Promise { ensureExportDir(); await cleanupExpiredJobs(); const id = uuidv4(); const columns = sanitizeExportColumns(request.columns); const maxRows = clampMaxRows(request.maxRows); const format = request.format === 'csv' ? 'csv' : 'xlsx'; const stamp = new Date().toISOString().slice(0, 10); const suffix = request.userIds?.length ? '_selected' : ''; const fileName = `users_export${suffix}_${stamp}.${format}`; const { columns: _c, maxRows: _m, format: _f, ...requestPayload } = request; const row = await adminExportJobRepository.createExportJob({ id, requestedBy, fileName, requestPayload, format, maxRows, columns, expiresAt: new Date(Date.now() + JOB_TTL_MS), }); kickExportWorker(); return rowToView(row); } export async function getAdminUserExportJob( jobId: string, requestedBy: string ): Promise { const row = await adminExportJobRepository.getExportJobForUser(jobId, requestedBy); return row ? rowToView(row) : null; } export async function getAdminUserExportFilePath( jobId: string, requestedBy: string ): Promise<{ filePath: string; fileName: string } | null> { const row = await adminExportJobRepository.getExportJobForUser(jobId, requestedBy); if (!row || row.status !== 'completed' || !row.file_path || !row.file_name) return null; if (!fs.existsSync(row.file_path)) return null; return { filePath: row.file_path, fileName: row.file_name }; } /** * Starts background worker: recovers stale jobs and polls the queue. */ export function startAdminExportWorker(): void { if (workerStarted) return; workerStarted = true; ensureExportDir(); setImmediate(async () => { try { const n = await adminExportJobRepository.requeueAllRunningJobs(); if (n > 0) { console.log(`[admin-export] Re-queued ${n} running job(s) after restart`); } await adminExportJobRepository.requeueStaleRunningJobs(STALE_JOB_MINUTES); } catch (e) { console.error('[admin-export] Worker startup recovery error:', e); } kickExportWorker(); }); setInterval(() => { void cleanupExpiredJobs(); }, POLL_CLEANUP_MS).unref(); setInterval(() => { kickExportWorker(); }, WORKER_POLL_MS).unref(); } function kickExportWorker(): void { if (workerBusy) return; workerBusy = true; void processNextExportJob().finally(() => { workerBusy = false; }); } export { kickExportWorker }; async function processNextExportJob(): Promise { const job = await adminExportJobRepository.claimNextQueuedJob(WORKER_ID); if (!job) return; if (job.job_kind === 'reports') { const { runReportExportJob } = await import('./reportExportJobService'); try { await runReportExportJob(job); } catch (e) { const msg = e instanceof Error ? e.message : 'Export failed'; await adminExportJobRepository.failExportJob(job.id, msg); } return; } const columns = sanitizeExportColumns( Array.isArray(job.columns) ? job.columns : (job.columns as unknown as string[]) ); try { await runExportJob(job, job.request_payload as AdminUserExportRequest, columns, job.max_rows, job.format as 'xlsx' | 'csv'); } catch (e) { const msg = e instanceof Error ? e.message : 'Export failed'; await adminExportJobRepository.failExportJob(job.id, msg); } } async function runExportJob( job: AdminExportJobRow, request: AdminUserExportRequest, columns: AdminUserExportColumn[], maxRows: number, format: 'xlsx' | 'csv' ): Promise { const countries = await countryRepository.listAll(); const countryMap = new Map(countries.map((c) => [c.code, c.name])); const countryName = (code: string | null | undefined) => { if (!code) return ''; return countryMap.get(code) ?? code; }; const allRows: unknown[][] = []; let cursor: string | undefined; let exported = 0; const reportProgress = async (totalEstimate?: number | null) => { await adminExportJobRepository.updateExportJobProgress( job.id, exported, exported, totalEstimate ); }; if (request.userIds?.length) { const ids = request.userIds; for (let i = 0; i < ids.length; i += PAGE_SIZE) { const batchIds = ids.slice(i, i + PAGE_SIZE); const result = await adminService.listUsers({ ...request, userIds: batchIds, limit: batchIds.length, includeTotal: false, }); const chunk = buildExportRows(result.users, columns, countryName); if (allRows.length === 0) allRows.push(...chunk); else allRows.push(...chunk.slice(1)); exported += result.users.length; await reportProgress(); } } else { let includeTotal = true; while (exported < maxRows) { const remaining = maxRows - exported; const limit = Math.min(PAGE_SIZE, remaining); const result = await adminService.listUsers({ ...request, limit, cursor, includeTotal, offset: cursor ? undefined : exported > 0 ? exported : undefined, }); if (includeTotal && result.total != null) { await reportProgress(Math.min(result.total, maxRows)); } includeTotal = false; if (result.users.length === 0) break; const chunk = buildExportRows(result.users, columns, countryName); if (allRows.length === 0) allRows.push(...chunk); else allRows.push(...chunk.slice(1)); exported += result.users.length; await reportProgress(); if (!result.hasMore || !result.nextCursor) break; cursor = result.nextCursor; } } const outPath = path.join(EXPORT_DIR, `${job.id}.${format}`); if (format === 'csv') { const csv = allRows .map((row) => row .map((cell) => { const s = cell === null || cell === undefined ? '' : String(cell); if (/[",\n\r]/.test(s)) return `"${s.replace(/"/g, '""')}"`; return s; }) .join(',') ) .join('\n'); fs.writeFileSync(outPath, `\uFEFF${csv}`, 'utf8'); } else { const ws = XLSX.utils.aoa_to_sheet(allRows); const wb = XLSX.utils.book_new(); XLSX.utils.book_append_sheet(wb, ws, 'Users'); XLSX.writeFile(wb, outPath); } await adminExportJobRepository.completeExportJob(job.id, outPath, exported); } async function cleanupExpiredJobs(): Promise { try { const removed = await adminExportJobRepository.deleteExpiredExportJobs(); for (const row of removed) { if (row.file_path && fs.existsSync(row.file_path)) { try { fs.unlinkSync(row.file_path); } catch { // ignore } } } } catch (e) { console.warn('[admin-export] Cleanup error:', e); } }