334 lines
9.6 KiB
TypeScript
334 lines
9.6 KiB
TypeScript
/**
|
|
* Async admin user export jobs (PostgreSQL persistence + local filesystem).
|
|
*/
|
|
|
|
import fs from 'fs';
|
|
import os from 'os';
|
|
import path from 'path';
|
|
import { v4 as uuidv4 } from 'uuid';
|
|
import * as XLSX from 'xlsx';
|
|
import * as adminService from './adminService';
|
|
import * as adminExportJobRepository from '../repositories/adminExportJobRepository';
|
|
import type { AdminExportJobRow } from '../repositories/adminExportJobRepository';
|
|
import * as countryRepository from '../repositories/countryRepository';
|
|
import {
|
|
sanitizeExportColumns,
|
|
type AdminUserExportColumn,
|
|
} from '../utils/adminUserListQuery';
|
|
import { buildExportRows } from './adminUserExportFormat';
|
|
import type { AdminSortBy } from '../repositories/userRepository';
|
|
|
|
const EXPORT_DIR = path.join(process.cwd(), 'data', 'exports');
|
|
const JOB_TTL_MS = 60 * 60 * 1000;
|
|
const PAGE_SIZE = 500;
|
|
const DEFAULT_MAX_ROWS = 100_000;
|
|
const ABSOLUTE_MAX_ROWS = 500_000;
|
|
const POLL_CLEANUP_MS = 15 * 60 * 1000;
|
|
const WORKER_POLL_MS = 1500;
|
|
const STALE_JOB_MINUTES = 30;
|
|
|
|
export type ExportJobStatus = 'queued' | 'running' | 'completed' | 'failed';
|
|
|
|
export type AdminUserExportRequest = {
|
|
search?: string;
|
|
roleName?: 'user' | 'moderator' | 'admin' | 'super_admin';
|
|
emailVerified?: boolean;
|
|
country?: string;
|
|
inactiveSinceDays?: number;
|
|
minMainBalance?: string;
|
|
maxMainBalance?: string;
|
|
minBonusReceived?: string;
|
|
maxBonusReceived?: string;
|
|
accountStatus?: 'active' | 'disabled' | 'suspended';
|
|
hasReceivedBonus?: boolean;
|
|
createdFrom?: string;
|
|
createdTo?: string;
|
|
minAge?: number;
|
|
maxAge?: number;
|
|
sortBy?: AdminSortBy;
|
|
sortDir?: 'asc' | 'desc';
|
|
columns?: string[];
|
|
maxRows?: number;
|
|
userIds?: string[];
|
|
format?: 'xlsx' | 'csv';
|
|
};
|
|
|
|
export type ExportJobView = {
|
|
id: string;
|
|
status: ExportJobStatus;
|
|
progress: number;
|
|
rowCount: number;
|
|
totalEstimate: number | null;
|
|
error: string | null;
|
|
fileName: string | null;
|
|
createdAt: string;
|
|
completedAt: string | null;
|
|
jobKind?: 'users' | 'reports';
|
|
reportKey?: string | null;
|
|
};
|
|
|
|
const WORKER_ID = `${os.hostname()}-${process.pid}-${uuidv4().slice(0, 8)}`;
|
|
let workerBusy = false;
|
|
let workerStarted = false;
|
|
|
|
function ensureExportDir(): void {
|
|
fs.mkdirSync(EXPORT_DIR, { recursive: true });
|
|
}
|
|
|
|
function clampMaxRows(maxRows?: number): number {
|
|
if (maxRows === undefined || maxRows === null || !Number.isFinite(maxRows) || maxRows <= 0) {
|
|
return DEFAULT_MAX_ROWS;
|
|
}
|
|
return Math.min(Math.floor(maxRows), ABSOLUTE_MAX_ROWS);
|
|
}
|
|
|
|
function rowToView(row: AdminExportJobRow): ExportJobView {
|
|
return {
|
|
id: row.id,
|
|
status: row.status,
|
|
progress: row.progress,
|
|
rowCount: row.row_count,
|
|
totalEstimate: row.total_estimate,
|
|
error: row.error,
|
|
fileName: row.file_name,
|
|
createdAt: row.created_at.toISOString(),
|
|
completedAt: row.completed_at?.toISOString() ?? null,
|
|
jobKind: row.job_kind,
|
|
reportKey: row.report_key,
|
|
};
|
|
}
|
|
|
|
export async function startAdminUserExport(
|
|
requestedBy: string,
|
|
request: AdminUserExportRequest
|
|
): Promise<ExportJobView> {
|
|
ensureExportDir();
|
|
await cleanupExpiredJobs();
|
|
|
|
const id = uuidv4();
|
|
const columns = sanitizeExportColumns(request.columns);
|
|
const maxRows = clampMaxRows(request.maxRows);
|
|
const format = request.format === 'csv' ? 'csv' : 'xlsx';
|
|
const stamp = new Date().toISOString().slice(0, 10);
|
|
const suffix = request.userIds?.length ? '_selected' : '';
|
|
const fileName = `users_export${suffix}_${stamp}.${format}`;
|
|
|
|
const { columns: _c, maxRows: _m, format: _f, ...requestPayload } = request;
|
|
|
|
const row = await adminExportJobRepository.createExportJob({
|
|
id,
|
|
requestedBy,
|
|
fileName,
|
|
requestPayload,
|
|
format,
|
|
maxRows,
|
|
columns,
|
|
expiresAt: new Date(Date.now() + JOB_TTL_MS),
|
|
});
|
|
|
|
kickExportWorker();
|
|
|
|
return rowToView(row);
|
|
}
|
|
|
|
export async function getAdminUserExportJob(
|
|
jobId: string,
|
|
requestedBy: string
|
|
): Promise<ExportJobView | null> {
|
|
const row = await adminExportJobRepository.getExportJobForUser(jobId, requestedBy);
|
|
return row ? rowToView(row) : null;
|
|
}
|
|
|
|
export async function getAdminUserExportFilePath(
|
|
jobId: string,
|
|
requestedBy: string
|
|
): Promise<{ filePath: string; fileName: string } | null> {
|
|
const row = await adminExportJobRepository.getExportJobForUser(jobId, requestedBy);
|
|
if (!row || row.status !== 'completed' || !row.file_path || !row.file_name) return null;
|
|
if (!fs.existsSync(row.file_path)) return null;
|
|
return { filePath: row.file_path, fileName: row.file_name };
|
|
}
|
|
|
|
/**
|
|
* Starts background worker: recovers stale jobs and polls the queue.
|
|
*/
|
|
export function startAdminExportWorker(): void {
|
|
if (workerStarted) return;
|
|
workerStarted = true;
|
|
ensureExportDir();
|
|
|
|
setImmediate(async () => {
|
|
try {
|
|
const n = await adminExportJobRepository.requeueAllRunningJobs();
|
|
if (n > 0) {
|
|
console.log(`[admin-export] Re-queued ${n} running job(s) after restart`);
|
|
}
|
|
await adminExportJobRepository.requeueStaleRunningJobs(STALE_JOB_MINUTES);
|
|
} catch (e) {
|
|
console.error('[admin-export] Worker startup recovery error:', e);
|
|
}
|
|
kickExportWorker();
|
|
});
|
|
|
|
setInterval(() => {
|
|
void cleanupExpiredJobs();
|
|
}, POLL_CLEANUP_MS).unref();
|
|
|
|
setInterval(() => {
|
|
kickExportWorker();
|
|
}, WORKER_POLL_MS).unref();
|
|
}
|
|
|
|
function kickExportWorker(): void {
|
|
if (workerBusy) return;
|
|
workerBusy = true;
|
|
void processNextExportJob().finally(() => {
|
|
workerBusy = false;
|
|
});
|
|
}
|
|
|
|
export { kickExportWorker };
|
|
|
|
async function processNextExportJob(): Promise<void> {
|
|
const job = await adminExportJobRepository.claimNextQueuedJob(WORKER_ID);
|
|
if (!job) return;
|
|
|
|
if (job.job_kind === 'reports') {
|
|
const { runReportExportJob } = await import('./reportExportJobService');
|
|
try {
|
|
await runReportExportJob(job);
|
|
} catch (e) {
|
|
const msg = e instanceof Error ? e.message : 'Export failed';
|
|
await adminExportJobRepository.failExportJob(job.id, msg);
|
|
}
|
|
return;
|
|
}
|
|
|
|
const columns = sanitizeExportColumns(
|
|
Array.isArray(job.columns) ? job.columns : (job.columns as unknown as string[])
|
|
);
|
|
|
|
try {
|
|
await runExportJob(job, job.request_payload as AdminUserExportRequest, columns, job.max_rows, job.format as 'xlsx' | 'csv');
|
|
} catch (e) {
|
|
const msg = e instanceof Error ? e.message : 'Export failed';
|
|
await adminExportJobRepository.failExportJob(job.id, msg);
|
|
}
|
|
}
|
|
|
|
async function runExportJob(
|
|
job: AdminExportJobRow,
|
|
request: AdminUserExportRequest,
|
|
columns: AdminUserExportColumn[],
|
|
maxRows: number,
|
|
format: 'xlsx' | 'csv'
|
|
): Promise<void> {
|
|
const countries = await countryRepository.listAll();
|
|
const countryMap = new Map(countries.map((c) => [c.code, c.name]));
|
|
const countryName = (code: string | null | undefined) => {
|
|
if (!code) return '';
|
|
return countryMap.get(code) ?? code;
|
|
};
|
|
|
|
const allRows: unknown[][] = [];
|
|
let cursor: string | undefined;
|
|
let exported = 0;
|
|
|
|
const reportProgress = async (totalEstimate?: number | null) => {
|
|
await adminExportJobRepository.updateExportJobProgress(
|
|
job.id,
|
|
exported,
|
|
exported,
|
|
totalEstimate
|
|
);
|
|
};
|
|
|
|
if (request.userIds?.length) {
|
|
const ids = request.userIds;
|
|
for (let i = 0; i < ids.length; i += PAGE_SIZE) {
|
|
const batchIds = ids.slice(i, i + PAGE_SIZE);
|
|
const result = await adminService.listUsers({
|
|
...request,
|
|
userIds: batchIds,
|
|
limit: batchIds.length,
|
|
includeTotal: false,
|
|
});
|
|
const chunk = buildExportRows(result.users, columns, countryName);
|
|
if (allRows.length === 0) allRows.push(...chunk);
|
|
else allRows.push(...chunk.slice(1));
|
|
exported += result.users.length;
|
|
await reportProgress();
|
|
}
|
|
} else {
|
|
let includeTotal = true;
|
|
while (exported < maxRows) {
|
|
const remaining = maxRows - exported;
|
|
const limit = Math.min(PAGE_SIZE, remaining);
|
|
const result = await adminService.listUsers({
|
|
...request,
|
|
limit,
|
|
cursor,
|
|
includeTotal,
|
|
offset: cursor ? undefined : exported > 0 ? exported : undefined,
|
|
});
|
|
|
|
if (includeTotal && result.total != null) {
|
|
await reportProgress(Math.min(result.total, maxRows));
|
|
}
|
|
includeTotal = false;
|
|
|
|
if (result.users.length === 0) break;
|
|
|
|
const chunk = buildExportRows(result.users, columns, countryName);
|
|
if (allRows.length === 0) allRows.push(...chunk);
|
|
else allRows.push(...chunk.slice(1));
|
|
|
|
exported += result.users.length;
|
|
await reportProgress();
|
|
|
|
if (!result.hasMore || !result.nextCursor) break;
|
|
cursor = result.nextCursor;
|
|
}
|
|
}
|
|
|
|
const outPath = path.join(EXPORT_DIR, `${job.id}.${format}`);
|
|
if (format === 'csv') {
|
|
const csv = allRows
|
|
.map((row) =>
|
|
row
|
|
.map((cell) => {
|
|
const s = cell === null || cell === undefined ? '' : String(cell);
|
|
if (/[",\n\r]/.test(s)) return `"${s.replace(/"/g, '""')}"`;
|
|
return s;
|
|
})
|
|
.join(',')
|
|
)
|
|
.join('\n');
|
|
fs.writeFileSync(outPath, `\uFEFF${csv}`, 'utf8');
|
|
} else {
|
|
const ws = XLSX.utils.aoa_to_sheet(allRows);
|
|
const wb = XLSX.utils.book_new();
|
|
XLSX.utils.book_append_sheet(wb, ws, 'Users');
|
|
XLSX.writeFile(wb, outPath);
|
|
}
|
|
|
|
await adminExportJobRepository.completeExportJob(job.id, outPath, exported);
|
|
}
|
|
|
|
async function cleanupExpiredJobs(): Promise<void> {
|
|
try {
|
|
const removed = await adminExportJobRepository.deleteExpiredExportJobs();
|
|
for (const row of removed) {
|
|
if (row.file_path && fs.existsSync(row.file_path)) {
|
|
try {
|
|
fs.unlinkSync(row.file_path);
|
|
} catch {
|
|
// ignore
|
|
}
|
|
}
|
|
}
|
|
} catch (e) {
|
|
console.warn('[admin-export] Cleanup error:', e);
|
|
}
|
|
}
|