games_app/backend/src/services/adminUserExportService.ts
2026-07-02 09:21:05 +02:00

334 lines
9.6 KiB
TypeScript

/**
* Async admin user export jobs (PostgreSQL persistence + local filesystem).
*/
import fs from 'fs';
import os from 'os';
import path from 'path';
import { v4 as uuidv4 } from 'uuid';
import * as XLSX from 'xlsx';
import * as adminService from './adminService';
import * as adminExportJobRepository from '../repositories/adminExportJobRepository';
import type { AdminExportJobRow } from '../repositories/adminExportJobRepository';
import * as countryRepository from '../repositories/countryRepository';
import {
sanitizeExportColumns,
type AdminUserExportColumn,
} from '../utils/adminUserListQuery';
import { buildExportRows } from './adminUserExportFormat';
import type { AdminSortBy } from '../repositories/userRepository';
const EXPORT_DIR = path.join(process.cwd(), 'data', 'exports');
const JOB_TTL_MS = 60 * 60 * 1000;
const PAGE_SIZE = 500;
const DEFAULT_MAX_ROWS = 100_000;
const ABSOLUTE_MAX_ROWS = 500_000;
const POLL_CLEANUP_MS = 15 * 60 * 1000;
const WORKER_POLL_MS = 1500;
const STALE_JOB_MINUTES = 30;
export type ExportJobStatus = 'queued' | 'running' | 'completed' | 'failed';
export type AdminUserExportRequest = {
search?: string;
roleName?: 'user' | 'moderator' | 'admin' | 'super_admin';
emailVerified?: boolean;
country?: string;
inactiveSinceDays?: number;
minMainBalance?: string;
maxMainBalance?: string;
minBonusReceived?: string;
maxBonusReceived?: string;
accountStatus?: 'active' | 'disabled' | 'suspended';
hasReceivedBonus?: boolean;
createdFrom?: string;
createdTo?: string;
minAge?: number;
maxAge?: number;
sortBy?: AdminSortBy;
sortDir?: 'asc' | 'desc';
columns?: string[];
maxRows?: number;
userIds?: string[];
format?: 'xlsx' | 'csv';
};
export type ExportJobView = {
id: string;
status: ExportJobStatus;
progress: number;
rowCount: number;
totalEstimate: number | null;
error: string | null;
fileName: string | null;
createdAt: string;
completedAt: string | null;
jobKind?: 'users' | 'reports';
reportKey?: string | null;
};
const WORKER_ID = `${os.hostname()}-${process.pid}-${uuidv4().slice(0, 8)}`;
let workerBusy = false;
let workerStarted = false;
function ensureExportDir(): void {
fs.mkdirSync(EXPORT_DIR, { recursive: true });
}
function clampMaxRows(maxRows?: number): number {
if (maxRows === undefined || maxRows === null || !Number.isFinite(maxRows) || maxRows <= 0) {
return DEFAULT_MAX_ROWS;
}
return Math.min(Math.floor(maxRows), ABSOLUTE_MAX_ROWS);
}
function rowToView(row: AdminExportJobRow): ExportJobView {
return {
id: row.id,
status: row.status,
progress: row.progress,
rowCount: row.row_count,
totalEstimate: row.total_estimate,
error: row.error,
fileName: row.file_name,
createdAt: row.created_at.toISOString(),
completedAt: row.completed_at?.toISOString() ?? null,
jobKind: row.job_kind,
reportKey: row.report_key,
};
}
export async function startAdminUserExport(
requestedBy: string,
request: AdminUserExportRequest
): Promise<ExportJobView> {
ensureExportDir();
await cleanupExpiredJobs();
const id = uuidv4();
const columns = sanitizeExportColumns(request.columns);
const maxRows = clampMaxRows(request.maxRows);
const format = request.format === 'csv' ? 'csv' : 'xlsx';
const stamp = new Date().toISOString().slice(0, 10);
const suffix = request.userIds?.length ? '_selected' : '';
const fileName = `users_export${suffix}_${stamp}.${format}`;
const { columns: _c, maxRows: _m, format: _f, ...requestPayload } = request;
const row = await adminExportJobRepository.createExportJob({
id,
requestedBy,
fileName,
requestPayload,
format,
maxRows,
columns,
expiresAt: new Date(Date.now() + JOB_TTL_MS),
});
kickExportWorker();
return rowToView(row);
}
export async function getAdminUserExportJob(
jobId: string,
requestedBy: string
): Promise<ExportJobView | null> {
const row = await adminExportJobRepository.getExportJobForUser(jobId, requestedBy);
return row ? rowToView(row) : null;
}
export async function getAdminUserExportFilePath(
jobId: string,
requestedBy: string
): Promise<{ filePath: string; fileName: string } | null> {
const row = await adminExportJobRepository.getExportJobForUser(jobId, requestedBy);
if (!row || row.status !== 'completed' || !row.file_path || !row.file_name) return null;
if (!fs.existsSync(row.file_path)) return null;
return { filePath: row.file_path, fileName: row.file_name };
}
/**
* Starts background worker: recovers stale jobs and polls the queue.
*/
export function startAdminExportWorker(): void {
if (workerStarted) return;
workerStarted = true;
ensureExportDir();
setImmediate(async () => {
try {
const n = await adminExportJobRepository.requeueAllRunningJobs();
if (n > 0) {
console.log(`[admin-export] Re-queued ${n} running job(s) after restart`);
}
await adminExportJobRepository.requeueStaleRunningJobs(STALE_JOB_MINUTES);
} catch (e) {
console.error('[admin-export] Worker startup recovery error:', e);
}
kickExportWorker();
});
setInterval(() => {
void cleanupExpiredJobs();
}, POLL_CLEANUP_MS).unref();
setInterval(() => {
kickExportWorker();
}, WORKER_POLL_MS).unref();
}
function kickExportWorker(): void {
if (workerBusy) return;
workerBusy = true;
void processNextExportJob().finally(() => {
workerBusy = false;
});
}
export { kickExportWorker };
async function processNextExportJob(): Promise<void> {
const job = await adminExportJobRepository.claimNextQueuedJob(WORKER_ID);
if (!job) return;
if (job.job_kind === 'reports') {
const { runReportExportJob } = await import('./reportExportJobService');
try {
await runReportExportJob(job);
} catch (e) {
const msg = e instanceof Error ? e.message : 'Export failed';
await adminExportJobRepository.failExportJob(job.id, msg);
}
return;
}
const columns = sanitizeExportColumns(
Array.isArray(job.columns) ? job.columns : (job.columns as unknown as string[])
);
try {
await runExportJob(job, job.request_payload as AdminUserExportRequest, columns, job.max_rows, job.format as 'xlsx' | 'csv');
} catch (e) {
const msg = e instanceof Error ? e.message : 'Export failed';
await adminExportJobRepository.failExportJob(job.id, msg);
}
}
async function runExportJob(
job: AdminExportJobRow,
request: AdminUserExportRequest,
columns: AdminUserExportColumn[],
maxRows: number,
format: 'xlsx' | 'csv'
): Promise<void> {
const countries = await countryRepository.listAll();
const countryMap = new Map(countries.map((c) => [c.code, c.name]));
const countryName = (code: string | null | undefined) => {
if (!code) return '';
return countryMap.get(code) ?? code;
};
const allRows: unknown[][] = [];
let cursor: string | undefined;
let exported = 0;
const reportProgress = async (totalEstimate?: number | null) => {
await adminExportJobRepository.updateExportJobProgress(
job.id,
exported,
exported,
totalEstimate
);
};
if (request.userIds?.length) {
const ids = request.userIds;
for (let i = 0; i < ids.length; i += PAGE_SIZE) {
const batchIds = ids.slice(i, i + PAGE_SIZE);
const result = await adminService.listUsers({
...request,
userIds: batchIds,
limit: batchIds.length,
includeTotal: false,
});
const chunk = buildExportRows(result.users, columns, countryName);
if (allRows.length === 0) allRows.push(...chunk);
else allRows.push(...chunk.slice(1));
exported += result.users.length;
await reportProgress();
}
} else {
let includeTotal = true;
while (exported < maxRows) {
const remaining = maxRows - exported;
const limit = Math.min(PAGE_SIZE, remaining);
const result = await adminService.listUsers({
...request,
limit,
cursor,
includeTotal,
offset: cursor ? undefined : exported > 0 ? exported : undefined,
});
if (includeTotal && result.total != null) {
await reportProgress(Math.min(result.total, maxRows));
}
includeTotal = false;
if (result.users.length === 0) break;
const chunk = buildExportRows(result.users, columns, countryName);
if (allRows.length === 0) allRows.push(...chunk);
else allRows.push(...chunk.slice(1));
exported += result.users.length;
await reportProgress();
if (!result.hasMore || !result.nextCursor) break;
cursor = result.nextCursor;
}
}
const outPath = path.join(EXPORT_DIR, `${job.id}.${format}`);
if (format === 'csv') {
const csv = allRows
.map((row) =>
row
.map((cell) => {
const s = cell === null || cell === undefined ? '' : String(cell);
if (/[",\n\r]/.test(s)) return `"${s.replace(/"/g, '""')}"`;
return s;
})
.join(',')
)
.join('\n');
fs.writeFileSync(outPath, `\uFEFF${csv}`, 'utf8');
} else {
const ws = XLSX.utils.aoa_to_sheet(allRows);
const wb = XLSX.utils.book_new();
XLSX.utils.book_append_sheet(wb, ws, 'Users');
XLSX.writeFile(wb, outPath);
}
await adminExportJobRepository.completeExportJob(job.id, outPath, exported);
}
async function cleanupExpiredJobs(): Promise<void> {
try {
const removed = await adminExportJobRepository.deleteExpiredExportJobs();
for (const row of removed) {
if (row.file_path && fs.existsSync(row.file_path)) {
try {
fs.unlinkSync(row.file_path);
} catch {
// ignore
}
}
}
} catch (e) {
console.warn('[admin-export] Cleanup error:', e);
}
}