291 lines
11 KiB
JavaScript
291 lines
11 KiB
JavaScript
"use strict";
|
|
/**
|
|
* Async admin user export jobs (PostgreSQL persistence + local filesystem).
|
|
*/
|
|
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
if (k2 === undefined) k2 = k;
|
|
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
}
|
|
Object.defineProperty(o, k2, desc);
|
|
}) : (function(o, m, k, k2) {
|
|
if (k2 === undefined) k2 = k;
|
|
o[k2] = m[k];
|
|
}));
|
|
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
}) : function(o, v) {
|
|
o["default"] = v;
|
|
});
|
|
var __importStar = (this && this.__importStar) || (function () {
|
|
var ownKeys = function(o) {
|
|
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
var ar = [];
|
|
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
return ar;
|
|
};
|
|
return ownKeys(o);
|
|
};
|
|
return function (mod) {
|
|
if (mod && mod.__esModule) return mod;
|
|
var result = {};
|
|
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
__setModuleDefault(result, mod);
|
|
return result;
|
|
};
|
|
})();
|
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
};
|
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
exports.startAdminUserExport = startAdminUserExport;
|
|
exports.getAdminUserExportJob = getAdminUserExportJob;
|
|
exports.getAdminUserExportFilePath = getAdminUserExportFilePath;
|
|
exports.startAdminExportWorker = startAdminExportWorker;
|
|
exports.kickExportWorker = kickExportWorker;
|
|
const fs_1 = __importDefault(require("fs"));
|
|
const os_1 = __importDefault(require("os"));
|
|
const path_1 = __importDefault(require("path"));
|
|
const uuid_1 = require("uuid");
|
|
const XLSX = __importStar(require("xlsx"));
|
|
const adminService = __importStar(require("./adminService"));
|
|
const adminExportJobRepository = __importStar(require("../repositories/adminExportJobRepository"));
|
|
const countryRepository = __importStar(require("../repositories/countryRepository"));
|
|
const adminUserListQuery_1 = require("../utils/adminUserListQuery");
|
|
const adminUserExportFormat_1 = require("./adminUserExportFormat");
|
|
const EXPORT_DIR = path_1.default.join(process.cwd(), 'data', 'exports');
|
|
const JOB_TTL_MS = 60 * 60 * 1000;
|
|
const PAGE_SIZE = 500;
|
|
const DEFAULT_MAX_ROWS = 100000;
|
|
const ABSOLUTE_MAX_ROWS = 500000;
|
|
const POLL_CLEANUP_MS = 15 * 60 * 1000;
|
|
const WORKER_POLL_MS = 1500;
|
|
const STALE_JOB_MINUTES = 30;
|
|
const WORKER_ID = `${os_1.default.hostname()}-${process.pid}-${(0, uuid_1.v4)().slice(0, 8)}`;
|
|
let workerBusy = false;
|
|
let workerStarted = false;
|
|
function ensureExportDir() {
|
|
fs_1.default.mkdirSync(EXPORT_DIR, { recursive: true });
|
|
}
|
|
function clampMaxRows(maxRows) {
|
|
if (maxRows === undefined || maxRows === null || !Number.isFinite(maxRows) || maxRows <= 0) {
|
|
return DEFAULT_MAX_ROWS;
|
|
}
|
|
return Math.min(Math.floor(maxRows), ABSOLUTE_MAX_ROWS);
|
|
}
|
|
function rowToView(row) {
|
|
return {
|
|
id: row.id,
|
|
status: row.status,
|
|
progress: row.progress,
|
|
rowCount: row.row_count,
|
|
totalEstimate: row.total_estimate,
|
|
error: row.error,
|
|
fileName: row.file_name,
|
|
createdAt: row.created_at.toISOString(),
|
|
completedAt: row.completed_at?.toISOString() ?? null,
|
|
jobKind: row.job_kind,
|
|
reportKey: row.report_key,
|
|
};
|
|
}
|
|
async function startAdminUserExport(requestedBy, request) {
|
|
ensureExportDir();
|
|
await cleanupExpiredJobs();
|
|
const id = (0, uuid_1.v4)();
|
|
const columns = (0, adminUserListQuery_1.sanitizeExportColumns)(request.columns);
|
|
const maxRows = clampMaxRows(request.maxRows);
|
|
const format = request.format === 'csv' ? 'csv' : 'xlsx';
|
|
const stamp = new Date().toISOString().slice(0, 10);
|
|
const suffix = request.userIds?.length ? '_selected' : '';
|
|
const fileName = `users_export${suffix}_${stamp}.${format}`;
|
|
const { columns: _c, maxRows: _m, format: _f, ...requestPayload } = request;
|
|
const row = await adminExportJobRepository.createExportJob({
|
|
id,
|
|
requestedBy,
|
|
fileName,
|
|
requestPayload,
|
|
format,
|
|
maxRows,
|
|
columns,
|
|
expiresAt: new Date(Date.now() + JOB_TTL_MS),
|
|
});
|
|
kickExportWorker();
|
|
return rowToView(row);
|
|
}
|
|
async function getAdminUserExportJob(jobId, requestedBy) {
|
|
const row = await adminExportJobRepository.getExportJobForUser(jobId, requestedBy);
|
|
return row ? rowToView(row) : null;
|
|
}
|
|
async function getAdminUserExportFilePath(jobId, requestedBy) {
|
|
const row = await adminExportJobRepository.getExportJobForUser(jobId, requestedBy);
|
|
if (!row || row.status !== 'completed' || !row.file_path || !row.file_name)
|
|
return null;
|
|
if (!fs_1.default.existsSync(row.file_path))
|
|
return null;
|
|
return { filePath: row.file_path, fileName: row.file_name };
|
|
}
|
|
/**
|
|
* Starts background worker: recovers stale jobs and polls the queue.
|
|
*/
|
|
function startAdminExportWorker() {
|
|
if (workerStarted)
|
|
return;
|
|
workerStarted = true;
|
|
ensureExportDir();
|
|
setImmediate(async () => {
|
|
try {
|
|
const n = await adminExportJobRepository.requeueAllRunningJobs();
|
|
if (n > 0) {
|
|
console.log(`[admin-export] Re-queued ${n} running job(s) after restart`);
|
|
}
|
|
await adminExportJobRepository.requeueStaleRunningJobs(STALE_JOB_MINUTES);
|
|
}
|
|
catch (e) {
|
|
console.error('[admin-export] Worker startup recovery error:', e);
|
|
}
|
|
kickExportWorker();
|
|
});
|
|
setInterval(() => {
|
|
void cleanupExpiredJobs();
|
|
}, POLL_CLEANUP_MS).unref();
|
|
setInterval(() => {
|
|
kickExportWorker();
|
|
}, WORKER_POLL_MS).unref();
|
|
}
|
|
function kickExportWorker() {
|
|
if (workerBusy)
|
|
return;
|
|
workerBusy = true;
|
|
void processNextExportJob().finally(() => {
|
|
workerBusy = false;
|
|
});
|
|
}
|
|
async function processNextExportJob() {
|
|
const job = await adminExportJobRepository.claimNextQueuedJob(WORKER_ID);
|
|
if (!job)
|
|
return;
|
|
if (job.job_kind === 'reports') {
|
|
const { runReportExportJob } = await Promise.resolve().then(() => __importStar(require('./reportExportJobService')));
|
|
try {
|
|
await runReportExportJob(job);
|
|
}
|
|
catch (e) {
|
|
const msg = e instanceof Error ? e.message : 'Export failed';
|
|
await adminExportJobRepository.failExportJob(job.id, msg);
|
|
}
|
|
return;
|
|
}
|
|
const columns = (0, adminUserListQuery_1.sanitizeExportColumns)(Array.isArray(job.columns) ? job.columns : job.columns);
|
|
try {
|
|
await runExportJob(job, job.request_payload, columns, job.max_rows, job.format);
|
|
}
|
|
catch (e) {
|
|
const msg = e instanceof Error ? e.message : 'Export failed';
|
|
await adminExportJobRepository.failExportJob(job.id, msg);
|
|
}
|
|
}
|
|
async function runExportJob(job, request, columns, maxRows, format) {
|
|
const countries = await countryRepository.listAll();
|
|
const countryMap = new Map(countries.map((c) => [c.code, c.name]));
|
|
const countryName = (code) => {
|
|
if (!code)
|
|
return '';
|
|
return countryMap.get(code) ?? code;
|
|
};
|
|
const allRows = [];
|
|
let cursor;
|
|
let exported = 0;
|
|
const reportProgress = async (totalEstimate) => {
|
|
await adminExportJobRepository.updateExportJobProgress(job.id, exported, exported, totalEstimate);
|
|
};
|
|
if (request.userIds?.length) {
|
|
const ids = request.userIds;
|
|
for (let i = 0; i < ids.length; i += PAGE_SIZE) {
|
|
const batchIds = ids.slice(i, i + PAGE_SIZE);
|
|
const result = await adminService.listUsers({
|
|
...request,
|
|
userIds: batchIds,
|
|
limit: batchIds.length,
|
|
includeTotal: false,
|
|
});
|
|
const chunk = (0, adminUserExportFormat_1.buildExportRows)(result.users, columns, countryName);
|
|
if (allRows.length === 0)
|
|
allRows.push(...chunk);
|
|
else
|
|
allRows.push(...chunk.slice(1));
|
|
exported += result.users.length;
|
|
await reportProgress();
|
|
}
|
|
}
|
|
else {
|
|
let includeTotal = true;
|
|
while (exported < maxRows) {
|
|
const remaining = maxRows - exported;
|
|
const limit = Math.min(PAGE_SIZE, remaining);
|
|
const result = await adminService.listUsers({
|
|
...request,
|
|
limit,
|
|
cursor,
|
|
includeTotal,
|
|
offset: cursor ? undefined : exported > 0 ? exported : undefined,
|
|
});
|
|
if (includeTotal && result.total != null) {
|
|
await reportProgress(Math.min(result.total, maxRows));
|
|
}
|
|
includeTotal = false;
|
|
if (result.users.length === 0)
|
|
break;
|
|
const chunk = (0, adminUserExportFormat_1.buildExportRows)(result.users, columns, countryName);
|
|
if (allRows.length === 0)
|
|
allRows.push(...chunk);
|
|
else
|
|
allRows.push(...chunk.slice(1));
|
|
exported += result.users.length;
|
|
await reportProgress();
|
|
if (!result.hasMore || !result.nextCursor)
|
|
break;
|
|
cursor = result.nextCursor;
|
|
}
|
|
}
|
|
const outPath = path_1.default.join(EXPORT_DIR, `${job.id}.${format}`);
|
|
if (format === 'csv') {
|
|
const csv = allRows
|
|
.map((row) => row
|
|
.map((cell) => {
|
|
const s = cell === null || cell === undefined ? '' : String(cell);
|
|
if (/[",\n\r]/.test(s))
|
|
return `"${s.replace(/"/g, '""')}"`;
|
|
return s;
|
|
})
|
|
.join(','))
|
|
.join('\n');
|
|
fs_1.default.writeFileSync(outPath, `\uFEFF${csv}`, 'utf8');
|
|
}
|
|
else {
|
|
const ws = XLSX.utils.aoa_to_sheet(allRows);
|
|
const wb = XLSX.utils.book_new();
|
|
XLSX.utils.book_append_sheet(wb, ws, 'Users');
|
|
XLSX.writeFile(wb, outPath);
|
|
}
|
|
await adminExportJobRepository.completeExportJob(job.id, outPath, exported);
|
|
}
|
|
async function cleanupExpiredJobs() {
|
|
try {
|
|
const removed = await adminExportJobRepository.deleteExpiredExportJobs();
|
|
for (const row of removed) {
|
|
if (row.file_path && fs_1.default.existsSync(row.file_path)) {
|
|
try {
|
|
fs_1.default.unlinkSync(row.file_path);
|
|
}
|
|
catch {
|
|
// ignore
|
|
}
|
|
}
|
|
}
|
|
}
|
|
catch (e) {
|
|
console.warn('[admin-export] Cleanup error:', e);
|
|
}
|
|
}
|
|
//# sourceMappingURL=adminUserExportService.js.map
|