669 lines
32 KiB
TypeScript
669 lines
32 KiB
TypeScript
/**
|
|
* Withdrawal state machine v2 against a real PostgreSQL database.
|
|
* Blockchain + hot-wallet balances are stubbed; ledger, locks and row-level concurrency are real.
|
|
* Run with RUN_DB_TESTS=1 and DATABASE_URL pointing to a disposable database.
|
|
*/
|
|
|
|
import { after, before, beforeEach, describe, test } from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import crypto from 'node:crypto';
|
|
|
|
import { query, runMigrations } from '../db/connection';
|
|
import { ensureTestMigrations, isDbTestEnabled } from './helpers/testDb';
|
|
import { encrypt } from '../utils/crypto';
|
|
import * as settingsRepository from '../repositories/settingsRepository';
|
|
import * as hotWalletRepository from '../repositories/hotWalletRepository';
|
|
import * as withdrawalRepository from '../repositories/withdrawalRepository';
|
|
import * as authService from '../services/authService';
|
|
import * as hotLiquidityService from '../services/hotLiquidityService';
|
|
import * as payout from '../services/withdrawals/withdrawalPayoutService';
|
|
import type {
|
|
BroadcastOutcome,
|
|
PayoutChainStatus,
|
|
SignedPayout,
|
|
} from '../services/withdrawals/withdrawalPayoutService';
|
|
import * as walletNotificationService from '../services/walletNotificationService';
|
|
import * as referralService from '../services/referralService';
|
|
import * as userAdminStatsService from '../services/userAdminStatsService';
|
|
import * as securityService from '../services/securityService';
|
|
import * as withdrawalService from '../services/withdrawalService';
|
|
|
|
const dbDescribe = isDbTestEnabled() ? describe : describe.skip;
|
|
|
|
const USDT = 1_000_000n;
|
|
const FEE = 1n * USDT;
|
|
const TRON_DEST = `T${'A'.repeat(33)}`;
|
|
|
|
type Mutable<T> = { -readonly [K in keyof T]: T[K] };
|
|
const patch = <T extends object>(mod: T) => mod as Mutable<T> & Record<string, unknown>;
|
|
|
|
const chain = {
|
|
usdt: 0n,
|
|
native: 0n,
|
|
signCalls: 0,
|
|
broadcastCalls: 0,
|
|
rebroadcastCalls: 0,
|
|
signError: null as Error | null,
|
|
broadcastOutcome: { kind: 'accepted' } as BroadcastOutcome,
|
|
statuses: new Map<string, PayoutChainStatus>(),
|
|
expiresInMs: 60_000,
|
|
lastHash: '',
|
|
autoRetryEnabled: false,
|
|
};
|
|
const notifications: Array<{ fn: string; userId: string }> = [];
|
|
let referralCalls = 0;
|
|
let walletTypeId = 0;
|
|
|
|
function resetChain() {
|
|
chain.usdt = 1_000_000n * USDT;
|
|
chain.native = 10n ** 20n;
|
|
chain.signCalls = 0;
|
|
chain.broadcastCalls = 0;
|
|
chain.rebroadcastCalls = 0;
|
|
chain.signError = null;
|
|
chain.broadcastOutcome = { kind: 'accepted' };
|
|
chain.expiresInMs = 60_000;
|
|
chain.autoRetryEnabled = false;
|
|
notifications.length = 0;
|
|
referralCalls = 0;
|
|
}
|
|
|
|
async function createFundedUser(availableUsdt: bigint): Promise<string> {
|
|
const id = crypto.randomUUID();
|
|
await query(
|
|
`INSERT INTO users (id, email_hash, password_hash, role_id) VALUES ($1, $2, 'test-hash', 1)`,
|
|
[id, crypto.createHash('sha256').update(id).digest('hex')]
|
|
);
|
|
const acc = await query<{ id: string }[]>(
|
|
`INSERT INTO ledger_accounts (user_id, wallet_type_id, account_kind, available_balance, locked_balance)
|
|
VALUES ($1, $2, 'main', $3, 0) RETURNING id`,
|
|
[id, walletTypeId, (availableUsdt * USDT).toString()]
|
|
);
|
|
await query(
|
|
`INSERT INTO ledger_entries (account_id, type, amount, balance_after, reference_type, reference_id)
|
|
VALUES ($1, 'deposit', $2, $2, 'test', $3)`,
|
|
[acc[0]!.id, (availableUsdt * USDT).toString(), id]
|
|
);
|
|
return id;
|
|
}
|
|
|
|
async function balances(userId: string) {
|
|
const rows = await query<{ available: string; locked: string; entries: string }[]>(
|
|
`SELECT a.available_balance::text AS available, a.locked_balance::text AS locked,
|
|
COALESCE((SELECT SUM(e.amount) FROM ledger_entries e WHERE e.account_id = a.id), 0)::text AS entries
|
|
FROM ledger_accounts a
|
|
WHERE a.user_id = $1 AND a.wallet_type_id = $2 AND a.account_kind = 'main'`,
|
|
[userId, walletTypeId]
|
|
);
|
|
const r = rows[0]!;
|
|
const available = BigInt(r.available);
|
|
const locked = BigInt(r.locked);
|
|
// Ledger integrity invariant used by the platform monitor.
|
|
assert.equal(available + locked, BigInt(r.entries), 'available + locked must equal ledger entries');
|
|
return { available, locked };
|
|
}
|
|
|
|
async function locksFor(txId: string): Promise<bigint> {
|
|
const rows = await query<{ s: string }[]>(
|
|
`SELECT COALESCE(SUM(amount), 0)::text AS s FROM withdrawal_locks WHERE transaction_id = $1`,
|
|
[txId]
|
|
);
|
|
return BigInt(rows[0]!.s);
|
|
}
|
|
|
|
async function historyOf(txId: string): Promise<string[]> {
|
|
const rows = await withdrawalRepository.listHistory(txId);
|
|
return rows.map((h) => `${h.from_status ?? '∅'}>${h.to_status}`);
|
|
}
|
|
|
|
async function submit(userId: string, amountUsdt: bigint, key?: string) {
|
|
return withdrawalService.requestWithdrawal(userId, 'USDT_TRC20', TRON_DEST, amountUsdt * USDT, undefined, {
|
|
autoProcess: false,
|
|
idempotencyKey: key,
|
|
});
|
|
}
|
|
|
|
const ADMIN = { actorType: 'admin' as const, actorUserId: null };
|
|
|
|
async function expectCode(p: Promise<unknown>, code: string) {
|
|
await assert.rejects(p, (e: unknown) => {
|
|
assert.equal((e as { code?: string }).code, code);
|
|
return true;
|
|
});
|
|
}
|
|
|
|
dbDescribe('withdrawal lifecycle v2 (database)', () => {
|
|
before(async () => {
|
|
await ensureTestMigrations();
|
|
const wt = await query<{ id: number }[]>(`SELECT id FROM wallet_types WHERE code = 'USDT_TRC20'`);
|
|
walletTypeId = wt[0]!.id;
|
|
|
|
const realGetSettings = settingsRepository.getSettings;
|
|
patch(settingsRepository).getSettings = async () => ({
|
|
...(await realGetSettings()),
|
|
allow_withdrawals: true,
|
|
network_usdt_trc20_withdraw_enabled: true,
|
|
network_usdt_bep20_withdraw_enabled: true,
|
|
daily_withdrawal_limit_usd: 1_000_000,
|
|
large_withdrawal_threshold_usd: 1_000_000,
|
|
withdrawal_network_fee_usdt_micro: Number(FEE),
|
|
withdrawal_auto_retry_enabled: chain.autoRetryEnabled,
|
|
withdrawal_auto_retry_interval_minutes: 1,
|
|
});
|
|
patch(authService).assertWithdrawalSecondFactor = async () => undefined;
|
|
patch(hotWalletRepository).getActiveHotWallet = async (id: number) =>
|
|
({
|
|
id: '00000000-0000-4000-8000-00000000h0t1',
|
|
wallet_type_id: id,
|
|
address: `T${'H'.repeat(33)}`,
|
|
encrypted_private_key: encrypt('11'.repeat(32)),
|
|
is_active: true,
|
|
}) as unknown as Awaited<ReturnType<typeof hotWalletRepository.getActiveHotWallet>>;
|
|
patch(hotLiquidityService).readHotBalances = async () => ({
|
|
usdtBalanceRaw: chain.usdt,
|
|
nativeBalanceRaw: chain.native,
|
|
});
|
|
patch(payout).signPayout = async (input: { network: 'TRON' | 'BSC' }): Promise<SignedPayout> => {
|
|
chain.signCalls++;
|
|
if (chain.signError) throw chain.signError;
|
|
const hash = `hash_${crypto.randomUUID()}`;
|
|
chain.lastHash = hash;
|
|
return {
|
|
network: input.network,
|
|
txHash: hash,
|
|
payload: `signed:${hash}`,
|
|
expiresAt: new Date(Date.now() + chain.expiresInMs),
|
|
fromAddress: `T${'H'.repeat(33)}`,
|
|
};
|
|
};
|
|
patch(payout).broadcastPayout = async () => {
|
|
chain.broadcastCalls++;
|
|
return chain.broadcastOutcome;
|
|
};
|
|
patch(payout).rebroadcastPayout = async () => {
|
|
chain.rebroadcastCalls++;
|
|
};
|
|
patch(payout).getPayoutStatus = async (_n: string, hash: string) =>
|
|
chain.statuses.get(hash) ?? { kind: 'pending' };
|
|
patch(payout).isBscNonceConsumed = async () => false;
|
|
|
|
for (const fn of [
|
|
'notifyWithdrawalSubmitted',
|
|
'notifyWithdrawalProcessing',
|
|
'notifyWithdrawalCompleted',
|
|
'notifyWithdrawalFailed',
|
|
'notifyWithdrawalRejected',
|
|
'notifyWithdrawalCancelled',
|
|
]) {
|
|
patch(walletNotificationService)[fn] = async (userId: string) => {
|
|
notifications.push({ fn, userId });
|
|
};
|
|
}
|
|
patch(referralService).processWithdrawalReferralCommissions = async () => {
|
|
referralCalls++;
|
|
return undefined as never;
|
|
};
|
|
patch(userAdminStatsService).queueUserAdminStatsRefresh = () => undefined;
|
|
patch(securityService).raiseAlert = async () => ({}) as never;
|
|
});
|
|
|
|
beforeEach(() => resetChain());
|
|
|
|
after(async () => {
|
|
// Dedicated test database: rows are left for inspection.
|
|
});
|
|
|
|
test('12.1 request is recorded as pending and reserves amount + fee immediately', async () => {
|
|
const userId = await createFundedUser(100n);
|
|
const res = await submit(userId, 10n);
|
|
assert.equal(res.status, 'pending');
|
|
assert.equal(res.totalDebited, (11n * USDT).toString());
|
|
assert.deepEqual(await balances(userId), { available: 89n * USDT, locked: 11n * USDT });
|
|
assert.equal(await locksFor(res.transactionId), 11n * USDT);
|
|
assert.deepEqual(await historyOf(res.transactionId), ['∅>pending']);
|
|
assert.ok(notifications.some((n) => n.fn === 'notifyWithdrawalSubmitted'));
|
|
});
|
|
|
|
test('12.1 reserved amount cannot be reused (no double spend)', async () => {
|
|
const userId = await createFundedUser(15n);
|
|
await submit(userId, 10n); // 11 reserved
|
|
await expectCode(submit(userId, 4n), 'INSUFFICIENT_BALANCE'); // 4 + 1 > 4 left
|
|
assert.deepEqual(await balances(userId), { available: 4n * USDT, locked: 11n * USDT });
|
|
});
|
|
|
|
test('12.1 / 4.2 hot wallet short of USDT: no user error, stays pending (pending>processing>pending)', async () => {
|
|
chain.usdt = 0n;
|
|
const userId = await createFundedUser(50n);
|
|
const res = await submit(userId, 10n);
|
|
const out = await withdrawalService.processWithdrawal(res.transactionId, ADMIN);
|
|
assert.equal(out.status, 'pending');
|
|
assert.equal(out.pendingReason, 'awaiting_liquidity');
|
|
assert.equal(chain.signCalls, 0);
|
|
const row = await withdrawalRepository.findWithdrawal(res.transactionId);
|
|
assert.equal(row!.status, 'pending');
|
|
assert.equal(row!.pending_reason, 'awaiting_liquidity');
|
|
assert.deepEqual(await historyOf(res.transactionId), ['∅>pending', 'pending>processing', 'processing>pending']);
|
|
assert.deepEqual(await balances(userId), { available: 39n * USDT, locked: 11n * USDT });
|
|
assert.ok(!notifications.some((n) => n.fn === 'notifyWithdrawalFailed' || n.fn === 'notifyWithdrawalRejected'));
|
|
});
|
|
|
|
test('12.1 hot wallet short of gas: stays pending (awaiting_gas)', async () => {
|
|
chain.native = 0n;
|
|
const userId = await createFundedUser(50n);
|
|
const res = await submit(userId, 10n);
|
|
const out = await withdrawalService.processWithdrawal(res.transactionId, ADMIN);
|
|
assert.equal(out.pendingReason, 'awaiting_gas');
|
|
assert.equal((await withdrawalRepository.findWithdrawal(res.transactionId))!.status, 'pending');
|
|
});
|
|
|
|
test('7 / 12.3 after replenishment the admin processes it; completed only after confirmations', async () => {
|
|
chain.usdt = 0n;
|
|
const userId = await createFundedUser(50n);
|
|
const res = await submit(userId, 10n);
|
|
await withdrawalService.processWithdrawal(res.transactionId, ADMIN); // pending (no liquidity)
|
|
|
|
chain.usdt = 1_000n * USDT; // replenished
|
|
const out = await withdrawalService.processWithdrawal(res.transactionId, ADMIN);
|
|
assert.equal(out.status, 'processing');
|
|
assert.equal(chain.signCalls, 1);
|
|
assert.equal(chain.broadcastCalls, 1);
|
|
const hash = out.blockchainTxId!;
|
|
await query(`UPDATE transactions SET required_confirmations = 20 WHERE id = $1`, [res.transactionId]);
|
|
|
|
// Broadcast but not yet confirmed: still processing, nothing debited yet.
|
|
chain.statuses.set(hash, { kind: 'mined', success: true, confirmations: 3 });
|
|
let row = (await withdrawalRepository.findWithdrawal(res.transactionId))!;
|
|
assert.equal(await withdrawalService.verifyBroadcastWithdrawal(row), 'confirming');
|
|
row = (await withdrawalRepository.findWithdrawal(res.transactionId))!;
|
|
assert.equal(row.status, 'processing');
|
|
assert.equal(row.confirmations, 3);
|
|
assert.deepEqual(await balances(userId), { available: 39n * USDT, locked: 11n * USDT });
|
|
|
|
chain.statuses.set(hash, { kind: 'mined', success: true, confirmations: 20 });
|
|
assert.equal(await withdrawalService.verifyBroadcastWithdrawal(row), 'completed');
|
|
row = (await withdrawalRepository.findWithdrawal(res.transactionId))!;
|
|
assert.equal(row.status, 'completed');
|
|
assert.deepEqual(await balances(userId), { available: 39n * USDT, locked: 0n });
|
|
assert.equal(await locksFor(res.transactionId), 0n);
|
|
assert.equal(referralCalls, 1);
|
|
assert.ok(notifications.some((n) => n.fn === 'notifyWithdrawalCompleted'));
|
|
assert.deepEqual(await historyOf(res.transactionId), [
|
|
'∅>pending',
|
|
'pending>processing',
|
|
'processing>pending',
|
|
'pending>processing',
|
|
'processing>completed',
|
|
]);
|
|
|
|
// Re-verifying a completed payout never debits twice.
|
|
assert.equal(await withdrawalService.verifyBroadcastWithdrawal(row), 'skipped');
|
|
assert.deepEqual(await balances(userId), { available: 39n * USDT, locked: 0n });
|
|
|
|
// 12.2 completed is final.
|
|
await expectCode(withdrawalService.processWithdrawal(res.transactionId, ADMIN), 'WITHDRAWAL_INVALID_TRANSITION');
|
|
await expectCode(
|
|
withdrawalService.rejectWithdrawal(res.transactionId, { actorId: userId, reason: 'too late' }),
|
|
'WITHDRAWAL_INVALID_TRANSITION'
|
|
);
|
|
assert.deepEqual(await balances(userId), { available: 39n * USDT, locked: 0n });
|
|
});
|
|
|
|
test('4.3 technical failure -> failed (funds stay reserved), then retried and requeued', async () => {
|
|
const userId = await createFundedUser(50n);
|
|
const res = await submit(userId, 10n);
|
|
chain.signError = new Error('RPC 500 while building transaction');
|
|
const out = await withdrawalService.processWithdrawal(res.transactionId, ADMIN);
|
|
assert.equal(out.status, 'failed');
|
|
assert.deepEqual(await balances(userId), { available: 39n * USDT, locked: 11n * USDT });
|
|
assert.ok(notifications.some((n) => n.fn === 'notifyWithdrawalFailed'));
|
|
|
|
// Failed -> Pending (admin requeue) -> Processing (retry) without a second debit.
|
|
await withdrawalService.requeueWithdrawal(res.transactionId, userId);
|
|
assert.equal((await withdrawalRepository.findWithdrawal(res.transactionId))!.status, 'pending');
|
|
chain.signError = null;
|
|
const retry = await withdrawalService.processWithdrawal(res.transactionId, ADMIN);
|
|
assert.equal(retry.status, 'processing');
|
|
const row = (await withdrawalRepository.findWithdrawal(res.transactionId))!;
|
|
assert.equal(row.processing_attempts, 2);
|
|
assert.deepEqual(await balances(userId), { available: 39n * USDT, locked: 11n * USDT });
|
|
await expectCode(withdrawalService.requeueWithdrawal(res.transactionId, userId), 'WITHDRAWAL_INVALID_TRANSITION');
|
|
});
|
|
|
|
test('4.3 failed -> processing directly (admin retry)', async () => {
|
|
const userId = await createFundedUser(50n);
|
|
const res = await submit(userId, 5n);
|
|
chain.signError = new Error('boom');
|
|
await withdrawalService.processWithdrawal(res.transactionId, ADMIN);
|
|
chain.signError = null;
|
|
const retry = await withdrawalService.processWithdrawal(res.transactionId, ADMIN);
|
|
assert.equal(retry.status, 'processing');
|
|
});
|
|
|
|
test('8 / 12.4 reject pending: full refund exactly once, history + notification', async () => {
|
|
const userId = await createFundedUser(50n);
|
|
const res = await submit(userId, 10n);
|
|
const row = await withdrawalService.rejectWithdrawal(res.transactionId, {
|
|
actorId: userId,
|
|
reason: 'Destination flagged by compliance',
|
|
});
|
|
assert.equal(row.status, 'rejected');
|
|
assert.equal(row.rejection_reason, 'Destination flagged by compliance');
|
|
assert.ok(row.refunded_at);
|
|
assert.equal((row.metadata as { refund?: { amount?: string } }).refund?.amount, (11n * USDT).toString());
|
|
assert.deepEqual(await balances(userId), { available: 50n * USDT, locked: 0n });
|
|
assert.ok(notifications.some((n) => n.fn === 'notifyWithdrawalRejected'));
|
|
|
|
await expectCode(
|
|
withdrawalService.rejectWithdrawal(res.transactionId, { actorId: userId, reason: 'again' }),
|
|
'WITHDRAWAL_INVALID_TRANSITION'
|
|
);
|
|
await expectCode(withdrawalService.processWithdrawal(res.transactionId, ADMIN), 'WITHDRAWAL_INVALID_TRANSITION');
|
|
assert.deepEqual(await balances(userId), { available: 50n * USDT, locked: 0n });
|
|
});
|
|
|
|
test('4.4 reject failed request after verification', async () => {
|
|
const userId = await createFundedUser(50n);
|
|
const res = await submit(userId, 10n);
|
|
chain.signError = new Error('boom');
|
|
await withdrawalService.processWithdrawal(res.transactionId, ADMIN);
|
|
const row = await withdrawalService.rejectWithdrawal(res.transactionId, { actorId: userId, reason: 'Not retried' });
|
|
assert.equal(row.status, 'rejected');
|
|
assert.deepEqual(await balances(userId), { available: 50n * USDT, locked: 0n });
|
|
});
|
|
|
|
test('4.4 a processing request (payout broadcast) cannot be rejected or cancelled', async () => {
|
|
const userId = await createFundedUser(50n);
|
|
const res = await submit(userId, 10n);
|
|
await withdrawalService.processWithdrawal(res.transactionId, ADMIN);
|
|
await expectCode(
|
|
withdrawalService.rejectWithdrawal(res.transactionId, { actorId: userId, reason: 'stop' }),
|
|
'WITHDRAWAL_INVALID_TRANSITION'
|
|
);
|
|
await expectCode(withdrawalService.cancelWithdrawal(res.transactionId, { userId }), 'WITHDRAWAL_INVALID_TRANSITION');
|
|
assert.deepEqual(await balances(userId), { available: 39n * USDT, locked: 11n * USDT });
|
|
});
|
|
|
|
test('3 user cancels own pending request: refunded; other users cannot cancel it', async () => {
|
|
const userId = await createFundedUser(30n);
|
|
const other = await createFundedUser(1n);
|
|
const res = await submit(userId, 10n);
|
|
await expectCode(withdrawalService.cancelWithdrawal(res.transactionId, { userId: other }), 'WITHDRAWAL_NOT_FOUND');
|
|
const row = await withdrawalService.cancelWithdrawal(res.transactionId, { userId });
|
|
assert.equal(row.status, 'cancelled');
|
|
assert.deepEqual(await balances(userId), { available: 30n * USDT, locked: 0n });
|
|
await expectCode(withdrawalService.cancelWithdrawal(res.transactionId, { userId }), 'WITHDRAWAL_INVALID_TRANSITION');
|
|
assert.ok(notifications.some((n) => n.fn === 'notifyWithdrawalCancelled'));
|
|
});
|
|
|
|
test('10 / 12.5 two admins process simultaneously: one payout only', async () => {
|
|
const userId = await createFundedUser(50n);
|
|
const res = await submit(userId, 10n);
|
|
const results = await Promise.allSettled([
|
|
withdrawalService.processWithdrawal(res.transactionId, ADMIN),
|
|
withdrawalService.processWithdrawal(res.transactionId, ADMIN),
|
|
withdrawalService.processWithdrawal(res.transactionId, ADMIN),
|
|
]);
|
|
assert.equal(results.filter((r) => r.status === 'fulfilled').length, 1);
|
|
assert.equal(chain.signCalls, 1);
|
|
assert.equal(chain.broadcastCalls, 1);
|
|
});
|
|
|
|
test('10 / 12.5 concurrent rejects refund exactly once', async () => {
|
|
const userId = await createFundedUser(50n);
|
|
const res = await submit(userId, 10n);
|
|
const results = await Promise.allSettled(
|
|
[1, 2, 3, 4].map(() =>
|
|
withdrawalService.rejectWithdrawal(res.transactionId, { actorId: userId, reason: 'dup click' })
|
|
)
|
|
);
|
|
assert.equal(results.filter((r) => r.status === 'fulfilled').length, 1);
|
|
assert.deepEqual(await balances(userId), { available: 50n * USDT, locked: 0n });
|
|
});
|
|
|
|
test('10 validate vs reject at the same time: exactly one wins', async () => {
|
|
const userId = await createFundedUser(50n);
|
|
const res = await submit(userId, 10n);
|
|
const [processed, rejected] = await Promise.allSettled([
|
|
withdrawalService.processWithdrawal(res.transactionId, ADMIN),
|
|
withdrawalService.rejectWithdrawal(res.transactionId, { actorId: userId, reason: 'race' }),
|
|
]);
|
|
const wins = [processed, rejected].filter((r) => r.status === 'fulfilled').length;
|
|
assert.equal(wins, 1);
|
|
const row = (await withdrawalRepository.findWithdrawal(res.transactionId))!;
|
|
const b = await balances(userId);
|
|
if (row.status === 'rejected') {
|
|
assert.deepEqual(b, { available: 50n * USDT, locked: 0n });
|
|
assert.equal(chain.broadcastCalls, 0);
|
|
} else {
|
|
assert.equal(row.status, 'processing');
|
|
assert.deepEqual(b, { available: 39n * USDT, locked: 11n * USDT });
|
|
}
|
|
});
|
|
|
|
test('10 double click / repeated request with the same idempotency key creates one request', async () => {
|
|
const userId = await createFundedUser(50n);
|
|
const key = `test-${crypto.randomUUID()}`;
|
|
const results = await Promise.all([submit(userId, 10n, key), submit(userId, 10n, key), submit(userId, 10n, key)]);
|
|
assert.equal(new Set(results.map((r) => r.transactionId)).size, 1);
|
|
assert.equal(results.filter((r) => r.duplicate).length, 2);
|
|
assert.deepEqual(await balances(userId), { available: 39n * USDT, locked: 11n * USDT });
|
|
});
|
|
|
|
test('5 lost broadcast response: stays processing, verified by hash, same tx re-sent, never re-signed', async () => {
|
|
const userId = await createFundedUser(50n);
|
|
const res = await submit(userId, 10n);
|
|
chain.broadcastOutcome = { kind: 'uncertain', message: 'ECONNRESET' };
|
|
const out = await withdrawalService.processWithdrawal(res.transactionId, ADMIN);
|
|
assert.equal(out.status, 'processing');
|
|
let row = (await withdrawalRepository.findWithdrawal(res.transactionId))!;
|
|
assert.equal(row.processing_stage, 'broadcast_uncertain');
|
|
const hash = row.blockchain_tx_id!;
|
|
|
|
// Retry by an admin is refused while the hash is being verified.
|
|
await expectCode(withdrawalService.processWithdrawal(res.transactionId, ADMIN), 'WITHDRAWAL_INVALID_TRANSITION');
|
|
|
|
chain.statuses.set(hash, { kind: 'not_found' });
|
|
assert.equal(await withdrawalService.verifyBroadcastWithdrawal(row), 'rebroadcast');
|
|
assert.equal(chain.rebroadcastCalls, 1);
|
|
assert.equal(chain.signCalls, 1);
|
|
|
|
chain.statuses.set(hash, { kind: 'mined', success: true, confirmations: 999 });
|
|
row = (await withdrawalRepository.findWithdrawal(res.transactionId))!;
|
|
assert.equal(await withdrawalService.verifyBroadcastWithdrawal(row), 'completed');
|
|
assert.equal(chain.signCalls, 1);
|
|
assert.deepEqual(await balances(userId), { available: 39n * USDT, locked: 0n });
|
|
});
|
|
|
|
test('5 payout never included and expired: failed, hash archived, retry signs a new tx', async () => {
|
|
const userId = await createFundedUser(50n);
|
|
const res = await submit(userId, 10n);
|
|
chain.expiresInMs = -10 * 60_000;
|
|
await withdrawalService.processWithdrawal(res.transactionId, ADMIN);
|
|
let row = (await withdrawalRepository.findWithdrawal(res.transactionId))!;
|
|
const firstHash = row.blockchain_tx_id!;
|
|
chain.statuses.set(firstHash, { kind: 'not_found' });
|
|
assert.equal(await withdrawalService.verifyBroadcastWithdrawal(row), 'failed');
|
|
row = (await withdrawalRepository.findWithdrawal(res.transactionId))!;
|
|
assert.equal(row.status, 'failed');
|
|
assert.equal(row.blockchain_tx_id, null);
|
|
assert.deepEqual(row.abandoned_tx_hashes, [firstHash]);
|
|
assert.deepEqual(await balances(userId), { available: 39n * USDT, locked: 11n * USDT });
|
|
|
|
chain.expiresInMs = 60_000;
|
|
const retry = await withdrawalService.processWithdrawal(res.transactionId, ADMIN);
|
|
assert.equal(retry.status, 'processing');
|
|
assert.notEqual(retry.blockchainTxId, firstHash);
|
|
});
|
|
|
|
test('5 payout reverted on-chain: failed, funds still reserved for retry', async () => {
|
|
const userId = await createFundedUser(50n);
|
|
const res = await submit(userId, 10n);
|
|
await withdrawalService.processWithdrawal(res.transactionId, ADMIN);
|
|
const row = (await withdrawalRepository.findWithdrawal(res.transactionId))!;
|
|
chain.statuses.set(row.blockchain_tx_id!, { kind: 'mined', success: false, confirmations: 30, message: 'REVERT' });
|
|
assert.equal(await withdrawalService.verifyBroadcastWithdrawal(row), 'failed');
|
|
assert.deepEqual(await balances(userId), { available: 39n * USDT, locked: 11n * USDT });
|
|
});
|
|
|
|
test('4.2 node refuses the signed payout for lack of gas: back to pending, nothing sent', async () => {
|
|
const userId = await createFundedUser(50n);
|
|
const res = await submit(userId, 10n);
|
|
chain.broadcastOutcome = { kind: 'not_sent', failure: 'liquidity_gas', message: 'BANDWITH_ERROR' };
|
|
const out = await withdrawalService.processWithdrawal(res.transactionId, ADMIN);
|
|
assert.equal(out.status, 'pending');
|
|
const row = (await withdrawalRepository.findWithdrawal(res.transactionId))!;
|
|
assert.equal(row.pending_reason, 'awaiting_gas');
|
|
assert.equal(row.blockchain_tx_id, null);
|
|
assert.equal(row.abandoned_tx_hashes?.length, 1);
|
|
});
|
|
|
|
test('interrupted processing (no signed tx) is recovered to pending by the monitor', async () => {
|
|
const userId = await createFundedUser(50n);
|
|
const res = await submit(userId, 10n);
|
|
await withdrawalRepository.claimForProcessing(res.transactionId, ADMIN);
|
|
await query(
|
|
`UPDATE transactions SET processing_started_at = NOW() - INTERVAL '30 minutes' WHERE id = $1`,
|
|
[res.transactionId]
|
|
);
|
|
await withdrawalService.runWithdrawalMonitorCycle();
|
|
const row = (await withdrawalRepository.findWithdrawal(res.transactionId))!;
|
|
assert.equal(row.status, 'pending');
|
|
assert.equal(row.pending_reason, 'processing_interrupted');
|
|
});
|
|
|
|
test('large withdrawal awaiting approval cannot be processed from the queue', async () => {
|
|
const userId = await createFundedUser(50n);
|
|
const res = await submit(userId, 10n);
|
|
await withdrawalRepository.patchPendingDetails(res.transactionId, { pendingReason: 'awaiting_approval' });
|
|
await expectCode(withdrawalService.processWithdrawal(res.transactionId, ADMIN), 'WITHDRAWAL_AWAITING_APPROVAL');
|
|
const approved = await withdrawalService.processApprovedWithdrawal(res.transactionId, userId);
|
|
assert.equal(approved.status, 'processing');
|
|
});
|
|
|
|
test('daily limit counts failed (still reserved) but not rejected/cancelled requests', async () => {
|
|
const userId = await createFundedUser(500n);
|
|
const realGet = settingsRepository.getSettings;
|
|
patch(settingsRepository).getSettings = async () => ({ ...(await realGet()), daily_withdrawal_limit_usd: 25 });
|
|
try {
|
|
const a = await submit(userId, 10n); // 11 used
|
|
chain.signError = new Error('boom');
|
|
await withdrawalService.processWithdrawal(a.transactionId, ADMIN); // failed, still counts
|
|
const b = await submit(userId, 10n); // 22 used
|
|
await expectCode(submit(userId, 5n), 'DAILY_WITHDRAWAL_LIMIT_EXCEEDED');
|
|
await withdrawalService.cancelWithdrawal(b.transactionId, { userId }); // frees 11
|
|
const c = await submit(userId, 5n);
|
|
assert.equal(c.status, 'pending');
|
|
} finally {
|
|
patch(settingsRepository).getSettings = realGet;
|
|
}
|
|
});
|
|
|
|
/** Isolates auto-retry tests from liquidity-pending rows left by earlier tests. */
|
|
async function parkOtherAutoRetryRows() {
|
|
await query(
|
|
`UPDATE transactions SET pending_reason = 'queued'
|
|
WHERE type = 'withdrawal' AND status = 'pending'
|
|
AND pending_reason IN ('awaiting_liquidity', 'awaiting_gas', 'hot_wallet_unavailable')`
|
|
);
|
|
}
|
|
|
|
test('auto-retry disabled: liquidity-pending requests are left for the admin', async () => {
|
|
await parkOtherAutoRetryRows();
|
|
chain.usdt = 0n;
|
|
const userId = await createFundedUser(50n);
|
|
const res = await submit(userId, 10n);
|
|
await withdrawalService.processWithdrawal(res.transactionId, ADMIN); // -> awaiting_liquidity
|
|
chain.usdt = 1_000n * USDT; // replenished, but the option is off
|
|
const before = await historyOf(res.transactionId);
|
|
const r = await withdrawalService.runWithdrawalAutoRetryCycle();
|
|
assert.deepEqual(r, { attempted: 0, skipped: 0 });
|
|
assert.equal((await withdrawalRepository.findWithdrawal(res.transactionId))!.status, 'pending');
|
|
assert.deepEqual(await historyOf(res.transactionId), before);
|
|
});
|
|
|
|
test('auto-retry enabled but hot wallet still empty: no claim, no history churn', async () => {
|
|
await parkOtherAutoRetryRows();
|
|
chain.usdt = 0n;
|
|
const userId = await createFundedUser(50n);
|
|
const res = await submit(userId, 10n);
|
|
await withdrawalService.processWithdrawal(res.transactionId, ADMIN);
|
|
chain.autoRetryEnabled = true;
|
|
const before = await historyOf(res.transactionId);
|
|
const r = await withdrawalService.runWithdrawalAutoRetryCycle();
|
|
assert.equal(r.attempted, 0);
|
|
assert.equal(chain.signCalls, 0);
|
|
assert.deepEqual(await historyOf(res.transactionId), before);
|
|
});
|
|
|
|
test('auto-retry enabled after replenishment: processed automatically (system actor)', async () => {
|
|
await parkOtherAutoRetryRows();
|
|
chain.native = 0n;
|
|
const userId = await createFundedUser(50n);
|
|
const res = await submit(userId, 10n);
|
|
await withdrawalService.processWithdrawal(res.transactionId, ADMIN); // -> awaiting_gas
|
|
chain.native = 10n ** 20n;
|
|
chain.autoRetryEnabled = true;
|
|
const r = await withdrawalService.runWithdrawalAutoRetryCycle();
|
|
assert.equal(r.attempted, 1);
|
|
const row = (await withdrawalRepository.findWithdrawal(res.transactionId))!;
|
|
assert.equal(row.status, 'processing');
|
|
assert.ok(row.blockchain_tx_id);
|
|
const history = await withdrawalRepository.listHistory(res.transactionId);
|
|
assert.equal(history[history.length - 1]!.actor_type, 'system');
|
|
assert.deepEqual(await balances(userId), { available: 39n * USDT, locked: 11n * USDT });
|
|
});
|
|
|
|
test('auto-retry is FIFO and never exceeds what the hot wallet can pay', async () => {
|
|
await parkOtherAutoRetryRows();
|
|
chain.usdt = 0n;
|
|
const userId = await createFundedUser(100n);
|
|
const first = await submit(userId, 10n);
|
|
const second = await submit(userId, 10n);
|
|
await withdrawalService.processWithdrawal(first.transactionId, ADMIN);
|
|
await withdrawalService.processWithdrawal(second.transactionId, ADMIN);
|
|
// Payouts still in flight from earlier tests are reserved against the hot wallet too.
|
|
const inFlight = await withdrawalRepository.sumInFlightPayouts('TRON', first.transactionId);
|
|
chain.usdt = inFlight + 15n * USDT; // enough for one 10 USDT payout, not two
|
|
chain.autoRetryEnabled = true;
|
|
const r = await withdrawalService.runWithdrawalAutoRetryCycle();
|
|
assert.equal(r.attempted, 1);
|
|
assert.equal((await withdrawalRepository.findWithdrawal(first.transactionId))!.status, 'processing');
|
|
assert.equal((await withdrawalRepository.findWithdrawal(second.transactionId))!.status, 'pending');
|
|
});
|
|
|
|
test('auto-retry never touches requests awaiting approval or set aside by an admin', async () => {
|
|
await parkOtherAutoRetryRows();
|
|
const userId = await createFundedUser(50n);
|
|
const a = await submit(userId, 5n);
|
|
const b = await submit(userId, 5n);
|
|
await withdrawalRepository.patchPendingDetails(a.transactionId, { pendingReason: 'awaiting_approval' });
|
|
chain.signError = new Error('boom');
|
|
await withdrawalService.processWithdrawal(b.transactionId, ADMIN); // failed
|
|
chain.signError = null;
|
|
await withdrawalService.requeueWithdrawal(b.transactionId, userId); // requeued_by_admin
|
|
chain.autoRetryEnabled = true;
|
|
const r = await withdrawalService.runWithdrawalAutoRetryCycle();
|
|
assert.equal(r.attempted, 0);
|
|
assert.equal((await withdrawalRepository.findWithdrawal(a.transactionId))!.status, 'pending');
|
|
assert.equal((await withdrawalRepository.findWithdrawal(b.transactionId))!.status, 'pending');
|
|
});
|
|
|
|
test('legacy refunded failures become read-only failed_refunded and are never retried', async () => {
|
|
const userId = await createFundedUser(5n);
|
|
const legacy = await query<{ id: string }[]>(
|
|
`INSERT INTO transactions (user_id, wallet_type_id, type, amount, fee, status, to_address)
|
|
VALUES ($1, $2, 'withdrawal', $3, 0, 'failed', $4) RETURNING id`,
|
|
[userId, walletTypeId, (3n * USDT).toString(), TRON_DEST]
|
|
);
|
|
await runMigrations();
|
|
const row = (await withdrawalRepository.findWithdrawal(legacy[0]!.id))!;
|
|
assert.equal(row.status, 'failed_refunded');
|
|
await expectCode(withdrawalService.processWithdrawal(row.id, ADMIN), 'WITHDRAWAL_INVALID_TRANSITION');
|
|
await expectCode(
|
|
withdrawalService.rejectWithdrawal(row.id, { actorId: userId, reason: 'x' }),
|
|
'WITHDRAWAL_INVALID_TRANSITION'
|
|
);
|
|
assert.deepEqual(await balances(userId), { available: 5n * USDT, locked: 0n });
|
|
});
|
|
});
|