diff --git a/src/integration/blockchain/shared/evm/evm-client.ts b/src/integration/blockchain/shared/evm/evm-client.ts index 5c5cd775b9..077204588a 100644 --- a/src/integration/blockchain/shared/evm/evm-client.ts +++ b/src/integration/blockchain/shared/evm/evm-client.ts @@ -168,6 +168,11 @@ export abstract class EvmClient extends BlockchainClient { return this.provider.getBlockNumber(); } + async getLatestBlockTimestamp(): Promise { + const block = await this.provider.getBlock('latest'); + return block.timestamp; + } + async getTransactionCount(address: string): Promise { return this.provider.getTransactionCount(address); } diff --git a/src/shared/auth/role.guard.ts b/src/shared/auth/role.guard.ts index 16c2b53c14..e6ae1ede1d 100644 --- a/src/shared/auth/role.guard.ts +++ b/src/shared/auth/role.guard.ts @@ -11,6 +11,8 @@ class RoleGuardClass implements CanActivate { UserRole.BETA, UserRole.ADMIN, UserRole.SUPER_ADMIN, + UserRole.SUPPORT, + UserRole.COMPLIANCE, ], [UserRole.USER]: [UserRole.VIP, UserRole.BETA, UserRole.ADMIN, UserRole.SUPER_ADMIN, UserRole.CUSTODY], [UserRole.VIP]: [UserRole.ADMIN, UserRole.SUPER_ADMIN], diff --git a/src/subdomains/generic/support/dto/user-data-support.dto.ts b/src/subdomains/generic/support/dto/user-data-support.dto.ts index a2bc529581..68ce06fa13 100644 --- a/src/subdomains/generic/support/dto/user-data-support.dto.ts +++ b/src/subdomains/generic/support/dto/user-data-support.dto.ts @@ -85,6 +85,15 @@ export class KycFileListEntry { id: number; amlAccountType?: string; verifiedName?: string; + country?: { name: string }; + allBeneficialOwnersDomicile?: string; + amlListAddedDate?: Date; + amlListExpiredDate?: Date; + amlListReactivatedDate?: Date; + highRisk?: boolean; + pep?: boolean; + complexOrgStructure?: boolean; + totalVolumeChfAuditPeriod?: number; } export class KycFileYearlyStats { diff --git a/src/subdomains/generic/support/support.service.ts b/src/subdomains/generic/support/support.service.ts index 08ab50f775..495ecaab22 100644 --- a/src/subdomains/generic/support/support.service.ts +++ b/src/subdomains/generic/support/support.service.ts @@ -152,6 +152,15 @@ export class SupportService { id: userData.id, amlAccountType: userData.amlAccountType, verifiedName: userData.verifiedName, + country: userData.country ? { name: userData.country.name } : undefined, + allBeneficialOwnersDomicile: userData.allBeneficialOwnersDomicile, + amlListAddedDate: userData.amlListAddedDate, + amlListExpiredDate: userData.amlListExpiredDate, + amlListReactivatedDate: userData.amlListReactivatedDate, + highRisk: userData.highRisk, + pep: userData.pep, + complexOrgStructure: userData.complexOrgStructure, + totalVolumeChfAuditPeriod: userData.totalVolumeChfAuditPeriod, }; } diff --git a/src/subdomains/generic/user/models/user-data/user-data.controller.ts b/src/subdomains/generic/user/models/user-data/user-data.controller.ts index 0653eb9183..0a98cfe738 100644 --- a/src/subdomains/generic/user/models/user-data/user-data.controller.ts +++ b/src/subdomains/generic/user/models/user-data/user-data.controller.ts @@ -171,11 +171,12 @@ export class UserDataController { @ApiExcludeEndpoint() @UseGuards(AuthGuard(), RoleGuard(UserRole.COMPLIANCE), UserActiveGuard()) async downloadUserData(@Body() data: DownloadUserDataDto, @Res({ passthrough: true }) res): Promise { - const zipContent = await this.userDataService.downloadUserData(data.userDataIds); + const zipContent = await this.userDataService.downloadUserData(data.userDataIds, data.checkOnly); + const prefix = data.checkOnly ? 'DFX_check' : 'DFX_export'; res.set({ 'Content-Type': 'application/zip', - 'Content-Disposition': `attachment; filename="DFX_export_${Util.filenameDate()}.zip"`, + 'Content-Disposition': `attachment; filename="${prefix}_${Util.filenameDate()}.zip"`, }); return new StreamableFile(zipContent); diff --git a/src/subdomains/generic/user/models/user-data/user-data.service.ts b/src/subdomains/generic/user/models/user-data/user-data.service.ts index 87e3d9e97a..e435cdedec 100644 --- a/src/subdomains/generic/user/models/user-data/user-data.service.ts +++ b/src/subdomains/generic/user/models/user-data/user-data.service.ts @@ -221,7 +221,22 @@ export class UserDataService { async getUserDatasWithKycFile(): Promise { return this.userDataRepo .createQueryBuilder('userData') - .select(['userData.id', 'userData.kycFileId', 'userData.amlAccountType', 'userData.verifiedName']) + .leftJoinAndSelect('userData.country', 'country') + .select([ + 'userData.id', + 'userData.kycFileId', + 'userData.amlAccountType', + 'userData.verifiedName', + 'userData.allBeneficialOwnersDomicile', + 'userData.amlListAddedDate', + 'userData.amlListExpiredDate', + 'userData.amlListReactivatedDate', + 'userData.highRisk', + 'userData.pep', + 'userData.complexOrgStructure', + 'userData.totalVolumeChfAuditPeriod', + 'country.name', + ]) .where('userData.kycFileId > 0') .orderBy('userData.kycFileId', 'ASC') .getMany(); @@ -328,29 +343,39 @@ export class UserDataService { return userData; } - async downloadUserData(userDataIds: number[]): Promise { + async downloadUserData(userDataIds: number[], checkOnly = false): Promise { let count = userDataIds.length; const zip = new JSZip(); const downloadTargets = Config.fileDownloadConfig.reverse(); - let errorLog = ''; + const errors: { userDataId: number; errorType: string; folder: string; details: string }[] = []; + + const escapeCsvValue = (value: string): string => { + if (value.includes(';') || value.includes('"') || value.includes('\n')) { + return `"${value.replace(/"/g, '""')}"`; + } + return value; + }; for (const userDataId of userDataIds.reverse()) { const userData = await this.getUserData(userDataId, { kycSteps: true }); - if (!userData?.verifiedName) { - errorLog += !userData - ? `Error: UserData ${userDataId} not found\n` - : `Error: UserData ${userDataId} has no verifiedName\n`; + if (!userData) { + errors.push({ userDataId, errorType: 'NotFound', folder: '', details: '' }); + continue; + } + + if (!userData.verifiedName) { + errors.push({ userDataId, errorType: 'NoVerifiedName', folder: '', details: '' }); continue; } const baseFolderName = `${(count--).toString().padStart(2, '0')}_${String(userDataId)}_${ userData.verifiedName }`.replace(/\./g, ''); - const parentFolder = zip.folder(baseFolderName); + const parentFolder = checkOnly ? null : zip.folder(baseFolderName); - if (!parentFolder) { - errorLog += `Error: Failed to create folder for UserData ${userDataId}\n`; + if (!checkOnly && !parentFolder) { + errors.push({ userDataId, errorType: 'FolderCreationFailed', folder: baseFolderName, details: '' }); continue; } @@ -363,25 +388,31 @@ export class UserDataService { for (const { id, name, files: fileConfig } of applicableTargets) { const folderName = `${id.toString().padStart(2, '0')}_${name}`; - const subFolder = parentFolder.folder(folderName); + const subFolder = checkOnly ? null : parentFolder?.folder(folderName); - if (!subFolder) { - errorLog += `Error: Failed to create folder '${folderName}' for UserData ${userDataId}\n`; + if (!checkOnly && !subFolder) { + errors.push({ userDataId, errorType: 'FolderCreationFailed', folder: folderName, details: '' }); continue; } - for (const { name: fileName, fileTypes, prefixes, filter, handleFileNotFound, sort } of fileConfig) { + for (const { name: fileNameFn, fileTypes, prefixes, filter, handleFileNotFound, sort } of fileConfig) { const files = allFiles .filter((f) => prefixes(userData).some((p) => f.path.startsWith(p))) .filter((f) => !fileTypes || fileTypes.some((t) => f.contentType.startsWith(t))) .filter((f) => !filter || filter(f, userData)); if (!files.length) { - if (handleFileNotFound && handleFileNotFound(subFolder, userData)) continue; - errorLog += `Error: File missing for folder '${folderName}' for UserData ${userDataId}\n`; + if (handleFileNotFound) { + // Evaluate handleFileNotFound with a temp folder to check if it would handle the case + const tempFolder = checkOnly ? new JSZip().folder('temp') : subFolder; + if (handleFileNotFound(tempFolder, userData)) continue; + } + errors.push({ userDataId, errorType: 'FileMissing', folder: folderName, details: '' }); continue; } + if (checkOnly) continue; + const selectedFile = files.reduce((l, c) => (sort ? sort(l, c) : l.updated > c.updated ? l : c)); try { @@ -391,16 +422,21 @@ export class UserDataService { selectedFile.type, selectedFile.name, ); - const filePath = `${userDataId}-${fileName?.(selectedFile) ?? name}.${selectedFile.name.split('.').pop()}`; + const filePath = `${userDataId}-${fileNameFn?.(selectedFile) ?? name}.${selectedFile.name.split('.').pop()}`; subFolder.file(filePath, fileData.data); } catch { - errorLog += `Error: Failed to download file '${selectedFile.name}' for UserData ${userDataId}\n`; + errors.push({ userDataId, errorType: 'DownloadFailed', folder: folderName, details: selectedFile.name }); } } } } - if (errorLog) zip.file('error_log.txt', errorLog); + const csvHeader = 'UserDataId;ErrorType;Folder;Details'; + const csvRows = errors.map( + (e) => `${e.userDataId};${e.errorType};${escapeCsvValue(e.folder)};${escapeCsvValue(e.details)}`, + ); + const csvContent = errors.length > 0 ? [csvHeader, ...csvRows].join('\n') : 'No_Error'; + zip.file('error_log.csv', csvContent); return zip.generateAsync({ type: 'nodebuffer' }); } @@ -1056,6 +1092,7 @@ export class UserDataService { .catch((e) => this.logger.critical(`Error in document copy files for master ${master.id}:`, e)); // optional master updates + if (master.status === UserDataStatus.KYC_ONLY && slave.users.length && slave.wallet) master.wallet = slave.wallet; if ([UserDataStatus.KYC_ONLY, UserDataStatus.DEACTIVATED].includes(master.status)) master.status = slave.status; if (!master.amlListAddedDate && slave.amlListAddedDate) { master.amlListAddedDate = slave.amlListAddedDate; @@ -1071,6 +1108,11 @@ export class UserDataService { } if (!master.verifiedName && slave.verifiedName) master.verifiedName = slave.verifiedName; master.mail = mail ?? slave.mail ?? master.mail; + if (!master.tradeApprovalDate && slave.tradeApprovalDate) master.tradeApprovalDate = slave.tradeApprovalDate; + + const pendingRecommendation = master.kycSteps.find((k) => k.name === KycStepName.RECOMMENDATION && !k.isDone); + if (master.tradeApprovalDate && pendingRecommendation) + await this.kycAdminService.updateKycStepInternal(pendingRecommendation.update(ReviewStatus.COMPLETED)); // Adapt user used refs for (const user of master.users) { diff --git a/src/subdomains/generic/user/models/user/dto/download-user-data.dto.ts b/src/subdomains/generic/user/models/user/dto/download-user-data.dto.ts index 980448b394..952c5c583e 100644 --- a/src/subdomains/generic/user/models/user/dto/download-user-data.dto.ts +++ b/src/subdomains/generic/user/models/user/dto/download-user-data.dto.ts @@ -1,9 +1,14 @@ -import { ApiProperty } from '@nestjs/swagger'; -import { IsArray, IsNotEmpty } from 'class-validator'; +import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; +import { IsArray, IsBoolean, IsNotEmpty, IsOptional } from 'class-validator'; export class DownloadUserDataDto { @ApiProperty() @IsNotEmpty() @IsArray() userDataIds: number[]; + + @ApiPropertyOptional() + @IsOptional() + @IsBoolean() + checkOnly?: boolean; } diff --git a/src/subdomains/supporting/payout/services/payout-evm.service.ts b/src/subdomains/supporting/payout/services/payout-evm.service.ts index efe50ef50f..d0c80227c6 100644 --- a/src/subdomains/supporting/payout/services/payout-evm.service.ts +++ b/src/subdomains/supporting/payout/services/payout-evm.service.ts @@ -37,10 +37,15 @@ export abstract class PayoutEvmService { } async isTxExpired(txHash: string): Promise { - const receipt = await this.client.getTxReceipt(txHash); - if (receipt) return false; // TX was mined (success or fail) + if (!(await this.isRpcSynced())) return false; const tx = await this.client.getTx(txHash); - return tx === null; // TX does not exist anymore -> expired + return tx === null; + } + + private async isRpcSynced(maxAgeSeconds = 300): Promise { + const blockTimestamp = await this.client.getLatestBlockTimestamp(); + const blockAge = Date.now() / 1000 - blockTimestamp; + return blockAge < maxAgeSeconds; } } diff --git a/src/subdomains/supporting/payout/strategies/payout/impl/base/evm.strategy.ts b/src/subdomains/supporting/payout/strategies/payout/impl/base/evm.strategy.ts index 291659cd06..4a6e9f4fe0 100644 --- a/src/subdomains/supporting/payout/strategies/payout/impl/base/evm.strategy.ts +++ b/src/subdomains/supporting/payout/strategies/payout/impl/base/evm.strategy.ts @@ -3,6 +3,7 @@ import { Asset } from 'src/shared/models/asset/asset.entity'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { DisabledProcess, Process } from 'src/shared/services/process.service'; import { AsyncCache, CacheItemResetPeriod } from 'src/shared/utils/async-cache'; +import { Util } from 'src/shared/utils/util'; import { FeeResult } from 'src/subdomains/supporting/payout/interfaces'; import { PriceCurrency, PriceValidity } from 'src/subdomains/supporting/pricing/services/pricing.service'; import { PayoutOrder } from '../../../../entities/payout-order.entity'; @@ -82,9 +83,11 @@ export abstract class EvmStrategy extends PayoutStrategy { } } - // Whitelisted failure type: Flashbots expired transactions (TX does not exist on-chain) override async canRetryFailedPayout(order: PayoutOrder): Promise { if (!order.payoutTxId) return false; + + if (Util.hoursDiff(order.updated) < 1) return false; + return this.payoutEvmService.isTxExpired(order.payoutTxId); } }