feat: add DataqueryModule — company lookup + sanctions screening
Adds /v1/dataquery/* endpoints proxying to dataquery-api (ClickHouse) and Yente/OpenSanctions (Hetzner): POST /v1/dataquery/company/lookup — search RO/DE/UK/US registries POST /v1/dataquery/sanctions — OFAC SDN + OpenSanctions (HIT/CLEAR) GET /v1/dataquery/:group/schema — ClickHouse group schema POST /v1/dataquery/:group/query — raw SELECT within group scope SQL injection prevention via sanitize() + dataquery-api table-scope enforcement. Timeouts: 12s per upstream call. Promise.allSettled for parallel fan-out.
This commit is contained in:
parent
2af9b5c94a
commit
d13bd5d0de
5 changed files with 269 additions and 0 deletions
|
|
@ -26,6 +26,7 @@ import { IntelligenceModule } from './intelligence/intelligence.module';
|
|||
import { SegmentsModule } from './segments/segments.module';
|
||||
import { ResearchBriefsModule } from './research-briefs/research-briefs.module';
|
||||
import { BriefingModule } from './briefing/briefing.module';
|
||||
import { DataqueryModule } from './dataquery/dataquery.module';
|
||||
import { SessionGuard } from './auth/session.guard';
|
||||
import { TenantGuard } from './auth/tenant.guard';
|
||||
|
||||
|
|
@ -68,6 +69,7 @@ function parseRedisConnection(redisUrl: string | undefined) {
|
|||
SegmentsModule,
|
||||
ResearchBriefsModule,
|
||||
BriefingModule,
|
||||
DataqueryModule,
|
||||
],
|
||||
controllers: [HealthController],
|
||||
providers: [
|
||||
|
|
|
|||
39
src/dataquery/dataquery.controller.ts
Normal file
39
src/dataquery/dataquery.controller.ts
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
import { Body, Controller, Get, Param, Post } from '@nestjs/common';
|
||||
import { CompanyLookupDto, RawQueryDto, SanctionsCheckDto } from './dataquery.dto';
|
||||
import { DataqueryService } from './dataquery.service';
|
||||
|
||||
/**
|
||||
* Proxy catre dataquery-api (ClickHouse grupe pe Netcup)
|
||||
* si Yente/OpenSanctions (Hetzner).
|
||||
*
|
||||
* Toate rutele necesita sesiune si tenant activ — fara @Public().
|
||||
* Prefix global v1 → rutele sunt /v1/dataquery/...
|
||||
*/
|
||||
@Controller('dataquery')
|
||||
export class DataqueryController {
|
||||
constructor(private readonly svc: DataqueryService) {}
|
||||
|
||||
/** Cauta compania in registrele nationale. country: ro|de|uk|us|all (default all). */
|
||||
@Post('company/lookup')
|
||||
companyLookup(@Body() dto: CompanyLookupDto) {
|
||||
return this.svc.companyLookup(dto.name, dto.country ?? 'all');
|
||||
}
|
||||
|
||||
/** Screening sanctiuni OFAC + OpenSanctions. Returneaza verdict: HIT | CLEAR. */
|
||||
@Post('sanctions')
|
||||
sanctionsCheck(@Body() dto: SanctionsCheckDto) {
|
||||
return this.svc.sanctionsCheck(dto.name);
|
||||
}
|
||||
|
||||
/** Schema (coloane + randuri) pentru orice grupa ClickHouse. */
|
||||
@Get(':group/schema')
|
||||
schema(@Param('group') group: string) {
|
||||
return this.svc.groupSchema(group);
|
||||
}
|
||||
|
||||
/** Query SELECT arbitrar intr-o grupa (scoped la tabelele grupei). */
|
||||
@Post(':group/query')
|
||||
query(@Param('group') group: string, @Body() dto: RawQueryDto) {
|
||||
return this.svc.groupQuery(group, dto.sql);
|
||||
}
|
||||
}
|
||||
28
src/dataquery/dataquery.dto.ts
Normal file
28
src/dataquery/dataquery.dto.ts
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
import { IsIn, IsNotEmpty, IsOptional, IsString, MaxLength, MinLength } from 'class-validator';
|
||||
|
||||
export class CompanyLookupDto {
|
||||
@IsString()
|
||||
@IsNotEmpty()
|
||||
@MinLength(2)
|
||||
@MaxLength(200)
|
||||
name: string;
|
||||
|
||||
@IsOptional()
|
||||
@IsIn(['ro', 'de', 'uk', 'us', 'all'])
|
||||
country?: 'ro' | 'de' | 'uk' | 'us' | 'all';
|
||||
}
|
||||
|
||||
export class SanctionsCheckDto {
|
||||
@IsString()
|
||||
@IsNotEmpty()
|
||||
@MinLength(2)
|
||||
@MaxLength(300)
|
||||
name: string;
|
||||
}
|
||||
|
||||
export class RawQueryDto {
|
||||
@IsString()
|
||||
@IsNotEmpty()
|
||||
@MaxLength(2000)
|
||||
sql: string;
|
||||
}
|
||||
10
src/dataquery/dataquery.module.ts
Normal file
10
src/dataquery/dataquery.module.ts
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
import { Module } from '@nestjs/common';
|
||||
import { DataqueryController } from './dataquery.controller';
|
||||
import { DataqueryService } from './dataquery.service';
|
||||
|
||||
@Module({
|
||||
controllers: [DataqueryController],
|
||||
providers: [DataqueryService],
|
||||
exports: [DataqueryService],
|
||||
})
|
||||
export class DataqueryModule {}
|
||||
190
src/dataquery/dataquery.service.ts
Normal file
190
src/dataquery/dataquery.service.ts
Normal file
|
|
@ -0,0 +1,190 @@
|
|||
import { Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
const TIMEOUT_MS = 12_000;
|
||||
|
||||
/** Sanitize user input for LIKE queries — keeps characters valid in company names. */
|
||||
function sanitize(raw: string): string {
|
||||
return raw.replace(/[;\\]/g, '').replace(/'/g, "''").slice(0, 200);
|
||||
}
|
||||
|
||||
const COUNTRY_CONFIGS = {
|
||||
ro: {
|
||||
group: 'romania',
|
||||
label: 'România (ONRC)',
|
||||
sql: (name: string) =>
|
||||
`SELECT DENUMIRE, CUI, COD_INMATRICULARE, DATA_INMATRICULARE, FORMA_JURIDICA, ADR_JUDET, ADR_LOCALITATE, ADR_DEN_STRADA, WEB FROM romania_onrc_firme_raw WHERE lower(DENUMIRE) LIKE lower('%${sanitize(name)}%') LIMIT 25`,
|
||||
},
|
||||
de: {
|
||||
group: 'germania',
|
||||
label: 'Germania (Handelsregister)',
|
||||
sql: (name: string) =>
|
||||
`SELECT companyId, name, foundedDate, dissolutionDate, fullAddress, capitalAmount, capitalCurrency, objective, courtName FROM germany_handelsregister_companies_current WHERE lower(name) LIKE lower('%${sanitize(name)}%') LIMIT 25`,
|
||||
},
|
||||
uk: {
|
||||
group: 'uk_export',
|
||||
label: 'UK (Companies House)',
|
||||
sql: (name: string) =>
|
||||
`SELECT CompanyName, CompanyNumber, CompanyStatus, CompanyCategory, IncorporationDate, DissolutionDate, RegAddress_AddressLine1, RegAddress_PostTown, RegAddress_Country FROM companies_house_uk_raw WHERE lower(CompanyName) LIKE lower('%${sanitize(name)}%') LIMIT 25`,
|
||||
},
|
||||
us: {
|
||||
group: 'us',
|
||||
label: 'SUA',
|
||||
sql: (name: string) =>
|
||||
`SELECT BUSINESS_NAME, MAILING_CITY, MAILING_STATE, SALES_VOLUME, NUMBER_OF_EMPLOYEES, PUBLIC_PRIVATE_COMPANY FROM us_business_data_raw WHERE lower(BUSINESS_NAME) LIKE lower('%${sanitize(name)}%') LIMIT 25`,
|
||||
},
|
||||
} as const;
|
||||
|
||||
type CountryKey = keyof typeof COUNTRY_CONFIGS;
|
||||
|
||||
@Injectable()
|
||||
export class DataqueryService {
|
||||
private readonly logger = new Logger(DataqueryService.name);
|
||||
|
||||
private readonly dqBase =
|
||||
process.env.DATAQUERY_API_URL ?? 'http://152.53.112.35:9420';
|
||||
|
||||
private readonly yenteBase =
|
||||
process.env.YENTE_BASE_URL ?? 'https://yente.91.98.39.120.sslip.io';
|
||||
|
||||
// ── Company lookup across national registries ─────────────────────────────
|
||||
async companyLookup(name: string, country: CountryKey | 'all' = 'all') {
|
||||
const targets: CountryKey[] =
|
||||
country === 'all'
|
||||
? (Object.keys(COUNTRY_CONFIGS) as CountryKey[])
|
||||
: [country];
|
||||
|
||||
const settled = await Promise.allSettled(
|
||||
targets.map(async (c) => {
|
||||
const cfg = COUNTRY_CONFIGS[c];
|
||||
const data = await this.dqPost(`${cfg.group}/query`, { sql: cfg.sql(name) });
|
||||
return {
|
||||
country: c,
|
||||
label: cfg.label,
|
||||
count: (data.row_count as number) ?? 0,
|
||||
columns: (data.columns as string[]) ?? [],
|
||||
rows: (data.rows as unknown[][]) ?? [],
|
||||
};
|
||||
}),
|
||||
);
|
||||
|
||||
const results = settled
|
||||
.filter((r): r is PromiseFulfilledResult<ReturnType<typeof Object.assign>> => r.status === 'fulfilled')
|
||||
.map((r) => r.value);
|
||||
|
||||
return {
|
||||
query: name,
|
||||
results,
|
||||
errors: settled
|
||||
.filter((r): r is PromiseRejectedResult => r.status === 'rejected')
|
||||
.map((r) => String(r.reason)),
|
||||
total_hits: results.reduce((sum, r) => sum + r.count, 0),
|
||||
};
|
||||
}
|
||||
|
||||
// ── Sanctions screening: OFAC SDN + Yente OpenSanctions ──────────────────
|
||||
async sanctionsCheck(name: string) {
|
||||
const safe = sanitize(name);
|
||||
const ofacSql = `SELECT ent_num, sdn_name, sdn_type, program, title, remarks FROM ofac_sdn_raw WHERE lower(sdn_name) LIKE lower('%${safe}%') LIMIT 15`;
|
||||
|
||||
const [ofacResult, yenteResult] = await Promise.allSettled([
|
||||
this.dqPost('finantare_risc/query', { sql: ofacSql }),
|
||||
this.yenteSearch(name),
|
||||
]);
|
||||
|
||||
const ofacRows =
|
||||
ofacResult.status === 'fulfilled' ? ((ofacResult.value.rows as unknown[][]) ?? []) : [];
|
||||
const ofacCols =
|
||||
ofacResult.status === 'fulfilled' ? ((ofacResult.value.columns as string[]) ?? []) : [];
|
||||
const yenteResults =
|
||||
yenteResult.status === 'fulfilled' ? ((yenteResult.value.results as unknown[]) ?? []) : [];
|
||||
|
||||
const verdict = ofacRows.length > 0 || yenteResults.length > 0 ? 'HIT' : 'CLEAR';
|
||||
|
||||
return {
|
||||
verdict,
|
||||
name,
|
||||
ofac: {
|
||||
source: 'OFAC SDN List',
|
||||
hits: ofacRows.length,
|
||||
columns: ofacCols,
|
||||
rows: ofacRows,
|
||||
error: ofacResult.status === 'rejected' ? String(ofacResult.reason) : null,
|
||||
},
|
||||
yente: {
|
||||
source: 'OpenSanctions (Yente)',
|
||||
hits: yenteResults.length,
|
||||
results: yenteResults.slice(0, 10).map((r) => {
|
||||
const hit = r as Record<string, unknown>;
|
||||
return {
|
||||
id: hit['id'],
|
||||
caption: hit['caption'],
|
||||
schema: hit['schema'],
|
||||
datasets: hit['datasets'],
|
||||
score: hit['score'],
|
||||
};
|
||||
}),
|
||||
error: yenteResult.status === 'rejected' ? String(yenteResult.reason) : null,
|
||||
},
|
||||
checked_at: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
// ── Schema proxy ──────────────────────────────────────────────────────────
|
||||
async groupSchema(group: string) {
|
||||
return this.dqGet(`${group}/schema`);
|
||||
}
|
||||
|
||||
// ── Raw query proxy ───────────────────────────────────────────────────────
|
||||
async groupQuery(group: string, sql: string) {
|
||||
return this.dqPost(`${group}/query`, { sql });
|
||||
}
|
||||
|
||||
// ── HTTP helpers ──────────────────────────────────────────────────────────
|
||||
private async dqGet(path: string) {
|
||||
return this.dqFetch(path, 'GET');
|
||||
}
|
||||
|
||||
private async dqPost(path: string, body: unknown) {
|
||||
return this.dqFetch(path, 'POST', body);
|
||||
}
|
||||
|
||||
private async dqFetch(
|
||||
path: string,
|
||||
method: 'GET' | 'POST',
|
||||
body?: unknown,
|
||||
): Promise<Record<string, unknown>> {
|
||||
const ctrl = new AbortController();
|
||||
const t = setTimeout(() => ctrl.abort(), TIMEOUT_MS);
|
||||
try {
|
||||
const res = await fetch(`${this.dqBase}/${path}`, {
|
||||
method,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: body !== undefined ? JSON.stringify(body) : undefined,
|
||||
signal: ctrl.signal,
|
||||
});
|
||||
if (!res.ok) {
|
||||
const text = await res.text();
|
||||
this.logger.warn(`dataquery ${path} → ${res.status}: ${text.slice(0, 200)}`);
|
||||
throw new Error(`dataquery-api ${res.status}: ${text.slice(0, 200)}`);
|
||||
}
|
||||
return res.json() as Promise<Record<string, unknown>>;
|
||||
} finally {
|
||||
clearTimeout(t);
|
||||
}
|
||||
}
|
||||
|
||||
private async yenteSearch(name: string): Promise<{ results: unknown[] }> {
|
||||
const ctrl = new AbortController();
|
||||
const t = setTimeout(() => ctrl.abort(), TIMEOUT_MS);
|
||||
try {
|
||||
const res = await fetch(
|
||||
`${this.yenteBase}/search/sanctions?q=${encodeURIComponent(name)}&limit=15`,
|
||||
{ signal: ctrl.signal },
|
||||
);
|
||||
if (!res.ok) throw new Error(`yente ${res.status}`);
|
||||
return res.json() as Promise<{ results: unknown[] }>;
|
||||
} finally {
|
||||
clearTimeout(t);
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue