import { createHash } from "node:crypto"; import * as XLSX from "xlsx"; import type { AssociationDirectoryEntry, InsertAssociation } from "../drizzle/schema"; import { TRPCError } from "@trpc/server"; import { normalizeLegalRepresentativeRole, serializeAssociationGovernance } from "@shared/associationGovernance"; type ParsedDirectoryRow = { sheetName: string; rowNumber: number; nomAssociation: string; emailOfficiel: string | null; emailOfficielNormalise: string | null; siret: string | null; rna: string | null; adresse: string | null; codePostal: string | null; ville: string | null; telephone: string | null; siteWeb: string | null; facebookUrl: string | null; instagramUrl: string | null; dateCreation: Date | null; objetAssociation: string | null; statutJuridique: "association_loi_1901" | "association_reconnue_utilite_publique" | "fondation" | "autre"; nomRepresentant: string | null; fonctionRepresentant: string | null; sourceFingerprint: string; }; type PreviewRow = { sheetName: string; rowNumber: number; nomAssociation: string; emailOfficiel: string | null; ville: string | null; status: "valid" | "missing_email" | "duplicate_email"; message: string; }; export type DuplicateEmailGroup = { emailOfficielNormalise: string; emailOfficiel: string; rowNumbers: number[]; options: Array<{ sheetName: string; rowNumber: number; nomAssociation: string; ville: string | null; }>; }; export type DirectoryPreviewResult = { fileName: string; totalRows: number; validRows: number; missingEmailRows: number; duplicateEmailRows: number; previewRows: PreviewRow[]; duplicateGroups: DuplicateEmailGroup[]; allRows: ParsedDirectoryRow[]; importableRows: ParsedDirectoryRow[]; }; const headerAliases: Record = { nomAssociation: ["nom association", "association", "nom", "raison sociale", "nom de la structure"], emailOfficiel: ["email officiel", "email", "mail", "courriel", "adresse email"], siret: ["siret", "numéro siret", "numero siret"], rna: ["rna", "numéro rna", "numero rna"], adresse: ["adresse", "adresse siège", "adresse siege", "adresse du siège"], codePostal: ["code postal", "cp"], ville: ["ville", "commune"], telephone: ["telephone", "téléphone", "tel", "tél", "tel.", "port.", "port"], siteWeb: ["site web", "site", "website", "url site"], facebookUrl: ["facebook", "facebook url", "facebook link", "lien facebook", "url facebook"], instagramUrl: ["instagram", "instagram url", "instagram link", "lien instagram", "url instagram"], dateCreation: ["date création", "date creation", "creation", "date de création"], objetAssociation: ["objet", "objet association", "activité", "activités", "activite"], statutJuridique: ["statut", "statut juridique"], nomRepresentant: ["nom représentant", "nom representant", "président", "president", "responsable", "president"], fonctionRepresentant: ["fonction représentant", "fonction representant", "fonction", "qualité", "qualite", "secretaire", "secrétaire"], }; function normalizeHeader(value: string) { return value .normalize("NFD") .replace(/[\u0300-\u036f]/g, "") .trim() .toLowerCase(); } function cleanString(value: unknown) { if (value === null || value === undefined) return null; const text = String(value).trim(); return text ? text : null; } function extractFirstEmail(value: unknown) { const text = cleanString(value); if (!text) return null; const match = text.match(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/i); return match?.[0] ?? null; } function normalizeEmail(value: string | null) { return value ? value.trim().toLowerCase() : null; } function normalizeSiret(value: string | null) { return value ? value.replace(/\D/g, "") || null : null; } function normalizeRna(value: string | null) { return value ? value.replace(/\s+/g, "").toUpperCase() : null; } function normalizeSiteWeb(value: string | null) { if (!value) return null; if (/^https?:\/\//i.test(value)) return value; return `https://${value}`; } function normalizeTelephone(value: string | null) { if (!value) return null; const first = value .split(/[\/;,]/) .map((part) => part.trim()) .find(Boolean); return first ? first.slice(0, 20) : null; } function normalizeDate(value: unknown) { if (value === null || value === undefined || value === "") return null; if (value instanceof Date && !Number.isNaN(value.getTime())) { return value; } if (typeof value === "number") { const parsed = XLSX.SSF.parse_date_code(value); if (parsed) { return new Date(Date.UTC(parsed.y, parsed.m - 1, parsed.d)); } } const parsed = new Date(String(value)); if (Number.isNaN(parsed.getTime())) return null; return parsed; } function normalizeStatut(value: string | null): ParsedDirectoryRow["statutJuridique"] { if (!value) return "association_loi_1901"; const normalized = normalizeHeader(value); if (normalized.includes("utilite publique")) return "association_reconnue_utilite_publique"; if (normalized.includes("fondation")) return "fondation"; if (normalized.includes("1901") || normalized.includes("association")) return "association_loi_1901"; return "autre"; } function computeFingerprint(row: Omit) { return createHash("sha256").update(JSON.stringify(row)).digest("hex"); } function getCommuneFromSheetName(sheetName: string) { const normalized = normalizeHeader(sheetName).replace(/[-_]/g, " "); if (normalized.includes("kourou")) return "Kourou"; if (normalized.includes("sinnamary")) return "Sinnamary"; if (normalized.includes("iracoubo")) return "Iracoubo"; if (normalized.includes("st elie") || normalized.includes("saint elie")) return "Saint-Élie"; return null; } function resolveColumnIndex(headers: string[], field: keyof typeof headerAliases) { const aliases = headerAliases[field]; return headers.findIndex(header => aliases.includes(normalizeHeader(header))); } function readCell(row: unknown[], headers: string[], field: keyof typeof headerAliases) { const index = resolveColumnIndex(headers, field); if (index === -1) return null; return row[index]; } function resolveEmailValue(row: unknown[], headers: string[]) { const direct = extractFirstEmail(readCell(row, headers, "emailOfficiel")); if (direct) return direct; for (const cell of row) { const extracted = extractFirstEmail(cell); if (extracted) return extracted; } return null; } export function parseAssociationDirectoryWorkbook(fileBuffer: Buffer, fileName: string): DirectoryPreviewResult { const workbook = XLSX.read(fileBuffer, { type: "buffer", cellDates: true }); if (workbook.SheetNames.length === 0) { throw new TRPCError({ code: "BAD_REQUEST", message: "Le fichier Excel ne contient aucune feuille exploitable" }); } const importableRows: ParsedDirectoryRow[] = []; const previewRows: PreviewRow[] = []; const emailRowMap = new Map(); let hasAnyUsableSheet = false; workbook.SheetNames.forEach((sheetName) => { const sheet = workbook.Sheets[sheetName]; if (!sheet) return; const rows = XLSX.utils.sheet_to_json(sheet, { header: 1, defval: null }); if (rows.length < 2) return; const headers = (rows[0] || []).map(value => String(value ?? "")); if (resolveColumnIndex(headers, "nomAssociation") === -1) return; hasAnyUsableSheet = true; const communeFromSheet = getCommuneFromSheetName(sheetName); rows.slice(1).forEach((rawRow, index) => { const rowNumber = index + 2; const nomAssociation = cleanString(readCell(rawRow, headers, "nomAssociation")); if (!nomAssociation) { return; } const emailOfficiel = resolveEmailValue(rawRow, headers); const emailOfficielNormalise = normalizeEmail(emailOfficiel); const rawAdresse = cleanString(readCell(rawRow, headers, "adresse")); const parsedRowBase = { sheetName, rowNumber, nomAssociation, emailOfficiel, emailOfficielNormalise, siret: normalizeSiret(cleanString(readCell(rawRow, headers, "siret"))), rna: normalizeRna(cleanString(readCell(rawRow, headers, "rna"))), adresse: rawAdresse && extractFirstEmail(rawAdresse) ? null : rawAdresse, codePostal: cleanString(readCell(rawRow, headers, "codePostal")), ville: communeFromSheet || cleanString(readCell(rawRow, headers, "ville")), telephone: normalizeTelephone(cleanString(readCell(rawRow, headers, "telephone"))), siteWeb: normalizeSiteWeb(cleanString(readCell(rawRow, headers, "siteWeb"))), facebookUrl: normalizeSiteWeb(cleanString(readCell(rawRow, headers, "facebookUrl"))), instagramUrl: normalizeSiteWeb(cleanString(readCell(rawRow, headers, "instagramUrl"))), dateCreation: normalizeDate(readCell(rawRow, headers, "dateCreation")), objetAssociation: cleanString(readCell(rawRow, headers, "objetAssociation")), statutJuridique: normalizeStatut(cleanString(readCell(rawRow, headers, "statutJuridique"))), nomRepresentant: cleanString(readCell(rawRow, headers, "nomRepresentant")), fonctionRepresentant: cleanString(readCell(rawRow, headers, "fonctionRepresentant")), }; const parsedRow: ParsedDirectoryRow = { ...parsedRowBase, sourceFingerprint: computeFingerprint(parsedRowBase), }; importableRows.push(parsedRow); if (emailOfficielNormalise) { const refs = emailRowMap.get(emailOfficielNormalise) || []; refs.push(rowNumber); emailRowMap.set(emailOfficielNormalise, refs); } }); }); if (!hasAnyUsableSheet || importableRows.length === 0) { throw new TRPCError({ code: "BAD_REQUEST", message: "Le fichier Excel ne contient pas de feuille exploitable avec une colonne de nom d'association", }); } importableRows.forEach(row => { const duplicateRows = row.emailOfficielNormalise ? emailRowMap.get(row.emailOfficielNormalise) || [] : []; if (!row.emailOfficielNormalise) { previewRows.push({ sheetName: row.sheetName, rowNumber: row.rowNumber, nomAssociation: row.nomAssociation, emailOfficiel: row.emailOfficiel, ville: row.ville, status: "missing_email", message: `Feuille ${row.sheetName} : email officiel manquant, la ligne ne pourra pas être rattachée automatiquement`, }); return; } if (duplicateRows.length > 1) { previewRows.push({ sheetName: row.sheetName, rowNumber: row.rowNumber, nomAssociation: row.nomAssociation, emailOfficiel: row.emailOfficiel, ville: row.ville, status: "duplicate_email", message: `Feuille ${row.sheetName} : email dupliqué dans le fichier (lignes ${duplicateRows.join(", ")})`, }); return; } previewRows.push({ sheetName: row.sheetName, rowNumber: row.rowNumber, nomAssociation: row.nomAssociation, emailOfficiel: row.emailOfficiel, ville: row.ville, status: "valid", message: `Feuille ${row.sheetName} : ligne prête à être importée`, }); }); const importableRowKeys = new Set( previewRows .filter(row => row.status !== "duplicate_email") .map(row => `${row.rowNumber}::${row.nomAssociation}`) ); const duplicateGroups: DuplicateEmailGroup[] = Array.from(emailRowMap.entries()) .filter(([, rowNumbers]) => rowNumbers.length > 1) .map(([emailOfficielNormalise, rowNumbers]) => { const options = importableRows .filter(row => row.emailOfficielNormalise === emailOfficielNormalise) .map(row => ({ sheetName: row.sheetName, rowNumber: row.rowNumber, nomAssociation: row.nomAssociation, ville: row.ville, })); return { emailOfficielNormalise, emailOfficiel: options.length > 0 ? importableRows.find(row => row.emailOfficielNormalise === emailOfficielNormalise)?.emailOfficiel || emailOfficielNormalise : emailOfficielNormalise, rowNumbers, options, }; }); return { fileName, totalRows: importableRows.length, validRows: previewRows.filter(row => row.status === "valid").length, missingEmailRows: previewRows.filter(row => row.status === "missing_email").length, duplicateEmailRows: previewRows.filter(row => row.status === "duplicate_email").length, previewRows, duplicateGroups, allRows: importableRows, importableRows: importableRows.filter(row => importableRowKeys.has(`${row.rowNumber}::${row.nomAssociation}`)), }; } export function resolveAssociationDirectoryImportRows( preview: DirectoryPreviewResult, duplicateSelections?: Record, ) { const selectedDuplicateKeys = new Set(); Object.entries(duplicateSelections || {}).forEach(([email, rowNumber]) => { const numericRow = Number(rowNumber); if (Number.isFinite(numericRow) && numericRow > 0) { selectedDuplicateKeys.add(`${email}::${numericRow}`); } }); return preview.allRows.filter((row) => { if (!row.emailOfficielNormalise) { return true; } const isDuplicate = preview.duplicateGroups.some(group => group.emailOfficielNormalise === row.emailOfficielNormalise); if (!isDuplicate) { return true; } return selectedDuplicateKeys.has(`${row.emailOfficielNormalise}::${row.rowNumber}`); }); } export function createAssociationProfileFromDirectoryEntry(userId: number, entry: AssociationDirectoryEntry): InsertAssociation { const governance = entry.nomRepresentant || entry.fonctionRepresentant ? serializeAssociationGovernance({ representantLegal: { nom: entry.nomRepresentant || "", prenom: "", email: "", telephone: "", fonction: normalizeLegalRepresentativeRole(entry.fonctionRepresentant), }, membres: [], }) : null; return { userId, sourceDirectoryEntryId: entry.id, nomAssociation: entry.nomAssociation, siret: entry.siret ?? null, rna: entry.rna ?? null, thematique: entry.thematique ?? null, adresse: entry.adresse ?? null, codePostal: entry.codePostal ?? null, ville: entry.ville ?? null, telephone: entry.telephone ?? null, emailContact: entry.emailOfficiel ?? null, siteWeb: entry.siteWeb ?? null, facebookUrl: entry.facebookUrl ?? null, instagramUrl: entry.instagramUrl ?? null, dateCreation: entry.dateCreation ?? null, objetAssociation: entry.objetAssociation ?? null, statutJuridique: entry.statutJuridique ?? "association_loi_1901", nomRepresentant: entry.nomRepresentant ?? null, fonctionRepresentant: entry.fonctionRepresentant ?? null, gouvernance: governance, profileComplete: Boolean(entry.nomAssociation && entry.adresse && entry.ville), isActive: true, }; }