diff --git a/frontend/src/api/transactions.ts b/frontend/src/api/transactions.ts index 42bd59d..2a0bac2 100644 --- a/frontend/src/api/transactions.ts +++ b/frontend/src/api/transactions.ts @@ -26,6 +26,25 @@ export function fetchTransactions(filters: TransactionFilters = {}) { return api.get(`/transactions?${params.toString()}`); } +// Used by CSV import's duplicate check (see pages/CsvImportPage.tsx) — needs +// every transaction in the imported date range for one account, not just a +// page of them, to know what's already there. Capped at 25 pages (5000 rows +// at the API's max page_size) so a huge date range can't turn one import +// into an unbounded fetch loop; beyond that, duplicate detection just +// silently covers less of the range instead of hanging. +const MAX_DUPLICATE_CHECK_PAGES = 25; + +export async function fetchAllTransactionsInRange(filters: TransactionFilters): Promise { + const pageSize = 200; + const items: Transaction[] = []; + for (let page = 1; page <= MAX_DUPLICATE_CHECK_PAGES; page++) { + const result = await fetchTransactions({ ...filters, page, page_size: pageSize }); + items.push(...result.items); + if (items.length >= result.total || result.items.length < pageSize) break; + } + return items; +} + /** Full range of years to offer in the year picker, from the earliest * transaction through the current year (see backend for the "gap year" * rationale). */ diff --git a/frontend/src/hooks/useTransactions.ts b/frontend/src/hooks/useTransactions.ts index 6451574..86d5add 100644 --- a/frontend/src/hooks/useTransactions.ts +++ b/frontend/src/hooks/useTransactions.ts @@ -3,6 +3,7 @@ import { bulkCreateTransactions, createTransaction, deleteTransaction, + fetchAllTransactionsInRange, fetchTransactions, fetchTransactionYears, updateTransaction, @@ -33,6 +34,21 @@ export function useTransactions(filters: TransactionFilters) { }); } +/** CSV import's duplicate check (see pages/CsvImportPage.tsx) — every + * existing transaction for one account within the imported date range, so + * re-importing an overlapping bank export can be told apart from new rows. */ +export function useTransactionsForDuplicateCheck( + accountId: number | null, + startDate: string | null, + endDate: string | null +) { + return useQuery({ + queryKey: ["transactions-duplicate-check", accountId, startDate, endDate], + queryFn: () => fetchAllTransactionsInRange({ account_id: accountId!, start_date: startDate!, end_date: endDate! }), + enabled: accountId !== null && startDate !== null && endDate !== null, + }); +} + export function useTransactionYears() { return useQuery({ queryKey: ["transaction-years"], diff --git a/frontend/src/lib/csv.ts b/frontend/src/lib/csv.ts index d039a0d..e6a66f6 100644 --- a/frontend/src/lib/csv.ts +++ b/frontend/src/lib/csv.ts @@ -86,21 +86,53 @@ export function parseDateWithFormat(raw: string, format: DateFormat): string | n return `${String(year).padStart(4, "0")}-${String(month).padStart(2, "0")}-${String(day).padStart(2, "0")}`; } -/** Parses a bank-export amount string — handles thousands separators and - * either "," or "." as the decimal mark by assuming whichever comes last is - * the decimal separator (e.g. "1.234,56" and "1,234.56" both work). */ -export function parseAmount(raw: string): number | null { +export const AMOUNT_FORMATS = ["auto", "dot-decimal", "comma-decimal"] as const; +export type AmountFormat = (typeof AMOUNT_FORMATS)[number]; + +/** Parses a bank-export amount string. `format` lets the user resolve what + * "auto" can't: a single "," or "." with nothing else in the string is + * genuinely ambiguous (thousands separator vs. decimal mark) — there's no + * way to tell "1,234" (one thousand two hundred thirty-four) from "1,234" + * (one point two three four) without knowing the exporting bank's locale. + * "auto" keeps the historical last-separator-wins heuristic for that case + * (e.g. "1.234,56" and "1,234.56" both resolve correctly since both + * separators are present), but no longer corrupts amounts that have + * *multiple* thousands separators and no decimal part at all (e.g. + * "1,234,567" or "1.234.567") — a number can have at most one decimal + * point, so 2+ occurrences of the same separator can only be thousands + * grouping. */ +export function parseAmount(raw: string, format: AmountFormat = "auto"): number | null { let value = raw.trim().replace(/[\s ]/g, ""); if (!value) return null; - const hasComma = value.includes(","); - const hasDot = value.includes("."); - if (hasComma && hasDot) { - value = value.lastIndexOf(",") > value.lastIndexOf(".") ? value.replace(/\./g, "").replace(",", ".") : value.replace(/,/g, ""); - } else if (hasComma) { - value = value.replace(",", "."); + if (format === "dot-decimal") { + value = value.replace(/,/g, ""); + } else if (format === "comma-decimal") { + value = value.replace(/\./g, "").replace(",", "."); + } else { + const commaCount = (value.match(/,/g) ?? []).length; + const dotCount = (value.match(/\./g) ?? []).length; + if (commaCount > 0 && dotCount > 0) { + value = value.lastIndexOf(",") > value.lastIndexOf(".") ? value.replace(/\./g, "").replace(",", ".") : value.replace(/,/g, ""); + } else if (commaCount > 1) { + value = value.replace(/,/g, ""); + } else if (commaCount === 1) { + value = value.replace(",", "."); + } else if (dotCount > 1) { + value = value.replace(/\./g, ""); + } } const num = Number(value); return Number.isFinite(num) ? num : null; } + +/** Stable key for spotting a re-imported bank export that overlaps a + * previous one — see pages/CsvImportPage.tsx's duplicate check. Matches on + * date + type + amount (2dp) + description, the same fields the API + * actually stores, so it works whether the "existing" side comes from a + * Transaction or the "candidate" side from a freshly parsed CSV row. */ +export function transactionDedupeKey(date: string, type: string, amount: number | string, description: string): string { + const normalizedAmount = typeof amount === "number" ? amount : Number(amount); + return `${date}|${type}|${normalizedAmount.toFixed(2)}|${description.trim().toLowerCase()}`; +} diff --git a/frontend/src/lib/i18n.ts b/frontend/src/lib/i18n.ts index cf3d72e..49d2a15 100644 --- a/frontend/src/lib/i18n.ts +++ b/frontend/src/lib/i18n.ts @@ -252,6 +252,16 @@ const ru = { "transactions.import.errorBadDate": "не удалось распознать дату «{{value}}»", "transactions.import.errorBadAmount": "не удалось распознать сумму «{{value}}»", "transactions.import.errorNoDescription": "нет описания", + "transactions.import.encodingLabel": "Кодировка файла", + "transactions.import.encodingHint": "Определяется автоматически; если вместо русских букв — кракозябры, попробуйте выбрать вручную.", + "transactions.import.sampleValueLabel": "Пример из файла: {{value}}", + "transactions.import.amountFormatLabel": "Разделитель дробной части", + "transactions.import.amountFormat.auto": "Автоматически", + "transactions.import.amountFormat.dot-decimal": "Точка (1234.56), запятая — тысячи", + "transactions.import.amountFormat.comma-decimal": "Запятая (1234,56), точка — тысячи", + "transactions.import.duplicateCheckLoading": "Проверяем, нет ли уже таких транзакций…", + "transactions.import.duplicatesFound": "Найдено похожих на уже существующие: {{count}}. Они не будут импортированы.", + "transactions.import.includeDuplicates": "Всё равно импортировать их", "transactions.allTypes": "Все типы", "transactions.expense": "Расходы", "transactions.income": "Доходы", @@ -759,6 +769,16 @@ const en: Record = { "transactions.import.errorBadDate": "couldn't parse the date \"{{value}}\"", "transactions.import.errorBadAmount": "couldn't parse the amount \"{{value}}\"", "transactions.import.errorNoDescription": "no description", + "transactions.import.encodingLabel": "File encoding", + "transactions.import.encodingHint": "Auto-detected; if you see garbled characters instead of letters, try picking one manually.", + "transactions.import.sampleValueLabel": "Example from file: {{value}}", + "transactions.import.amountFormatLabel": "Decimal separator", + "transactions.import.amountFormat.auto": "Automatic", + "transactions.import.amountFormat.dot-decimal": "Dot (1234.56), comma is thousands", + "transactions.import.amountFormat.comma-decimal": "Comma (1234,56), dot is thousands", + "transactions.import.duplicateCheckLoading": "Checking for transactions already on this account…", + "transactions.import.duplicatesFound": "Found {{count}} that look like duplicates of existing transactions. They won't be imported.", + "transactions.import.includeDuplicates": "Import them anyway", "transactions.allTypes": "All types", "transactions.expense": "Expenses", "transactions.income": "Income", diff --git a/frontend/src/pages/CsvImportPage.tsx b/frontend/src/pages/CsvImportPage.tsx index fcf96ee..8d20586 100644 --- a/frontend/src/pages/CsvImportPage.tsx +++ b/frontend/src/pages/CsvImportPage.tsx @@ -6,9 +6,18 @@ import { Button } from "@/components/ui/Button"; import { Label, Select } from "@/components/ui/Input"; import { useAccounts } from "@/hooks/useAccounts"; import { useCategories } from "@/hooks/useCategories"; -import { useBulkCreateTransactions } from "@/hooks/useTransactions"; +import { useBulkCreateTransactions, useTransactionsForDuplicateCheck } from "@/hooks/useTransactions"; import { translateCategoryName } from "@/lib/categoryLabels"; -import { DATE_FORMATS, parseAmount, parseCsv, parseDateWithFormat, type DateFormat } from "@/lib/csv"; +import { + AMOUNT_FORMATS, + DATE_FORMATS, + parseAmount, + parseCsv, + parseDateWithFormat, + transactionDedupeKey, + type AmountFormat, + type DateFormat, +} from "@/lib/csv"; import { formatCurrency } from "@/lib/format"; import { ApiError } from "@/api/client"; import { useTranslation } from "@/lib/i18n"; @@ -18,6 +27,14 @@ type Step = "upload" | "map" | "preview"; const NONE = ""; +// WHATWG encoding labels TextDecoder understands. UTF-8 covers most modern +// exports; the rest are here because bank CSVs — especially older or +// Russian-bank ones — are routinely still in a legacy Windows codepage, and +// decoding those as UTF-8 silently turns every non-ASCII character into +// mojibake instead of failing loudly. +const ENCODINGS = ["utf-8", "windows-1251", "windows-1252", "koi8-r"] as const; +type Encoding = (typeof ENCODINGS)[number]; + interface Mapping { date: string; amount: string; @@ -43,14 +60,28 @@ export function CsvImportPage() { const [accountId, setAccountId] = useState(""); const [step, setStep] = useState("upload"); const [fileName, setFileName] = useState(""); - const [headers, setHeaders] = useState([]); - const [dataRows, setDataRows] = useState([]); + const [fileBuffer, setFileBuffer] = useState(null); + const [encoding, setEncoding] = useState("utf-8"); const [mapping, setMapping] = useState({ date: "", amount: "", description: "", merchant: "", notes: "", category: "" }); const [dateFormat, setDateFormat] = useState("YYYY-MM-DD"); + const [amountFormat, setAmountFormat] = useState("auto"); + const [includeDuplicates, setIncludeDuplicates] = useState(false); const [parseError, setParseError] = useState(null); const [importError, setImportError] = useState(null); const [createdCount, setCreatedCount] = useState(null); + // Re-decoded whenever `encoding` changes, so picking a different encoding + // on the map screen (because the auto-detected guess came out wrong) + // re-parses the same upload without asking the user to re-select the file. + const { headers, dataRows } = useMemo(() => { + if (!fileBuffer) return { headers: [] as string[], dataRows: [] as string[][] }; + const text = new TextDecoder(encoding).decode(fileBuffer); + const rows = parseCsv(text); + if (rows.length < 2) return { headers: [] as string[], dataRows: [] as string[][] }; + const [headerRow, ...rest] = rows; + return { headers: headerRow, dataRows: rest }; + }, [fileBuffer, encoding]); + const categoryLookup = useMemo(() => { const map: Record<"income" | "expense", Map> = { income: new Map(), expense: new Map() }; for (const category of categories ?? []) { @@ -64,8 +95,54 @@ export function CsvImportPage() { return amount < 0 ? "expense" : "income"; } - const { valid, skipped } = useMemo(() => { - if (step !== "preview") return { valid: [] as TransactionInput[], skipped: [] as SkippedRow[] }; + /** First non-empty raw value under `headerName`, shown next to a mapping + * dropdown so the user can confirm they picked the right column before + * committing to a full preview — a bank's own header names (or a header + * row that's just "Column1", "Column2"…) aren't always self-explanatory. */ + function sampleValue(headerName: string): string { + if (!headerName) return ""; + const idx = headers.indexOf(headerName); + if (idx < 0) return ""; + const row = dataRows.find((cells) => (cells[idx] ?? "").trim() !== ""); + return (row?.[idx] ?? "").trim(); + } + + // Bounds the duplicate-check query (see useTransactionsForDuplicateCheck + // below) to the date range actually present in the file, instead of every + // transaction ever recorded on the account. + const dateBounds = useMemo(() => { + if (step !== "preview" || !mapping.date) return null; + const dateIdx = headers.indexOf(mapping.date); + if (dateIdx < 0) return null; + let min: string | null = null; + let max: string | null = null; + for (const cells of dataRows) { + const iso = parseDateWithFormat(cells[dateIdx] ?? "", dateFormat); + if (!iso) continue; + if (min === null || iso < min) min = iso; + if (max === null || iso > max) max = iso; + } + return min && max ? { min, max } : null; + }, [step, headers, dataRows, mapping.date, dateFormat]); + + const existingTransactions = useTransactionsForDuplicateCheck( + accountId ? Number(accountId) : null, + dateBounds?.min ?? null, + dateBounds?.max ?? null + ); + + // Keyed on date+type+amount+description — see lib/csv.ts's + // transactionDedupeKey doc comment for why those fields and not an ID. + const existingKeys = useMemo(() => { + const keys = new Set(); + for (const tx of existingTransactions.data ?? []) { + keys.add(transactionDedupeKey(tx.date, tx.type, tx.amount, tx.description)); + } + return keys; + }, [existingTransactions.data]); + + const { valid, skipped, duplicateCount } = useMemo(() => { + if (step !== "preview") return { valid: [] as TransactionInput[], skipped: [] as SkippedRow[], duplicateCount: 0 }; const dateIdx = headers.indexOf(mapping.date); const amountIdx = headers.indexOf(mapping.amount); @@ -74,7 +151,8 @@ export function CsvImportPage() { const notesIdx = mapping.notes ? headers.indexOf(mapping.notes) : -1; const categoryIdx = mapping.category ? headers.indexOf(mapping.category) : -1; - const validRows: TransactionInput[] = []; + const freshRows: TransactionInput[] = []; + const duplicateRows: TransactionInput[] = []; const skippedRows: SkippedRow[] = []; dataRows.forEach((cells, index) => { @@ -91,7 +169,7 @@ export function CsvImportPage() { skippedRows.push({ row: rowNumber, reason: t("transactions.import.errorBadDate", { value: rawDate || "—" }) }); return; } - const amount = parseAmount(rawAmount); + const amount = parseAmount(rawAmount, amountFormat); if (amount === null || amount === 0) { skippedRows.push({ row: rowNumber, reason: t("transactions.import.errorBadAmount", { value: rawAmount || "—" }) }); return; @@ -105,7 +183,7 @@ export function CsvImportPage() { const type = resolveType(amount); const categoryId = rawCategory ? categoryLookup[type].get(rawCategory.toLowerCase()) ?? null : null; - validRows.push({ + const candidate: TransactionInput = { account_id: Number(accountId), category_id: categoryId, transfer_account_id: null, @@ -115,53 +193,73 @@ export function CsvImportPage() { merchant: rawMerchant || null, notes: rawNotes || null, date: isoDate, - }); + }; + + const key = transactionDedupeKey(candidate.date, candidate.type, candidate.amount, candidate.description); + if (existingKeys.has(key)) { + duplicateRows.push(candidate); + } else { + freshRows.push(candidate); + } }); - return { valid: validRows, skipped: skippedRows }; + return { + valid: includeDuplicates ? [...freshRows, ...duplicateRows] : freshRows, + skipped: skippedRows, + duplicateCount: duplicateRows.length, + }; // eslint-disable-next-line react-hooks/exhaustive-deps - }, [step, headers, dataRows, mapping, dateFormat, categoryLookup, accountId, t]); + }, [step, headers, dataRows, mapping, dateFormat, amountFormat, categoryLookup, accountId, existingKeys, includeDuplicates, t]); - function handleFileSelected(event: React.ChangeEvent) { + async function handleFileSelected(event: React.ChangeEvent) { const file = event.target.files?.[0]; event.target.value = ""; if (!file) return; setParseError(null); - const reader = new FileReader(); - reader.onload = () => { - const text = String(reader.result ?? ""); - const rows = parseCsv(text); - if (rows.length < 2) { - setParseError(t("transactions.import.errorEmptyFile")); - return; - } - const [headerRow, ...rest] = rows; - setFileName(file.name); - setHeaders(headerRow); - setDataRows(rest); - // Best-effort auto-mapping by common header names — the user can - // still correct any of these on the next screen. - const guess = (...candidates: string[]) => - headerRow.find((h) => candidates.includes(h.trim().toLowerCase())) ?? ""; - setMapping({ - date: guess("date", "дата"), - amount: guess("amount", "сумма"), - description: guess("description", "описание", "назначение платежа"), - merchant: guess("merchant", "payee", "получатель"), - notes: guess("notes", "заметка", "примечание"), - category: guess("category", "категория"), - }); - setStep("map"); - }; - reader.readAsText(file); + const buffer = await file.arrayBuffer(); + // Invalid-UTF-8 byte sequences almost always mean a Cyrillic bank export + // in the classic Windows codepage — a far better first guess than + // staying on UTF-8 and showing every letter as mojibake. The user can + // still override this from the dropdown on the next screen either way. + let detectedEncoding: Encoding = "utf-8"; + try { + new TextDecoder("utf-8", { fatal: true }).decode(buffer); + } catch { + detectedEncoding = "windows-1251"; + } + + const rows = parseCsv(new TextDecoder(detectedEncoding).decode(buffer)); + if (rows.length < 2) { + setParseError(t("transactions.import.errorEmptyFile")); + return; + } + const [headerRow] = rows; + setFileName(file.name); + setFileBuffer(buffer); + setEncoding(detectedEncoding); + // Best-effort auto-mapping by common header names — the user can still + // correct any of these on the next screen. + const guess = (...candidates: string[]) => + headerRow.find((h) => candidates.includes(h.trim().toLowerCase())) ?? ""; + setMapping({ + date: guess("date", "дата"), + amount: guess("amount", "сумма"), + description: guess("description", "описание", "назначение платежа"), + merchant: guess("merchant", "payee", "получатель"), + notes: guess("notes", "заметка", "примечание"), + category: guess("category", "категория"), + }); + setAmountFormat("auto"); + setIncludeDuplicates(false); + setStep("map"); } function startOver() { setStep("upload"); setFileName(""); - setHeaders([]); - setDataRows([]); + setFileBuffer(null); + setEncoding("utf-8"); setCreatedCount(null); setImportError(null); } @@ -235,6 +333,18 @@ export function CsvImportPage() { {fileName} +
+ + +

{t("transactions.import.encodingHint")}

+
+
@@ -248,6 +358,11 @@ export function CsvImportPage() { ))} + {mapping.date && ( +

+ {t("transactions.import.sampleValueLabel", { value: sampleValue(mapping.date) || "—" })} +

+ )}
@@ -272,6 +387,21 @@ export function CsvImportPage() { ))}

{t("transactions.import.amountHint")}

+ {mapping.amount && ( +

+ {t("transactions.import.sampleValueLabel", { value: sampleValue(mapping.amount) || "—" })} +

+ )} +
+
+ +
@@ -289,6 +419,11 @@ export function CsvImportPage() { ))} + {mapping.description && ( +

+ {t("transactions.import.sampleValueLabel", { value: sampleValue(mapping.description) || "—" })} +

+ )}
@@ -300,6 +435,11 @@ export function CsvImportPage() { ))} + {mapping.merchant && ( +

+ {t("transactions.import.sampleValueLabel", { value: sampleValue(mapping.merchant) || "—" })} +

+ )}
@@ -311,6 +451,11 @@ export function CsvImportPage() { ))} + {mapping.notes && ( +

+ {t("transactions.import.sampleValueLabel", { value: sampleValue(mapping.notes) || "—" })} +

+ )}
@@ -323,6 +468,11 @@ export function CsvImportPage() { ))}

{t("transactions.import.categoryHint")}

+ {mapping.category && ( +

+ {t("transactions.import.sampleValueLabel", { value: sampleValue(mapping.category) || "—" })} +

+ )}
@@ -355,6 +505,27 @@ export function CsvImportPage() { {t("transactions.import.summary", { valid: valid.length, skipped: skipped.length })}

+ {existingTransactions.isLoading && ( +

{t("transactions.import.duplicateCheckLoading")}

+ )} + + {duplicateCount > 0 && ( +
+ +
+ )} + {valid.length > 0 && (