Coder/aurum
1
0
Fork 0
mirror of https://github.com/Zproger/Aurum.git synced 2026-10-13 02:56:39 +07:00
Code Issues Packages Projects Releases Wiki Activity

feat: улучшить импорт CSV — дедупликация, кодировка, парсинг сумм

Browse Source 
Пять доработок импорта транзакций из CSV:
1. Проверка дублей при повторном импорте — сверка с уже существующими
   транзакциями на счёте за диапазон дат файла (по дате+типу+сумме+
   описанию), похожие исключаются из импорта по умолчанию с опцией
   всё равно их добавить.
2. Автоопределение кодировки файла (UTF-8 с откатом на Windows-1251)
   вместо жёстко зашитого UTF-8 — раньше кириллица из банковских
   выгрузок превращалась в кракозябры. Есть ручной выбор на случай
   неверного автоопределения.
3. Исправлен баг в parseAmount: суммы с несколькими разделителями
   тысяч и без дробной части (например "1,234,567") теряли цифры и
   отбрасывались как нераспознанные — теперь распознаются верно.
4. Добавлен явный выбор разделителя дробной части (точка/запятая) на
   случай, когда автоопределение не может разрешить неоднозначность.
5. Пример значения из файла показывается прямо под каждым выбором
   колонки при сопоставлении — не нужно доходить до превью, чтобы
   понять, что в колонке.
This commit is contained in:
ZProger
2026-09-10 20:06:41 +03:00
parent 78826db33e
commit 777e3af725
5 changed files with 310 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
frontend/src/api/transactions.ts
View File

@@ -26,6 +26,25 @@ export function fetchTransactions(filters: TransactionFilters = {}) {
return api.get<TransactionPage>(`/transactions?${params.toString()}`);
}
// Used by CSV import's duplicate check (see pages/CsvImportPage.tsx) — needs
// every transaction in the imported date range for one account, not just a
// page of them, to know what's already there. Capped at 25 pages (5000 rows
// at the API's max page_size) so a huge date range can't turn one import
// into an unbounded fetch loop; beyond that, duplicate detection just
// silently covers less of the range instead of hanging.
const MAX_DUPLICATE_CHECK_PAGES = 25;
export async function fetchAllTransactionsInRange(filters: TransactionFilters): Promise<Transaction[]> {
const pageSize = 200;
const items: Transaction[] = [];
for (let page = 1; page <= MAX_DUPLICATE_CHECK_PAGES; page++) {
const result = await fetchTransactions({ ...filters, page, page_size: pageSize });
items.push(...result.items);
if (items.length >= result.total || result.items.length < pageSize) break;
}
return items;
}
/** Full range of years to offer in the year picker, from the earliest
* transaction through the current year (see backend for the "gap year"
* rationale). */

16
frontend/src/hooks/useTransactions.ts
View File

@@ -3,6 +3,7 @@ import {
bulkCreateTransactions,
createTransaction,
deleteTransaction,
fetchAllTransactionsInRange,
fetchTransactions,
fetchTransactionYears,
updateTransaction,
@@ -33,6 +34,21 @@ export function useTransactions(filters: TransactionFilters) {
});
}
/** CSV import's duplicate check (see pages/CsvImportPage.tsx) — every
* existing transaction for one account within the imported date range, so
* re-importing an overlapping bank export can be told apart from new rows. */
export function useTransactionsForDuplicateCheck(
accountId: number | null,
startDate: string | null,
endDate: string | null
) {
return useQuery({
queryKey: ["transactions-duplicate-check", accountId, startDate, endDate],
queryFn: () => fetchAllTransactionsInRange({ account_id: accountId!, start_date: startDate!, end_date: endDate! }),
enabled: accountId !== null && startDate !== null && endDate !== null,
});
}
export function useTransactionYears() {
return useQuery({
queryKey: ["transaction-years"],

52
frontend/src/lib/csv.ts
View File

@@ -86,21 +86,53 @@ export function parseDateWithFormat(raw: string, format: DateFormat): string | n
return `${String(year).padStart(4, "0")}-${String(month).padStart(2, "0")}-${String(day).padStart(2, "0")}`;
}
/** Parses a bank-export amount string — handles thousands separators and
* either "," or "." as the decimal mark by assuming whichever comes last is
* the decimal separator (e.g. "1.234,56" and "1,234.56" both work). */
export function parseAmount(raw: string): number | null {
export const AMOUNT_FORMATS = ["auto", "dot-decimal", "comma-decimal"] as const;
export type AmountFormat = (typeof AMOUNT_FORMATS)[number];
/** Parses a bank-export amount string. `format` lets the user resolve what
* "auto" can't: a single "," or "." with nothing else in the string is
* genuinely ambiguous (thousands separator vs. decimal mark) — there's no
* way to tell "1,234" (one thousand two hundred thirty-four) from "1,234"
* (one point two three four) without knowing the exporting bank's locale.
* "auto" keeps the historical last-separator-wins heuristic for that case
* (e.g. "1.234,56" and "1,234.56" both resolve correctly since both
* separators are present), but no longer corrupts amounts that have
* *multiple* thousands separators and no decimal part at all (e.g.
* "1,234,567" or "1.234.567") — a number can have at most one decimal
* point, so 2+ occurrences of the same separator can only be thousands
* grouping. */
export function parseAmount(raw: string, format: AmountFormat = "auto"): number | null {
let value = raw.trim().replace(/[\s ]/g, "");
if (!value) return null;
const hasComma = value.includes(",");
const hasDot = value.includes(".");
if (hasComma && hasDot) {
value = value.lastIndexOf(",") > value.lastIndexOf(".") ? value.replace(/\./g, "").replace(",", ".") : value.replace(/,/g, "");
} else if (hasComma) {
value = value.replace(",", ".");
if (format === "dot-decimal") {
value = value.replace(/,/g, "");
} else if (format === "comma-decimal") {
value = value.replace(/\./g, "").replace(",", ".");
} else {
const commaCount = (value.match(/,/g) ?? []).length;
const dotCount = (value.match(/\./g) ?? []).length;
if (commaCount > 0 && dotCount > 0) {
value = value.lastIndexOf(",") > value.lastIndexOf(".") ? value.replace(/\./g, "").replace(",", ".") : value.replace(/,/g, "");
} else if (commaCount > 1) {
value = value.replace(/,/g, "");
} else if (commaCount === 1) {
value = value.replace(",", ".");
} else if (dotCount > 1) {
value = value.replace(/\./g, "");
}
}
const num = Number(value);
return Number.isFinite(num) ? num : null;
}
/** Stable key for spotting a re-imported bank export that overlaps a
* previous one — see pages/CsvImportPage.tsx's duplicate check. Matches on
* date + type + amount (2dp) + description, the same fields the API
* actually stores, so it works whether the "existing" side comes from a
* Transaction or the "candidate" side from a freshly parsed CSV row. */
export function transactionDedupeKey(date: string, type: string, amount: number | string, description: string): string {
const normalizedAmount = typeof amount === "number" ? amount : Number(amount);
return `${date}|${type}|${normalizedAmount.toFixed(2)}|${description.trim().toLowerCase()}`;
}

20
frontend/src/lib/i18n.ts
View File

@@ -252,6 +252,16 @@ const ru = {
"transactions.import.errorBadDate": "не удалось распознать дату «{{value}}»",
"transactions.import.errorBadAmount": "не удалось распознать сумму «{{value}}»",
"transactions.import.errorNoDescription": "нет описания",
"transactions.import.encodingLabel": "Кодировка файла",
"transactions.import.encodingHint": "Определяется автоматически; если вместо русских букв — кракозябры, попробуйте выбрать вручную.",
"transactions.import.sampleValueLabel": "Пример из файла: {{value}}",
"transactions.import.amountFormatLabel": "Разделитель дробной части",
"transactions.import.amountFormat.auto": "Автоматически",
"transactions.import.amountFormat.dot-decimal": "Точка (1234.56), запятая — тысячи",
"transactions.import.amountFormat.comma-decimal": "Запятая (1234,56), точка — тысячи",
"transactions.import.duplicateCheckLoading": "Проверяем, нет ли уже таких транзакций…",
"transactions.import.duplicatesFound": "Найдено похожих на уже существующие: {{count}}. Они не будут импортированы.",
"transactions.import.includeDuplicates": "Всё равно импортировать их",
"transactions.allTypes": "Все типы",
"transactions.expense": "Расходы",
"transactions.income": "Доходы",
@@ -759,6 +769,16 @@ const en: Record<keyof typeof ru, string> = {
"transactions.import.errorBadDate": "couldn't parse the date \"{{value}}\"",
"transactions.import.errorBadAmount": "couldn't parse the amount \"{{value}}\"",
"transactions.import.errorNoDescription": "no description",
"transactions.import.encodingLabel": "File encoding",
"transactions.import.encodingHint": "Auto-detected; if you see garbled characters instead of letters, try picking one manually.",
"transactions.import.sampleValueLabel": "Example from file: {{value}}",
"transactions.import.amountFormatLabel": "Decimal separator",
"transactions.import.amountFormat.auto": "Automatic",
"transactions.import.amountFormat.dot-decimal": "Dot (1234.56), comma is thousands",
"transactions.import.amountFormat.comma-decimal": "Comma (1234,56), dot is thousands",
"transactions.import.duplicateCheckLoading": "Checking for transactions already on this account…",
"transactions.import.duplicatesFound": "Found {{count}} that look like duplicates of existing transactions. They won't be imported.",
"transactions.import.includeDuplicates": "Import them anyway",
"transactions.allTypes": "All types",
"transactions.expense": "Expenses",
"transactions.income": "Income",

255
frontend/src/pages/CsvImportPage.tsx
View File

@@ -6,9 +6,18 @@ import { Button } from "@/components/ui/Button";
import { Label, Select } from "@/components/ui/Input";
import { useAccounts } from "@/hooks/useAccounts";
import { useCategories } from "@/hooks/useCategories";
import { useBulkCreateTransactions } from "@/hooks/useTransactions";
import { useBulkCreateTransactions, useTransactionsForDuplicateCheck } from "@/hooks/useTransactions";
import { translateCategoryName } from "@/lib/categoryLabels";
import { DATE_FORMATS, parseAmount, parseCsv, parseDateWithFormat, type DateFormat } from "@/lib/csv";
import {
AMOUNT_FORMATS,
DATE_FORMATS,
parseAmount,
parseCsv,
parseDateWithFormat,
transactionDedupeKey,
type AmountFormat,
type DateFormat,
} from "@/lib/csv";
import { formatCurrency } from "@/lib/format";
import { ApiError } from "@/api/client";
import { useTranslation } from "@/lib/i18n";
@@ -18,6 +27,14 @@ type Step = "upload" | "map" | "preview";
const NONE = "";
// WHATWG encoding labels TextDecoder understands. UTF-8 covers most modern
// exports; the rest are here because bank CSVs — especially older or
// Russian-bank ones — are routinely still in a legacy Windows codepage, and
// decoding those as UTF-8 silently turns every non-ASCII character into
// mojibake instead of failing loudly.
const ENCODINGS = ["utf-8", "windows-1251", "windows-1252", "koi8-r"] as const;
type Encoding = (typeof ENCODINGS)[number];
interface Mapping {
date: string;
amount: string;
@@ -43,14 +60,28 @@ export function CsvImportPage() {
const [accountId, setAccountId] = useState("");
const [step, setStep] = useState<Step>("upload");
const [fileName, setFileName] = useState("");
const [headers, setHeaders] = useState<string[]>([]);
const [dataRows, setDataRows] = useState<string[][]>([]);
const [fileBuffer, setFileBuffer] = useState<ArrayBuffer | null>(null);
const [encoding, setEncoding] = useState<Encoding>("utf-8");
const [mapping, setMapping] = useState<Mapping>({ date: "", amount: "", description: "", merchant: "", notes: "", category: "" });
const [dateFormat, setDateFormat] = useState<DateFormat>("YYYY-MM-DD");
const [amountFormat, setAmountFormat] = useState<AmountFormat>("auto");
const [includeDuplicates, setIncludeDuplicates] = useState(false);
const [parseError, setParseError] = useState<string | null>(null);
const [importError, setImportError] = useState<string | null>(null);
const [createdCount, setCreatedCount] = useState<number | null>(null);
// Re-decoded whenever `encoding` changes, so picking a different encoding
// on the map screen (because the auto-detected guess came out wrong)
// re-parses the same upload without asking the user to re-select the file.
const { headers, dataRows } = useMemo(() => {
if (!fileBuffer) return { headers: [] as string[], dataRows: [] as string[][] };
const text = new TextDecoder(encoding).decode(fileBuffer);
const rows = parseCsv(text);
if (rows.length < 2) return { headers: [] as string[], dataRows: [] as string[][] };
const [headerRow, ...rest] = rows;
return { headers: headerRow, dataRows: rest };
}, [fileBuffer, encoding]);
const categoryLookup = useMemo(() => {
const map: Record<"income" | "expense", Map<string, number>> = { income: new Map(), expense: new Map() };
for (const category of categories ?? []) {
@@ -64,8 +95,54 @@ export function CsvImportPage() {
return amount < 0 ? "expense" : "income";
}
const { valid, skipped } = useMemo(() => {
if (step !== "preview") return { valid: [] as TransactionInput[], skipped: [] as SkippedRow[] };
/** First non-empty raw value under `headerName`, shown next to a mapping
* dropdown so the user can confirm they picked the right column before
* committing to a full preview — a bank's own header names (or a header
* row that's just "Column1", "Column2"…) aren't always self-explanatory. */
function sampleValue(headerName: string): string {
if (!headerName) return "";
const idx = headers.indexOf(headerName);
if (idx < 0) return "";
const row = dataRows.find((cells) => (cells[idx] ?? "").trim() !== "");
return (row?.[idx] ?? "").trim();
}
// Bounds the duplicate-check query (see useTransactionsForDuplicateCheck
// below) to the date range actually present in the file, instead of every
// transaction ever recorded on the account.
const dateBounds = useMemo(() => {
if (step !== "preview" || !mapping.date) return null;
const dateIdx = headers.indexOf(mapping.date);
if (dateIdx < 0) return null;
let min: string | null = null;
let max: string | null = null;
for (const cells of dataRows) {
const iso = parseDateWithFormat(cells[dateIdx] ?? "", dateFormat);
if (!iso) continue;
if (min === null || iso < min) min = iso;
if (max === null || iso > max) max = iso;
}
return min && max ? { min, max } : null;
}, [step, headers, dataRows, mapping.date, dateFormat]);
const existingTransactions = useTransactionsForDuplicateCheck(
accountId ? Number(accountId) : null,
dateBounds?.min ?? null,
dateBounds?.max ?? null
);
// Keyed on date+type+amount+description — see lib/csv.ts's
// transactionDedupeKey doc comment for why those fields and not an ID.
const existingKeys = useMemo(() => {
const keys = new Set<string>();
for (const tx of existingTransactions.data ?? []) {
keys.add(transactionDedupeKey(tx.date, tx.type, tx.amount, tx.description));
}
return keys;
}, [existingTransactions.data]);
const { valid, skipped, duplicateCount } = useMemo(() => {
if (step !== "preview") return { valid: [] as TransactionInput[], skipped: [] as SkippedRow[], duplicateCount: 0 };
const dateIdx = headers.indexOf(mapping.date);
const amountIdx = headers.indexOf(mapping.amount);
@@ -74,7 +151,8 @@ export function CsvImportPage() {
const notesIdx = mapping.notes ? headers.indexOf(mapping.notes) : -1;
const categoryIdx = mapping.category ? headers.indexOf(mapping.category) : -1;
const validRows: TransactionInput[] = [];
const freshRows: TransactionInput[] = [];
const duplicateRows: TransactionInput[] = [];
const skippedRows: SkippedRow[] = [];
dataRows.forEach((cells, index) => {
@@ -91,7 +169,7 @@ export function CsvImportPage() {
skippedRows.push({ row: rowNumber, reason: t("transactions.import.errorBadDate", { value: rawDate || "—" }) });
return;
}
const amount = parseAmount(rawAmount);
const amount = parseAmount(rawAmount, amountFormat);
if (amount === null || amount === 0) {
skippedRows.push({ row: rowNumber, reason: t("transactions.import.errorBadAmount", { value: rawAmount || "—" }) });
return;
@@ -105,7 +183,7 @@ export function CsvImportPage() {
const type = resolveType(amount);
const categoryId = rawCategory ? categoryLookup[type].get(rawCategory.toLowerCase()) ?? null : null;
validRows.push({
const candidate: TransactionInput = {
account_id: Number(accountId),
category_id: categoryId,
transfer_account_id: null,
@@ -115,53 +193,73 @@ export function CsvImportPage() {
merchant: rawMerchant || null,
notes: rawNotes || null,
date: isoDate,
});
};
const key = transactionDedupeKey(candidate.date, candidate.type, candidate.amount, candidate.description);
if (existingKeys.has(key)) {
duplicateRows.push(candidate);
} else {
freshRows.push(candidate);
}
});
return { valid: validRows, skipped: skippedRows };
return {
valid: includeDuplicates ? [...freshRows, ...duplicateRows] : freshRows,
skipped: skippedRows,
duplicateCount: duplicateRows.length,
};
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [step, headers, dataRows, mapping, dateFormat, categoryLookup, accountId, t]);
}, [step, headers, dataRows, mapping, dateFormat, amountFormat, categoryLookup, accountId, existingKeys, includeDuplicates, t]);
function handleFileSelected(event: React.ChangeEvent<HTMLInputElement>) {
async function handleFileSelected(event: React.ChangeEvent<HTMLInputElement>) {
const file = event.target.files?.[0];
event.target.value = "";
if (!file) return;
setParseError(null);
const reader = new FileReader();
reader.onload = () => {
const text = String(reader.result ?? "");
const rows = parseCsv(text);
if (rows.length < 2) {
setParseError(t("transactions.import.errorEmptyFile"));
return;
}
const [headerRow, ...rest] = rows;
setFileName(file.name);
setHeaders(headerRow);
setDataRows(rest);
// Best-effort auto-mapping by common header names — the user can
// still correct any of these on the next screen.
const guess = (...candidates: string[]) =>
headerRow.find((h) => candidates.includes(h.trim().toLowerCase())) ?? "";
setMapping({
date: guess("date", "дата"),
amount: guess("amount", "сумма"),
description: guess("description", "описание", "назначение платежа"),
merchant: guess("merchant", "payee", "получатель"),
notes: guess("notes", "заметка", "примечание"),
category: guess("category", "категория"),
});
setStep("map");
};
reader.readAsText(file);
const buffer = await file.arrayBuffer();
// Invalid-UTF-8 byte sequences almost always mean a Cyrillic bank export
// in the classic Windows codepage — a far better first guess than
// staying on UTF-8 and showing every letter as mojibake. The user can
// still override this from the dropdown on the next screen either way.
let detectedEncoding: Encoding = "utf-8";
try {
new TextDecoder("utf-8", { fatal: true }).decode(buffer);
} catch {
detectedEncoding = "windows-1251";
}
const rows = parseCsv(new TextDecoder(detectedEncoding).decode(buffer));
if (rows.length < 2) {
setParseError(t("transactions.import.errorEmptyFile"));
return;
}
const [headerRow] = rows;
setFileName(file.name);
setFileBuffer(buffer);
setEncoding(detectedEncoding);
// Best-effort auto-mapping by common header names — the user can still
// correct any of these on the next screen.
const guess = (...candidates: string[]) =>
headerRow.find((h) => candidates.includes(h.trim().toLowerCase())) ?? "";
setMapping({
date: guess("date", "дата"),
amount: guess("amount", "сумма"),
description: guess("description", "описание", "назначение платежа"),
merchant: guess("merchant", "payee", "получатель"),
notes: guess("notes", "заметка", "примечание"),
category: guess("category", "категория"),
});
setAmountFormat("auto");
setIncludeDuplicates(false);
setStep("map");
}
function startOver() {
setStep("upload");
setFileName("");
setHeaders([]);
setDataRows([]);
setFileBuffer(null);
setEncoding("utf-8");
setCreatedCount(null);
setImportError(null);
}
@@ -235,6 +333,18 @@ export function CsvImportPage() {
<span className="text-xs text-text-muted">{fileName}</span>
</CardHeader>
<CardContent className="space-y-3">
<div>
<Label htmlFor="map-encoding">{t("transactions.import.encodingLabel")}</Label>
<Select id="map-encoding" value={encoding} onChange={(event) => setEncoding(event.target.value as Encoding)} className="sm:w-56">
{ENCODINGS.map((enc) => (
<option key={enc} value={enc}>
{enc.toUpperCase()}
</option>
))}
</Select>
<p className="mt-1 text-xs text-text-muted">{t("transactions.import.encodingHint")}</p>
</div>
<div className="grid grid-cols-1 gap-3 sm:grid-cols-2">
<div>
<Label htmlFor="map-date">{t("transactions.import.dateColumnLabel")}</Label>
@@ -248,6 +358,11 @@ export function CsvImportPage() {
</option>
))}
</Select>
{mapping.date && (
<p className="mt-1 text-xs text-text-muted">
{t("transactions.import.sampleValueLabel", { value: sampleValue(mapping.date) || "—" })}
</p>
)}
</div>
<div>
<Label htmlFor="map-date-format">{t("transactions.import.dateFormatLabel")}</Label>
@@ -272,6 +387,21 @@ export function CsvImportPage() {
))}
</Select>
<p className="mt-1 text-xs text-text-muted">{t("transactions.import.amountHint")}</p>
{mapping.amount && (
<p className="text-xs text-text-muted">
{t("transactions.import.sampleValueLabel", { value: sampleValue(mapping.amount) || "—" })}
</p>
)}
</div>
<div>
<Label htmlFor="map-amount-format">{t("transactions.import.amountFormatLabel")}</Label>
<Select id="map-amount-format" value={amountFormat} onChange={(event) => setAmountFormat(event.target.value as AmountFormat)}>
{AMOUNT_FORMATS.map((format) => (
<option key={format} value={format}>
{t(`transactions.import.amountFormat.${format}`)}
</option>
))}
</Select>
</div>
<div>
<Label htmlFor="map-description">{t("transactions.import.descriptionColumnLabel")}</Label>
@@ -289,6 +419,11 @@ export function CsvImportPage() {
</option>
))}
</Select>
{mapping.description && (
<p className="mt-1 text-xs text-text-muted">
{t("transactions.import.sampleValueLabel", { value: sampleValue(mapping.description) || "—" })}
</p>
)}
</div>
<div>
<Label htmlFor="map-merchant">{t("transactions.form.merchantLabel")}</Label>
@@ -300,6 +435,11 @@ export function CsvImportPage() {
</option>
))}
</Select>
{mapping.merchant && (
<p className="mt-1 text-xs text-text-muted">
{t("transactions.import.sampleValueLabel", { value: sampleValue(mapping.merchant) || "—" })}
</p>
)}
</div>
<div>
<Label htmlFor="map-notes">{t("transactions.form.notesLabel")}</Label>
@@ -311,6 +451,11 @@ export function CsvImportPage() {
</option>
))}
</Select>
{mapping.notes && (
<p className="mt-1 text-xs text-text-muted">
{t("transactions.import.sampleValueLabel", { value: sampleValue(mapping.notes) || "—" })}
</p>
)}
</div>
<div>
<Label htmlFor="map-category">{t("transactions.form.categoryLabel")}</Label>
@@ -323,6 +468,11 @@ export function CsvImportPage() {
))}
</Select>
<p className="mt-1 text-xs text-text-muted">{t("transactions.import.categoryHint")}</p>
{mapping.category && (
<p className="text-xs text-text-muted">
{t("transactions.import.sampleValueLabel", { value: sampleValue(mapping.category) || "—" })}
</p>
)}
</div>
</div>
@@ -355,6 +505,27 @@ export function CsvImportPage() {
{t("transactions.import.summary", { valid: valid.length, skipped: skipped.length })}
</p>
{existingTransactions.isLoading && (
<p className="text-xs text-text-muted">{t("transactions.import.duplicateCheckLoading")}</p>
)}
{duplicateCount > 0 && (
<div className="flex items-start gap-2 rounded-lg border border-border bg-surface-2 p-3 text-xs text-text-muted">
<label className="flex items-start gap-2">
<input
type="checkbox"
checked={includeDuplicates}
onChange={(event) => setIncludeDuplicates(event.target.checked)}
className="mt-0.5 h-3.5 w-3.5 accent-text-primary"
/>
<span>
{t("transactions.import.duplicatesFound", { count: duplicateCount })}{" "}
{t("transactions.import.includeDuplicates")}
</span>
</label>
</div>
)}
{valid.length > 0 && (
<div className="overflow-x-auto rounded-lg border border-border">
<table className="w-full text-left text-sm">