fix: report omitted bytes against retained text, not budget (codex round 2)

finish() derived the exact omitted count from the pre-trim prefix/suffix
budget, but the UTF-8 boundary trims drop additional partial-codepoint bytes,
so an "Omitted N bytes" notice overstated what was kept (head maxBytes:2 over
'a€b' returned 'a' but claimed exact 3 when 4 bytes are absent). Derive the
count from the bytes actually returned (total − keptPrefix − keptSuffix) so
exact metadata matches the text and atLeast stays a valid lower bound.
This commit is contained in:
Dudu-0223
2026-07-07 10:01:03 +08:00
parent cbfdf9d092
commit f4acb1cf05
2 changed files with 40 additions and 13 deletions

View File

@@ -388,23 +388,33 @@ export class TextRetainer {
finish(): RetainedText {
const prefixLen = Math.min(this.total, this.prefixCap)
const suffixLen = Math.min(this.total - prefixLen, this.suffixCap)
const omitted = this.omittedAt(this.total)
const truncated = omitted > 0
const prefix = concat(this.prefixChunks) // exactly prefixLen bytes (prefixHeld === prefixLen)
const suffix = concat(this.suffixChunks).subarray(this.suffixHeld - suffixLen)
// With nothing omitted, prefix and suffix are ADJACENT slices of one stream
// (prefixLen + suffixLen === total), so the head|tail split is artificial: a
// codepoint may span it. Decode the contiguous whole as one buffer — trimming
// or decoding the halves separately here would corrupt a boundary-spanning
// codepoint though no content was actually dropped. Only a real omitted gap
// makes each side a true cut: trim each to a UTF-8 boundary and decode
// separately so a codepoint is never reconstructed across the gap.
const text = truncated
? decoder.decode(trimTrailingPartialUtf8(prefix)) + decoder.decode(trimLeadingContinuationUtf8(suffix))
// With nothing omitted by budget, prefix and suffix are ADJACENT slices of
// one stream (prefixLen + suffixLen === total), so the head|tail split is
// artificial: a codepoint may span it. Decode the contiguous whole as one
// buffer — trimming or decoding the halves separately here would corrupt a
// boundary-spanning codepoint though no content was dropped. Only a real
// omitted gap makes each side a true cut: trim each to a UTF-8 boundary and
// decode separately so a codepoint is never reconstructed across the gap.
const budgetOmitted = this.omittedAt(this.total)
const [keptPrefix, keptSuffix] = budgetOmitted > 0
? [trimTrailingPartialUtf8(prefix), trimLeadingContinuationUtf8(suffix)]
: [prefix, suffix]
const text = budgetOmitted > 0
? decoder.decode(keptPrefix) + decoder.decode(keptSuffix)
: decoder.decode(concat([prefix, suffix]))
// Report omission against the bytes ACTUALLY returned, not the pre-trim
// budget: a boundary trim drops partial-codepoint bytes too, so an exact
// count derived from the budget alone would overstate the retained text (and
// any "Omitted N bytes" notice built from it would be a lie). total_seen
// retained stays a valid lower bound under `atLeast` (true total ≥ seen).
const omitted = this.total - keptPrefix.length - keptSuffix.length
const truncated = omitted > 0
return {
text,
truncated,

View File

@@ -223,8 +223,10 @@ describe('TextRetainer — UTF-8 boundary handling', () => {
const result = r.finish()
expect(result.text).toBe('a') // partial '€' dropped, no U+FFFD
expect(result.text).not.toContain('<27>')
// Omission counts BYTES not kept by retention: 5 total 2 prefix = 3.
expect(result.omittedBytes).toEqual<Omitted>({ kind: 'exact', count: 3 })
// Omission counts bytes ACTUALLY absent from the returned text, including
// the partial 'E2' the boundary trim dropped: 5 total 1 retained = 4
// (not the pre-trim budget of 3, which would overstate what was kept).
expect(result.omittedBytes).toEqual<Omitted>({ kind: 'exact', count: 4 })
})
it('trims a leading partial codepoint at the tail cut', () => {
@@ -236,6 +238,21 @@ describe('TextRetainer — UTF-8 boundary handling', () => {
const result = r.finish()
expect(result.text).toBe('b') // partial '€' at the front dropped
expect(result.text).not.toContain('<27>')
// Honest count: 5 total 1 retained ('b') = 4, including the trimmed AC.
expect(result.omittedBytes).toEqual<Omitted>({ kind: 'exact', count: 4 })
})
it('omitted count matches the bytes actually absent, across a headTail boundary trim', () => {
// Regression: the exact count must equal total retained (post-trim), never
// the pre-trim budget. 'a€€b' is 8 bytes (61 E2828C… ×2 61? no: 61 E2 82 AC
// E2 82 AC 62). headBytes 2 keeps 'a'+partial-E2 → trims to 'a' (1 byte);
// tailBytes 2 keeps partial-AC+'b' → trims to 'b' (1 byte). Retained text is
// 2 bytes, so omitted must be 8 2 = 6 — not the budget's 8 2 2 = 4.
const r = new TextRetainer({ kind: 'headTail', headBytes: 2, tailBytes: 2 })
r.push('a€€b')
const result = r.finish()
const retainedBytes = new TextEncoder().encode(result.text).length
expect(result.omittedBytes).toEqual<Omitted>({ kind: 'exact', count: 8 - retainedBytes })
})
it('preserves a whole multibyte codepoint that fits exactly', () => {