fix: report omitted bytes against retained text, not budget (codex round 2)
finish() derived the exact omitted count from the pre-trim prefix/suffix budget, but the UTF-8 boundary trims drop additional partial-codepoint bytes, so an "Omitted N bytes" notice overstated what was kept (head maxBytes:2 over 'a€b' returned 'a' but claimed exact 3 when 4 bytes are absent). Derive the count from the bytes actually returned (total − keptPrefix − keptSuffix) so exact metadata matches the text and atLeast stays a valid lower bound.
This commit is contained in:
@@ -388,23 +388,33 @@ export class TextRetainer {
|
||||
finish(): RetainedText {
|
||||
const prefixLen = Math.min(this.total, this.prefixCap)
|
||||
const suffixLen = Math.min(this.total - prefixLen, this.suffixCap)
|
||||
const omitted = this.omittedAt(this.total)
|
||||
const truncated = omitted > 0
|
||||
|
||||
const prefix = concat(this.prefixChunks) // exactly prefixLen bytes (prefixHeld === prefixLen)
|
||||
const suffix = concat(this.suffixChunks).subarray(this.suffixHeld - suffixLen)
|
||||
|
||||
// With nothing omitted, prefix and suffix are ADJACENT slices of one stream
|
||||
// (prefixLen + suffixLen === total), so the head|tail split is artificial: a
|
||||
// codepoint may span it. Decode the contiguous whole as one buffer — trimming
|
||||
// or decoding the halves separately here would corrupt a boundary-spanning
|
||||
// codepoint though no content was actually dropped. Only a real omitted gap
|
||||
// makes each side a true cut: trim each to a UTF-8 boundary and decode
|
||||
// separately so a codepoint is never reconstructed across the gap.
|
||||
const text = truncated
|
||||
? decoder.decode(trimTrailingPartialUtf8(prefix)) + decoder.decode(trimLeadingContinuationUtf8(suffix))
|
||||
// With nothing omitted by budget, prefix and suffix are ADJACENT slices of
|
||||
// one stream (prefixLen + suffixLen === total), so the head|tail split is
|
||||
// artificial: a codepoint may span it. Decode the contiguous whole as one
|
||||
// buffer — trimming or decoding the halves separately here would corrupt a
|
||||
// boundary-spanning codepoint though no content was dropped. Only a real
|
||||
// omitted gap makes each side a true cut: trim each to a UTF-8 boundary and
|
||||
// decode separately so a codepoint is never reconstructed across the gap.
|
||||
const budgetOmitted = this.omittedAt(this.total)
|
||||
const [keptPrefix, keptSuffix] = budgetOmitted > 0
|
||||
? [trimTrailingPartialUtf8(prefix), trimLeadingContinuationUtf8(suffix)]
|
||||
: [prefix, suffix]
|
||||
const text = budgetOmitted > 0
|
||||
? decoder.decode(keptPrefix) + decoder.decode(keptSuffix)
|
||||
: decoder.decode(concat([prefix, suffix]))
|
||||
|
||||
// Report omission against the bytes ACTUALLY returned, not the pre-trim
|
||||
// budget: a boundary trim drops partial-codepoint bytes too, so an exact
|
||||
// count derived from the budget alone would overstate the retained text (and
|
||||
// any "Omitted N bytes" notice built from it would be a lie). total_seen −
|
||||
// retained stays a valid lower bound under `atLeast` (true total ≥ seen).
|
||||
const omitted = this.total - keptPrefix.length - keptSuffix.length
|
||||
const truncated = omitted > 0
|
||||
|
||||
return {
|
||||
text,
|
||||
truncated,
|
||||
|
||||
@@ -223,8 +223,10 @@ describe('TextRetainer — UTF-8 boundary handling', () => {
|
||||
const result = r.finish()
|
||||
expect(result.text).toBe('a') // partial '€' dropped, no U+FFFD
|
||||
expect(result.text).not.toContain('<27>')
|
||||
// Omission counts BYTES not kept by retention: 5 total − 2 prefix = 3.
|
||||
expect(result.omittedBytes).toEqual<Omitted>({ kind: 'exact', count: 3 })
|
||||
// Omission counts bytes ACTUALLY absent from the returned text, including
|
||||
// the partial 'E2' the boundary trim dropped: 5 total − 1 retained = 4
|
||||
// (not the pre-trim budget of 3, which would overstate what was kept).
|
||||
expect(result.omittedBytes).toEqual<Omitted>({ kind: 'exact', count: 4 })
|
||||
})
|
||||
|
||||
it('trims a leading partial codepoint at the tail cut', () => {
|
||||
@@ -236,6 +238,21 @@ describe('TextRetainer — UTF-8 boundary handling', () => {
|
||||
const result = r.finish()
|
||||
expect(result.text).toBe('b') // partial '€' at the front dropped
|
||||
expect(result.text).not.toContain('<27>')
|
||||
// Honest count: 5 total − 1 retained ('b') = 4, including the trimmed AC.
|
||||
expect(result.omittedBytes).toEqual<Omitted>({ kind: 'exact', count: 4 })
|
||||
})
|
||||
|
||||
it('omitted count matches the bytes actually absent, across a headTail boundary trim', () => {
|
||||
// Regression: the exact count must equal total − retained (post-trim), never
|
||||
// the pre-trim budget. 'a€€b' is 8 bytes (61 E2828C… ×2 61? no: 61 E2 82 AC
|
||||
// E2 82 AC 62). headBytes 2 keeps 'a'+partial-E2 → trims to 'a' (1 byte);
|
||||
// tailBytes 2 keeps partial-AC+'b' → trims to 'b' (1 byte). Retained text is
|
||||
// 2 bytes, so omitted must be 8 − 2 = 6 — not the budget's 8 − 2 − 2 = 4.
|
||||
const r = new TextRetainer({ kind: 'headTail', headBytes: 2, tailBytes: 2 })
|
||||
r.push('a€€b')
|
||||
const result = r.finish()
|
||||
const retainedBytes = new TextEncoder().encode(result.text).length
|
||||
expect(result.omittedBytes).toEqual<Omitted>({ kind: 'exact', count: 8 - retainedBytes })
|
||||
})
|
||||
|
||||
it('preserves a whole multibyte codepoint that fits exactly', () => {
|
||||
|
||||
Reference in New Issue
Block a user