diff --git a/packages/util/retention/src/index.ts b/packages/util/retention/src/index.ts index 91f6412691..f0f40ba7af 100644 --- a/packages/util/retention/src/index.ts +++ b/packages/util/retention/src/index.ts @@ -388,23 +388,33 @@ export class TextRetainer { finish(): RetainedText { const prefixLen = Math.min(this.total, this.prefixCap) const suffixLen = Math.min(this.total - prefixLen, this.suffixCap) - const omitted = this.omittedAt(this.total) - const truncated = omitted > 0 const prefix = concat(this.prefixChunks) // exactly prefixLen bytes (prefixHeld === prefixLen) const suffix = concat(this.suffixChunks).subarray(this.suffixHeld - suffixLen) - // With nothing omitted, prefix and suffix are ADJACENT slices of one stream - // (prefixLen + suffixLen === total), so the head|tail split is artificial: a - // codepoint may span it. Decode the contiguous whole as one buffer — trimming - // or decoding the halves separately here would corrupt a boundary-spanning - // codepoint though no content was actually dropped. Only a real omitted gap - // makes each side a true cut: trim each to a UTF-8 boundary and decode - // separately so a codepoint is never reconstructed across the gap. - const text = truncated - ? decoder.decode(trimTrailingPartialUtf8(prefix)) + decoder.decode(trimLeadingContinuationUtf8(suffix)) + // With nothing omitted by budget, prefix and suffix are ADJACENT slices of + // one stream (prefixLen + suffixLen === total), so the head|tail split is + // artificial: a codepoint may span it. Decode the contiguous whole as one + // buffer — trimming or decoding the halves separately here would corrupt a + // boundary-spanning codepoint though no content was dropped. Only a real + // omitted gap makes each side a true cut: trim each to a UTF-8 boundary and + // decode separately so a codepoint is never reconstructed across the gap. + const budgetOmitted = this.omittedAt(this.total) + const [keptPrefix, keptSuffix] = budgetOmitted > 0 + ? [trimTrailingPartialUtf8(prefix), trimLeadingContinuationUtf8(suffix)] + : [prefix, suffix] + const text = budgetOmitted > 0 + ? decoder.decode(keptPrefix) + decoder.decode(keptSuffix) : decoder.decode(concat([prefix, suffix])) + // Report omission against the bytes ACTUALLY returned, not the pre-trim + // budget: a boundary trim drops partial-codepoint bytes too, so an exact + // count derived from the budget alone would overstate the retained text (and + // any "Omitted N bytes" notice built from it would be a lie). total_seen − + // retained stays a valid lower bound under `atLeast` (true total ≥ seen). + const omitted = this.total - keptPrefix.length - keptSuffix.length + const truncated = omitted > 0 + return { text, truncated, diff --git a/packages/util/retention/tests/retention.spec.ts b/packages/util/retention/tests/retention.spec.ts index fd512508be..ec424595cb 100644 --- a/packages/util/retention/tests/retention.spec.ts +++ b/packages/util/retention/tests/retention.spec.ts @@ -223,8 +223,10 @@ describe('TextRetainer — UTF-8 boundary handling', () => { const result = r.finish() expect(result.text).toBe('a') // partial '€' dropped, no U+FFFD expect(result.text).not.toContain('�') - // Omission counts BYTES not kept by retention: 5 total − 2 prefix = 3. - expect(result.omittedBytes).toEqual({ kind: 'exact', count: 3 }) + // Omission counts bytes ACTUALLY absent from the returned text, including + // the partial 'E2' the boundary trim dropped: 5 total − 1 retained = 4 + // (not the pre-trim budget of 3, which would overstate what was kept). + expect(result.omittedBytes).toEqual({ kind: 'exact', count: 4 }) }) it('trims a leading partial codepoint at the tail cut', () => { @@ -236,6 +238,21 @@ describe('TextRetainer — UTF-8 boundary handling', () => { const result = r.finish() expect(result.text).toBe('b') // partial '€' at the front dropped expect(result.text).not.toContain('�') + // Honest count: 5 total − 1 retained ('b') = 4, including the trimmed AC. + expect(result.omittedBytes).toEqual({ kind: 'exact', count: 4 }) + }) + + it('omitted count matches the bytes actually absent, across a headTail boundary trim', () => { + // Regression: the exact count must equal total − retained (post-trim), never + // the pre-trim budget. 'a€€b' is 8 bytes (61 E2828C… ×2 61? no: 61 E2 82 AC + // E2 82 AC 62). headBytes 2 keeps 'a'+partial-E2 → trims to 'a' (1 byte); + // tailBytes 2 keeps partial-AC+'b' → trims to 'b' (1 byte). Retained text is + // 2 bytes, so omitted must be 8 − 2 = 6 — not the budget's 8 − 2 − 2 = 4. + const r = new TextRetainer({ kind: 'headTail', headBytes: 2, tailBytes: 2 }) + r.push('a€€b') + const result = r.finish() + const retainedBytes = new TextEncoder().encode(result.text).length + expect(result.omittedBytes).toEqual({ kind: 'exact', count: 8 - retainedBytes }) }) it('preserves a whole multibyte codepoint that fits exactly', () => {