fix: address codex review round 1

- Re-validate redirect targets through validateFetchUrl before following, so a
  same-origin Location carrying credentials (or a non-http(s)/over-long URL)
  cannot bypass the transport hygiene a direct request enforces.
- Treat only DROPPED bytes as truncation: a body exactly at maxResponseBytes is
  no longer falsely flagged truncated (which emitted a spurious footer).
- Honor the declared response charset: parse the Content-Type charset and decode
  with it (rejecting unsupported labels as WEB_UNSUPPORTED_CONTENT_TYPE) instead
  of always assuming UTF-8 and returning replacement characters.
- Catalog the web seam vocabulary in docs/core-data-structures/web.md with
  type-equiv blocks + manifest entries, per the core-data-structures rule.

packages/web/web-fetch-local/src/index.ts

@@ -18,7 +18,7 @@ export {
   LocalFetchProvider,
 } from './provider.ts'
 export type { LocalFetchLimits } from './provider.ts'
-export { classifyContentType, isSameOrigin, validateFetchUrl } from './policy.ts'
+export { classifyContentType, decoderForCharset, isSameOrigin, parseCharset, validateFetchUrl } from './policy.ts'
 export type { FetchableKind } from './policy.ts'
 
 /** Default `User-Agent`: an explicit product agent, never a browser disguise. */

packages/web/web-fetch-local/src/policy.ts

@@ -57,3 +57,29 @@ export function classifyContentType(contentType: string | null): FetchableKind |
   if (mime === 'application/json' || mime === 'application/xml' || mime.endsWith('+json') || mime.endsWith('+xml')) return 'text'
   return undefined
 }
+
+/**
+ * Extract the `charset` parameter from a response `Content-Type`, lower-cased,
+ * or `undefined` when absent. The provider feeds this label to `TextDecoder`
+ * so a non-UTF-8 response is decoded with its declared encoding rather than
+ * silently mangled into replacement characters.
+ */
+export function parseCharset(contentType: string | null): string | undefined {
+  const match = /;\s*charset\s*=\s*"?([^";]+)"?/i.exec(contentType ?? '')
+  return match?.[1]?.trim().toLowerCase()
+}
+
+/**
+ * Build a `TextDecoder` for the declared charset, falling back to UTF-8 when
+ * none is declared. Throws {@link WebError} `WEB_UNSUPPORTED_CONTENT_TYPE` when
+ * the label is present but not a charset `TextDecoder` recognizes — better to
+ * fail loudly than return mojibake.
+ */
+export function decoderForCharset(charset: string | undefined): TextDecoder {
+  if (charset === undefined) return new TextDecoder('utf-8')
+  try {
+    return new TextDecoder(charset)
+  } catch (error: unknown) {
+    throw new WebError(`unsupported charset "${charset}"`, 'WEB_UNSUPPORTED_CONTENT_TYPE', { cause: error })
+  }
+}

packages/web/web-fetch-local/src/provider.ts

@@ -21,7 +21,7 @@
 
 import { WebError } from '@deepseek-ai/dsh-web'
 import type { WebFetchBody, WebFetchProvider, WebFetchRequest, WebFetchResult, WebProviderStatus } from '@deepseek-ai/dsh-web'
-import { classifyContentType, isSameOrigin, validateFetchUrl } from './policy.ts'
+import { classifyContentType, decoderForCharset, isSameOrigin, parseCharset, validateFetchUrl } from './policy.ts'
 
 /** Resolved provider limits (the plugin's schemastery Config supplies defaults). */
 export interface LocalFetchLimits {
@@ -92,14 +92,18 @@ export class LocalFetchProvider implements WebFetchProvider {
           throw new WebError(`redirect response (HTTP ${response.status}) without a Location header`, 'WEB_PROVIDER_ERROR')
         }
         const target = resolveRedirect(location, currentUrl)
-        if (!isSameOrigin(target, currentUrl)) {
+        // Re-validate the target against the same transport hygiene a direct
+        // request gets: a redirect must not be a back door to a credentialed,
+        // non-http(s), or over-long URL that validateFetchUrl would reject.
+        const validatedTarget = validateFetchUrl(target.toString(), this.limits.maxUrlLength)
+        if (!isSameOrigin(validatedTarget, currentUrl)) {
           throw new WebError(
-            `cross-origin redirect to ${target.origin} is not followed automatically; retry against that URL directly`,
+            `cross-origin redirect to ${validatedTarget.origin} is not followed automatically; retry against that URL directly`,
             'WEB_REDIRECT_BLOCKED',
           )
         }
         await response.body?.cancel()
-        currentUrl = target
+        currentUrl = validatedTarget
         continue
       }
 
@@ -124,14 +128,18 @@ export class LocalFetchProvider implements WebFetchProvider {
 
   /** Read, byte-cap, classify, and decode the final response body. */
   private async readBody(response: Response, finalUrl: URL): Promise<WebFetchResult> {
-    const kind = classifyContentType(response.headers.get('content-type'))
+    const contentType = response.headers.get('content-type')
+    const kind = classifyContentType(contentType)
     if (kind === undefined) {
       await response.body?.cancel()
-      throw new WebError(`unsupported content type "${response.headers.get('content-type') ?? 'unknown'}"`, 'WEB_UNSUPPORTED_CONTENT_TYPE')
+      throw new WebError(`unsupported content type "${contentType ?? 'unknown'}"`, 'WEB_UNSUPPORTED_CONTENT_TYPE')
     }
 
+    // Resolve the decoder BEFORE reading the body so an unsupported charset
+    // fails without consuming the stream.
+    const decoder = decoderForCharset(parseCharset(contentType))
     const { bytes, truncatedByBytes } = await this.readCapped(response)
-    const decoded = new TextDecoder('utf-8').decode(bytes)
+    const decoded = decoder.decode(bytes)
     const truncatedByChars = decoded.length > this.limits.maxBodyChars
     const content = truncatedByChars ? decoded.slice(0, this.limits.maxBodyChars) : decoded
     const body: WebFetchBody = kind === 'html' ? { kind: 'html', content } : { kind: 'text', content }
@@ -173,7 +181,10 @@ export class LocalFetchProvider implements WebFetchProvider {
         const { done, value } = await reader.read()
         if (done) break
         const remaining = this.limits.maxResponseBytes - total
-        if (value.byteLength >= remaining) {
+        // Only DROPPED bytes count as truncation: a chunk that exactly fills the
+        // remaining capacity keeps all its bytes and we read on to observe EOF,
+        // so an exactly-at-cap body is not falsely flagged truncated.
+        if (value.byteLength > remaining) {
           chunks.push(value.subarray(0, remaining))
           total += remaining
           truncatedByBytes = true

packages/web/web-fetch-local/tests/fetch-local.spec.ts

@@ -3,7 +3,7 @@ import { createServer, type IncomingMessage, type Server, type ServerResponse }
 import { AddressInfo } from 'node:net'
 import { Context } from 'cordis'
 import WebService from '@deepseek-ai/dsh-web'
-import { LocalFetchProvider, LOCAL_FETCH_PROVIDER_ID, classifyContentType, isSameOrigin, validateFetchUrl } from '@deepseek-ai/dsh-web-fetch-local'
+import { LocalFetchProvider, LOCAL_FETCH_PROVIDER_ID, classifyContentType, decoderForCharset, isSameOrigin, parseCharset, validateFetchUrl } from '@deepseek-ai/dsh-web-fetch-local'
 import type { LocalFetchLimits } from '@deepseek-ai/dsh-web-fetch-local'
 import * as fetchPlugin from '@deepseek-ai/dsh-web-fetch-local'
 
@@ -62,6 +62,19 @@ describe('policy helpers', () => {
     expect(isSameOrigin(new URL('https://a.com'), new URL('https://b.com'))).toBe(false)
     expect(isSameOrigin(new URL('http://a.com'), new URL('https://a.com'))).toBe(false)
   })
+
+  it('parses the charset parameter', () => {
+    expect(parseCharset('text/html; charset=UTF-8')).toBe('utf-8')
+    expect(parseCharset('text/plain; charset="iso-8859-1"')).toBe('iso-8859-1')
+    expect(parseCharset('text/plain')).toBeUndefined()
+    expect(parseCharset(null)).toBeUndefined()
+  })
+
+  it('builds a decoder for a charset and defaults to UTF-8', () => {
+    expect(decoderForCharset(undefined).encoding).toBe('utf-8')
+    expect(decoderForCharset('iso-8859-1').encoding).toBe('windows-1252')
+    expect(() => decoderForCharset('not-a-charset')).toThrow(expect.objectContaining({ code: 'WEB_UNSUPPORTED_CONTENT_TYPE' }))
+  })
 })
 
 describe('LocalFetchProvider success', () => {
@@ -109,6 +122,13 @@ describe('LocalFetchProvider caps', () => {
     expect(result.truncated).toBe(true)
   })
 
+  it('does not flag a body that exactly fills the byte cap as truncated', async () => {
+    handler = (_req, res) => { res.writeHead(200, { 'content-type': 'text/plain' }); res.end('abcd') }
+    const result = await provider({ maxResponseBytes: 4 }).fetch({ url: base })
+    expect(result.body.content).toBe('abcd')
+    expect(result.truncated).toBe(false)
+  })
+
   it('truncates a decoded body past the character cap', async () => {
     handler = (_req, res) => { res.writeHead(200, { 'content-type': 'text/plain' }); res.end('abcdefghij') }
     const result = await provider({ maxBodyChars: 3 }).fetch({ url: base })
@@ -133,6 +153,19 @@ describe('LocalFetchProvider caps', () => {
     const result = await provider().fetch({ url: base })
     expect(result.body.content).toBe('sized')
   })
+
+  it('decodes a non-UTF-8 declared charset', async () => {
+    // 0xE9 is "é" in ISO-8859-1; decoded as UTF-8 it would be a replacement char.
+    handler = (_req, res) => { res.writeHead(200, { 'content-type': 'text/plain; charset=iso-8859-1' }); res.end(Buffer.from([0x63, 0x61, 0x66, 0xE9])) }
+    const result = await provider().fetch({ url: base })
+    expect(result.body.content).toBe('café')
+  })
+
+  it('rejects an unsupported declared charset', async () => {
+    handler = (_req, res) => { res.writeHead(200, { 'content-type': 'text/plain; charset=not-a-charset' }); res.end('x') }
+    await expect(provider().fetch({ url: base }))
+      .rejects.toThrow(expect.objectContaining({ code: 'WEB_UNSUPPORTED_CONTENT_TYPE' }))
+  })
 })
 
 describe('LocalFetchProvider redirects', () => {
@@ -152,6 +185,13 @@ describe('LocalFetchProvider redirects', () => {
       .rejects.toThrow(expect.objectContaining({ code: 'WEB_REDIRECT_BLOCKED' }))
   })
 
+  it('re-validates a redirect target, rejecting same-origin credentials in the Location', async () => {
+    const { port } = server.address() as AddressInfo
+    handler = (_req, res) => { res.writeHead(302, { location: `http://user:pass@127.0.0.1:${port}/` }); res.end() }
+    await expect(provider().fetch({ url: base }))
+      .rejects.toThrow(expect.objectContaining({ code: 'WEB_BLOCKED_URL' }))
+  })
+
   it('rejects exceeding the redirect hop cap', async () => {
     handler = (req, res) => {
       const n = Number(new URL(req.url ?? '/', base).searchParams.get('n') ?? '0')