/** * `LocalFetchProvider`: a `WebFetchProvider` that retrieves a concrete public * HTTP(S) URL with platform-native `fetch` at the repo's Node floor and returns a status * code plus bounded decoded content. It owns SAFE RESOURCE RETRIEVAL — URL * validation, redirect policy, timeout, abort, byte caps, charset decoding, * content-type classification, binary rejection — but NOT presentation * (HTML→markdown lives in `@deepseek-ai/dsh-tool-web`). * * Redirects are followed manually (`redirect: 'manual'`) so the provider can * enforce a same-origin-only policy: a cross-origin redirect is refused with * `WEB_REDIRECT_BLOCKED`, requiring a fresh tool call (Claude Code's WebFetch * uses the same model). It does NOT carry browser cookies, editor/git * credentials, or implicit access to private services. * * SSRF / private-network protection is DEFERRED (see the package RFC); until it * lands this provider is an SSRF primitive and must not be enabled where it can * reach sensitive internal targets. * * @module @deepseek-ai/dsh-web-fetch-local/provider */ import { WebError } from '@deepseek-ai/dsh-web' import type { WebFetchBody, WebFetchProvider, WebFetchRequest, WebFetchResult, WebProviderStatus } from '@deepseek-ai/dsh-web' import { clampTimeout, deadline, timeoutOf } from '@deepseek-ai/dsh-timeout' import { classifyContentType, decoderForCharset, isSameOrigin, parseCharset, validateFetchUrl } from './policy.ts' /** Resolved provider limits (the plugin's schemastery Config supplies defaults). */ export interface LocalFetchLimits { /** Maximum accepted request URL length. */ maxUrlLength: number /** Maximum response body size in bytes (read is aborted past this). */ maxResponseBytes: number /** Maximum decoded body length in characters (truncated past this). */ maxBodyChars: number /** Default fetch timeout in milliseconds. */ timeoutMs: number /** Upper bound for a per-request timeout override. */ maxTimeoutMs: number /** Maximum number of (same-origin) redirect hops to follow. */ maxRedirects: number /** `User-Agent` header sent on every request. */ userAgent: string } /** Stable id this provider registers under. */ export const LOCAL_FETCH_PROVIDER_ID = 'local-http' /** The anonymous public HTTP(S) fetch provider. */ export class LocalFetchProvider implements WebFetchProvider { readonly id = LOCAL_FETCH_PROVIDER_ID constructor(private readonly limits: LocalFetchLimits) {} /** No credentials to check — an anonymous public fetcher is always usable. */ status(): WebProviderStatus { return { available: true } } async fetch(request: WebFetchRequest, exec?: { readonly signal?: AbortSignal }): Promise { if (exec?.signal?.aborted) throw new WebError('web fetch aborted', 'WEB_ABORTED') const timeoutMs = clampTimeout(request.timeoutMs, this.limits.timeoutMs, this.limits.maxTimeoutMs) // One deadline signal fuses the caller's abort with our own timeout, so the // network request and the streaming read both stop on either. The timeout // abort carries a TimeoutReason we recover afterward to classify the cause // (translateAbortOrNetwork), instead of hand-rolling a controller + timer + // reason-recovery dance. using d = deadline(exec?.signal, timeoutMs, 'WEB_FETCH_TIMEOUT') return await this.followAndRead(request.url, d.signal) } /** Follow same-origin redirects up to the hop cap, then read the final response. */ private async followAndRead(initialUrl: string, signal: AbortSignal): Promise { let currentUrl = validateFetchUrl(initialUrl, this.limits.maxUrlLength) let redirectsFollowed = 0 for (;;) { const response = await this.requestOnce(currentUrl, signal) if (isRedirectStatus(response.status)) { // The redirect budget is enforced BEFORE this hop's target is resolved // or origin-checked, so `maxRedirects: N` follows at most N redirects // exactly: the (N+1)th redirect is refused as "exceeded" regardless of // where it points (a same-origin/cross-origin distinction on a hop we // are not allowed to follow would be the wrong diagnosis). if (redirectsFollowed >= this.limits.maxRedirects) { await response.body?.cancel() throw new WebError(`exceeded the maximum of ${this.limits.maxRedirects} redirects`, 'WEB_REDIRECT_BLOCKED') } const location = response.headers.get('location') if (location === null) { // A redirect status with no Location is not a usable resource. Cancel // the (possibly streaming) body before throwing so no socket leaks. await response.body?.cancel() throw new WebError(`redirect response (HTTP ${response.status}) without a Location header`, 'WEB_PROVIDER_ERROR') } const target = resolveRedirect(location, currentUrl) // Re-validate the target against the same transport hygiene a direct // request gets: a redirect must not be a back door to a credentialed, // non-http(s), or over-long URL that validateFetchUrl would reject. A // rejection here must still cancel the body first (see below). let validatedTarget: URL try { validatedTarget = validateFetchUrl(target.toString(), this.limits.maxUrlLength) if (!isSameOrigin(validatedTarget, currentUrl)) { throw new WebError( `cross-origin redirect to ${validatedTarget.origin} is not followed automatically; retry against that URL directly`, 'WEB_REDIRECT_BLOCKED', ) } } catch (error: unknown) { await response.body?.cancel() throw error } await response.body?.cancel() currentUrl = validatedTarget redirectsFollowed++ continue } return await this.readBody(response, currentUrl, signal) } } private async requestOnce(url: URL, signal: AbortSignal): Promise { try { return await fetch(url, { method: 'GET', redirect: 'manual', headers: { 'user-agent': this.limits.userAgent, 'accept': 'text/html,application/xhtml+xml,text/*;q=0.9,application/json;q=0.8' }, signal, }) } catch (error: unknown) { throw translateAbortOrNetwork(error, signal) } } /** Read, byte-cap, classify, and decode the final response body. */ private async readBody(response: Response, finalUrl: URL, signal: AbortSignal): Promise { const contentType = response.headers.get('content-type') const kind = classifyContentType(contentType) if (kind === undefined) { await response.body?.cancel() throw new WebError(`unsupported content type "${contentType ?? 'unknown'}"`, 'WEB_UNSUPPORTED_CONTENT_TYPE') } // Resolve the decoder BEFORE reading the body so an unsupported charset // fails without consuming the stream — but cancel the body on that failure // so the socket does not leak (matching the unsupported-content-type path). let decoder: TextDecoder try { decoder = decoderForCharset(parseCharset(contentType)) } catch (error: unknown) { await response.body?.cancel() throw error } const { bytes, truncatedByBytes } = await this.readCapped(response, signal) const decoded = decoder.decode(bytes) const truncatedByChars = decoded.length > this.limits.maxBodyChars const content = truncatedByChars ? decoded.slice(0, this.limits.maxBodyChars) : decoded const body: WebFetchBody = kind === 'html' ? { kind: 'html', content } : { kind: 'text', content } return { providerId: this.id, url: finalUrl.toString(), statusCode: response.status, body, truncated: truncatedByBytes || truncatedByChars, } } /** * Read the response stream up to `maxResponseBytes`. A `Content-Length` over * the cap rejects immediately with `WEB_FETCH_TOO_LARGE`; a stream that grows * past the cap is cut short (`truncatedByBytes`) rather than rejected, so a * server that under-reports still yields a bounded usable body. */ private async readCapped(response: Response, signal: AbortSignal): Promise<{ bytes: Uint8Array; truncatedByBytes: boolean }> { const declared = response.headers.get('content-length') if (declared !== null) { const length = Number(declared) if (Number.isFinite(length) && length > this.limits.maxResponseBytes) { await response.body?.cancel() throw new WebError(`response exceeds the maximum of ${this.limits.maxResponseBytes} bytes`, 'WEB_FETCH_TOO_LARGE') } } /* v8 ignore next -- a 2xx Response from fetch always exposes a body stream; the null guard is defensive. */ if (response.body === null) return { bytes: new Uint8Array(0), truncatedByBytes: false } const chunks: Uint8Array[] = [] let total = 0 let truncatedByBytes = false const reader = response.body.getReader() try { for (;;) { const { done, value } = await reader.read() if (done) break const remaining = this.limits.maxResponseBytes - total // Only DROPPED bytes count as truncation: a chunk that exactly fills the // remaining capacity keeps all its bytes and we read on to observe EOF, // so an exactly-at-cap body is not falsely flagged truncated. if (value.byteLength > remaining) { chunks.push(value.subarray(0, remaining)) total += remaining truncatedByBytes = true break } chunks.push(value) total += value.byteLength } } catch (error: unknown) { /* v8 ignore next -- mid-stream read fault needs a network drop after headers; translate path covered by request-phase tests. */ throw translateAbortOrNetwork(error, signal) } finally { /* v8 ignore next 4 -- cancel() after a completed/broken read settles without rejecting; unobserved best-effort cleanup. */ await reader.cancel().catch(() => { // Cancel after a successful read (or after we broke past the cap) is // best-effort cleanup; the bytes we need are already collected. }) } const bytes = new Uint8Array(total) let offset = 0 for (const chunk of chunks) { bytes.set(chunk, offset) offset += chunk.byteLength } return { bytes, truncatedByBytes } } } /** HTTP redirect status codes that carry a `Location`. */ function isRedirectStatus(status: number): boolean { return status === 301 || status === 302 || status === 303 || status === 307 || status === 308 } /** Resolve a (possibly relative) `Location` against the current URL. */ function resolveRedirect(location: string, base: URL): URL { try { return new URL(location, base) } catch (error: unknown) { /* v8 ignore next 2 -- URL resolution against a valid absolute base effectively never throws; defensive guard. */ throw new WebError(`invalid redirect Location "${location}"`, 'WEB_PROVIDER_ERROR', { cause: error }) } } /** * Translate a thrown fetch/stream error into a `WebError`, classified by the * deadline signal rather than the error's shape (which differs by phase: the * request-phase `fetch` rejects with the abort reason, while the read-phase * reader surfaces a bare `AbortError`). `timeoutOf(signal, 'WEB_FETCH_TIMEOUT')` * recovering OUR reason means our timeout fired (`WEB_FETCH_TIMEOUT`); any other * abort — an upstream cancel, or a foreign/outer deadline's timeout under * nesting — is `WEB_ABORTED`; a throw with the signal NOT aborted is a * transport/network failure (`WEB_PROVIDER_ERROR`). */ function translateAbortOrNetwork(error: unknown, signal: AbortSignal): WebError { const timeout = timeoutOf(signal, 'WEB_FETCH_TIMEOUT') if (timeout !== undefined) return new WebError('web fetch timed out', 'WEB_FETCH_TIMEOUT', { cause: timeout }) if (signal.aborted) return new WebError('web fetch aborted', 'WEB_ABORTED', { cause: error }) return new WebError(`web fetch failed: ${String(error)}`, 'WEB_PROVIDER_ERROR', { cause: error }) }