Mirror: A Node.js fetch shim using built-in Request, Response, and Headers (but without native fetch)
1import { Stream, Readable, pipeline } from 'node:stream';
2import { Socket } from 'node:net';
3import * as https from 'node:https';
4import * as http from 'node:http';
5import * as url from 'node:url';
6
7import { extractBody } from './body';
8import { createContentDecoder } from './encoding';
9import {
10 URL,
11 Request,
12 RequestInit,
13 Response,
14 HeadersInit,
15 Headers,
16} from './webstd';
17import { getHttpsAgent, getHttpAgent } from './agent';
18
19/** Maximum allowed redirects (matching Chromium's limit) */
20const MAX_REDIRECTS = 20;
21
22const DEFAULT_TIMEOUT = 30_000;
23
24const parseURL = (input: string, base?: string | URL): URL | null => {
25 try {
26 return new URL(input, base);
27 } catch {
28 return null;
29 }
30};
31
32const isHeaders = (x: unknown): x is Headers =>
33 x != null &&
34 typeof x === 'object' &&
35 (('append' in x && typeof x.append === 'function') || x instanceof Headers);
36
37/** Convert Node.js raw headers array to Headers */
38const headersOfRawHeaders = (rawHeaders: readonly string[]): Headers => {
39 const headers = new Headers();
40 for (let i = 0; i < rawHeaders.length; i += 2)
41 headers.append(rawHeaders[i], rawHeaders[i + 1]);
42 return headers;
43};
44
45type HeadersDict = Record<string, string | readonly string[]>;
46
47/** Assign Headers to a Node.js OutgoingMessage (request) */
48const assignOutgoingMessageHeaders = (
49 outgoing: http.OutgoingMessage,
50 headers: HeadersInit
51): HeadersDict => {
52 // Preassemble array headers, mostly only for Set-Cookie
53 // We're avoiding `getSetCookie` since support is unclear in Node 18
54 let collection: HeadersDict;
55 if (!Array.isArray(headers) && !isHeaders(headers)) {
56 collection = headers;
57 } else {
58 collection = Object.create(null);
59 const canonicalNames = new Map();
60 for (const [name, value] of headers) {
61 const lowerKey = name.toLowerCase();
62 let key = canonicalNames.get(lowerKey) ?? name;
63 if (!canonicalNames.has(lowerKey)) canonicalNames.set(lowerKey, name);
64 if (Array.isArray(collection[key])) {
65 collection[key].push(value);
66 } else if (collection[key] != undefined) {
67 collection[key] = [collection[key] as string, value];
68 } else {
69 collection[key] = value;
70 }
71 }
72 }
73 // We don't use `setHeaders` due to a Bun bug (Fix: https://github.com/oven-sh/bun/pull/27050)
74 for (const key in collection) {
75 outgoing.setHeader(key, collection[key]);
76 }
77 return collection;
78};
79
80const stripRedirectHeaders = (headers: HeadersDict | undefined) => {
81 if (headers) {
82 for (const key in headers) {
83 switch (key.toLowerCase()) {
84 case 'content-length':
85 case 'content-type':
86 delete headers[key];
87 }
88 }
89 }
90};
91
92/** Normalize methods and disallow special methods */
93const toRedirectOption = (
94 redirect: string | undefined
95): 'follow' | 'manual' | 'error' => {
96 switch (redirect) {
97 case 'follow':
98 case 'manual':
99 case 'error':
100 return redirect;
101 case undefined:
102 return 'follow';
103 default:
104 throw new TypeError(
105 `Request constructor: ${redirect} is not an accepted type. Expected one of follow, manual, error.`
106 );
107 }
108};
109
110/** Normalize methods and disallow special methods */
111const methodToHttpOption = (method: string | undefined): string => {
112 const normalized = method?.toUpperCase();
113 switch (normalized) {
114 case 'CONNECT':
115 case 'TRACE':
116 case 'TRACK':
117 throw new TypeError(
118 `Failed to construct 'Request': '${method}' HTTP method is unsupported.`
119 );
120 case 'DELETE':
121 case 'GET':
122 case 'HEAD':
123 case 'OPTIONS':
124 case 'POST':
125 case 'PUT':
126 return normalized;
127 default:
128 return method ?? 'GET';
129 }
130};
131
132/** Convert URL to Node.js HTTP request options and disallow unsupported protocols */
133const urlToHttpOptions = (input: URL) => {
134 const _url = new URL(input);
135 switch (_url.protocol) {
136 // TODO: 'file:' and 'data:' support
137 case 'http:':
138 case 'https:':
139 return url.urlToHttpOptions(_url);
140 default:
141 throw new TypeError(`URL scheme "${_url.protocol}" is not supported.`);
142 }
143};
144
145/** Returns if `input` is a Request object */
146const isRequest = (input: any): input is Request =>
147 input != null && typeof input === 'object' && 'body' in input;
148
149/** Returns if status `code` is a redirect code */
150const isRedirectCode = (
151 code: number | undefined
152): code is 301 | 302 | 303 | 307 | 308 =>
153 code === 301 || code === 302 || code === 303 || code === 307 || code === 308;
154
155function createResponse(
156 body: ConstructorParameters<typeof Response>[0] | null,
157 init: ResponseInit,
158 params: {
159 url: string;
160 redirected: boolean;
161 type: 'basic' | 'cors' | 'default' | 'error' | 'opaque' | 'opaqueredirect';
162 }
163) {
164 const response = new Response(body, init);
165 Object.defineProperty(response, 'url', { value: params.url });
166 if (params.type !== 'default')
167 Object.defineProperty(response, 'type', { value: params.type });
168 if (params.redirected)
169 Object.defineProperty(response, 'redirected', { value: params.redirected });
170 return response;
171}
172
173function attachRefLifetime(body: Readable, socket: Socket): void {
174 const { _read } = body;
175 body.on('close', () => {
176 socket.unref();
177 });
178 body._read = function _readRef(...args: Parameters<Readable['_read']>) {
179 body._read = _read;
180 socket.ref();
181 return _read.apply(this, args);
182 };
183}
184
185async function _fetch(
186 input: string | URL | Request,
187 init?: RequestInit
188): Promise<Response> {
189 const initFromRequest = isRequest(input);
190 const initUrl = initFromRequest ? input.url : input;
191 const initBody = init?.body ?? (initFromRequest ? input.body : null);
192 const signal = init?.signal ?? (initFromRequest ? input.signal : undefined);
193 const redirect = toRedirectOption(
194 init?.redirect ?? (initFromRequest ? input.redirect : undefined)
195 );
196
197 let requestUrl = new URL(initUrl);
198 let requestBody = extractBody(initBody);
199 let redirects = 0;
200
201 let requestHeaders =
202 init?.headers ?? (initFromRequest ? input.headers : undefined);
203
204 const requestOptions = {
205 ...urlToHttpOptions(requestUrl),
206 timeout: init?.connectTimeout ?? DEFAULT_TIMEOUT,
207 method: methodToHttpOption(initFromRequest ? input.method : init?.method),
208 signal,
209 } satisfies http.RequestOptions;
210
211 function _call(
212 resolve: (response: Response | Promise<Response>) => void,
213 reject: (reason?: any) => void
214 ) {
215 requestOptions.agent =
216 requestOptions.protocol === 'https:'
217 ? getHttpsAgent(requestOptions)
218 : getHttpAgent(requestOptions);
219 const method = requestOptions.method;
220 const protocol = requestOptions.protocol === 'https:' ? https : http;
221 const outgoing = protocol.request(requestOptions);
222
223 let incoming: http.IncomingMessage | undefined;
224
225 const destroy = (reason?: any) => {
226 if (reason) {
227 outgoing?.destroy(signal?.aborted ? signal.reason : reason);
228 incoming?.destroy(signal?.aborted ? signal.reason : reason);
229 reject(signal?.aborted ? signal.reason : reason);
230 }
231 signal?.removeEventListener('abort', destroy);
232 };
233
234 signal?.addEventListener('abort', destroy);
235
236 outgoing.on('timeout', () => {
237 if (!incoming) {
238 const error = new Error('Request timed out') as NodeJS.ErrnoException;
239 error.code = 'ETIMEDOUT';
240 destroy(error);
241 }
242 });
243
244 outgoing.on('response', _incoming => {
245 if (signal?.aborted) {
246 return;
247 }
248
249 incoming = _incoming;
250 incoming.setTimeout(0); // Forcefully disable timeout
251 incoming.socket.unref();
252 incoming.on('error', destroy);
253
254 const responseInit = {
255 status: incoming.statusCode,
256 statusText: incoming.statusMessage,
257 headers: headersOfRawHeaders(incoming.rawHeaders),
258 } satisfies ResponseInit;
259
260 if (isRedirectCode(responseInit.status)) {
261 const location = responseInit.headers.get('Location');
262 const locationURL =
263 location != null ? parseURL(location, requestUrl) : null;
264 if (redirect === 'error') {
265 reject(
266 new Error(
267 'URI requested responds with a redirect, redirect mode is set to error'
268 )
269 );
270 return;
271 } else if (redirect === 'manual' && location) {
272 responseInit.headers.set('Location', locationURL?.href ?? location);
273 } else if (redirect === 'follow') {
274 if (locationURL === null) {
275 reject(
276 new Error('URI requested responds with an invalid redirect URL')
277 );
278 return;
279 } else if (++redirects > MAX_REDIRECTS) {
280 reject(new Error(`maximum redirect reached at: ${requestUrl}`));
281 return;
282 } else if (
283 locationURL.protocol !== 'http:' &&
284 locationURL.protocol !== 'https:'
285 ) {
286 // TODO: do we need a special Error instance here?
287 reject(new Error('URL scheme must be a HTTP(S) scheme'));
288 return;
289 }
290
291 if (
292 responseInit.status === 303 ||
293 ((responseInit.status === 301 || responseInit.status === 302) &&
294 method === 'POST')
295 ) {
296 requestBody = extractBody(null);
297 requestOptions.method = 'GET';
298 stripRedirectHeaders(requestHeaders as HeadersDict);
299 } else if (
300 requestBody.body != null &&
301 requestBody.contentLength == null
302 ) {
303 reject(new Error('Cannot follow redirect with a streamed body'));
304 return;
305 } else {
306 requestBody = extractBody(initBody);
307 }
308
309 Object.assign(
310 requestOptions,
311 urlToHttpOptions((requestUrl = locationURL))
312 );
313 return _call(resolve, reject);
314 }
315 }
316
317 let body: Readable | null = incoming;
318 const encoding = responseInit.headers
319 .get('Content-Encoding')
320 ?.toLowerCase();
321 if (
322 method === 'HEAD' ||
323 responseInit.status === 204 ||
324 responseInit.status === 304
325 ) {
326 body = null;
327 } else if (encoding != null) {
328 responseInit.headers.set('Content-Encoding', encoding);
329 body = pipeline(body, createContentDecoder(encoding), destroy);
330 outgoing.on('error', destroy);
331 }
332
333 // Re-ref the socket when the body starts being consumed to prevent
334 // early process exit, then unref when done to allow normal exit.
335 if (body != null) {
336 attachRefLifetime(body, incoming.socket);
337 }
338
339 resolve(
340 createResponse(body, responseInit, {
341 type: 'default',
342 url: requestUrl.toString(),
343 redirected: redirects > 0,
344 })
345 );
346 });
347
348 outgoing.on('error', destroy);
349
350 if (requestHeaders) {
351 requestHeaders = assignOutgoingMessageHeaders(outgoing, requestHeaders);
352 }
353
354 if (!outgoing.hasHeader('Accept')) {
355 outgoing.setHeader('Accept', '*/*');
356 }
357 if (!outgoing.hasHeader('Content-Type') && requestBody.contentType) {
358 outgoing.setHeader('Content-Type', requestBody.contentType);
359 }
360
361 if (
362 requestBody.body == null &&
363 (method === 'POST' || method === 'PUT' || method === 'PATCH')
364 ) {
365 outgoing.setHeader('Content-Length', '0');
366 } else if (requestBody.body != null && requestBody.contentLength != null) {
367 outgoing.setHeader('Content-Length', `${requestBody.contentLength}`);
368 }
369
370 if (requestBody.body == null) {
371 outgoing.end();
372 } else if (requestBody.body instanceof Uint8Array) {
373 outgoing.write(requestBody.body);
374 outgoing.end();
375 } else {
376 const body =
377 requestBody.body instanceof Stream
378 ? requestBody.body
379 : Readable.fromWeb(requestBody.body);
380 pipeline(body, outgoing, destroy);
381 }
382 }
383
384 return await new Promise(_call);
385}
386
387export { _fetch as fetch };