forked from
nekomimi.pet/wisp.place-monorepo
Monorepo for wisp.place. A static site hosting service built on top of the AT Protocol.
expo backoff, fix concurrency
+159
-30
+5
-1
apps/hosting-service/src/index.ts
+5
-1
apps/hosting-service/src/index.ts
···
16
16
17
17
const PORT = process.env.PORT ? parseInt(process.env.PORT) : 3001;
18
18
const CACHE_DIR = process.env.CACHE_DIR || './cache/sites';
19
+
const BACKFILL_CONCURRENCY = process.env.BACKFILL_CONCURRENCY
20
+
? parseInt(process.env.BACKFILL_CONCURRENCY)
21
+
: undefined; // Let backfill.ts default (10) apply
19
22
20
23
// Parse CLI arguments
21
24
const args = process.argv.slice(2);
···
52
55
console.log('🔄 Backfill requested, starting cache backfill...');
53
56
backfillCache({
54
57
skipExisting: true,
55
-
concurrency: 3,
58
+
concurrency: BACKFILL_CONCURRENCY,
56
59
}).then((stats) => {
57
60
console.log('✅ Cache backfill completed');
58
61
}).catch((err) => {
···
83
86
Cache: ${CACHE_DIR}
84
87
Firehose: Connected to Firehose
85
88
Cache-Only: ${CACHE_ONLY_MODE ? 'ENABLED (no DB writes)' : 'DISABLED'}
89
+
Backfill: ${backfillOnStartup ? `ENABLED (concurrency: ${BACKFILL_CONCURRENCY || 10})` : 'DISABLED'}
86
90
`);
87
91
88
92
// Graceful shutdown
+26
-2
apps/hosting-service/src/lib/utils.ts
+26
-2
apps/hosting-service/src/lib/utils.ts
···
90
90
export async function fetchSiteRecord(did: string, rkey: string): Promise<{ record: WispFsRecord; cid: string } | null> {
91
91
try {
92
92
const pdsEndpoint = await getPdsForDid(did);
93
-
if (!pdsEndpoint) return null;
93
+
if (!pdsEndpoint) {
94
+
console.error('[hosting-service] Failed to get PDS endpoint for DID', { did, rkey });
95
+
return null;
96
+
}
94
97
95
98
const url = `${pdsEndpoint}/xrpc/com.atproto.repo.getRecord?repo=${encodeURIComponent(did)}&collection=place.wisp.fs&rkey=${encodeURIComponent(rkey)}`;
96
99
const data = await safeFetchJson(url);
···
100
103
cid: data.cid || ''
101
104
};
102
105
} catch (err) {
103
-
console.error('Failed to fetch site record', did, rkey, err);
106
+
const errorCode = (err as any)?.code;
107
+
const errorMsg = err instanceof Error ? err.message : String(err);
108
+
109
+
// Better error logging to distinguish between network errors and 404s
110
+
if (errorMsg.includes('HTTP 404') || errorMsg.includes('Not Found')) {
111
+
console.log('[hosting-service] Site record not found', { did, rkey });
112
+
} else if (errorCode && ['ECONNRESET', 'ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR', 'ETIMEDOUT'].includes(errorCode)) {
113
+
console.error('[hosting-service] Network/SSL error fetching site record (after retries)', {
114
+
did,
115
+
rkey,
116
+
error: errorMsg,
117
+
code: errorCode
118
+
});
119
+
} else {
120
+
console.error('[hosting-service] Failed to fetch site record', {
121
+
did,
122
+
rkey,
123
+
error: errorMsg,
124
+
code: errorCode
125
+
});
126
+
}
127
+
104
128
return null;
105
129
}
106
130
}
+128
-27
packages/@wisp/safe-fetch/src/index.ts
+128
-27
packages/@wisp/safe-fetch/src/index.ts
···
28
28
const MAX_BLOB_SIZE = 500 * 1024 * 1024; // 500MB
29
29
const MAX_REDIRECTS = 10;
30
30
31
+
// Retry configuration
32
+
const MAX_RETRIES = 3;
33
+
const INITIAL_RETRY_DELAY = 1000; // 1 second
34
+
const MAX_RETRY_DELAY = 10000; // 10 seconds
35
+
31
36
function isBlockedHost(hostname: string): boolean {
32
37
const lowerHost = hostname.toLowerCase();
33
38
···
44
49
return false;
45
50
}
46
51
52
+
/**
53
+
* Check if an error is retryable (network/SSL errors, not HTTP errors)
54
+
*/
55
+
function isRetryableError(err: unknown): boolean {
56
+
if (!(err instanceof Error)) return false;
57
+
58
+
// Network errors (ECONNRESET, ENOTFOUND, etc.)
59
+
const errorCode = (err as any).code;
60
+
if (errorCode) {
61
+
const retryableCodes = [
62
+
'ECONNRESET',
63
+
'ECONNREFUSED',
64
+
'ETIMEDOUT',
65
+
'ENOTFOUND',
66
+
'ENETUNREACH',
67
+
'EAI_AGAIN',
68
+
'EPIPE',
69
+
'ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR', // SSL/TLS handshake failures
70
+
'ERR_SSL_WRONG_VERSION_NUMBER',
71
+
'UNABLE_TO_VERIFY_LEAF_SIGNATURE',
72
+
];
73
+
if (retryableCodes.includes(errorCode)) {
74
+
return true;
75
+
}
76
+
}
77
+
78
+
// Timeout errors
79
+
if (err.name === 'AbortError' || err.message.includes('timeout')) {
80
+
return true;
81
+
}
82
+
83
+
// Fetch failures (generic network errors)
84
+
if (err.message.includes('fetch failed')) {
85
+
return true;
86
+
}
87
+
88
+
return false;
89
+
}
90
+
91
+
/**
92
+
* Sleep for a given number of milliseconds
93
+
*/
94
+
function sleep(ms: number): Promise<void> {
95
+
return new Promise(resolve => setTimeout(resolve, ms));
96
+
}
97
+
98
+
/**
99
+
* Retry a function with exponential backoff
100
+
*/
101
+
async function withRetry<T>(
102
+
fn: () => Promise<T>,
103
+
options: { maxRetries?: number; initialDelay?: number; maxDelay?: number; context?: string } = {}
104
+
): Promise<T> {
105
+
const maxRetries = options.maxRetries ?? MAX_RETRIES;
106
+
const initialDelay = options.initialDelay ?? INITIAL_RETRY_DELAY;
107
+
const maxDelay = options.maxDelay ?? MAX_RETRY_DELAY;
108
+
const context = options.context ?? 'Request';
109
+
110
+
let lastError: unknown;
111
+
112
+
for (let attempt = 0; attempt <= maxRetries; attempt++) {
113
+
try {
114
+
return await fn();
115
+
} catch (err) {
116
+
lastError = err;
117
+
118
+
// Don't retry if this is the last attempt or error is not retryable
119
+
if (attempt === maxRetries || !isRetryableError(err)) {
120
+
throw err;
121
+
}
122
+
123
+
// Calculate delay with exponential backoff
124
+
const delay = Math.min(initialDelay * Math.pow(2, attempt), maxDelay);
125
+
126
+
const errorCode = (err as any)?.code;
127
+
const errorMsg = err instanceof Error ? err.message : String(err);
128
+
console.warn(
129
+
`${context} failed (attempt ${attempt + 1}/${maxRetries + 1}): ${errorMsg}${errorCode ? ` [${errorCode}]` : ''} - retrying in ${delay}ms`
130
+
);
131
+
132
+
await sleep(delay);
133
+
}
134
+
}
135
+
136
+
throw lastError;
137
+
}
138
+
47
139
export async function safeFetch(
48
140
url: string,
49
-
options?: RequestInit & { maxSize?: number; timeout?: number }
141
+
options?: RequestInit & { maxSize?: number; timeout?: number; retry?: boolean }
50
142
): Promise<Response> {
143
+
const shouldRetry = options?.retry !== false; // Default to true
51
144
const timeoutMs = options?.timeout ?? FETCH_TIMEOUT;
52
145
const maxSize = options?.maxSize ?? MAX_RESPONSE_SIZE;
53
146
54
-
// Parse and validate URL
147
+
// Parse and validate URL (done once, outside retry loop)
55
148
let parsedUrl: URL;
56
149
try {
57
150
parsedUrl = new URL(url);
···
68
161
throw new Error(`Blocked host: ${hostname}`);
69
162
}
70
163
71
-
const controller = new AbortController();
72
-
const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
164
+
const fetchFn = async () => {
165
+
const controller = new AbortController();
166
+
const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
167
+
168
+
try {
169
+
const response = await fetch(url, {
170
+
...options,
171
+
signal: controller.signal,
172
+
redirect: 'follow',
173
+
headers: {
174
+
'User-Agent': 'wisp-place hosting-service',
175
+
...(options?.headers || {}),
176
+
},
177
+
});
73
178
74
-
try {
75
-
const response = await fetch(url, {
76
-
...options,
77
-
signal: controller.signal,
78
-
redirect: 'follow',
79
-
headers: {
80
-
'User-Agent': 'wisp-place hosting-service',
81
-
...(options?.headers || {}),
82
-
},
83
-
});
179
+
const contentLength = response.headers.get('content-length');
180
+
if (contentLength && parseInt(contentLength, 10) > maxSize) {
181
+
throw new Error(`Response too large: ${contentLength} bytes`);
182
+
}
84
183
85
-
const contentLength = response.headers.get('content-length');
86
-
if (contentLength && parseInt(contentLength, 10) > maxSize) {
87
-
throw new Error(`Response too large: ${contentLength} bytes`);
184
+
return response;
185
+
} catch (err) {
186
+
if (err instanceof Error && err.name === 'AbortError') {
187
+
throw new Error(`Request timeout after ${timeoutMs}ms`);
188
+
}
189
+
throw err;
190
+
} finally {
191
+
clearTimeout(timeoutId);
88
192
}
193
+
};
89
194
90
-
return response;
91
-
} catch (err) {
92
-
if (err instanceof Error && err.name === 'AbortError') {
93
-
throw new Error(`Request timeout after ${timeoutMs}ms`);
94
-
}
95
-
throw err;
96
-
} finally {
97
-
clearTimeout(timeoutId);
195
+
if (shouldRetry) {
196
+
return withRetry(fetchFn, { context: `Fetch ${parsedUrl.hostname}` });
197
+
} else {
198
+
return fetchFn();
98
199
}
99
200
}
100
201
101
202
export async function safeFetchJson<T = any>(
102
203
url: string,
103
-
options?: RequestInit & { maxSize?: number; timeout?: number }
204
+
options?: RequestInit & { maxSize?: number; timeout?: number; retry?: boolean }
104
205
): Promise<T> {
105
206
const maxJsonSize = options?.maxSize ?? MAX_JSON_SIZE;
106
207
const response = await safeFetch(url, { ...options, maxSize: maxJsonSize });
···
146
247
147
248
export async function safeFetchBlob(
148
249
url: string,
149
-
options?: RequestInit & { maxSize?: number; timeout?: number }
250
+
options?: RequestInit & { maxSize?: number; timeout?: number; retry?: boolean }
150
251
): Promise<Uint8Array> {
151
252
const maxBlobSize = options?.maxSize ?? MAX_BLOB_SIZE;
152
253
const timeoutMs = options?.timeout ?? FETCH_TIMEOUT_BLOB;