Mirror of
fix: parsing issues and add backfill for today midday
+231
-121
+9
-10
scripts/backfill.ts
+9
-10
scripts/backfill.ts
···
1
1
import { z } from "zod";
2
2
import { spawn } from "child_process";
3
3
4
-
const END_TIME_CONSTANT = "2026-02-03T06:00:00Z";
4
+
const END_TIME_CONSTANT = "2026-02-03T14:00:00Z";
5
5
const WINDOW_HOURS = 8;
6
6
7
7
const EventSchema = z.object({
···
64
64
const startIso = start.toISOString().split(".")[0] + "Z";
65
65
const endIso = end.toISOString().split(".")[0] + "Z";
66
66
67
-
const repository = `repo:${owner}/${repo}`;
68
-
const timeRange = `closed:${startIso}..${endIso}`;
69
-
const filters = "is:closed reason:completed -is:unmerged";
70
-
71
-
const query = encodeURIComponent(`${repository} ${filters} ${timeRange}`);
67
+
const query = encodeURIComponent(
68
+
`repo:${owner}/${repo} is:closed reason:completed -is:unmerged closed:${startIso}..${endIso}`,
69
+
);
72
70
73
71
const headers = {
74
72
Accept: "application/vnd.github.v3+json",
···
149
147
const postId = post.uri.split("/").pop();
150
148
const isRepost = !!item.reason;
151
149
152
-
const titlePrefix = isRepost ? `[Repost from @${authorHandle}] ` : "";
153
-
const title = `${titlePrefix}${post.record.text.substring(0, 80)}`;
154
-
155
150
events.push({
156
151
source: "bluesky",
157
-
title,
152
+
title: `${isRepost ? `[Repost from @${authorHandle}] ` : ""}${post.record.text.substring(0, 80)}`,
158
153
description: post.record.text,
159
154
url: `https://bsky.app/profile/${authorHandle}/post/${postId}`,
160
155
timestamp: itemDate.toISOString(),
···
170
165
}
171
166
172
167
async function run() {
168
+
// Ensuring MODELS_TOKEN exists even if not used for fetching,
169
+
// to maintain consistency with the lib requirement.
170
+
getRequiredEnv("MODELS_TOKEN");
171
+
173
172
const end = new Date(END_TIME_CONSTANT);
174
173
const start = new Date(end.getTime() - WINDOW_HOURS * 60 * 60 * 1000);
175
174
+121
src/content/posts/2026-02-03-midday.json
+121
src/content/posts/2026-02-03-midday.json
···
1
+
{
2
+
"title": "npmx slashes CI times and expands i18n",
3
+
"date": "2026-02-03T14:00:00.000Z",
4
+
"type": "midday",
5
+
"topics": [
6
+
{
7
+
"title": "CI/CD Performance Optimization",
8
+
"summary": "Major overhaul of CI pipelines to reduce bottlenecks. Highlights include switching to chromium-headless-shell for Playwright, matrix-splitting a11y tests by theme, and implementing a 'pooling' strategy for axe-core runs that cuts unit test time by 60%. Documentation was also added to guide contributors on linking issues to PRs for automated workflow management.",
9
+
"relevanceScore": 10,
10
+
"sources": [
11
+
{
12
+
"platform": "github",
13
+
"url": "https://github.com/npmx-dev/npmx.dev/pull/815"
14
+
},
15
+
{
16
+
"platform": "github",
17
+
"url": "https://github.com/npmx-dev/npmx.dev/pull/819"
18
+
},
19
+
{
20
+
"platform": "github",
21
+
"url": "https://github.com/npmx-dev/npmx.dev/pull/833"
22
+
},
23
+
{
24
+
"platform": "github",
25
+
"url": "https://github.com/npmx-dev/npmx.dev/pull/818"
26
+
}
27
+
]
28
+
},
29
+
{
30
+
"title": "Global Expansion and i18n Fixes",
31
+
"summary": "npmx significantly expanded its language support with the addition of Marathi and en-GB locales. Updates were also merged for Chinese, French, and Polish translations. Technical fixes addressed hydration mismatches in contributor counts and pluralization logic for the about page.",
32
+
"relevanceScore": 8,
33
+
"sources": [
34
+
{
35
+
"platform": "github",
36
+
"url": "https://github.com/npmx-dev/npmx.dev/pull/800"
37
+
},
38
+
{
39
+
"platform": "github",
40
+
"url": "https://github.com/npmx-dev/npmx.dev/pull/820"
41
+
},
42
+
{
43
+
"platform": "github",
44
+
"url": "https://github.com/npmx-dev/npmx.dev/pull/825"
45
+
}
46
+
]
47
+
},
48
+
{
49
+
"title": "UI/UX Accessibility and Navigation",
50
+
"summary": "Resolved critical mobile navigation bugs where the hamburger menu obstructed search inputs and caused page shifts. Enhanced accessibility by refactoring Call-to-Action cards with semantic H3 headings and optimized z-index layering for better link focus. Smooth scrolling was disabled to prevent 'jerky' navigation transitions.",
51
+
"relevanceScore": 8,
52
+
"sources": [
53
+
{
54
+
"platform": "github",
55
+
"url": "https://github.com/npmx-dev/npmx.dev/pull/804"
56
+
},
57
+
{
58
+
"platform": "github",
59
+
"url": "https://github.com/npmx-dev/npmx.dev/pull/799"
60
+
},
61
+
{
62
+
"platform": "github",
63
+
"url": "https://github.com/npmx-dev/npmx.dev/pull/823"
64
+
}
65
+
]
66
+
},
67
+
{
68
+
"title": "Architectural Refactoring",
69
+
"summary": "Internal code cleanup involved separating 'npm' composables to reduce technical debt, moving non-composable logic to utilities, and standardizing directory casing (renaming 'compare' to 'Compare').",
70
+
"relevanceScore": 7,
71
+
"sources": [
72
+
{
73
+
"platform": "github",
74
+
"url": "https://github.com/npmx-dev/npmx.dev/pull/827"
75
+
},
76
+
{
77
+
"platform": "github",
78
+
"url": "https://github.com/npmx-dev/npmx.dev/pull/814"
79
+
}
80
+
]
81
+
},
82
+
{
83
+
"title": "SSR and Performance Tuning",
84
+
"summary": "Optimized server-side rendering by preventing unnecessary hits to the OAuth session endpoint during SSR. Build efficiency was improved by updating vue-data-ui, which significantly reduced the chunk count. Session cleanup logic was refactored to run in parallel for better performance.",
85
+
"relevanceScore": 7,
86
+
"sources": [
87
+
{
88
+
"platform": "github",
89
+
"url": "https://github.com/npmx-dev/npmx.dev/pull/807"
90
+
},
91
+
{
92
+
"platform": "github",
93
+
"url": "https://github.com/npmx-dev/npmx.dev/pull/836"
94
+
},
95
+
{
96
+
"platform": "github",
97
+
"url": "https://github.com/npmx-dev/npmx.dev/pull/837"
98
+
}
99
+
]
100
+
},
101
+
{
102
+
"title": "Stability Reversions and Hotfixes",
103
+
"summary": "Reverted recent commits that introduced a missing 'defu' dependency and triggered latent CI failures in the main branch. A regression affecting the '?' keyboard shortcut was fixed to allow modifier key access, and missing imports for charts were restored.",
104
+
"relevanceScore": 6,
105
+
"sources": [
106
+
{
107
+
"platform": "github",
108
+
"url": "https://github.com/npmx-dev/npmx.dev/pull/797"
109
+
},
110
+
{
111
+
"platform": "github",
112
+
"url": "https://github.com/npmx-dev/npmx.dev/pull/809"
113
+
},
114
+
{
115
+
"platform": "github",
116
+
"url": "https://github.com/npmx-dev/npmx.dev/pull/842"
117
+
}
118
+
]
119
+
}
120
+
]
121
+
}
+101
-111
src/lib/events.ts
+101
-111
src/lib/events.ts
···
1
1
import { z } from "astro/zod";
2
2
import { TopicSchema, type Topic } from "../lib/schema";
3
3
4
+
const INFERENCE_URL = "https://models.inference.ai.azure.com/chat/completions";
5
+
4
6
const EventSchema = z.object({
5
7
source: z.enum(["github", "bluesky"]),
6
8
title: z.string(),
···
31
33
return text.replace(/npmx/gi, "npmx");
32
34
}
33
35
36
+
async function requestInference(payload: object) {
37
+
const token = getRequiredEnv("MODELS_TOKEN");
38
+
const response = await fetch(INFERENCE_URL, {
39
+
method: "POST",
40
+
headers: {
41
+
"Content-Type": "application/json",
42
+
Authorization: `Bearer ${token}`,
43
+
"User-Agent": "npmx-digest-bot",
44
+
},
45
+
body: JSON.stringify(payload),
46
+
});
47
+
48
+
if (!response.ok) {
49
+
const errorBody = await response.text();
50
+
throw new Error(`Inference failed [${response.status}]: ${errorBody}`);
51
+
}
52
+
53
+
return response.json();
54
+
}
55
+
34
56
export async function fetchGitHubEvents(since: Date): Promise<Event[]> {
35
57
const owner = "npmx-dev";
36
58
const repo = "npmx.dev";
···
38
60
const events: Event[] = [];
39
61
40
62
const startIso = since.toISOString().split(".")[0] + "Z";
41
-
const now = new Date();
42
-
const endIso = now.toISOString().split(".")[0] + "Z";
63
+
const endIso = new Date().toISOString().split(".")[0] + "Z";
43
64
44
-
const repository = `repo:${owner}/${repo}`;
45
-
const timeRange = `closed:${startIso}..${endIso}`;
46
-
const filters = "is:closed reason:completed -is:unmerged";
47
-
48
-
const query = encodeURIComponent(`${repository} ${filters} ${timeRange}`);
49
-
50
-
const headers = {
51
-
Accept: "application/vnd.github.v3+json",
52
-
"User-Agent": "npmx-digest-bot",
53
-
Authorization: `Bearer ${token}`,
54
-
};
65
+
const query = encodeURIComponent(
66
+
`repo:${owner}/${repo} is:closed reason:completed -is:unmerged closed:${startIso}..${endIso}`,
67
+
);
55
68
56
69
try {
57
70
const response = await fetch(
58
71
`https://api.github.com/search/issues?q=${query}`,
59
-
{ headers },
72
+
{
73
+
headers: {
74
+
Accept: "application/vnd.github.v3+json",
75
+
"User-Agent": "npmx-digest-bot",
76
+
Authorization: `Bearer ${token}`,
77
+
},
78
+
},
60
79
);
61
80
62
81
if (response.ok) {
···
64
83
const items = data.items || [];
65
84
66
85
items.forEach((item: any) => {
67
-
const isPR = !!item.pull_request;
68
86
events.push({
69
87
source: "github",
70
-
title: `${isPR ? "Merged PR" : "Closed Issue"} #${item.number}: ${item.title}`,
88
+
title: `${!!item.pull_request ? "Merged PR" : "Closed Issue"} #${item.number}: ${item.title}`,
71
89
description: item.body || "No description provided",
72
90
url: item.html_url,
73
91
timestamp: item.closed_at || item.created_at,
···
101
119
const { feed } = await feedRes.json();
102
120
103
121
const posts = feed.reduce((acc: Event[], item: any) => {
104
-
const actionTimestamp = item.reason?.indexedAt || item.post.indexedAt;
105
-
const itemDate = new Date(actionTimestamp);
106
-
107
-
if (itemDate >= since) {
108
-
const authorHandle = item.post.author.handle;
109
-
const isRepost = !!item.reason;
110
-
const postText = item.post.record.text;
111
-
const postId = item.post.uri.split("/").pop();
112
-
122
+
const timestamp = item.reason?.indexedAt || item.post.indexedAt;
123
+
if (new Date(timestamp) >= since) {
124
+
const author = item.post.author.handle;
113
125
acc.push({
114
126
source: "bluesky",
115
-
title: `${isRepost ? `[Repost from @${authorHandle}] ` : ""}${postText.substring(0, 80)}`,
116
-
description: postText,
117
-
url: `https://bsky.app/profile/${authorHandle}/post/${postId}`,
118
-
timestamp: actionTimestamp,
127
+
title: `${item.reason ? `[Repost from @${author}] ` : ""}${item.post.record.text.substring(0, 80)}`,
128
+
description: item.post.record.text,
129
+
url: `https://bsky.app/profile/${author}/post/${item.post.uri.split("/").pop()}`,
130
+
timestamp,
119
131
});
120
132
}
121
133
return acc;
···
131
143
}
132
144
133
145
export async function generateSmartDigest(events: Event[]): Promise<Topic[]> {
134
-
const token = getRequiredEnv("GITHUB_TOKEN");
135
-
if (!token || events.length === 0) return [];
146
+
if (events.length === 0) return [];
136
147
137
148
LOG.ai(
138
149
`Clustering ${events.length} signals into topics (Prioritizing Bluesky)...`,
139
150
);
140
151
141
-
const prompt = `You are a technical analyst for npmx. Group the following events into 5-6 logical "Topics".
152
+
const prompt = `You are a technical analyst for npmx. Group these events into 5-6 logical "Topics".
153
+
Return ONLY JSON: { "topics": Topic[] }.
142
154
143
-
STRATEGY:
144
-
1. Community Focus: Treat "bluesky" events as high-signal community interests.
145
-
2. Inclusive Clustering: Ensure that "bluesky" posts are not sidelined; weave them into relevant technical topics where possible.
146
-
3. Topic Weight: If a topic includes a "bluesky" post, it should generally have a higher relevanceScore.
147
-
148
-
Sort by relevanceScore (1-10). Refer to the project strictly as "npmx" (lowercase).
149
-
Return ONLY a JSON array with this structure: { "topics": Topic[] }.
155
+
Topic Structure:
156
+
{
157
+
"title": "string",
158
+
"summary": "string",
159
+
"relevanceScore": number (1-10),
160
+
"sources": [{ "platform": "github" | "bluesky", "url": "string" }]
161
+
}
150
162
151
163
Events: ${JSON.stringify(events)}`;
152
164
153
165
try {
154
-
const response = await fetch(
155
-
"https://api.github.com/models/chat/completions",
156
-
{
157
-
method: "POST",
158
-
headers: {
159
-
"Content-Type": "application/json",
160
-
Authorization: `Bearer ${token}`,
161
-
},
162
-
body: JSON.stringify({
163
-
messages: [{ role: "user", content: prompt }],
164
-
model: "gpt-4o-mini",
165
-
temperature: 0.3,
166
-
response_format: { type: "json_object" },
167
-
}),
168
-
},
169
-
);
166
+
const data = await requestInference({
167
+
messages: [
168
+
{ role: "system", content: "You are a JSON-only generator. No prose." },
169
+
{ role: "user", content: prompt },
170
+
],
171
+
model: "gpt-4o-mini",
172
+
temperature: 0.1,
173
+
response_format: { type: "json_object" },
174
+
});
170
175
171
-
if (response.ok) {
172
-
const data = await response.json();
173
-
const content = data.choices[0].message.content;
174
-
const parsed = JSON.parse(content);
176
+
// Cast the parsed content to our interface
177
+
const parsed = JSON.parse(data.choices[0].message.content) as {
178
+
topics: Topic[];
179
+
};
175
180
176
-
const rawTopics = parsed.topics.map((t: any) => {
177
-
const validated = TopicSchema.parse(t);
178
-
const hasBluesky = events.some(
179
-
(e) =>
180
-
e.source === "bluesky" &&
181
-
(t.summary.includes(e.title.substring(0, 15)) ||
182
-
t.title.includes(e.source)),
183
-
);
181
+
// Verify the topics array exists before processing
182
+
if (!parsed.topics || !Array.isArray(parsed.topics)) {
183
+
throw new Error("AI response missing 'topics' array");
184
+
}
184
185
185
-
return {
186
-
...validated,
187
-
title: sanitizeBrand(validated.title),
188
-
summary: sanitizeBrand(validated.summary),
189
-
relevanceScore: hasBluesky
190
-
? Math.min(10, validated.relevanceScore + 1)
191
-
: validated.relevanceScore,
192
-
};
193
-
});
186
+
const topics = parsed.topics.map((t) => {
187
+
// Step 1: Runtime validation via Zod
188
+
// This is where it would have failed if the AI output was "messy"
189
+
const validated = TopicSchema.parse(t);
194
190
195
-
const topics = rawTopics.sort(
196
-
(a: Topic, b: Topic) => b.relevanceScore - a.relevanceScore,
191
+
// Step 2: Logic & Refinement
192
+
const hasBluesky = validated.sources.some(
193
+
(s) => s.platform === "bluesky",
197
194
);
198
195
199
-
LOG.success(
200
-
`Successfully clustered into ${topics.length} topics with Bluesky priority.`,
201
-
);
202
-
return topics;
203
-
}
204
-
} catch {
205
-
LOG.error("AI Clustering failed.");
196
+
return {
197
+
...validated,
198
+
title: sanitizeBrand(validated.title),
199
+
summary: sanitizeBrand(validated.summary),
200
+
relevanceScore: hasBluesky
201
+
? Math.min(10, validated.relevanceScore + 1)
202
+
: validated.relevanceScore,
203
+
};
204
+
});
205
+
206
+
LOG.success(
207
+
`Successfully clustered into ${topics.length} topics with Bluesky priority.`,
208
+
);
209
+
return topics.sort((a, b) => b.relevanceScore - a.relevanceScore);
210
+
} catch (err: any) {
211
+
// If Zod fails, err.errors will contain exactly which field was missing/wrong
212
+
LOG.error(`AI Clustering failed: ${err.message}`);
213
+
return [];
206
214
}
207
-
return [];
208
215
}
209
216
210
217
export async function generateCatchyTitle(topic: Topic): Promise<string> {
211
-
const token = getRequiredEnv("GITHUB_TOKEN");
212
-
if (!token) return "New Update";
213
-
214
218
const prompt = `You are a tech journalist for npmx. Create a very short (max 5-7 words), catchy headline for this topic.
215
219
Return ONLY the text, no quotes. Topic: ${topic.title} - ${topic.summary}`;
216
220
217
221
try {
218
-
const response = await fetch(
219
-
"https://api.github.com/models/chat/completions",
220
-
{
221
-
method: "POST",
222
-
headers: {
223
-
"Content-Type": "application/json",
224
-
Authorization: `Bearer ${token}`,
225
-
},
226
-
body: JSON.stringify({
227
-
messages: [{ role: "user", content: prompt }],
228
-
model: "gpt-4o-mini",
229
-
temperature: 0.8,
230
-
max_tokens: 30,
231
-
}),
232
-
},
233
-
);
222
+
const data = await requestInference({
223
+
messages: [{ role: "user", content: prompt }],
224
+
model: "gpt-4o-mini",
225
+
temperature: 0.8,
226
+
max_tokens: 30,
227
+
});
234
228
235
-
if (response.ok) {
236
-
const data = await response.json();
237
-
const title = data.choices[0].message.content.trim();
238
-
return sanitizeBrand(title);
239
-
}
229
+
return sanitizeBrand(data.choices[0].message.content.trim());
240
230
} catch {
241
231
LOG.error("Failed to generate title");
232
+
return sanitizeBrand(topic.title);
242
233
}
243
-
return sanitizeBrand(topic.title);
244
234
}