Openstatus
www.openstatus.dev
🐛 improve notification (#1216)
* 🐛 improve notification
* Apply suggestions from code review
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
---------
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
authored by
Thibault Le Ouay
Copilot
and committed by
GitHub
89f744f5
74c6746a
+71
-57
+71
-57
apps/workflows/src/checker/index.ts
+71
-57
apps/workflows/src/checker/index.ts
···
96
96
if (affectedRegion.count >= numberOfRegions / 2 || numberOfRegions === 1) {
97
97
switch (status) {
98
98
case "active": {
99
-
if (monitor.status !== "active") {
100
-
await db
101
-
.update(schema.monitor)
102
-
.set({ status: "active" })
103
-
.where(eq(schema.monitor.id, monitor.id));
104
-
}
105
-
106
-
const incident = await db
107
-
.select()
108
-
.from(incidentTable)
109
-
.where(
110
-
and(
111
-
eq(incidentTable.monitorId, Number(monitorId)),
112
-
isNull(incidentTable.resolvedAt),
113
-
isNull(incidentTable.acknowledgedAt),
114
-
),
115
-
)
116
-
.get();
117
-
118
-
if (!incident) {
119
-
// it was just a single failure not a proper incident
120
-
break;
121
-
}
122
-
if (incident?.resolvedAt) {
123
-
// incident is already resolved
99
+
// it's been resolved
100
+
if (monitor.status === "active") {
124
101
break;
125
102
}
126
103
127
-
console.log(`🤓 recovering incident ${incident.id}`);
104
+
console.log(`🔄 update monitorStatus ${monitor.id} status: ACTIVE`);
128
105
await db
129
-
.update(incidentTable)
130
-
.set({
131
-
resolvedAt: new Date(cronTimestamp),
132
-
autoResolved: true,
133
-
})
134
-
.where(eq(incidentTable.id, incident.id))
135
-
.run();
106
+
.update(schema.monitor)
107
+
.set({ status: "active" })
108
+
.where(eq(schema.monitor.id, monitor.id));
109
+
110
+
// we can't have a monitor in error without an incident
111
+
if (monitor.status === "error") {
112
+
const incident = await db
113
+
.select()
114
+
.from(incidentTable)
115
+
.where(
116
+
and(
117
+
eq(incidentTable.monitorId, Number(monitorId)),
118
+
isNull(incidentTable.resolvedAt),
119
+
isNull(incidentTable.acknowledgedAt),
120
+
),
121
+
)
122
+
.get();
123
+
124
+
if (!incident) {
125
+
// it was just a single failure not a proper incident
126
+
break;
127
+
}
128
+
if (incident?.resolvedAt) {
129
+
// incident is already resolved
130
+
break;
131
+
}
132
+
133
+
console.log(`🤓 recovering incident ${incident.id}`);
134
+
await db
135
+
.update(incidentTable)
136
+
.set({
137
+
resolvedAt: new Date(cronTimestamp),
138
+
autoResolved: true,
139
+
})
140
+
.where(eq(incidentTable.id, incident.id))
141
+
.run();
142
+
}
136
143
137
144
await triggerNotifications({
138
145
monitorId,
···
155
162
break;
156
163
}
157
164
case "degraded":
158
-
if (monitor.status !== "degraded") {
159
-
console.log(
160
-
`🔄 update monitorStatus ${monitor.id} status: DEGRADED}`,
161
-
);
162
-
await db
163
-
.update(schema.monitor)
164
-
.set({ status: "degraded" })
165
-
.where(eq(schema.monitor.id, monitor.id));
166
-
// figure how to send the notification once
167
-
await triggerNotifications({
168
-
monitorId,
169
-
statusCode,
170
-
message,
171
-
notifType: "degraded",
172
-
cronTimestamp,
173
-
latency,
174
-
region,
175
-
incidentId: `${cronTimestamp}`,
176
-
});
165
+
if (monitor.status === "degraded") {
166
+
// already degraded let's return early
167
+
break;
177
168
}
169
+
console.log(`🔄 update monitorStatus ${monitor.id} status: DEGRADED`);
170
+
await db
171
+
.update(schema.monitor)
172
+
.set({ status: "degraded" })
173
+
.where(eq(schema.monitor.id, monitor.id));
174
+
// figure how to send the notification once
175
+
await triggerNotifications({
176
+
monitorId,
177
+
statusCode,
178
+
message,
179
+
notifType: "degraded",
180
+
cronTimestamp,
181
+
latency,
182
+
region,
183
+
incidentId: `${cronTimestamp}`,
184
+
});
185
+
178
186
await checkerAudit.publishAuditLog({
179
187
id: `monitor:${monitorId}`,
180
188
action: "monitor.degraded",
···
183
191
});
184
192
break;
185
193
case "error":
186
-
if (monitor.status !== "error") {
187
-
await db
188
-
.update(schema.monitor)
189
-
.set({ status: "error" })
190
-
.where(eq(schema.monitor.id, monitor.id));
194
+
if (monitor.status === "error") {
195
+
// already in error let's return early
196
+
break;
191
197
}
198
+
199
+
console.log(`🔄 update monitorStatus ${monitor.id} status: ERROR`);
200
+
await db
201
+
.update(schema.monitor)
202
+
.set({ status: "error" })
203
+
.where(eq(schema.monitor.id, monitor.id));
204
+
192
205
try {
193
206
const incident = await db
194
207
.select()
···
258
271
} catch {
259
272
console.log("incident was already created");
260
273
}
274
+
261
275
break;
262
276
default:
263
277
console.log("should not happen");