Openstatus
www.openstatus.dev
feat: tb update schema (#472)
* chore: update tb schema
* 💽 tb
* chore: update tb client and zod schema
* docs: changelog typo
* fix: hydration error
* chore: clean up
* fix: wrong imports
* 💽 tb update
* 💽 tb update
---------
Co-authored-by: Thibault Le Ouay <thibaultleouay@gmail.Com>
authored by
Maximilian Kaske
Thibault Le Ouay
and committed by
GitHub
4d213e8f
fa4c7d63
+198
-66
+20
-3
apps/server/src/checker/checker.ts
+20
-3
apps/server/src/checker/checker.ts
···
10
10
payload,
11
11
latency,
12
12
statusCode,
13
+
message,
13
14
}: PublishPingType) => {
14
15
try {
15
16
console.log(
···
17
18
payload,
18
19
)} with latency ${latency} and status code ${statusCode}`,
19
20
);
20
-
await publishPing({ payload, statusCode, latency });
21
+
await publishPing({
22
+
payload,
23
+
statusCode,
24
+
latency,
25
+
message,
26
+
});
21
27
} catch {
22
28
try {
23
29
console.log(
24
30
"2️⃣ try publish ping to tb - attempt 2 ",
25
31
JSON.stringify(payload),
26
32
);
27
-
await publishPing({ payload, statusCode, latency });
33
+
34
+
await publishPing({
35
+
payload,
36
+
statusCode,
37
+
latency,
38
+
message,
39
+
});
28
40
} catch (e) {
29
41
throw e;
30
42
}
···
40
52
let startTime = 0;
41
53
let endTime = 0;
42
54
let res = null;
55
+
let message = undefined;
43
56
// We are doing these for wrong urls
44
57
try {
45
58
startTime = Date.now();
···
47
60
endTime = Date.now();
48
61
} catch (e) {
49
62
endTime = Date.now();
63
+
message = `${e}`;
50
64
console.log(
51
65
`🚨 error on pingEndpoint for ${JSON.stringify(data)} error: `,
52
66
e,
···
59
73
payload: data,
60
74
latency,
61
75
statusCode: res.status,
76
+
message: undefined,
62
77
});
63
78
if (data?.status === "error") {
64
79
await updateMonitorStatus({
···
80
95
await publishPingRetryPolicy({
81
96
payload: data,
82
97
latency,
83
-
statusCode: res?.status || 0,
98
+
statusCode: res?.status,
99
+
message: message,
84
100
});
101
+
85
102
if (data?.status === "active") {
86
103
await updateMonitorStatus({
87
104
monitorId: data.monitorId,
+5
-5
apps/server/src/checker/ping.ts
+5
-5
apps/server/src/checker/ping.ts
···
1
-
import { nanoid } from "nanoid";
2
-
3
1
import { publishPingResponse } from "@openstatus/tinybird";
4
2
5
3
import { env } from "../env";
···
40
38
41
39
export type PublishPingType = {
42
40
payload: Payload;
43
-
statusCode: number;
44
41
latency: number;
42
+
statusCode?: number | undefined;
43
+
message?: string | undefined;
45
44
};
46
45
47
46
export async function publishPing({
48
47
payload,
49
-
statusCode,
50
48
latency,
49
+
message,
50
+
statusCode,
51
51
}: PublishPingType) {
52
52
const { monitorId, cronTimestamp, url, workspaceId } = payload;
53
53
···
56
56
process.env.NODE_ENV === "test"
57
57
) {
58
58
const res = await publishPingResponse({
59
-
id: nanoid(), // TBD: we don't need it
60
59
timestamp: Date.now(),
61
60
statusCode,
62
61
latency,
···
65
64
monitorId,
66
65
cronTimestamp,
67
66
workspaceId,
67
+
message: message,
68
68
});
69
69
if (res.successful_rows === 0) {
70
70
throw new Error(`error 0 rows on publish ping for ${payload.monitorId}`);
+2
-1
apps/server/src/checker/utils.ts
+2
-1
apps/server/src/checker/utils.ts
+1
-1
apps/web/src/app/_components/input-search.tsx
+1
-1
apps/web/src/app/_components/input-search.tsx
-1
apps/web/src/app/api/checker/regions/_checker.ts
-1
apps/web/src/app/api/checker/regions/_checker.ts
+1
-1
apps/web/src/app/monitor/[id]/page.tsx
+1
-1
apps/web/src/app/monitor/[id]/page.tsx
+1
-1
apps/web/src/app/play/@modal/(..)monitor/[id]/page.tsx
+1
-1
apps/web/src/app/play/@modal/(..)monitor/[id]/page.tsx
+2
-2
apps/web/src/components/marketing/cards.tsx
+2
-2
apps/web/src/components/marketing/cards.tsx
···
20
20
return (
21
21
<li key={i}>
22
22
<p className="flex flex-col">
23
-
<p>
23
+
<span>
24
24
<FeatureIcon className="text-foreground/80 mb-1 mr-1.5 inline-flex h-4 w-4" />
25
25
<span className="text-foreground font-medium">
26
26
{feature.catchline.replace(".", "")}
27
27
</span>{" "}
28
-
</p>
28
+
</span>
29
29
<span className="text-muted-foreground">
30
30
{feature.description}
31
31
</span>
+1
apps/web/src/components/status-page/status-check.tsx
+1
apps/web/src/components/status-page/status-check.tsx
···
52
52
function calcStatus() {
53
53
const { count, ok } = monitorsData.flat(1).reduce(
54
54
(prev, curr) => {
55
+
if (!curr.statusCode) return prev; // TODO: handle this better
55
56
const isOk = curr.statusCode <= 299 && curr.statusCode >= 200;
56
57
return { count: prev.count + 1, ok: prev.ok + (isOk ? 1 : 0) };
57
58
},
+1
-1
apps/web/src/content/changelog/sms-notification.mdx
+1
-1
apps/web/src/content/changelog/sms-notification.mdx
···
1
1
---
2
2
title: Get alerted on your cell phone
3
3
description:
4
-
Receive alerts on your cell phone via SMS when your monitor is done.
4
+
Receive alerts on your cell phone via SMS when your monitor is down.
5
5
publishedAt: 2023-11-13
6
6
image: /assets/changelog/sms-notification.png
7
7
---
+6
-2
packages/notifications/discord/src/index.ts
+6
-2
packages/notifications/discord/src/index.ts
···
20
20
notification,
21
21
region,
22
22
statusCode,
23
+
message,
23
24
}: {
24
25
monitor: Monitor;
25
26
notification: Notification;
26
-
statusCode: number;
27
+
statusCode?: number;
27
28
region: string;
29
+
message?: string;
28
30
}) => {
29
31
const notificationData = JSON.parse(notification.data);
30
32
const { discord: webhookUrl } = notificationData; // webhook url
···
34
36
await postToWebhook(
35
37
`Your monitor ${name} is down 🚨
36
38
37
-
Your monitor with url ${monitor.url} is down in ${region} with status code ${statusCode}.`,
39
+
Your monitor with url ${monitor.url} is down in ${region} with ${
40
+
statusCode ? `status code ${statusCode}` : `error message ${message}`
41
+
}.`,
38
42
webhookUrl,
39
43
);
40
44
} catch (err) {
+10
-2
packages/notifications/email/src/index.ts
+10
-2
packages/notifications/email/src/index.ts
···
8
8
notification,
9
9
region,
10
10
statusCode,
11
+
message,
11
12
}: {
12
13
monitor: Monitor;
13
14
notification: Notification;
14
-
statusCode: number;
15
+
statusCode?: number;
15
16
region: string;
17
+
message?: string;
16
18
}) => {
17
19
// FIXME:
18
20
const config = EmailConfigurationSchema.parse(JSON.parse(notification.data));
···
29
31
from: "Notifications <ping@openstatus.dev>",
30
32
31
33
subject: `Your monitor ${monitor.name} is down 🚨`,
32
-
html: `<p>Hi,<br><br>Your monitor ${monitor.name} is down in ${region}. </p><p>URL : ${monitor.url}</p><p>Status Code: ${statusCode}</p><p>OpenStatus 🏓 </p>`,
34
+
html: `<p>Hi,<br><br>Your monitor ${
35
+
monitor.name
36
+
} is down in ${region}. </p><p>URL : ${monitor.url}</p> ${
37
+
statusCode
38
+
? `<p>Status Code: ${statusCode}</p>`
39
+
: `<p>Error message: ${message}</p>`
40
+
}<p>OpenStatus 🏓 </p>`,
33
41
}),
34
42
});
35
43
+10
-2
packages/notifications/slack/src/index.ts
+10
-2
packages/notifications/slack/src/index.ts
···
17
17
notification,
18
18
region,
19
19
statusCode,
20
+
message,
20
21
}: {
21
22
monitor: Monitor;
22
23
notification: Notification;
23
-
statusCode: number;
24
+
statusCode?: number;
24
25
region: string;
26
+
message?: string;
25
27
}) => {
26
28
const notificationData = JSON.parse(notification.data);
27
29
const { slack: webhookUrl } = notificationData; // webhook url
···
35
37
type: "section",
36
38
text: {
37
39
type: "mrkdwn",
38
-
text: `Your monitor <${monitor.url}/|${name}> is down in ${region} with status code ${statusCode} 🚨 \n\n Powered by <https://www.openstatus.dev/|OpenStatus>.`,
40
+
text: `Your monitor <${
41
+
monitor.url
42
+
}/|${name}> is down in ${region} with ${
43
+
statusCode
44
+
? `status code ${statusCode}`
45
+
: `error message ${message}`
46
+
} 🚨 \n\n Powered by <https://www.openstatus.dev/|OpenStatus>.`,
39
47
},
40
48
accessory: {
41
49
type: "button",
+6
-2
packages/notifications/twillio-sms/src/index.ts
+6
-2
packages/notifications/twillio-sms/src/index.ts
···
8
8
notification,
9
9
region,
10
10
statusCode,
11
+
message,
11
12
}: {
12
13
monitor: Monitor;
13
14
notification: Notification;
14
-
statusCode: number;
15
+
statusCode?: number;
15
16
region: string;
17
+
message?: string;
16
18
}) => {
17
19
const notificationData = SmsConfigurationSchema.parse(
18
20
JSON.parse(notification.data),
···
24
26
body.set("From", "+14807252613");
25
27
body.set(
26
28
"Body",
27
-
`Your monitor ${name} / ${monitor.url} is down in ${region} with status code ${statusCode}`,
29
+
`Your monitor ${name} / ${monitor.url} is down in ${region} with ${
30
+
statusCode ? `status code ${statusCode}` : `error: ${message}`
31
+
}`,
28
32
);
29
33
30
34
try {
+35
packages/tinybird/README.md
+35
packages/tinybird/README.md
···
1
+
### A guide on how to migrate your tinybird datasource
2
+
3
+
> What to do when you want to add/remove/update a column in your `datasource`.
4
+
5
+
The `_migration` folder includes:
6
+
7
+
- `ping_response__v4.datasource` which represents the `VERSION 4` of our
8
+
datasource
9
+
- `ping_response.datasource` which has the upgraded schema and a new `VERSION 5`
10
+
- `tb_backfill_populate.pipe` will fill the datasource with all the data until a
11
+
given timestamp
12
+
- `tb_materialized_until_change_ingest.pipe` will fill the data from a given
13
+
timestamp
14
+
15
+
```
16
+
tb push _migration/ping_response.datasource
17
+
tb push _migration/tb_materialized_until_change_ingest.pipe
18
+
# after the given ts, it is time to run the backfill populate
19
+
tb push _migration/tb_backfill_populate.pipe --populate --wait
20
+
# after populate ends, it is time to remove the pipe
21
+
tb pipe rm tb_backfill_populate --yes
22
+
```
23
+
24
+
Check if all the rows have been migrated:
25
+
26
+
```
27
+
tb pipe _migration/tb_datasource_union.pipe
28
+
# after checking the result of the pipe
29
+
tb pipe rm tb_datasource_union.pipe --yes
30
+
```
31
+
32
+
---
33
+
34
+
Link to the [issue](https://github.com/openstatusHQ/openstatus/issues/278) from
35
+
Gonzalo as reference.
+16
packages/tinybird/_migration/ping_response.datasource
+16
packages/tinybird/_migration/ping_response.datasource
···
1
+
VERSION 5
2
+
3
+
SCHEMA >
4
+
`latency` Int64 `json:$.latency`,
5
+
`monitorId` String `json:$.monitorId`,
6
+
`region` LowCardinality(String) `json:$.region`,
7
+
`statusCode` Nullable(Int16) `json:$.statusCode`,
8
+
`timestamp` Int64 `json:$.timestamp`,
9
+
`url` String `json:$.url`,
10
+
`workspaceId` String `json:$.workspaceId`,
11
+
`cronTimestamp` Int64 `json:$.cronTimestamp`,
12
+
`message` Nullable(String) `json:$.message`
13
+
14
+
ENGINE "MergeTree"
15
+
ENGINE_SORTING_KEY "monitorId, cronTimestamp"
16
+
ENGINE_PARTITION_KEY "toYYYYMM(fromUnixTimestamp64Milli(cronTimestamp))"
+16
packages/tinybird/_migration/ping_response__v4.datasource
+16
packages/tinybird/_migration/ping_response__v4.datasource
···
1
+
VERSION 4
2
+
3
+
SCHEMA >
4
+
`id` String `json:$.id`,
5
+
`latency` Int16 `json:$.latency`,
6
+
`monitorId` String `json:$.monitorId`,
7
+
`region` LowCardinality(String) `json:$.region`,
8
+
`statusCode` Int16 `json:$.statusCode`,
9
+
`timestamp` Int64 `json:$.timestamp`,
10
+
`url` String `json:$.url`,
11
+
`workspaceId` String `json:$.workspaceId`,
12
+
`cronTimestamp` Int64 `json:$.cronTimestamp`
13
+
14
+
ENGINE "MergeTree"
15
+
ENGINE_SORTING_KEY "monitorId, cronTimestamp"
16
+
ENGINE_PARTITION_KEY "toYYYYMM(fromUnixTimestamp64Milli(cronTimestamp))"
+17
packages/tinybird/_migration/tb_backfill_populate.pipe
+17
packages/tinybird/_migration/tb_backfill_populate.pipe
···
1
+
NODE mat_node
2
+
SQL >
3
+
4
+
SELECT
5
+
latency,
6
+
monitorId,
7
+
toLowCardinality(region) region,
8
+
statusCode,
9
+
timestamp,
10
+
url,
11
+
workspaceId,
12
+
cronTimestamp,
13
+
FROM ping_response__v4
14
+
WHERE fromUnixTimestamp64Milli(cronTimestamp) <= '2023-11-13 18:16:00.000'
15
+
16
+
TYPE materialized
17
+
DATASOURCE ping_response__v5
+6
packages/tinybird/_migration/tb_datasource_union.pipe
+6
packages/tinybird/_migration/tb_datasource_union.pipe
+17
packages/tinybird/_migration/tb_materialized_until_change_ingest.pipe
+17
packages/tinybird/_migration/tb_materialized_until_change_ingest.pipe
···
1
+
NODE mat_node
2
+
SQL >
3
+
4
+
SELECT
5
+
latency,
6
+
monitorId,
7
+
toLowCardinality(region) region,
8
+
statusCode,
9
+
timestamp,
10
+
url,
11
+
workspaceId,
12
+
cronTimestamp,
13
+
FROM ping_response__v4
14
+
WHERE fromUnixTimestamp64Milli(cronTimestamp) > '2023-11-13 18:16:00.000'
15
+
16
+
TYPE materialized
17
+
DATASOURCE ping_response__v5
+1
packages/tinybird/package.json
+1
packages/tinybird/package.json
+1
-1
packages/tinybird/pipes/home_stats.pipe
+1
-1
packages/tinybird/pipes/home_stats.pipe
+3
-3
packages/tinybird/pipes/response_list.pipe
+3
-3
packages/tinybird/pipes/response_list.pipe
···
1
-
VERSION 1
1
+
VERSION 2
2
2
3
3
NODE response_list_0
4
4
SQL >
5
5
6
6
%
7
-
SELECT id, latency, monitorId, region, statusCode, timestamp, url, workspaceId, cronTimestamp
8
-
FROM ping_response
7
+
SELECT latency, monitorId, region, statusCode, timestamp, url, workspaceId, cronTimestamp
8
+
FROM ping_response__v5
9
9
WHERE monitorId = {{ String(monitorId, 'openstatusPing') }}
10
10
{% if defined(region) %}
11
11
AND region = {{ String(region) }}
+1
-1
packages/tinybird/pipes/status_timezone.pipe
+1
-1
packages/tinybird/pipes/status_timezone.pipe
+2
-2
packages/tinybird/src/client.ts
+2
-2
packages/tinybird/src/client.ts
···
14
14
const tb = new Tinybird({ token: process.env.TINY_BIRD_API_KEY! });
15
15
16
16
export const publishPingResponse = tb.buildIngestEndpoint({
17
-
datasource: "ping_response__v4",
17
+
datasource: "ping_response__v5",
18
18
event: tbIngestPingResponse,
19
19
});
20
20
21
21
export function getResponseList(tb: Tinybird) {
22
22
return tb.buildPipe({
23
-
pipe: "response_list__v1",
23
+
pipe: "response_list__v2",
24
24
parameters: tbParameterResponseList,
25
25
data: tbBuildResponseList,
26
26
opts: {
+12
-35
packages/tinybird/src/validation.ts
+12
-35
packages/tinybird/src/validation.ts
···
1
1
import * as z from "zod";
2
2
3
-
export const vercelRegions = [
4
-
"arn1",
5
-
"bom1",
6
-
"cdg1",
7
-
"cle1",
8
-
"cpt1",
9
-
"dub1",
10
-
"fra1",
11
-
"gru1",
12
-
"hkg1",
13
-
"hnd1",
14
-
"iad1",
15
-
"icn1",
16
-
"kix1",
17
-
"lhr1",
18
-
"pdx1",
19
-
"sfo1",
20
-
"sin1",
21
-
"syd1",
22
-
] as const;
23
-
24
-
export const flyRegions = ["ams", "iad", "hkg", "jnb", "syd", "gru"] as const;
25
-
26
-
export const availableRegions = [...vercelRegions, ...flyRegions] as const;
3
+
import { availableRegions } from "@openstatus/utils";
27
4
28
5
/**
29
-
* Values for the datasource ping_response__v4
6
+
* Values for the datasource ping_response
30
7
*/
31
8
export const tbIngestPingResponse = z.object({
32
-
id: z.string(),
33
9
workspaceId: z.string(),
34
10
monitorId: z.string(),
35
11
timestamp: z.number().int(),
36
-
statusCode: z.number().int(),
12
+
statusCode: z.number().int().nullable().optional(),
37
13
latency: z.number(), // in ms
38
14
cronTimestamp: z.number().int().optional().nullable().default(Date.now()),
39
15
url: z.string().url(),
40
16
region: z.string().min(3).max(4), // REMINDER: won't work on fy
17
+
message: z.string().nullable().optional(),
41
18
});
42
19
43
20
/**
44
-
* Values from the pipe response_list__v1
21
+
* Values from the pipe response_list
45
22
*/
46
23
export const tbBuildResponseList = z.object({
47
-
id: z.string(),
48
24
workspaceId: z.string(),
49
25
monitorId: z.string(),
50
26
timestamp: z.number().int(),
51
-
statusCode: z.number().int(),
27
+
statusCode: z.number().int().nullable().default(null),
52
28
latency: z.number().int(), // in ms
53
29
cronTimestamp: z.number().int().nullable().default(Date.now()),
54
30
url: z.string().url(),
55
31
region: z.enum(availableRegions),
32
+
message: z.string().nullable().optional(),
56
33
});
57
34
58
35
/**
59
-
* Params for pipe response_list__v1
36
+
* Params for pipe response_list
60
37
*/
61
38
export const tbParameterResponseList = z.object({
62
39
monitorId: z.string().default(""), // REMINDER: remove default once alpha
···
68
45
});
69
46
70
47
/**
71
-
* Params for pipe status_timezone__v0
48
+
* Params for pipe status_timezone
72
49
*/
73
50
export const tbParameterMonitorList = z.object({
74
51
monitorId: z.string(),
···
77
54
});
78
55
79
56
/**
80
-
* Values from the pipe status_timezone__v0
57
+
* Values from the pipe status_timezone
81
58
*/
82
59
export const tbBuildMonitorList = z.object({
83
60
count: z.number().int(),
···
87
64
});
88
65
89
66
/**
90
-
* Params for pipe home_stats__v0
67
+
* Params for pipe home_stats
91
68
*/
92
69
export const tbParameterHomeStats = z.object({
93
70
cronTimestamp: z.number().int().optional(),
···
95
72
});
96
73
97
74
/**
98
-
* Values from the pipe home_stats__v0
75
+
* Values from the pipe home_stats
99
76
*/
100
77
export const tbBuildHomeStats = z.object({
101
78
count: z.number().int(),
+2
packages/utils/index.ts
+2
packages/utils/index.ts