data endpoint for entity 90008 (aka. a website)
fix: filter legit page visits properly, dont count api, rss, prefetch
+101
-51
+1
-1
src/app.html
+1
-1
src/app.html
···
6
6
<meta name="viewport" content="width=device-width, initial-scale=1" />
7
7
%sveltekit.head%
8
8
</head>
9
-
<body class="md:overflow-hidden" data-sveltekit-preload-data="hover">
9
+
<body class="md:overflow-hidden">
10
10
<div style="display: contents">%sveltekit.body%</div>
11
11
</body>
12
12
</html>
+2
-2
src/components/pet.svelte
+2
-2
src/components/pet.svelte
···
77
77
let bounciness = 0.8; // How much energy is preserved on bounce
78
78
79
79
const sendBounceMetrics = () => {
80
-
fetch('/pet/bounce');
80
+
fetch('/_api/pet/bounce');
81
81
};
82
82
83
83
let deltaTravelled = 0.0;
···
94
94
};
95
95
96
96
const sendTotalDistance = () => {
97
-
fetch('/pet/distance', {
97
+
fetch('/_api/pet/distance', {
98
98
method: 'POST',
99
99
body: deltaTravelledTotal.toString()
100
100
});
+62
-1
src/hooks.server.ts
+62
-1
src/hooks.server.ts
···
3
3
import { steamUpdateNowPlaying } from '$lib/steam';
4
4
import { updateCommits } from '$lib/activity';
5
5
import { cancelJob, scheduleJob, scheduledJobs } from 'node-schedule';
6
-
import { sendAllMetrics } from '$lib/metrics';
6
+
import {
7
+
incrementFakeVisitCount,
8
+
incrementLegitVisitCount,
9
+
pushMetric,
10
+
sendAllMetrics
11
+
} from '$lib/metrics';
12
+
import {
13
+
addLastVisitor,
14
+
decrementVisitCount,
15
+
incrementVisitCount,
16
+
notifyDarkVisitors,
17
+
removeLastVisitor
18
+
} from '$lib/visits';
19
+
import { testUa } from '$lib/robots';
20
+
import { error } from '@sveltejs/kit';
7
21
8
22
const UPDATE_LAST_JOB_NAME = 'update steam game, lastfm track, bsky posts, git activity';
9
23
···
27
41
console.log(`error while running ${UPDATE_LAST_JOB_NAME} job: ${err}`);
28
42
}
29
43
}).invoke(); // invoke once immediately
44
+
45
+
export const handle = async ({ event, resolve }) => {
46
+
notifyDarkVisitors(event.url, event.request); // no await so it doesnt block
47
+
48
+
const isPrefetch = () => {
49
+
return (
50
+
event.request.headers.get('Sec-Purpose')?.includes('prefetch') ||
51
+
event.request.headers.get('Purpose')?.includes('prefetch') ||
52
+
event.request.headers.get('x-purpose')?.includes('preview') ||
53
+
event.request.headers.get('x-moz')?.includes('prefetch')
54
+
);
55
+
};
56
+
const isApi = () => {
57
+
return event.url.pathname.startsWith('/_api');
58
+
};
59
+
const isRss = () => {
60
+
return event.url.pathname.endsWith('/_rss');
61
+
};
62
+
63
+
// block any requests if the user agent is disallowed by our robots txt
64
+
const isFakeVisit =
65
+
(await testUa(event.url.toString(), event.request.headers.get('user-agent') ?? '')) === false;
66
+
if (isFakeVisit) {
67
+
pushMetric({ gazesys_visit_fake_total: incrementFakeVisitCount() });
68
+
throw error(403, 'get a better user agent silly');
69
+
}
70
+
71
+
// only push metric if legit page visit (still want rss to count here though)
72
+
const isPageVisit = !isApi() && !isPrefetch();
73
+
if (isPageVisit) pushMetric({ gazesys_visit_real_total: incrementLegitVisitCount() });
74
+
75
+
// only add visitors if its a "legit" page visit
76
+
let id = null;
77
+
let valid = false;
78
+
if (isPageVisit && !isRss()) {
79
+
id = addLastVisitor(event.request, event.cookies);
80
+
valid = incrementVisitCount(event.request, event.cookies);
81
+
}
82
+
83
+
const resp = await resolve(event);
84
+
// remove visitors if it was a 404
85
+
if (resp.status === 404) {
86
+
if (id !== null) removeLastVisitor(id);
87
+
if (valid) decrementVisitCount();
88
+
}
89
+
return resp;
90
+
};
+26
-14
src/lib/visits.ts
+26
-14
src/lib/visits.ts
···
6
6
import { get, writable } from 'svelte/store';
7
7
8
8
const visitCountFile = `${env.WEBSITE_DATA_DIR}/visitcount`;
9
-
const visitCount = writable(
9
+
export const visitCount = writable(
10
10
parseInt(existsSync(visitCountFile) ? readFileSync(visitCountFile).toString() : '0')
11
11
);
12
12
13
13
type Visitor = { visits: number[] };
14
-
const lastVisitors = writable<Map<string, Visitor>>(new Map());
14
+
export const lastVisitors = writable<Map<string, Visitor>>(new Map());
15
15
const VISITOR_EXPIRY_SECONDS = 60 * 60; // an hour seems reasonable
16
16
17
+
export const decrementVisitCount = () => {
18
+
visitCount.set(get(visitCount) - 1);
19
+
};
20
+
17
21
export const incrementVisitCount = (request: Request, cookies: Cookies) => {
18
22
let currentVisitCount = get(visitCount);
19
23
// check whether the request is from a bot or not (this doesnt need to be accurate we just want to filter out honest bots)
20
-
if (isBot(request)) {
21
-
return currentVisitCount;
22
-
}
24
+
if (isBot(request)) return false;
23
25
const scopedCookies = scopeCookies(cookies, '/');
24
26
// parse the last visit timestamp from cookies if it exists
25
27
const visitedTimestamp = parseInt(scopedCookies.get('visitedTimestamp') || '0');
···
36
38
// write the visit count to a file so we can load it later again
37
39
writeFileSync(visitCountFile, currentVisitCount.toString());
38
40
}
39
-
return currentVisitCount;
41
+
return true;
42
+
};
43
+
44
+
export const removeLastVisitor = (id: string) => {
45
+
const visitors = get(lastVisitors);
46
+
if (visitors.has(id)) {
47
+
const visitor = visitors.get(id) ?? { visits: [] };
48
+
visitor?.visits.pop();
49
+
visitors.set(id, visitor);
50
+
}
51
+
lastVisitors.set(visitors);
40
52
};
41
53
42
54
export const addLastVisitor = (request: Request, cookies: Cookies) => {
43
-
let visitors = get(lastVisitors);
44
-
visitors = _addLastVisitor(visitors, request, cookies);
55
+
const { visitors, visitorId } = _addLastVisitor(get(lastVisitors), request, cookies);
45
56
lastVisitors.set(visitors);
46
-
return visitors;
57
+
return visitorId;
47
58
};
48
59
49
60
export const getVisitorId = (cookies: Cookies) => {
···
66
77
}
67
78
});
68
79
// check whether the request is from a bot or not (this doesnt need to be accurate we just want to filter out honest bots)
69
-
if (isBot(request)) {
70
-
return visitors;
71
-
}
80
+
if (isBot(request)) return { visitors, visitorId: null };
72
81
const scopedCookies = scopeCookies(cookies, '/');
73
82
// parse the last visit timestamp from cookies if it exists
74
83
let visitorId = scopedCookies.get('visitorId') || '';
···
83
92
// put new visit in the front
84
93
visitorEntry.visits = [currentTime].concat(visitorEntry.visits);
85
94
visitors.set(visitorId, visitorEntry);
86
-
return visitors;
95
+
return {
96
+
visitors,
97
+
visitorId
98
+
};
87
99
};
88
100
89
101
export const isBot = (request: Request) => {
···
113
125
.then(async (resp) => {
114
126
if (resp !== null) {
115
127
const msg = await resp.json();
116
-
const host = `(${request.headers.get('host')} ${request.headers.get('x-real-ip')})`;
128
+
const host = `(${request.headers.get('host')}|${request.headers.get('x-real-ip')}|${request.headers.get('user-agent')})`;
117
129
console.log(
118
130
`sent visitor analytic to dark visitors: ${resp.statusText}; ${msg.message ?? ''}${host}`
119
131
);
+7
-25
src/routes/+layout.server.ts
+7
-25
src/routes/+layout.server.ts
···
1
-
import {
2
-
bounceCount,
3
-
distanceTravelled,
4
-
incrementFakeVisitCount,
5
-
incrementLegitVisitCount,
6
-
pushMetric
7
-
} from '$lib/metrics.js';
8
-
import { testUa } from '$lib/robots.js';
9
-
import { addLastVisitor, incrementVisitCount, notifyDarkVisitors } from '$lib/visits.js';
10
-
import { error } from '@sveltejs/kit';
1
+
import { bounceCount, distanceTravelled } from '$lib/metrics.js';
2
+
import { lastVisitors, visitCount } from '$lib/visits.js';
11
3
import { localDistanceTravelled } from '../components/pet.svelte';
12
4
import { get } from 'svelte/store';
13
5
···
16
8
export const prerender = false;
17
9
export const trailingSlash = 'always';
18
10
19
-
export async function load({ request, cookies, url }) {
20
-
notifyDarkVisitors(url, request); // no await so it doesnt block load
21
-
22
-
// block any requests if the user agent is disallowed by our robots txt
23
-
if ((await testUa(url.toString(), request.headers.get('user-agent') ?? '')) === false) {
24
-
pushMetric({ gazesys_visit_fake_total: incrementFakeVisitCount() });
25
-
throw error(403, 'get a better user agent silly');
26
-
} else {
27
-
pushMetric({ gazesys_visit_real_total: incrementLegitVisitCount() });
28
-
}
29
-
30
-
const lastVisitors = addLastVisitor(request, cookies);
11
+
export async function load({ url }) {
12
+
const visitors = get(lastVisitors);
31
13
let recentVisitCount = 0;
32
-
for (const [, visitor] of lastVisitors) {
14
+
for (const [, visitor] of visitors) {
33
15
recentVisitCount += visitor.visits.length;
34
16
}
35
17
···
38
20
petTotalBounce: bounceCount.get(),
39
21
petTotalDistance: distanceTravelled.get(),
40
22
petLocalDistance: get(localDistanceTravelled),
41
-
visitCount: incrementVisitCount(request, cookies),
42
-
lastVisitors,
23
+
visitCount: get(visitCount),
24
+
lastVisitors: visitors,
43
25
recentVisitCount
44
26
};
45
27
}
+1
-1
src/routes/+page.svelte
+1
-1
src/routes/+page.svelte
···
372
372
event.preventDefault();
373
373
const data = new FormData(event.currentTarget);
374
374
try {
375
-
fetch(`${PUBLIC_BASE_URL}/pushnotif/?content=${data.get('content')}`);
375
+
fetch(`${PUBLIC_BASE_URL}/_api/pushnotif/?content=${data.get('content')}`);
376
376
} catch (err) {
377
377
console.log(`failed to send notif: ${err}`);
378
378
}
src/routes/pet/bounce/+server.ts
src/routes/_api/pet/bounce/+server.ts
src/routes/pet/bounce/+server.ts
src/routes/_api/pet/bounce/+server.ts
src/routes/pet/distance/+server.ts
src/routes/_api/pet/distance/+server.ts
src/routes/pet/distance/+server.ts
src/routes/_api/pet/distance/+server.ts
src/routes/pushnotif/+server.ts
src/routes/_api/pushnotif/+server.ts
src/routes/pushnotif/+server.ts
src/routes/_api/pushnotif/+server.ts