See the best posts from any Bluesky account
Add sitemap.xml, robots.txt, and SEO audit notes
Adds a dynamic sitemap index at /sitemap.xml pointing to chunked
sitemaps at /sitemaps/:n.xml (25k profiles per chunk) to support
scale past the 50k-URL-per-sitemap limit. Only backfilled,
non-deleted profiles are advertised.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
+369
+1
.adonisjs/server/controllers.ts
+1
.adonisjs/server/controllers.ts
···
11
11
OgImage: () => import('#controllers/og_image_controller'),
12
12
Profile: () => import('#controllers/profile_controller'),
13
13
Search: () => import('#controllers/search_controller'),
14
+
Sitemap: () => import('#controllers/sitemap_controller'),
14
15
Typeahead: () => import('#controllers/typeahead_controller'),
15
16
}
+6
.adonisjs/server/routes.d.ts
+6
.adonisjs/server/routes.d.ts
···
22
22
'api.repost.delete': { paramsTuple?: []; params?: {} }
23
23
'og.landing': { paramsTuple?: []; params?: {} }
24
24
'og.profile': { paramsTuple: [ParamValue]; params: {'handle': ParamValue} }
25
+
'sitemap.index': { paramsTuple?: []; params?: {} }
26
+
'sitemap.chunk': { paramsTuple: [ParamValue]; params: {'n': ParamValue} }
25
27
'health.live': { paramsTuple?: []; params?: {} }
26
28
'health.ready': { paramsTuple?: []; params?: {} }
27
29
}
···
39
41
'oauth.callback': { paramsTuple?: []; params?: {} }
40
42
'og.landing': { paramsTuple?: []; params?: {} }
41
43
'og.profile': { paramsTuple: [ParamValue]; params: {'handle': ParamValue} }
44
+
'sitemap.index': { paramsTuple?: []; params?: {} }
45
+
'sitemap.chunk': { paramsTuple: [ParamValue]; params: {'n': ParamValue} }
42
46
'health.live': { paramsTuple?: []; params?: {} }
43
47
'health.ready': { paramsTuple?: []; params?: {} }
44
48
}
···
56
60
'oauth.callback': { paramsTuple?: []; params?: {} }
57
61
'og.landing': { paramsTuple?: []; params?: {} }
58
62
'og.profile': { paramsTuple: [ParamValue]; params: {'handle': ParamValue} }
63
+
'sitemap.index': { paramsTuple?: []; params?: {} }
64
+
'sitemap.chunk': { paramsTuple: [ParamValue]; params: {'n': ParamValue} }
59
65
'health.live': { paramsTuple?: []; params?: {} }
60
66
'health.ready': { paramsTuple?: []; params?: {} }
61
67
}
+41
SEO-TODO.md
+41
SEO-TODO.md
···
1
+
# SEO Audit — favs.blue
2
+
3
+
## P0 — Critical
4
+
5
+
1. ~~**No `robots.txt` or `sitemap.xml`**~~ — **DONE.** `public/robots.txt` added; dynamic sitemap at `/sitemap.xml` (index) + `/sitemaps/:n.xml` (chunks, 25k profiles each). Only backfilled, non-deleted profiles included.
6
+
2. **No `<meta name="description">`** — only OG descriptions exist. Add to `resources/views/components/layout.edge` (or per page: `landing.edge`, `profile/show.edge`, `about.edge`). OG description is set at `app/controllers/profile_controller.ts:364` but not mirrored to the standard meta tag.
7
+
3. **Duplicate-content risk on handle variants** — `/profile/@alice` 301s to `/profile/alice/likes` (`profile_controller.ts:205-212`), but URL-encoded `%40` variants may still get crawled. Consider a route-level constraint or middleware.
8
+
9
+
## P1 — High
10
+
11
+
4. **No `X-Robots-Tag` on `/health`, `/api`, `/oauth`** — add middleware in `start/kernel.ts` to set `X-Robots-Tag: noindex` on these routes.
12
+
5. **Profile avatars use `alt=""`** (`resources/views/pages/profile/show.edge:22`) — use `alt="@{handle} avatar"` for SEO/a11y.
13
+
6. **Weak heading hierarchy** — display name is a styled `<div>` (`profile/show.edge:28`), no `<h1>`. Replace with semantic heading.
14
+
7. **No JSON-LD structured data** — add `Person` schema on profile pages, `ItemList` for post lists, to unlock rich results in Google SERPs.
15
+
16
+
## P2 — Medium
17
+
18
+
8. **Landing page title is generic** — uses the layout default (`resources/views/components/layout.edge:30`). Add a `@slot('title')` with a CTA-oriented title in `landing.edge` (e.g. "favs.blue — Find the best posts from any Bluesky account").
19
+
9. **Missing `hreflang`** — only `lang="en"` on `<html>` (`layout.edge:2`). Low impact but easy win: add `<link rel="hreflang" hreflang="en" href="{{ canonicalUrl }}" />`.
20
+
10. **No preload/dns-prefetch** for fonts or `cdn.bsky.app` — add to `layout.edge` head. Affects Core Web Vitals (a ranking signal). Preconnect exists but no `preload`/`dns-prefetch`.
21
+
11. **Inconsistent `Cache-Control`** — landing/about have none; profile pages set `max-age=60` (`profile_controller.ts:367`); OG images set `max-age=86400` (`og_image_controller.ts:10,18`). Document and align.
22
+
12. **No per-post OG images** — route `/og/profile/:handle.png` exists (`start/routes.ts:102`) but no `/og/post/:did/:rkey.png`; shares of specific posts miss rich previews.
23
+
24
+
## Already solid
25
+
26
+
- Dynamic OG/Twitter cards (commits e82e33c, 42c00e9)
27
+
- Canonical URLs set per page (`profile_controller.ts:359`, `layout.edge:33-34`)
28
+
- Handle normalization with 301 redirects (`profile_controller.ts:205-212`)
29
+
- `loading="lazy"` on post images (`profile/show.edge:106,119`)
30
+
- Server-rendered Edge templates (crawler-friendly)
31
+
- Strict CSP via shield (`config/shield.ts:16-21`)
32
+
- Vite build with tree-shaking
33
+
34
+
## Suggested implementation order
35
+
36
+
1. ~~robots.txt~~ + X-Robots-Tag middleware (~5 min)
37
+
2. Meta descriptions (~10 min)
38
+
3. Avatar alt text (~5 min)
39
+
4. JSON-LD Person + ItemList (~30 min)
40
+
5. Heading hierarchy fix (~10 min)
41
+
6. ~~Dynamic sitemap.xml~~ (~20 min) — **done**
+61
app/controllers/sitemap_controller.ts
+61
app/controllers/sitemap_controller.ts
···
1
+
import type { HttpContext } from '@adonisjs/core/http'
2
+
import env from '#start/env'
3
+
import TrackedProfile from '#models/tracked_profile'
4
+
import {
5
+
DEFAULT_CHUNK_SIZE,
6
+
buildSitemapIndex,
7
+
buildSitemapChunk,
8
+
countChunks,
9
+
} from '#services/sitemap_builder'
10
+
11
+
export default class SitemapController {
12
+
async index({ response }: HttpContext) {
13
+
const baseUrl = env.get('APP_URL')
14
+
const total = await this.countEligibleProfiles()
15
+
const chunks = countChunks(total, DEFAULT_CHUNK_SIZE)
16
+
const xml = buildSitemapIndex(chunks, baseUrl)
17
+
response.header('Content-Type', 'application/xml; charset=utf-8')
18
+
response.header('Cache-Control', 'public, max-age=3600')
19
+
return response.send(xml)
20
+
}
21
+
22
+
async chunk({ params, response }: HttpContext) {
23
+
const n = Number(params.n)
24
+
if (!Number.isInteger(n) || n < 1) {
25
+
return response.notFound('Sitemap chunk not found')
26
+
}
27
+
28
+
const baseUrl = env.get('APP_URL')
29
+
const total = await this.countEligibleProfiles()
30
+
const chunks = countChunks(total, DEFAULT_CHUNK_SIZE)
31
+
if (n > chunks) {
32
+
return response.notFound('Sitemap chunk not found')
33
+
}
34
+
35
+
const offset = (n - 1) * DEFAULT_CHUNK_SIZE
36
+
const profiles = await TrackedProfile.query()
37
+
.whereNotNull('backfilled_at')
38
+
.whereNull('deleted_at')
39
+
.orderBy('did', 'asc')
40
+
.offset(offset)
41
+
.limit(DEFAULT_CHUNK_SIZE)
42
+
43
+
const xml = buildSitemapChunk(
44
+
profiles.map((p) => p.handle),
45
+
baseUrl,
46
+
{ includeStatic: n === 1 }
47
+
)
48
+
response.header('Content-Type', 'application/xml; charset=utf-8')
49
+
response.header('Cache-Control', 'public, max-age=3600')
50
+
return response.send(xml)
51
+
}
52
+
53
+
private async countEligibleProfiles(): Promise<number> {
54
+
const result = await TrackedProfile.query()
55
+
.whereNotNull('backfilled_at')
56
+
.whereNull('deleted_at')
57
+
.count('* as total')
58
+
const row = result[0] as unknown as { $extras: { total: number | string } }
59
+
return Number(row.$extras.total)
60
+
}
61
+
}
+62
app/services/sitemap_builder.ts
+62
app/services/sitemap_builder.ts
···
1
+
export const DEFAULT_CHUNK_SIZE = 25_000
2
+
3
+
export function countChunks(profileCount: number, chunkSize: number): number {
4
+
if (profileCount <= 0) return 1
5
+
return Math.ceil(profileCount / chunkSize)
6
+
}
7
+
8
+
function stripTrailingSlash(url: string): string {
9
+
return url.endsWith('/') ? url.slice(0, -1) : url
10
+
}
11
+
12
+
function xmlEscape(value: string): string {
13
+
return value
14
+
.replaceAll('&', '&')
15
+
.replaceAll('<', '<')
16
+
.replaceAll('>', '>')
17
+
.replaceAll('"', '"')
18
+
.replaceAll("'", ''')
19
+
}
20
+
21
+
export function buildSitemapIndex(chunkCount: number, baseUrl: string): string {
22
+
const base = stripTrailingSlash(baseUrl)
23
+
const count = Math.max(1, chunkCount)
24
+
const entries: string[] = []
25
+
for (let n = 1; n <= count; n++) {
26
+
entries.push(` <sitemap><loc>${base}/sitemaps/${n}.xml</loc></sitemap>`)
27
+
}
28
+
return [
29
+
'<?xml version="1.0" encoding="UTF-8"?>',
30
+
'<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">',
31
+
...entries,
32
+
'</sitemapindex>',
33
+
].join('\n')
34
+
}
35
+
36
+
export interface ChunkOptions {
37
+
includeStatic: boolean
38
+
}
39
+
40
+
export function buildSitemapChunk(
41
+
handles: string[],
42
+
baseUrl: string,
43
+
options: ChunkOptions
44
+
): string {
45
+
const base = stripTrailingSlash(baseUrl)
46
+
const urls: string[] = []
47
+
if (options.includeStatic) {
48
+
urls.push(` <url><loc>${base}/</loc></url>`)
49
+
urls.push(` <url><loc>${base}/about</loc></url>`)
50
+
}
51
+
for (const handle of handles) {
52
+
const escaped = xmlEscape(handle)
53
+
urls.push(` <url><loc>${base}/profile/${escaped}/likes</loc></url>`)
54
+
urls.push(` <url><loc>${base}/profile/${escaped}/reposts</loc></url>`)
55
+
}
56
+
return [
57
+
'<?xml version="1.0" encoding="UTF-8"?>',
58
+
'<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">',
59
+
...urls,
60
+
'</urlset>',
61
+
].join('\n')
62
+
}
+8
public/robots.txt
+8
public/robots.txt
+8
start/routes.ts
+8
start/routes.ts
···
18
18
const EngagementController = () => import('#controllers/engagement_controller')
19
19
const HealthChecksController = () => import('#controllers/health_checks_controller')
20
20
const OgImageController = () => import('#controllers/og_image_controller')
21
+
const SitemapController = () => import('#controllers/sitemap_controller')
21
22
22
23
// ---------------------------------------------------------------------------
23
24
// Landing
···
100
101
router.get('/og/landing.png', [OgImageController, 'landing']).as('og.landing')
101
102
102
103
router.get('/og/profile/:handle.png', [OgImageController, 'profile']).as('og.profile')
104
+
105
+
// ---------------------------------------------------------------------------
106
+
// Sitemap
107
+
// ---------------------------------------------------------------------------
108
+
109
+
router.get('/sitemap.xml', [SitemapController, 'index']).as('sitemap.index')
110
+
router.get('/sitemaps/:n.xml', [SitemapController, 'chunk']).as('sitemap.chunk')
103
111
104
112
// ---------------------------------------------------------------------------
105
113
// Health checks
+90
tests/functional/sitemap.spec.ts
+90
tests/functional/sitemap.spec.ts
···
1
+
import { test } from '@japa/runner'
2
+
import testUtils from '@adonisjs/core/services/test_utils'
3
+
import TrackedProfile from '#models/tracked_profile'
4
+
5
+
test.group('Sitemap', (group) => {
6
+
group.each.setup(() => testUtils.db().withGlobalTransaction())
7
+
8
+
test('GET /sitemap.xml returns 200 with XML content-type', async ({ client, assert }) => {
9
+
const response = await client.get('/sitemap.xml')
10
+
response.assertStatus(200)
11
+
const contentType = response.header('content-type') ?? ''
12
+
assert.include(contentType, 'xml')
13
+
})
14
+
15
+
test('GET /sitemap.xml returns a sitemap index pointing to chunk files', async ({
16
+
client,
17
+
assert,
18
+
}) => {
19
+
const response = await client.get('/sitemap.xml')
20
+
const body = response.text()
21
+
assert.include(body, '<sitemapindex')
22
+
assert.include(body, '/sitemaps/1.xml')
23
+
})
24
+
25
+
test('GET /sitemaps/1.xml returns 200 with urlset', async ({ client, assert }) => {
26
+
const response = await client.get('/sitemaps/1.xml')
27
+
response.assertStatus(200)
28
+
assert.include(response.text(), '<urlset')
29
+
})
30
+
31
+
test('GET /sitemaps/1.xml includes landing and about URLs', async ({ client, assert }) => {
32
+
const response = await client.get('/sitemaps/1.xml')
33
+
const body = response.text()
34
+
assert.include(body, '/about')
35
+
// The trailing-slash root URL:
36
+
assert.match(body, /<loc>[^<]+\/<\/loc>/)
37
+
})
38
+
39
+
test('GET /sitemaps/1.xml includes backfilled profile /likes and /reposts URLs', async ({
40
+
client,
41
+
assert,
42
+
}) => {
43
+
await TrackedProfile.create({
44
+
did: 'did:plc:sm1',
45
+
handle: 'sitemap-user.bsky.social',
46
+
firstSeenAt: Date.now(),
47
+
backfilledAt: Date.now(),
48
+
})
49
+
50
+
const response = await client.get('/sitemaps/1.xml')
51
+
const body = response.text()
52
+
assert.include(body, '/profile/sitemap-user.bsky.social/likes')
53
+
assert.include(body, '/profile/sitemap-user.bsky.social/reposts')
54
+
})
55
+
56
+
test('GET /sitemaps/1.xml excludes non-backfilled profiles', async ({ client, assert }) => {
57
+
await TrackedProfile.create({
58
+
did: 'did:plc:sm2',
59
+
handle: 'not-backfilled.bsky.social',
60
+
firstSeenAt: Date.now(),
61
+
backfilledAt: null,
62
+
})
63
+
64
+
const response = await client.get('/sitemaps/1.xml')
65
+
assert.notInclude(response.text(), 'not-backfilled.bsky.social')
66
+
})
67
+
68
+
test('GET /sitemaps/1.xml excludes deleted profiles', async ({ client, assert }) => {
69
+
await TrackedProfile.create({
70
+
did: 'did:plc:sm3',
71
+
handle: 'deleted-user.bsky.social',
72
+
firstSeenAt: Date.now(),
73
+
backfilledAt: Date.now(),
74
+
deletedAt: Date.now(),
75
+
})
76
+
77
+
const response = await client.get('/sitemaps/1.xml')
78
+
assert.notInclude(response.text(), 'deleted-user.bsky.social')
79
+
})
80
+
81
+
test('GET /sitemaps/999.xml returns 404 for out-of-range chunk', async ({ client }) => {
82
+
const response = await client.get('/sitemaps/999.xml')
83
+
response.assertStatus(404)
84
+
})
85
+
86
+
test('GET /sitemaps/0.xml returns 404 (chunks are 1-indexed)', async ({ client }) => {
87
+
const response = await client.get('/sitemaps/0.xml')
88
+
response.assertStatus(404)
89
+
})
90
+
})
+92
tests/unit/sitemap_builder.spec.ts
+92
tests/unit/sitemap_builder.spec.ts
···
1
+
import { test } from '@japa/runner'
2
+
import { buildSitemapIndex, buildSitemapChunk, countChunks } from '#services/sitemap_builder'
3
+
4
+
test.group('buildSitemapIndex', () => {
5
+
test('returns XML declaration and sitemapindex root', ({ assert }) => {
6
+
const xml = buildSitemapIndex(1, 'https://favs.blue')
7
+
assert.match(xml, /^<\?xml version="1\.0" encoding="UTF-8"\?>/)
8
+
assert.include(xml, '<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">')
9
+
assert.include(xml, '</sitemapindex>')
10
+
})
11
+
12
+
test('lists one <sitemap> entry per chunk', ({ assert }) => {
13
+
const xml = buildSitemapIndex(3, 'https://favs.blue')
14
+
assert.include(xml, '<loc>https://favs.blue/sitemaps/1.xml</loc>')
15
+
assert.include(xml, '<loc>https://favs.blue/sitemaps/2.xml</loc>')
16
+
assert.include(xml, '<loc>https://favs.blue/sitemaps/3.xml</loc>')
17
+
const count = (xml.match(/<sitemap>/g) || []).length
18
+
assert.equal(count, 3)
19
+
})
20
+
21
+
test('always includes at least one chunk even with zero profiles', ({ assert }) => {
22
+
const xml = buildSitemapIndex(0, 'https://favs.blue')
23
+
assert.include(xml, '<loc>https://favs.blue/sitemaps/1.xml</loc>')
24
+
})
25
+
26
+
test('strips trailing slash from baseUrl', ({ assert }) => {
27
+
const xml = buildSitemapIndex(1, 'https://favs.blue/')
28
+
assert.include(xml, '<loc>https://favs.blue/sitemaps/1.xml</loc>')
29
+
assert.notInclude(xml, '//sitemaps/1.xml')
30
+
})
31
+
})
32
+
33
+
test.group('buildSitemapChunk', () => {
34
+
test('returns XML declaration and urlset root', ({ assert }) => {
35
+
const xml = buildSitemapChunk([], 'https://favs.blue', { includeStatic: false })
36
+
assert.match(xml, /^<\?xml version="1\.0" encoding="UTF-8"\?>/)
37
+
assert.include(xml, '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">')
38
+
assert.include(xml, '</urlset>')
39
+
})
40
+
41
+
test('emits /likes and /reposts URLs for each handle', ({ assert }) => {
42
+
const xml = buildSitemapChunk(['alice.bsky.social', 'bob.bsky.social'], 'https://favs.blue', {
43
+
includeStatic: false,
44
+
})
45
+
assert.include(xml, '<loc>https://favs.blue/profile/alice.bsky.social/likes</loc>')
46
+
assert.include(xml, '<loc>https://favs.blue/profile/alice.bsky.social/reposts</loc>')
47
+
assert.include(xml, '<loc>https://favs.blue/profile/bob.bsky.social/likes</loc>')
48
+
assert.include(xml, '<loc>https://favs.blue/profile/bob.bsky.social/reposts</loc>')
49
+
const urlCount = (xml.match(/<url>/g) || []).length
50
+
assert.equal(urlCount, 4)
51
+
})
52
+
53
+
test('includes static URLs (landing, about) when includeStatic is true', ({ assert }) => {
54
+
const xml = buildSitemapChunk([], 'https://favs.blue', { includeStatic: true })
55
+
assert.include(xml, '<loc>https://favs.blue/</loc>')
56
+
assert.include(xml, '<loc>https://favs.blue/about</loc>')
57
+
})
58
+
59
+
test('omits static URLs when includeStatic is false', ({ assert }) => {
60
+
const xml = buildSitemapChunk(['alice.bsky.social'], 'https://favs.blue', {
61
+
includeStatic: false,
62
+
})
63
+
assert.notInclude(xml, '<loc>https://favs.blue/</loc>')
64
+
assert.notInclude(xml, '<loc>https://favs.blue/about</loc>')
65
+
})
66
+
67
+
test('XML-escapes handles containing special characters', ({ assert }) => {
68
+
const xml = buildSitemapChunk(['a&b.example.com'], 'https://favs.blue', {
69
+
includeStatic: false,
70
+
})
71
+
assert.include(xml, 'a&b.example.com')
72
+
assert.notInclude(xml, '<loc>https://favs.blue/profile/a&b.example.com/likes</loc>')
73
+
})
74
+
})
75
+
76
+
test.group('countChunks', () => {
77
+
test('returns 1 for zero profiles (index always has at least one chunk)', ({ assert }) => {
78
+
assert.equal(countChunks(0, 25000), 1)
79
+
})
80
+
81
+
test('returns 1 when profile count equals chunk size', ({ assert }) => {
82
+
assert.equal(countChunks(25000, 25000), 1)
83
+
})
84
+
85
+
test('returns 2 when profile count is one over chunk size', ({ assert }) => {
86
+
assert.equal(countChunks(25001, 25000), 2)
87
+
})
88
+
89
+
test('rounds up partial chunks', ({ assert }) => {
90
+
assert.equal(countChunks(60000, 25000), 3)
91
+
})
92
+
})