ai cooking
robots.txt (#231)
Co-authored-by: paul miller <paul.miller>
authored by
Paul Miller
paul miller
and committed by
GitHub
1bd2a7eb
3676edbe
+39
-1
+21
-1
internal/sitemap/sitemap.go
+21
-1
internal/sitemap/sitemap.go
···
4
4
"careme/internal/cache"
5
5
"encoding/base64"
6
6
"encoding/xml"
7
+
"fmt"
7
8
"log/slog"
8
9
"net/http"
9
10
"strings"
···
13
14
cache cache.ListCache
14
15
}
15
16
17
+
const (
18
+
domain = "https://careme.cooking"
19
+
robots = `# Allow all search engines to crawl the site
20
+
User-agent: *
21
+
Allow: /
22
+
23
+
# Sitemap location
24
+
Sitemap: %s/sitemap.xml
25
+
`
26
+
)
27
+
16
28
func New(c cache.ListCache) *Server {
17
29
18
30
return &Server{cache: c}
···
20
32
21
33
func (s *Server) Register(mux *http.ServeMux) {
22
34
mux.HandleFunc("GET /sitemap.xml", s.handleSitemap)
35
+
mux.HandleFunc("GET /robots.txt", s.handleRobots)
23
36
}
24
37
25
38
type urlSet struct {
···
53
66
54
67
//this is going to get too big. at some point we need a real db to find latest
55
68
//or we track new entries and expire a lsit.
56
-
const domain = "https://careme.cooking" //TODO pull this from config?
57
69
for _, hash := range hashes {
58
70
if hash == "" || strings.Contains(hash, "/") || strings.HasSuffix(hash, ".params") {
59
71
continue
···
77
89
slog.ErrorContext(r.Context(), "failed to encode sitemap", "error", err)
78
90
}
79
91
}
92
+
93
+
func (s *Server) handleRobots(w http.ResponseWriter, r *http.Request) {
94
+
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
95
+
full := fmt.Sprintf(robots, domain)
96
+
if _, err := w.Write([]byte(full)); err != nil {
97
+
slog.ErrorContext(r.Context(), "failed to write robots.txt", "error", err)
98
+
}
99
+
}
+18
internal/sitemap/sitemap_test.go
+18
internal/sitemap/sitemap_test.go
···
64
64
}
65
65
}
66
66
67
+
func TestHandleRobotsReturnsExpectedContent(t *testing.T) {
68
+
server := &Server{}
69
+
rr := httptest.NewRecorder()
70
+
req := httptest.NewRequest(http.MethodGet, "/robots.txt", nil)
71
+
72
+
server.handleRobots(rr, req)
73
+
74
+
if rr.Code != http.StatusOK {
75
+
t.Fatalf("expected status 200, got %d", rr.Code)
76
+
}
77
+
if got := rr.Header().Get("Content-Type"); !strings.Contains(got, "text/plain") {
78
+
t.Fatalf("expected text content type, got %q", got)
79
+
}
80
+
if rr.Body.String() != fmt.Sprintf(robots, domain) {
81
+
t.Fatalf("unexpected robots.txt body:\n%s", rr.Body.String())
82
+
}
83
+
}
84
+
67
85
func containsSitemapURL(entries []urlEntry, want string) bool {
68
86
for _, entry := range entries {
69
87
if entry.Loc == want {