forked from
tangled.org/core
Monorepo for Tangled
appview/state: update robots.txt
Signed-off-by: Anirudh Oppiliappan <anirudh@tangled.org>
+8
-1
+8
-1
appview/state/state.go
+8
-1
appview/state/state.go
···
207
207
w.Header().Set("Content-Type", "text/plain")
208
208
w.Header().Set("Cache-Control", "public, max-age=86400") // one day
209
209
210
-
robotsTxt := `User-agent: *
210
+
robotsTxt := `# Hello, Tanglers!
211
+
User-agent: *
211
212
Allow: /
213
+
Disallow: /*/*/settings
214
+
Disallow: /settings
215
+
Disallow: /*/*/compare
216
+
Disallow: /*/*/fork
217
+
218
+
Crawl-delay: 1
212
219
`
213
220
w.Write([]byte(robotsTxt))
214
221
}