The code and data behind xeiaso.net
chore(k8s/xesite): update anubis to use yaml config
Signed-off-by: Xe Iaso <me@xeiaso.net>
+63
-2
+51
manifest/xesite/anubis/botPolicies.yaml
+51
manifest/xesite/anubis/botPolicies.yaml
···
1
+
## Anubis has the ability to let you import snippets of configuration into the main
2
+
## configuration file. This allows you to break up your config into smaller parts
3
+
## that get logically assembled into one big file.
4
+
##
5
+
## Of note, a bot rule can either have inline bot configuration or import a
6
+
## bot config snippet. You cannot do both in a single bot rule.
7
+
##
8
+
## Import paths can either be prefixed with (data) to import from the common/shared
9
+
## rules in the data folder in the Anubis source tree or will point to absolute/relative
10
+
## paths in your filesystem. If you don't have access to the Anubis source tree, check
11
+
## /usr/share/docs/anubis/data or in the tarball you extracted Anubis from.
12
+
13
+
bots:
14
+
# Pathological bots to deny
15
+
- # This correlates to data/bots/ai-robots-txt.yaml in the source tree
16
+
import: (data)/bots/ai-robots-txt.yaml
17
+
- import: (data)/bots/cloudflare-workers.yaml
18
+
- import: (data)/bots/headless-browsers.yaml
19
+
- import: (data)/bots/us-ai-scraper.yaml
20
+
21
+
# Search engines to allow
22
+
- import: (data)/crawlers/googlebot.yaml
23
+
- import: (data)/crawlers/bingbot.yaml
24
+
- import: (data)/crawlers/duckduckbot.yaml
25
+
- import: (data)/crawlers/qwantbot.yaml
26
+
- import: (data)/crawlers/internet-archive.yaml
27
+
- import: (data)/crawlers/kagibot.yaml
28
+
- import: (data)/crawlers/marginalia.yaml
29
+
- import: (data)/crawlers/mojeekbot.yaml
30
+
31
+
# Allow common "keeping the internet working" routes (well-known, favicon, robots.txt)
32
+
- import: (data)/common/keep-internet-working.yaml
33
+
- import: /xe/cfg/anubis/xesite-rss-feeds.yaml
34
+
35
+
# # Punish any bot with "bot" in the user-agent string
36
+
# # This is known to have a high false-positive rate, use at your own risk
37
+
# - name: generic-bot-catchall
38
+
# user_agent_regex: (?i:bot|crawler)
39
+
# action: CHALLENGE
40
+
# challenge:
41
+
# difficulty: 16 # impossible
42
+
# report_as: 4 # lie to the operator
43
+
# algorithm: slow # intentionally waste CPU cycles and time
44
+
45
+
# Generic catchall rule
46
+
- name: generic-browser
47
+
user_agent_regex: >
48
+
Mozilla|Opera
49
+
action: CHALLENGE
50
+
51
+
dnsbl: false
+9
manifest/xesite/anubis/xesite-rss-feeds.yaml
+9
manifest/xesite/anubis/xesite-rss-feeds.yaml
+1
-1
manifest/xesite/deployment.yaml
+1
-1
manifest/xesite/deployment.yaml