this repo has no description
clean up
+17
-24
-1
.build.yml
-1
.build.yml
+8
-19
_config.ts
+8
-19
_config.ts
···
1
+
import { load } from "https://deno.land/std@0.224.0/dotenv/mod.ts";
2
+
1
3
import lume from "lume/mod.ts";
2
4
import jsx_preact from "lume/plugins/jsx_preact.ts";
3
5
import attributes from "lume/plugins/attributes.ts";
···
13
15
import slugify_urls from "lume/plugins/slugify_urls.ts";
14
16
import toml from "lume/plugins/toml.ts";
15
17
16
-
import { load } from "https://deno.land/std@0.224.0/dotenv/mod.ts";
17
-
18
18
// mdx plugins
19
19
import rehypeKatex from "npm:rehype-katex";
20
20
import remarkMath from "npm:remark-math";
···
24
24
25
25
// components
26
26
import Header from "./src/_components/Header.tsx";
27
+
28
+
await load({
29
+
export: true,
30
+
});
27
31
28
32
const site = lume({
29
33
src: "./src",
···
59
63
);
60
64
site.use(toml());
61
65
62
-
// let envPath = Deno.env.get("ALTAGOS_ENV_PATH");
63
-
// if (envPath == undefined) {
64
-
// envPath = "./.env";
65
-
// }
66
-
const _env = await load({
67
-
// envPath: envPath,
68
-
export: true,
69
-
});
70
-
71
-
const api_key = Deno.env.get("DARKVISITORS_API_KEY");
72
-
if (api_key == undefined) {
73
-
console.error("DARKVISITORS_API_KEY is not defined in .env file");
74
-
Deno.exit(1);
75
-
}
76
-
77
66
site.use(robots({
78
-
api_key: api_key,
67
+
env_var: "DARKVISITORS_API_KEY",
79
68
agent_types: [
80
-
"AI Data Scraper", "AI Assistant", "AI Search Crawler",
69
+
"AI Data Scraper", "AI Assistant", "AI Search Crawler", "Undocumented AI Agent"
81
70
],
82
71
}));
83
72
+9
-4
src/robots.ts
+9
-4
src/robots.ts
···
1
1
import Site from "lume/core/site.ts";
2
2
3
3
interface Options {
4
-
api_key: string;
4
+
env_var: string;
5
5
agent_types: string[];
6
6
}
7
7
8
8
export default function (options: Options) {
9
9
return (site: Site) => {
10
10
site.addEventListener("beforeSave", async () => {
11
-
// console.log("Fetching robots.txt from DarkVisitors... (api_key: " + options.api_key + ")");
11
+
const api_key = Deno.env.get(options.env_var);
12
+
if (api_key == undefined) {
13
+
console.error(options.env_var + " is not defined");
14
+
Deno.exit(1);
15
+
}
16
+
12
17
const robotsTXT = await fetch("https://api.darkvisitors.com/robots-txts", {
13
18
method: "POST",
14
19
headers: {
15
20
"Content-Type": "application/json",
16
-
"Authorization": "Bearer " + options.api_key,
21
+
"Authorization": "Bearer " + api_key,
17
22
},
18
23
body: JSON.stringify({
19
24
agent_types: options.agent_types,
···
22
27
});
23
28
if (robotsTXT.status != 200) {
24
29
console.error("Failed to fetch robots.txt (error code: " + robotsTXT.status + " " + robotsTXT.statusText + ")");
25
-
return;
30
+
Deno.exit(1);;
26
31
}
27
32
28
33
const robots = await site.getOrCreatePage("robots.txt");