the home site for me: also iteration 3 or 4 of my site
feat: add a rehost script to move cdns
+182
-1
+180
tools/rehost-cdn.sh
+180
tools/rehost-cdn.sh
···
1
+
#!/usr/bin/env bash
2
+
set -euo pipefail
3
+
4
+
API_URL="https://cdn.hackclub.com/api/v3/new"
5
+
TOKEN="${HACKCLUB_CDN_TOKEN:-}"
6
+
if [[ -z "${TOKEN}" ]]; then
7
+
TOKEN="${1:-}"
8
+
fi
9
+
if [[ -z "${TOKEN}" ]]; then
10
+
echo "Usage: HACKCLUB_CDN_TOKEN=... $0 [token] [--dry-run] [paths...]" >&2
11
+
exit 1
12
+
fi
13
+
14
+
DRY_RUN=false
15
+
SKIP_CHECK=false
16
+
CACHED_URLS=()
17
+
ARGS=()
18
+
for a in "$@"; do
19
+
case "$a" in
20
+
--dry-run) DRY_RUN=true ;;
21
+
--skip-check) SKIP_CHECK=true ;;
22
+
*) ARGS+=("$a") ;;
23
+
esac
24
+
done
25
+
# remove token if passed as first arg
26
+
if [[ ${#ARGS[@]} -gt 0 && "${ARGS[0]}" != "--dry-run" ]]; then
27
+
ARGS=("${ARGS[@]:1}")
28
+
fi
29
+
30
+
PATHS=("content")
31
+
if [[ ${#ARGS[@]} -gt 0 ]]; then PATHS=("${ARGS[@]}"); fi
32
+
33
+
TMP_DIR=".crush/rehost-cdn"
34
+
MAP_FILE="$TMP_DIR/map.tsv"
35
+
mkdir -p "$TMP_DIR"
36
+
touch "$MAP_FILE"
37
+
38
+
collect_urls() {
39
+
# Markdown images: 
40
+
rg -n --no-heading -e '!\[[^\]]*\]\((https?://[^)\s]+)\)' -g '!**/*.map' -g '!**/*.lock' "${PATHS[@]}" 2>/dev/null |
41
+
awk -F: '{file=$1; sub(/^[^:]*:/, "", $0); match($0, /!\[[^\]]*\]\((https?:\/\/[^)\s]+)\)/, m); if(m[1]!="") print file"\t"m[1]}' |
42
+
# Zola shortcode variants:
43
+
# - {% img(id="URL", ...) %}
44
+
# - {{ img(id="URL", ...) }}
45
+
cat <( rg -n --no-heading -e '\{[%{]\s*img[^}%]*[}%]\}' "${PATHS[@]}" 2>/dev/null | \
46
+
awk -F: '{file=$1; sub(/^[^:]*:/, "", $0); if (match($0, /(id|src)[[:space:]]*=[[:space:]]*"(https?:\/\/[^"[:space:]]+)"/, m)) print file"\t"m[2]}' ) |
47
+
awk -F'\t' '{print $1"\t"$2}' |
48
+
grep -E '\.(png|jpe?g|gif|webp|svg|bmp|tiff?|avif)(\?.*)?$' -i |
49
+
grep -vE 'hc-cdn\.|cdn\.hackclub\.com'
50
+
}
51
+
52
+
batch_upload() {
53
+
payload=$(jq -sR 'split("\n")|map(select(length>0))' <(printf "%s\n" "$@"))
54
+
for attempt in 1 2 3; do
55
+
resp=$(curl -sS -w "\n%{http_code}" -X POST "$API_URL" \
56
+
-H "Authorization: Bearer $TOKEN" \
57
+
-H 'Content-Type: application/json' \
58
+
--data-raw "$payload" 2>&1) || true
59
+
body=$(printf "%s" "$resp" | sed '$d')
60
+
code=$(printf "%s" "$resp" | tail -n1)
61
+
if [[ "$code" == "200" ]]; then
62
+
printf "%s" "$body" | jq -r '.files[] | .sourceUrl? as $s | .deployedUrl + "\t" + ( $s // "" )'
63
+
return 0
64
+
fi
65
+
echo "Upload attempt $attempt failed with $code" >&2
66
+
echo "Response body:" >&2
67
+
printf "%s\n" "$body" >&2
68
+
echo "Payload:" >&2
69
+
printf "%s\n" "$payload" >&2
70
+
sleep $((attempt*2))
71
+
done
72
+
echo "Upload failed after retries" >&2
73
+
return 1
74
+
}
75
+
76
+
mapfile -t LINES < <(collect_urls | sort -u)
77
+
78
+
URLS_TO_SEND=()
79
+
FILES=()
80
+
total=${#LINES[@]}
81
+
idx=0
82
+
for line in "${LINES[@]}"; do
83
+
idx=$((idx+1))
84
+
file="${line%%$'\t'*}"
85
+
url="${line#*$'\t'}"
86
+
if grep -Fq "${url}" "$MAP_FILE" 2>/dev/null; then
87
+
echo "[$idx/$total] cached: $url -> will rewrite only"
88
+
CACHED_URLS+=("$url")
89
+
continue
90
+
fi
91
+
if $DRY_RUN; then
92
+
echo "[$idx/$total] queued: $url"
93
+
URLS_TO_SEND+=("$url")
94
+
FILES+=("$file")
95
+
else
96
+
if $SKIP_CHECK; then
97
+
echo "[$idx/$total] no-check: $url"
98
+
URLS_TO_SEND+=("$url")
99
+
FILES+=("$file")
100
+
else
101
+
echo -n "[$idx/$total] checking: $url ... "
102
+
code=$(curl -sS -o /dev/null -w '%{http_code}' -L "$url" || echo 000)
103
+
if [[ "$code" =~ ^2|3 ]]; then
104
+
echo "ok ($code)"
105
+
URLS_TO_SEND+=("$url")
106
+
FILES+=("$file")
107
+
else
108
+
echo "fail ($code)"; echo "Skipping: $url" >&2
109
+
fi
110
+
fi
111
+
fi
112
+
done
113
+
114
+
if [[ ${#URLS_TO_SEND[@]} -eq 0 ]]; then
115
+
echo "No new image URLs to process"; exit 0
116
+
fi
117
+
118
+
BATCH=50
119
+
start=0
120
+
# Rewrites for cached URLs from map without uploading
121
+
if [ "${#CACHED_URLS[@]}" -gt 0 ] 2>/dev/null; then
122
+
echo "Rewriting cached URLs from map without upload..."
123
+
for src in "${CACHED_URLS[@]}"; do
124
+
dst=$(awk -F'\t' -v s="$src" '$1==s{print $2}' "$MAP_FILE" | head -n1)
125
+
[[ -z "$dst" ]] && continue
126
+
rg -l --fixed-strings -- "$src" "${PATHS[@]}" 2>/dev/null | while read -r f; do
127
+
mkdir -p "$TMP_DIR/backup"
128
+
if [[ ! -e "$TMP_DIR/backup/$f" ]]; then
129
+
mkdir -p "$TMP_DIR/backup/$(dirname "$f")"
130
+
cp "$f" "$TMP_DIR/backup/$f"
131
+
fi
132
+
sed -i "s#$(printf '%s' "$src" | sed -e 's/[.[\*^$]/\\&/g' -e 's#/#\\/#g')#$(printf '%s' "$dst" | sed -e 's/[&]/\\&/g' -e 's#/#\\/#g')#g" "$f"
133
+
echo "Rewrote (cached): $f"
134
+
done
135
+
done
136
+
fi
137
+
138
+
while [[ $start -lt ${#URLS_TO_SEND[@]} ]]; do
139
+
end=$(( start + BATCH ))
140
+
if [[ $end -gt ${#URLS_TO_SEND[@]} ]]; then end=${#URLS_TO_SEND[@]}; fi
141
+
chunk=("${URLS_TO_SEND[@]:start:end-start}")
142
+
if $DRY_RUN; then
143
+
for u in "${chunk[@]}"; do echo "DRY: would upload $u"; done
144
+
else
145
+
echo "Uploading ${#chunk[@]} URLs..."
146
+
resp=$(batch_upload "${chunk[@]}") || { echo "Upload failed" >&2; exit 1; }
147
+
echo "Upload response:"; printf "%s\n" "$resp"
148
+
mapfile -t deployed_arr < <(printf "%s\n" "$resp" | awk '{print $0}')
149
+
for i in "${!chunk[@]}"; do
150
+
src="${chunk[$i]}"
151
+
dst="${deployed_arr[$i]:-}"
152
+
if [[ -n "$dst" ]]; then
153
+
printf "%s\t%s\n" "$src" "$dst" | tee -a "$MAP_FILE"
154
+
fi
155
+
done
156
+
fi
157
+
start=$end
158
+
done
159
+
160
+
if $DRY_RUN; then echo "DRY: skipping replacements"; exit 0; fi
161
+
162
+
# Replace in-place using map
163
+
if [[ -s "$MAP_FILE" ]]; then
164
+
cp "$MAP_FILE" "$TMP_DIR/map-$(date +%s).tsv"
165
+
while IFS=$'\t' read -r src dst; do
166
+
[[ -z "$src" || -z "$dst" ]] && continue
167
+
rg -l --fixed-strings -- "$src" "${PATHS[@]}" 2>/dev/null | while read -r f; do
168
+
mkdir -p "$TMP_DIR/backup"
169
+
if [[ ! -e "$TMP_DIR/backup/$f" ]]; then
170
+
mkdir -p "$TMP_DIR/backup/$(dirname "$f")"
171
+
cp "$f" "$TMP_DIR/backup/$f"
172
+
fi
173
+
sed -i "s#$(printf '%s' "$src" | sed -e 's/[.[\*^$]/\\&/g' -e 's#/#\\/#g')#$(printf '%s' "$dst" | sed -e 's/[&]/\\&/g' -e 's#/#\\/#g')#g" "$f"
174
+
echo "Rewrote: $f"
175
+
done
176
+
done < "$MAP_FILE"
177
+
echo "Backups in $TMP_DIR/backup"
178
+
fi
179
+
180
+
echo "Done"