Code and data for arewedecentralizedyet.online and related projects
Skip certain domains entirely
Currently used to handle some duplicates/internal sites.
I considered moving it to the fetch- side of things, but given that I
have better tooling here for dealing with one-off stuff these seems a
more natural place
+32
-2
+6
data-processing/fedi-nodeinfo/nodeinfo-quirks.yaml
+6
data-processing/fedi-nodeinfo/nodeinfo-quirks.yaml
···
197
197
# federating or not.
198
198
},
199
199
200
+
# Domains to skip entirely from output.
201
+
"ignore_domains": {
202
+
"bridgy-federated.uc.r.appspot.com": "Internal site, not meant to be used",
203
+
"pixelfed.de.": "I'm not sure why this version - note trailing dot - ends up in the lists"
204
+
},
205
+
200
206
# Lists of misskey and pleroma forks
201
207
"forks": {
202
208
"misskey": [
+26
-2
data-processing/fedi-nodeinfo/parse-nodeinfo.py
+26
-2
data-processing/fedi-nodeinfo/parse-nodeinfo.py
···
93
93
94
94
def _load_quirks_config(
95
95
path: str,
96
-
) -> Tuple[Dict[str, Set[str]], Set[str], Set[str], Set[str], Dict[str, Tuple[int, ...]]]:
96
+
) -> Tuple[
97
+
Dict[str, Set[str]],
98
+
Set[str],
99
+
Set[str],
100
+
Set[str],
101
+
Dict[str, Tuple[int, ...]],
102
+
Set[str],
103
+
]:
97
104
with open(path, "r", encoding="utf-8") as f:
98
105
data = yaml.safe_load(f)
99
106
if not isinstance(data, dict):
···
161
168
continue
162
169
minimum_versions[str(name).lower()] = parsed
163
170
164
-
return quirks_by_software, known_software, misskey_forks, pleroma_forks, minimum_versions
171
+
ignore_domains: Set[str] = set()
172
+
ignore_domains_section = data.get("ignore_domains", {})
173
+
if isinstance(ignore_domains_section, dict):
174
+
ignore_domains = {str(domain).lower() for domain in ignore_domains_section}
175
+
elif isinstance(ignore_domains_section, list):
176
+
ignore_domains = {str(domain).lower() for domain in ignore_domains_section}
177
+
178
+
return (
179
+
quirks_by_software,
180
+
known_software,
181
+
misskey_forks,
182
+
pleroma_forks,
183
+
minimum_versions,
184
+
ignore_domains,
185
+
)
165
186
166
187
def _get_quirks(
167
188
software_name: Optional[str],
···
304
325
misskey_forks,
305
326
pleroma_forks,
306
327
minimum_versions,
328
+
ignore_domains,
307
329
) = _load_quirks_config(QUIRKS_CONFIG_PATH)
308
330
configured_software = (
309
331
set(known_software)
···
343
365
protocols,
344
366
protocols_str,
345
367
) = extract_fields(wrapper)
368
+
if str(hostname).lower() in ignore_domains:
369
+
continue
346
370
347
371
if _metadata_federation_disabled(wrapper):
348
372
federation_disabled_count += 1