Fetches a list of regions from geofabrik.de
0
list_geofabrik_regions.ts
1/**
2 * Fetches a list of Geofabrik Regions
3 * Geofabrik provides data for building map applications.
4 * All regions here should have a `.poly` and a `-latest.osm.pbf` file associated with them.
5 *
6 * @reference https://www.geofabrik.de/data/download.html
7 */
8import * as cheerio from "npm:cheerio"
9
10const baseURL = "https://download.geofabrik.de/"
11const regions = new Set()
12
13// antarctica has no children, so not in baseRegions, but add to regions
14const baseRegions = [
15 "australia-oceania",
16 "asia",
17 "africa",
18 "central-america",
19 "europe",
20 "north-america",
21 "south-america",
22]
23
24baseRegions.concat(["antarctica"])
25 .forEach(region => regions.add(region))
26
27const responses = baseRegions.map(fetchRegion)
28
29await Promise.all(responses)
30const regionsArray = [...regions];
31Deno.writeTextFile('regions.txt', regionsArray.sort().join("\n") + "\n")
32
33async function fetchRegion(region: string) {
34 console.log('fetching: ', region)
35 const url = baseURL + "/" + region
36 const resp = await fetch(url)
37 const html = await resp.text()
38 const { directories, polyFiles } = await parseGeofabrikDirectory(html)
39 polyFiles
40 .forEach(file => regions.add(region + "/" + file.replace(".poly", "")))
41
42 const childrenResp = directories.map(
43 (name) => {
44 const childName = name.replace(/^\//, "").replace(/\/$/, "")
45 if (!regions.has(childName)) {
46 return fetchRegion(region + "/" + childName)
47 }
48 }
49 )
50 await Promise.all(childrenResp)
51}
52
53async function parseGeofabrikDirectory(html: string) {
54 const $ = cheerio.load(html)
55 const rows = $("tr")
56
57 const directories: string[] = []
58 const polyFiles: string[] = []
59
60 rows.each((_, row) => {
61 const href = $(row).find("td a").first().attr("href")
62
63 if (!href) return
64
65 if (href.endsWith("/") && (href !== "/") && !href.endsWith("-updates/")) {
66 directories.push(href)
67 }
68 else if (href.endsWith(".poly")) {
69 polyFiles.push(href.replace("/", ""))
70 }
71 })
72
73 return {
74 directories,
75 polyFiles
76 }
77}