The code and data behind xeiaso.net
parallelize markdown parsing
Signed-off-by: Christine Dodrill <me@christine.website>
+86
-41
+35
Cargo.lock
+35
Cargo.lock
···
574
574
dependencies = [
575
575
"futures-channel",
576
576
"futures-core",
577
+
"futures-executor",
577
578
"futures-io",
578
579
"futures-sink",
579
580
"futures-task",
···
597
598
checksum = "db8d3b0917ff63a2a96173133c02818fac4a746b0a57569d3baca9ec0e945e08"
598
599
599
600
[[package]]
601
+
name = "futures-executor"
602
+
version = "0.3.9"
603
+
source = "registry+https://github.com/rust-lang/crates.io-index"
604
+
checksum = "9ee9ca2f7eb4475772cf39dd1cd06208dce2670ad38f4d9c7262b3e15f127068"
605
+
dependencies = [
606
+
"futures-core",
607
+
"futures-task",
608
+
"futures-util",
609
+
]
610
+
611
+
[[package]]
600
612
name = "futures-io"
601
613
version = "0.3.9"
602
614
source = "registry+https://github.com/rust-lang/crates.io-index"
603
615
checksum = "e37c1a51b037b80922864b8eed90692c5cd8abd4c71ce49b77146caa47f3253b"
604
616
605
617
[[package]]
618
+
name = "futures-macro"
619
+
version = "0.3.9"
620
+
source = "registry+https://github.com/rust-lang/crates.io-index"
621
+
checksum = "0f8719ca0e1f3c5e34f3efe4570ef2c0610ca6da85ae7990d472e9cbfba13664"
622
+
dependencies = [
623
+
"proc-macro-hack",
624
+
"proc-macro2",
625
+
"quote",
626
+
"syn",
627
+
]
628
+
629
+
[[package]]
606
630
name = "futures-sink"
607
631
version = "0.3.9"
608
632
source = "registry+https://github.com/rust-lang/crates.io-index"
···
623
647
source = "registry+https://github.com/rust-lang/crates.io-index"
624
648
checksum = "036a2107cdeb57f6d7322f1b6c363dad67cd63ca3b7d1b925bdf75bd5d96cda9"
625
649
dependencies = [
650
+
"futures-channel",
626
651
"futures-core",
627
652
"futures-io",
653
+
"futures-macro",
628
654
"futures-sink",
629
655
"futures-task",
630
656
"memchr",
631
657
"pin-project-lite 0.2.3",
632
658
"pin-utils",
659
+
"proc-macro-hack",
660
+
"proc-macro-nested",
633
661
"slab",
634
662
]
635
663
···
1581
1609
version = "0.5.19"
1582
1610
source = "registry+https://github.com/rust-lang/crates.io-index"
1583
1611
checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5"
1612
+
1613
+
[[package]]
1614
+
name = "proc-macro-nested"
1615
+
version = "0.1.7"
1616
+
source = "registry+https://github.com/rust-lang/crates.io-index"
1617
+
checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086"
1584
1618
1585
1619
[[package]]
1586
1620
name = "proc-macro2"
···
2790
2824
"comrak",
2791
2825
"envy",
2792
2826
"eyre",
2827
+
"futures",
2793
2828
"glob",
2794
2829
"go_vanity",
2795
2830
"hyper 0.14.2",
+1
Cargo.toml
+1
Cargo.toml
+3
-3
src/app/mod.rs
+3
-3
src/app/mod.rs
···
70
70
let resume = fs::read_to_string(cfg.resume_fname.clone())?;
71
71
let resume: String = markdown::render(&resume)?;
72
72
let mi = mi::Client::new(cfg.mi_token.clone(), crate::APPLICATION_NAME.to_string())?;
73
-
let blog = crate::post::load("blog", Some(&mi)).await?;
74
-
let gallery = crate::post::load("gallery", None).await?;
75
-
let talks = crate::post::load("talks", None).await?;
73
+
let blog = crate::post::load("blog").await?;
74
+
let gallery = crate::post::load("gallery").await?;
75
+
let talks = crate::post::load("talks").await?;
76
76
let mut everything: Vec<Post> = vec![];
77
77
78
78
{
+47
-38
src/post/mod.rs
+47
-38
src/post/mod.rs
···
1
1
use chrono::prelude::*;
2
2
use color_eyre::eyre::{eyre, Result, WrapErr};
3
3
use glob::glob;
4
-
use std::{cmp::Ordering, fs};
4
+
use std::{cmp::Ordering, path::PathBuf};
5
+
use tokio::fs;
5
6
6
7
pub mod frontmatter;
7
8
···
70
71
}
71
72
}
72
73
73
-
pub async fn load(dir: &str, mi: Option<&mi::Client>) -> Result<Vec<Post>> {
74
-
let mut result: Vec<Post> = vec![];
74
+
async fn read_post(dir: &str, fname: PathBuf) -> Result<Post> {
75
+
let body = fs::read_to_string(fname.clone())
76
+
.await
77
+
.wrap_err_with(|| format!("can't read {:?}", fname))?;
78
+
let (front_matter, content_offset) = frontmatter::Data::parse(body.clone().as_str())
79
+
.wrap_err_with(|| format!("can't parse frontmatter of {:?}", fname))?;
80
+
let body = &body[content_offset..];
81
+
let date = NaiveDate::parse_from_str(&front_matter.clone().date, "%Y-%m-%d")
82
+
.map_err(|why| eyre!("error parsing date in {:?}: {}", fname, why))?;
83
+
let link = format!("{}/{}", dir, fname.file_stem().unwrap().to_str().unwrap());
84
+
let body_html = crate::app::markdown::render(&body)
85
+
.wrap_err_with(|| format!("can't parse markdown for {:?}", fname))?;
86
+
let body = body.to_string();
87
+
let date: DateTime<FixedOffset> =
88
+
DateTime::<Utc>::from_utc(NaiveDateTime::new(date, NaiveTime::from_hms(0, 0, 0)), Utc)
89
+
.with_timezone(&Utc)
90
+
.into();
75
91
76
-
for path in glob(&format!("{}/*.markdown", dir))?.filter_map(Result::ok) {
77
-
log::debug!("loading {:?}", path);
78
-
let body =
79
-
fs::read_to_string(path.clone()).wrap_err_with(|| format!("can't read {:?}", path))?;
80
-
let (fm, content_offset) = frontmatter::Data::parse(body.clone().as_str())
81
-
.wrap_err_with(|| format!("can't parse frontmatter of {:?}", path))?;
82
-
let markup = &body[content_offset..];
83
-
let date = NaiveDate::parse_from_str(&fm.clone().date, "%Y-%m-%d")
84
-
.map_err(|why| eyre!("error parsing date in {:?}: {}", path, why))?;
85
-
let link = format!("{}/{}", dir, path.file_stem().unwrap().to_str().unwrap());
86
-
let mentions: Vec<mi::WebMention> = match mi {
87
-
None => vec![],
88
-
Some(mi) => mi
89
-
.mentioners(format!("https://christine.website/{}", link))
90
-
.await
91
-
.map_err(|why| tracing::error!("error: can't load mentions for {}: {}", link, why))
92
-
.unwrap_or(vec![]),
93
-
};
92
+
let mentions: Vec<mi::WebMention> = match std::env::var("MI_TOKEN") {
93
+
Ok(token) => mi::Client::new(token.to_string(), crate::APPLICATION_NAME.to_string())?
94
+
.mentioners(format!("https://christine.website/{}", link))
95
+
.await
96
+
.map_err(|why| tracing::error!("error: can't load mentions for {}: {}", link, why))
97
+
.unwrap_or(vec![]),
98
+
Err(_) => vec![],
99
+
};
94
100
95
-
result.push(Post {
96
-
front_matter: fm,
97
-
link: link,
98
-
body: markup.to_string(),
99
-
body_html: crate::app::markdown::render(&markup)
100
-
.wrap_err_with(|| format!("can't parse markdown for {:?}", path))?,
101
-
date: {
102
-
DateTime::<Utc>::from_utc(
103
-
NaiveDateTime::new(date, NaiveTime::from_hms(0, 0, 0)),
104
-
Utc,
105
-
)
106
-
.with_timezone(&Utc)
107
-
.into()
108
-
},
109
-
mentions: mentions,
110
-
})
111
-
}
101
+
Ok(Post {
102
+
front_matter,
103
+
link,
104
+
body,
105
+
body_html,
106
+
date,
107
+
mentions,
108
+
})
109
+
}
110
+
111
+
pub async fn load(dir: &str) -> Result<Vec<Post>> {
112
+
let futs = glob(&format!("{}/*.markdown", dir))?
113
+
.filter_map(Result::ok)
114
+
.map(|fname| read_post(dir, fname));
115
+
116
+
let mut result: Vec<Post> = futures::future::join_all(futs)
117
+
.await
118
+
.into_iter()
119
+
.map(Result::unwrap)
120
+
.collect();
112
121
113
122
if result.len() == 0 {
114
123
Err(eyre!("no posts loaded"))