this repo has no description
✨ Add buffering system for preview mode
There's a weird double-buffering issue but it works and its past 4am anyways
authored by
Gwenn Le Bihan
and committed by
Ewen Le Bihan
1c457f6a
263ec1ce
+295
-79
+31
Cargo.lock
+31
Cargo.lock
···
33
33
checksum = "f538837af36e6f6a9be0faa67f9a314f8119e4e4b5867c6ab40ed60360142519"
34
34
35
35
[[package]]
36
+
name = "ascii"
37
+
version = "1.1.0"
38
+
source = "registry+https://github.com/rust-lang/crates.io-index"
39
+
checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16"
40
+
41
+
[[package]]
36
42
name = "autocfg"
37
43
version = "1.2.0"
38
44
source = "registry+https://github.com/rust-lang/crates.io-index"
···
87
93
dependencies = [
88
94
"chrono",
89
95
]
96
+
97
+
[[package]]
98
+
name = "chunked_transfer"
99
+
version = "1.5.0"
100
+
source = "registry+https://github.com/rust-lang/crates.io-index"
101
+
checksum = "6e4de3bc4ea267985becf712dc6d9eed8b04c953b3fcfb339ebc87acd9804901"
90
102
91
103
[[package]]
92
104
name = "console"
···
231
243
version = "3.5.1"
232
244
source = "registry+https://github.com/rust-lang/crates.io-index"
233
245
checksum = "62adaabb884c94955b19907d60019f4e145d091c75345379e70d1ee696f7854f"
246
+
247
+
[[package]]
248
+
name = "httpdate"
249
+
version = "1.0.3"
250
+
source = "registry+https://github.com/rust-lang/crates.io-index"
251
+
checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
234
252
235
253
[[package]]
236
254
name = "iana-time-zone"
···
592
610
"serde_cbor",
593
611
"serde_json",
594
612
"svg",
613
+
"tiny_http",
595
614
]
596
615
597
616
[[package]]
···
635
654
"proc-macro2",
636
655
"quote",
637
656
"syn",
657
+
]
658
+
659
+
[[package]]
660
+
name = "tiny_http"
661
+
version = "0.12.0"
662
+
source = "registry+https://github.com/rust-lang/crates.io-index"
663
+
checksum = "389915df6413a2e74fb181895f933386023c71110878cd0825588928e64cdc82"
664
+
dependencies = [
665
+
"ascii",
666
+
"chunked_transfer",
667
+
"httpdate",
668
+
"log",
638
669
]
639
670
640
671
[[package]]
+1
Cargo.toml
+1
Cargo.toml
+116
-5
preview/engine.js
+116
-5
preview/engine.js
···
1
+
/**
2
+
* @typedef {object} GlobalData
3
+
* @property {Map<number, HTMLDivElement>} [frames]
4
+
*/
5
+
6
+
/**
7
+
* @typedef {Window & GlobalData} WindowWithData
8
+
*/
9
+
1
10
function displayFrame() {
2
11
const ms = millisecondsSinceStart(window.videoStartedAt)
3
-
console.info("displayFrame", ms)
12
+
console.debug("displayFrame", ms)
4
13
5
14
if (window.previouslyRenderedFrame) {
6
15
let f = window.frames.get(window.previouslyRenderedFrame)
7
-
f.style.display = "none"
8
-
f.classList.toggle("shown")
16
+
if (f) {
17
+
f.style.display = "none"
18
+
f.classList.toggle("shown")
19
+
}
9
20
}
10
21
11
22
let frame = closestFrame(ms)
···
13
24
f.style.display = "block"
14
25
f.classList.toggle("shown")
15
26
27
+
console.debug("framestLeftCount", framesLeftCount())
28
+
if (framesLeftCount() < window.FRAMES_BUFFER_SIZE) {
29
+
console.warn(
30
+
window.updatingBuffer ? "already updating buffer" : "update buffer now"
31
+
)
32
+
33
+
if (
34
+
!window.updatingBuffer &&
35
+
window.previouslyRenderedFrame - window.lastBufferUpdateWasOn > 2000
36
+
) {
37
+
console.info(
38
+
`Updating buffer: ${framesLeftCount()} < ${
39
+
window.FRAMES_BUFFER_SIZE
40
+
} remaining and last update was ${
41
+
window.previouslyRenderedFrame - window.lastBufferUpdateWasOn
42
+
} > 2000 ms ago`
43
+
)
44
+
updateBuffer()
45
+
}
46
+
}
47
+
16
48
window.previouslyRenderedFrame = frame
17
49
}
18
50
51
+
/**
52
+
*
53
+
* @returns number
54
+
*/
55
+
function framesLeftCount() {
56
+
return [...window.frames.keys()].filter(
57
+
(key) => key > window.previouslyRenderedFrame
58
+
).length
59
+
}
60
+
61
+
/**
62
+
*
63
+
* @param {number} ms
64
+
* @returns {number}
65
+
*/
19
66
function closestFrame(ms) {
20
67
const closest = [...window.frames.keys()].reduce((a, b) =>
21
68
Math.abs(b - ms) < Math.abs(a - ms) ? b : a
22
69
)
23
-
console.info("closestFrame", ms, "is", closest)
70
+
console.debug("closestFrame", ms, "is", closest)
24
71
return closest
25
72
}
26
73
···
33
80
return new Date().getTime() - start
34
81
}
35
82
83
+
/**
84
+
*
85
+
* @returns {number}
86
+
*/
87
+
function lastLoadedFrame() {
88
+
return Math.max(...[...window.frames.keys()])
89
+
}
90
+
91
+
async function updateBuffer() {
92
+
console.time("fetchFrames")
93
+
console.log("set updatingBuffer to true")
94
+
window.updatingBuffer = true
95
+
console.info(
96
+
"updateBuffer",
97
+
4 * window.FRAMES_BUFFER_SIZE,
98
+
window.previouslyRenderedFrame
99
+
)
100
+
// request half the buffer size for the next frames
101
+
await fetch(
102
+
new URL(
103
+
"/frames?" +
104
+
new URLSearchParams({
105
+
next: 4 * window.FRAMES_BUFFER_SIZE,
106
+
from: window.previouslyRenderedFrame,
107
+
}),
108
+
window.SERVER_ORIGIN
109
+
)
110
+
).then((response) => {
111
+
if (!response.ok) {
112
+
console.error("Failed to fetch frames", response)
113
+
return
114
+
}
115
+
console.timeEnd("fetchFrames")
116
+
117
+
console.time("insertFramesToDOM")
118
+
response.text().then((frames) => {
119
+
document.body.insertAdjacentHTML("beforeend", frames)
120
+
})
121
+
console.timeEnd("insertFramesToDOM")
122
+
})
123
+
124
+
console.time("pruneFramesFromDOM")
125
+
// remove frames that are not needed anymore
126
+
;[...window.frames.keys()].forEach((key) => {
127
+
if (key < window.previouslyRenderedFrame) {
128
+
window.frames.get(key).remove()
129
+
}
130
+
})
131
+
console.timeEnd("pruneFramesFromDOM")
132
+
133
+
loadFramesFromDOM()
134
+
console.log("set updatingBuffer to false")
135
+
window.updatingBuffer = false
136
+
window.lastBufferUpdateWasOn = window.previouslyRenderedFrame
137
+
}
138
+
36
139
window.addEventListener("keypress", (e) => {
37
140
if (e.key === " ") {
38
141
if (window.intervalID) {
···
43
146
}
44
147
})
45
148
46
-
window.startVideo = () => {
149
+
function loadFramesFromDOM() {
150
+
console.time("loadFramesFromDOM")
47
151
window.frames = new Map(
48
152
[...document.querySelectorAll("[id^=frame-]")].map((el) => [
49
153
parseInt(el.id.replace("frame-", "")),
50
154
el,
51
155
])
52
156
)
157
+
console.timeEnd("loadFramesFromDOM")
158
+
}
159
+
160
+
window.startVideo = () => {
161
+
loadFramesFromDOM()
53
162
window.refreshRate = 50
54
163
window.videoStartedAt = new Date().getTime()
55
164
window.previouslyRenderedFrame = null
165
+
window.lastBufferUpdateWasOn = null
166
+
window.updatingBuffer = false
56
167
57
168
displayFrame()
58
169
window.intervalID = setInterval(displayFrame, window.refreshRate)
+2
preview/index.html.hbs
+2
preview/index.html.hbs
···
6
6
<meta name="viewport" content="width=device-width, initial-scale=1.0">
7
7
<title>Shapemaker Preview</title>
8
8
<script>
9
+
window.SERVER_ORIGIN = "{{ serverorigin }}";
10
+
window.FRAMES_BUFFER_SIZE = {{ framesbuffersize }};
9
11
{{{ enginesource }}}
10
12
</script>
11
13
{{!-- <script src="preview/engine.js"></script> --}}
+49
-59
src/lib.rs
+49
-59
src/lib.rs
···
274
274
}
275
275
276
276
pub fn build_video(&self, render_to: &str) -> Result<(), String> {
277
-
let result = std::process::Command::new("ffmpeg")
277
+
let mut command = std::process::Command::new("ffmpeg");
278
+
279
+
command
278
280
.args(["-hide_banner", "-loglevel", "error"])
279
281
.args(["-framerate", &self.fps.to_string()])
280
282
// .args(["-pattern_type", "glob"]) // not available on Windows
···
288
290
])
289
291
.args(["-i", self.audiofile.to_str().unwrap()])
290
292
.args(["-t", &format!("{}", self.duration_ms() as f32 / 1000.0)])
291
-
.args(["-vcodec", "png"])
293
+
.args(["-c:v", "libx264"])
294
+
.args(["-pix_fmt", "yuv420p"])
292
295
.arg("-y")
293
-
.arg(render_to)
294
-
.output();
296
+
.arg(render_to);
297
+
298
+
println!("Running command: {:?}", command);
295
299
296
-
match result {
300
+
match command.output() {
297
301
Err(e) => Err(format!("Failed to execute ffmpeg: {}", e)),
298
302
Ok(r) => {
299
303
println!("{}", std::str::from_utf8(&r.stdout).unwrap());
···
616
620
.unwrap()
617
621
}
618
622
619
-
// rendered_svg_frames should map ms timestamps to SVG strings
620
-
pub fn output_preview(
621
-
&self,
622
-
canvas: &Canvas,
623
-
rendered_svg_frames: HashMap<usize, String>,
624
-
output_file: PathBuf,
625
-
) -> Result<&Self> {
626
-
let contents =
627
-
preview::render_template(rendered_svg_frames, canvas, self.audiofile.clone());
628
-
fs::write(output_file, contents)?;
629
-
Ok(self)
623
+
pub fn preview_on(&self, port: usize) {
624
+
let mut rendered_frames: HashMap<usize, String> = HashMap::new();
625
+
let progress_bar = self.setup_progress_bar();
626
+
627
+
for (frame, _, ms) in self.render_frames(&progress_bar, vec!["*"], true) {
628
+
rendered_frames.insert(ms, frame);
629
+
}
630
+
631
+
progress_bar.finish_and_clear();
632
+
633
+
preview::output_preview(
634
+
&self.initial_canvas,
635
+
&rendered_frames,
636
+
port,
637
+
PathBuf::from(".").join("preview.html"),
638
+
self.audiofile.clone(),
639
+
);
640
+
preview::start_preview_server(port, rendered_frames);
630
641
}
631
642
632
-
pub fn render_to(
633
-
&mut self,
634
-
output_file: String,
635
-
workers_count: usize,
636
-
preview_only: bool,
637
-
) -> Result<&Self> {
638
-
self.render_composition(output_file, vec!["*"], true, workers_count, preview_only)
643
+
pub fn render_to(&self, output_file: String, workers_count: usize, preview_only: bool) -> () {
644
+
self.render_composition(output_file, vec!["*"], true, workers_count, preview_only);
639
645
}
640
646
641
-
pub fn render_layers_in(
642
-
&self,
643
-
output_directory: String,
644
-
workers_count: usize,
645
-
) -> Result<&Self> {
647
+
pub fn render_layers_in(&self, output_directory: String, workers_count: usize) -> () {
646
648
for composition in self
647
649
.initial_canvas
648
650
.layers
···
655
657
false,
656
658
workers_count,
657
659
false,
658
-
)?;
660
+
);
659
661
}
660
-
Ok(self)
661
662
}
662
663
663
664
// Returns a triple of (SVG content, frame number, millisecond at frame)
···
757
758
frames_to_write
758
759
}
759
760
761
+
fn setup_progress_bar(&self) -> ProgressBar {
762
+
indicatif::ProgressBar::new(self.total_frames() as u64).with_style(
763
+
indicatif::ProgressStyle::with_template(
764
+
&(PROGRESS_BARS_STYLE.to_owned() + " ({pos:.bold} frames out of {len})"),
765
+
)
766
+
.unwrap()
767
+
.progress_chars("== "),
768
+
)
769
+
}
770
+
760
771
pub fn render_composition(
761
772
&self,
762
773
output_file: String,
···
764
775
render_background: bool,
765
776
workers_count: usize,
766
777
preview_only: bool,
767
-
) -> Result<&Self> {
778
+
) -> () {
768
779
let mut frame_writer_threads = vec![];
769
780
let mut frames_to_write: Vec<(String, usize, usize)> = vec![];
770
781
···
772
783
create_dir(self.frames_output_directory).unwrap();
773
784
create_dir_all(Path::new(&output_file).parent().unwrap()).unwrap();
774
785
775
-
let progress_bar = indicatif::ProgressBar::new(self.total_frames() as u64).with_style(
776
-
indicatif::ProgressStyle::with_template(
777
-
&(PROGRESS_BARS_STYLE.to_owned() + " ({pos:.bold} frames out of {len})"),
778
-
)
779
-
.unwrap()
780
-
.progress_chars("== "),
781
-
);
782
786
let total_frames = self.total_frames();
783
787
let aspect_ratio =
784
788
self.initial_canvas.grid_size.0 as f32 / self.initial_canvas.grid_size.1 as f32;
785
789
let resolution = self.resolution;
786
-
let frames_output_directory = self.frames_output_directory;
790
+
791
+
let progress_bar = self.setup_progress_bar();
787
792
progress_bar.set_message("Rendering frames to SVG");
788
793
789
794
for (frame, no, ms) in self.render_frames(&progress_bar, composition, render_background) {
790
-
std::fs::write(format!("{}/{}.svg", frames_output_directory, no), &frame)?;
795
+
std::fs::write(
796
+
format!("{}/{}.svg", self.frames_output_directory, no),
797
+
&frame,
798
+
);
791
799
frames_to_write.push((frame, no, ms));
792
800
}
793
801
794
-
self.output_preview(
795
-
&self.initial_canvas,
796
-
frames_to_write
797
-
.iter()
798
-
.map(|(f, _, ms)| (ms.clone(), f.clone()))
799
-
.collect(),
800
-
PathBuf::from(&output_file)
801
-
.parent()
802
-
.unwrap()
803
-
.join("preview.html"),
804
-
);
805
-
806
802
progress_bar.println("Rendered frames to SVG");
807
-
808
-
if preview_only {
809
-
progress_bar.finish_and_clear();
810
-
return Ok(self);
811
-
}
812
-
813
803
progress_bar.set_message("Rendering SVG frames to PNG");
814
804
progress_bar.set_position(0);
815
805
816
806
let chunk_size = (frames_to_write.len() as f32 / workers_count as f32).ceil() as usize;
817
807
let frames_to_write = Arc::new(frames_to_write);
808
+
let frames_output_directory = self.frames_output_directory.clone();
818
809
for i in 0..workers_count {
819
810
let frames_to_write = Arc::clone(&frames_to_write);
820
811
let progress_bar = progress_bar.clone();
···
852
843
panic!("Failed to build video: {}", e);
853
844
}
854
845
spinner.end(&format!("Built video to {}", output_file));
855
-
Ok(self)
856
846
}
857
847
}
+8
-9
src/main.rs
+8
-9
src/main.rs
···
26
26
video.duration_override = args.flag_duration.map(|seconds| seconds * 1000);
27
27
video.fps = args.flag_fps.unwrap_or(30);
28
28
video.audiofile = args.flag_audio.unwrap().into();
29
-
30
-
video
29
+
video = video
31
30
.init(&|canvas: _, context: _| {
32
31
context.extra = State {
33
32
bass_pattern_at: Region::from_origin_and_size((6, 3), (3, 3)),
···
83
82
.command("remove", &|argumentsline, canvas, _| {
84
83
let args = argumentsline.splitn(3, ' ').collect::<Vec<_>>();
85
84
canvas.remove_object(args[0]);
86
-
})
87
-
.render_to(
88
-
args.arg_file,
89
-
args.flag_workers.unwrap_or(8),
90
-
args.flag_preview,
91
-
)
92
-
.unwrap();
85
+
});
86
+
87
+
if args.flag_preview {
88
+
video.preview_on(8888);
89
+
} else {
90
+
video.render_to(args.arg_file, args.flag_workers.unwrap_or(8), false);
91
+
}
93
92
}
94
93
95
94
fn update_stem_position(
+3
-3
src/midi.rs
+3
-3
src/midi.rs
···
60
60
// }
61
61
}
62
62
63
-
let duration_ms = notes_per_ms.keys().max().unwrap_or(&0);
63
+
let duration_ms = notes_per_ms.keys().max().unwrap_or(&0).clone();
64
64
let mut amplitudes = Vec::<f32>::new();
65
65
let mut last_amplitude = 0.0;
66
-
for i in 0..*duration_ms {
66
+
for i in 0..duration_ms {
67
67
if let Some(notes) = notes_per_ms.get(&i) {
68
68
last_amplitude = notes
69
69
.iter()
···
79
79
Stem {
80
80
amplitude_max: notes.iter().map(|n| n.vel).max().unwrap_or(0) as f32,
81
81
amplitude_db: amplitudes,
82
-
duration_ms: notes.iter().map(|n| n.tick).max().unwrap_or(0) as usize,
82
+
duration_ms: duration_ms,
83
83
notes: notes_per_ms,
84
84
name: name.clone(),
85
85
},
+85
-3
src/preview.rs
+85
-3
src/preview.rs
···
1
-
use std::{collections::HashMap, path::PathBuf};
1
+
use std::{collections::HashMap, fs, hash::Hash, path::PathBuf};
2
2
3
3
use handlebars::Handlebars;
4
+
use itertools::Itertools;
4
5
use serde_json::json;
5
6
6
7
use crate::{Canvas, ColorMapping};
7
8
9
+
const FRAMES_BUFFER_SIZE: usize = 500;
10
+
8
11
pub fn render_template(
9
-
frames: HashMap<usize, String>,
12
+
frames: &HashMap<usize, String>,
10
13
canvas: &Canvas,
11
14
path_to_audio_file: PathBuf,
15
+
port: usize,
12
16
) -> String {
13
17
let template = String::from_utf8_lossy(include_bytes!("../preview/index.html.hbs"));
14
18
let engine_js_source = String::from_utf8_lossy(include_bytes!("../preview/engine.js"));
···
20
24
"frames":frames,
21
25
"audiopath": path_to_audio_file,
22
26
"enginesource": engine_js_source,
23
-
"background": canvas.background.map_or("black".to_string(), |color| color.to_string(&canvas.colormap))
27
+
"background": canvas.background.map_or("black".to_string(), |color| color.to_string(&canvas.colormap)),
28
+
"serverorigin": format!("http://localhost:{}", port),
29
+
"framesbuffersize": FRAMES_BUFFER_SIZE,
24
30
}),
25
31
)
26
32
.unwrap()
27
33
}
34
+
35
+
// rendered_svg_frames should map ms timestamps to SVG strings
36
+
pub fn output_preview(
37
+
canvas: &Canvas,
38
+
rendered_svg_frames: &HashMap<usize, String>,
39
+
server_port: usize,
40
+
output_file: PathBuf,
41
+
audio_file: PathBuf,
42
+
) {
43
+
let first_frames = rendered_svg_frames
44
+
.iter()
45
+
// over 3000 loaded frames get really heavy on the browser (too much DOM nodes)
46
+
.sorted_by_key(|(ms, _)| *ms)
47
+
.take((2 * FRAMES_BUFFER_SIZE).min(10_000))
48
+
.map(|(ms, svg)| (*ms, svg.clone()))
49
+
.collect::<HashMap<usize, String>>();
50
+
51
+
let contents = render_template(&first_frames, canvas, audio_file, server_port);
52
+
fs::write(output_file, contents);
53
+
}
54
+
55
+
pub fn start_preview_server(port: usize, frames: HashMap<usize, String>) {
56
+
let server = tiny_http::Server::http(format!("0.0.0.0:{}", port)).unwrap();
57
+
println!("Preview server running on port {}", port);
58
+
let sorted_frames: Vec<(&usize, &String)> =
59
+
frames.iter().sorted_by_key(|(ms, _)| *ms).collect();
60
+
println!("{} frames available", sorted_frames.len());
61
+
62
+
for request in server.incoming_requests() {
63
+
let (frame_start_ms, requested_frames_count) = get_request_params(request.url());
64
+
65
+
println!(
66
+
"Request for {} frames @ {}ms",
67
+
requested_frames_count, frame_start_ms,
68
+
);
69
+
70
+
let contents = sorted_frames
71
+
.iter()
72
+
.filter(|(ms, _)| **ms >= frame_start_ms)
73
+
.take(requested_frames_count)
74
+
.map(|(ms, svg_string)| {
75
+
format!(
76
+
r#"<div style="display: none;" id="frame-{}" class="frame">{}</div>"#,
77
+
ms, svg_string
78
+
)
79
+
})
80
+
.join("\n");
81
+
82
+
request.respond(tiny_http::Response::from_string(contents).with_header(
83
+
tiny_http::Header {
84
+
field: "Access-Control-Allow-Origin".parse().unwrap(),
85
+
value: "*".parse().unwrap(),
86
+
},
87
+
));
88
+
}
89
+
}
90
+
91
+
// returns (ms timestamp of first frame to send, number of frames to send)
92
+
fn get_request_params(url: &str) -> (usize, usize) {
93
+
let mut first_frame_ms = 0;
94
+
let mut num_frames = 1;
95
+
96
+
let (_, querystring) = url.split_once("?").unwrap_or(("", ""));
97
+
for (key, value) in querystring
98
+
.split("&")
99
+
.map(|pair| pair.split_once("=").unwrap_or(("", "")))
100
+
{
101
+
match key {
102
+
"from" => first_frame_ms = value.parse().unwrap_or(0),
103
+
"next" => num_frames = value.parse().unwrap_or(1),
104
+
_ => (),
105
+
}
106
+
}
107
+
108
+
(first_frame_ms, num_frames)
109
+
}