MIRROR: javascript for 馃悳's, a tiny runtime with big ambitions
1const std = @import("std");
2const builtin = @import("builtin");
3const json = @import("json.zig");
4const debug = @import("debug.zig");
5
6const PARALLEL_LINK_THRESHOLD = 500;
7const LINK_THREAD_COUNT = 8;
8
9pub fn createSymlinkOrCopy(dir: std.fs.Dir, target: []const u8, link_name: []const u8) !void {
10 if (comptime builtin.os.tag == .windows) {
11 try createSymlinkWindows(dir, target, link_name);
12 } else try dir.symLink(target, link_name, .{});
13}
14
15pub fn createSymlinkAbsolute(target: []const u8, link_path: []const u8) void {
16 if (comptime builtin.os.tag == .windows) {
17 createSymlinkAbsoluteWindows(target, link_path);
18 } else std.posix.symlink(target, link_path) catch {};
19}
20
21fn createSymlinkWindows(dir: std.fs.Dir, target: []const u8, link_name: []const u8) !void {
22 if (comptime builtin.os.tag != .windows) return;
23 var target_utf16: [std.fs.max_path_bytes]u16 = undefined;
24 var link_utf16: [std.fs.max_path_bytes]u16 = undefined;
25 const target_len = try std.unicode.utf8ToUtf16Le(&target_utf16, target);
26 const link_len = try std.unicode.utf8ToUtf16Le(&link_utf16, link_name);
27 target_utf16[target_len] = 0;
28
29 _ = try std.os.windows.CreateSymbolicLink(
30 dir.fd, link_utf16[0..link_len],
31 target_utf16[0..target_len :0], false,
32 );
33}
34
35fn createSymlinkAbsoluteWindows(target: []const u8, link_path: []const u8) void {
36 if (comptime builtin.os.tag != .windows) return;
37 var target_utf16: [std.fs.max_path_bytes]u16 = undefined;
38 var link_utf16: [std.fs.max_path_bytes]u16 = undefined;
39 const target_len = std.unicode.utf8ToUtf16Le(&target_utf16, target) catch return;
40 const link_len = std.unicode.utf8ToUtf16Le(&link_utf16, link_path) catch return;
41 target_utf16[target_len] = 0;
42
43 _ = std.os.windows.CreateSymbolicLink(
44 null, link_utf16[0..link_len],
45 target_utf16[0..target_len :0], false,
46 ) catch {};
47}
48
49pub const LinkError = error{
50 IoError,
51 PathNotFound,
52 CrossDevice,
53 PermissionDenied,
54 OutOfMemory,
55 PathTooLong,
56};
57
58pub const StatsSnapshot = struct {
59 files_linked: u32,
60 files_copied: u32,
61 files_cloned: u32,
62 bytes_linked: u64,
63 bytes_copied: u64,
64 dirs_created: u32,
65 bins_linked: u32,
66 packages_installed: u32,
67 packages_skipped: u32,
68};
69
70pub const LinkStats = struct {
71 files_linked: std.atomic.Value(u32),
72 files_copied: std.atomic.Value(u32),
73 files_cloned: std.atomic.Value(u32),
74 bytes_linked: std.atomic.Value(u64),
75 bytes_copied: std.atomic.Value(u64),
76 dirs_created: std.atomic.Value(u32),
77 bins_linked: std.atomic.Value(u32),
78 packages_installed: std.atomic.Value(u32),
79 packages_skipped: std.atomic.Value(u32),
80
81 pub fn init() LinkStats {
82 return .{
83 .files_linked = std.atomic.Value(u32).init(0),
84 .files_copied = std.atomic.Value(u32).init(0),
85 .files_cloned = std.atomic.Value(u32).init(0),
86 .bytes_linked = std.atomic.Value(u64).init(0),
87 .bytes_copied = std.atomic.Value(u64).init(0),
88 .dirs_created = std.atomic.Value(u32).init(0),
89 .bins_linked = std.atomic.Value(u32).init(0),
90 .packages_installed = std.atomic.Value(u32).init(0),
91 .packages_skipped = std.atomic.Value(u32).init(0),
92 };
93 }
94
95 pub fn snapshot(self: *const LinkStats) StatsSnapshot {
96 return .{
97 .files_linked = self.files_linked.load(.acquire),
98 .files_copied = self.files_copied.load(.acquire),
99 .files_cloned = self.files_cloned.load(.acquire),
100 .bytes_linked = self.bytes_linked.load(.acquire),
101 .bytes_copied = self.bytes_copied.load(.acquire),
102 .dirs_created = self.dirs_created.load(.acquire),
103 .bins_linked = self.bins_linked.load(.acquire),
104 .packages_installed = self.packages_installed.load(.acquire),
105 .packages_skipped = self.packages_skipped.load(.acquire),
106 };
107 }
108};
109
110pub const PackageLink = struct {
111 cache_path: []const u8,
112 node_modules_path: []const u8,
113 name: []const u8,
114 parent_path: ?[]const u8 = null,
115 file_count: u32 = 0,
116 has_bin: bool = true,
117};
118
119pub const Linker = struct {
120 allocator: std.mem.Allocator,
121 stats: LinkStats,
122 node_modules_dir: ?std.fs.Dir,
123 bin_dir: ?std.fs.Dir,
124 node_modules_path: []const u8,
125 cross_device: std.atomic.Value(bool),
126
127 pub fn init(allocator: std.mem.Allocator) Linker {
128 return .{
129 .allocator = allocator,
130 .stats = LinkStats.init(),
131 .node_modules_dir = null,
132 .bin_dir = null,
133 .node_modules_path = "",
134 .cross_device = std.atomic.Value(bool).init(false),
135 };
136 }
137
138 pub fn deinit(self: *Linker) void {
139 if (self.node_modules_dir) |*d| d.close();
140 if (self.bin_dir) |*d| d.close();
141 if (self.node_modules_path.len > 0) self.allocator.free(self.node_modules_path);
142 }
143
144 pub fn setNodeModulesPath(self: *Linker, path: []const u8) !void {
145 std.fs.cwd().makePath(path) catch |err| switch (err) {
146 error.PathAlreadyExists => {},
147 else => return error.IoError,
148 };
149
150 var new_nm_dir = try std.fs.cwd().openDir(path, .{});
151 errdefer new_nm_dir.close();
152
153 const new_path = try self.allocator.dupe(u8, path);
154 errdefer self.allocator.free(new_path);
155
156 const bin_path = try std.fmt.allocPrint(self.allocator, "{s}/.bin", .{path});
157 defer self.allocator.free(bin_path);
158 std.fs.cwd().makePath(bin_path) catch |err| switch (err) {
159 error.PathAlreadyExists => {},
160 else => return error.IoError,
161 };
162
163 const new_bin_dir = try std.fs.cwd().openDir(bin_path, .{});
164
165 if (self.bin_dir) |*d| d.close();
166 if (self.node_modules_dir) |*d| d.close();
167 if (self.node_modules_path.len > 0) self.allocator.free(self.node_modules_path);
168
169 self.node_modules_dir = new_nm_dir;
170 self.node_modules_path = new_path;
171 self.bin_dir = new_bin_dir;
172 }
173
174 pub fn linkPackage(self: *Linker, pkg: PackageLink) !void {
175 const node_modules = self.node_modules_dir orelse return error.IoError;
176
177 const install_path = if (pkg.parent_path) |parent|
178 try std.fmt.allocPrint(self.allocator, "{s}/node_modules/{s}", .{ parent, pkg.name })
179 else
180 try self.allocator.dupe(u8, pkg.name);
181 defer self.allocator.free(install_path);
182
183 var source_dir = std.fs.cwd().openDir(pkg.cache_path, .{ .iterate = true }) catch {
184 return error.PathNotFound;
185 };
186 defer source_dir.close();
187
188 const source_version = readPackageVersion(self.allocator, source_dir);
189 defer if (source_version) |v| self.allocator.free(v);
190
191 var should_skip = false;
192 var has_existing_install = false;
193 var has_existing_package = false;
194
195 {
196 const existing = node_modules.openDir(install_path, .{ .iterate = true }) catch null;
197 if (existing) |dir| {
198 has_existing_install = true;
199 var installed_dir = dir;
200 defer installed_dir.close();
201
202 const installed_version = readPackageVersion(self.allocator, installed_dir);
203 defer if (installed_version) |v| self.allocator.free(v);
204 has_existing_package = installed_version != null;
205 should_skip = packageVersionsMatch(source_version, installed_version) and
206 installedFileCountMatches(installed_dir, pkg.file_count);
207 }
208 }
209
210 if (should_skip) {
211 _ = self.stats.packages_skipped.fetchAdd(1, .release);
212 return;
213 }
214
215 if (has_existing_install and has_existing_package) {
216 node_modules.deleteTree(install_path) catch return error.IoError;
217 }
218
219 node_modules.makePath(install_path) catch |err| switch (err) {
220 error.PathAlreadyExists => {},
221 else => return error.IoError,
222 };
223 _ = self.stats.dirs_created.fetchAdd(1, .release);
224
225 var dest_dir = node_modules.openDir(install_path, .{}) catch return error.IoError;
226 defer dest_dir.close();
227
228 self.linkDirectoryWithHint(source_dir, dest_dir, pkg.file_count, pkg.name) catch |err| return err;
229
230 if (pkg.parent_path == null and pkg.has_bin) try self.linkBinaries(pkg.name);
231 _ = self.stats.packages_installed.fetchAdd(1, .release);
232 }
233
234 fn linkBinaries(self: *Linker, pkg_name: []const u8) !void {
235 const bin_dir = self.bin_dir orelse return;
236 const node_modules = self.node_modules_dir orelse return;
237
238 var pkg_dir = node_modules.openDir(pkg_name, .{}) catch return;
239 defer pkg_dir.close();
240
241 const pkg_json_file = pkg_dir.openFile("package.json", .{}) catch return;
242 defer pkg_json_file.close();
243
244 const content = pkg_json_file.readToEndAlloc(self.allocator, 1024 * 1024) catch return;
245 defer self.allocator.free(content);
246
247 var doc = json.JsonDoc.parse(content) catch return;
248 defer doc.deinit();
249
250 const root_val = doc.root();
251
252 if (root_val.getObject("bin")) |bin_obj| {
253 var iter = bin_obj.objectIterator() orelse return;
254 while (iter.next()) |entry| {
255 const bin_path = entry.value.asString() orelse continue;
256 self.createBinSymlink(pkg_name, entry.key, bin_path, bin_dir) catch continue;
257 }
258 } else if (root_val.getString("bin")) |bin_path| {
259 const simple_name = if (std.mem.indexOf(u8, pkg_name, "/")) |slash|
260 pkg_name[slash + 1 ..]
261 else
262 pkg_name;
263 self.createBinSymlink(pkg_name, simple_name, bin_path, bin_dir) catch {};
264 }
265 }
266
267 fn readPackageVersion(allocator: std.mem.Allocator, dir: std.fs.Dir) ?[]const u8 {
268 const pkg_json = dir.openFile("package.json", .{}) catch return null;
269 defer pkg_json.close();
270
271 const content = pkg_json.readToEndAlloc(allocator, 256 * 1024) catch return null;
272 defer allocator.free(content);
273
274 var doc = json.JsonDoc.parse(content) catch return null;
275 defer doc.deinit();
276
277 const version = doc.root().getString("version") orelse return null;
278 return allocator.dupe(u8, version) catch null;
279 }
280
281 fn packageVersionsMatch(source_version: ?[]const u8, installed_version: ?[]const u8) bool {
282 const src = source_version orelse return false;
283 const dst = installed_version orelse return false;
284 return std.mem.eql(u8, src, dst);
285 }
286
287 fn installedFileCountMatches(dir: std.fs.Dir, expected_files: u32) bool {
288 if (expected_files == 0) return true;
289 const actual_files = countFilesRecursive(dir) catch return false;
290 return actual_files >= expected_files;
291 }
292
293 fn countFilesRecursive(dir: std.fs.Dir) !u32 {
294 var count: u32 = 0;
295 var iter = dir.iterate();
296 while (try iter.next()) |entry| {
297 switch (entry.kind) {
298 .file => count += 1,
299 .directory => {
300 var child = dir.openDir(entry.name, .{ .iterate = true }) catch continue;
301 defer child.close();
302 count += try countFilesRecursive(child);
303 },
304 else => {},
305 }}
306 return count;
307 }
308
309 fn createBinSymlink(self: *Linker, pkg_name: []const u8, cmd_name: []const u8, bin_path: []const u8, bin_dir: std.fs.Dir) !void {
310 var normalized_path = bin_path;
311 if (std.mem.startsWith(u8, normalized_path, "./")) {
312 normalized_path = normalized_path[2..];
313 }
314
315 const target = try std.fmt.allocPrint(self.allocator, "../{s}/{s}", .{ pkg_name, normalized_path });
316 defer self.allocator.free(target);
317
318 bin_dir.deleteFile(cmd_name) catch {};
319 try createSymlinkOrCopy(bin_dir, target, cmd_name);
320
321 _ = self.stats.bins_linked.fetchAdd(1, .release);
322 }
323
324 const FileWorkItem = struct {
325 source_path: []const u8,
326 dest_path: []const u8,
327 kind: std.fs.Dir.Entry.Kind,
328 link_target: ?[]const u8,
329 };
330
331 fn linkDirectory(self: *Linker, source: std.fs.Dir, dest: std.fs.Dir) !void {
332 try self.linkDirectorySequential(source, dest);
333 }
334
335 pub fn linkDirectoryWithHint(self: *Linker, source: std.fs.Dir, dest: std.fs.Dir, file_count: u32, name: []const u8) !void {
336 if (file_count >= PARALLEL_LINK_THRESHOLD) {
337 debug.log(" parallel link: {s} ({d} files)", .{ name, file_count });
338 try self.linkDirectoryParallel(source, dest);
339 } else try self.linkDirectorySequential(source, dest);
340 }
341
342 fn linkDirectorySequential(self: *Linker, source: std.fs.Dir, dest: std.fs.Dir) !void {
343 var iter = source.iterate();
344 while (try iter.next()) |entry| {
345 switch (entry.kind) {
346 .directory => {
347 dest.makePath(entry.name) catch |err| switch (err) {
348 error.PathAlreadyExists => {},
349 else => return error.IoError,
350 };
351 _ = self.stats.dirs_created.fetchAdd(1, .release);
352
353 var child_source = source.openDir(entry.name, .{ .iterate = true }) catch continue;
354 defer child_source.close();
355
356 var child_dest = dest.openDir(entry.name, .{}) catch continue;
357 defer child_dest.close();
358
359 try self.linkDirectorySequential(child_source, child_dest);
360 },
361 .file => try self.linkFile(source, dest, entry.name),
362 .sym_link => {
363 var link_buf: [std.fs.max_path_bytes]u8 = undefined;
364 const target = source.readLink(entry.name, &link_buf) catch continue;
365 createSymlinkOrCopy(dest, target, entry.name) catch {};
366 },
367 else => {},
368 }
369 }
370 }
371
372 const ParallelThreadContext = struct {
373 linker: *Linker,
374 items: []const FileWorkItem,
375 source_base: std.fs.Dir,
376 dest_base: std.fs.Dir,
377 };
378
379 fn linkDirectoryParallel(self: *Linker, source: std.fs.Dir, dest: std.fs.Dir) !void {
380 var work_items = std.ArrayListUnmanaged(FileWorkItem){};
381 defer {
382 for (work_items.items) |item| {
383 self.allocator.free(item.source_path);
384 self.allocator.free(item.dest_path);
385 if (item.link_target) |t| self.allocator.free(t);
386 }
387 work_items.deinit(self.allocator);
388 }
389
390 try self.collectWorkItems(source, dest, "", &work_items);
391 if (work_items.items.len == 0) return;
392
393 const items_slice = work_items.items;
394 const chunk_size = (items_slice.len + LINK_THREAD_COUNT - 1) / LINK_THREAD_COUNT;
395
396 var threads: [LINK_THREAD_COUNT]?std.Thread = undefined;
397 for (&threads) |*t| t.* = null;
398 var contexts: [LINK_THREAD_COUNT]ParallelThreadContext = undefined;
399
400 var offset: usize = 0;
401 for (0..LINK_THREAD_COUNT) |i| {
402 if (offset >= items_slice.len) break;
403 const end = @min(offset + chunk_size, items_slice.len);
404
405 contexts[i] = .{
406 .linker = self,
407 .items = items_slice[offset..end],
408 .source_base = source,
409 .dest_base = dest,
410 };
411
412 threads[i] = std.Thread.spawn(.{}, processWorkItems, .{&contexts[i]}) catch |err| blk: {
413 debug.log("Thread spawn failed for chunk {d}-{d}: {s}", .{ offset, end, @errorName(err) });
414 processWorkItems(&contexts[i]);
415 break :blk null;
416 }; offset = end;
417 }
418
419 for (&threads) |*t| if (t.*) |thread| thread.join();
420 }
421
422 fn processWorkItems(ctx: *const ParallelThreadContext) void {
423 for (ctx.items) |item| {
424 switch (item.kind) {
425 .file => {
426 const src_dir_path = std.fs.path.dirname(item.source_path) orelse "";
427 const dst_dir_path = std.fs.path.dirname(item.dest_path) orelse "";
428 const filename = std.fs.path.basename(item.source_path);
429
430 var src_dir = if (src_dir_path.len > 0)
431 ctx.source_base.openDir(src_dir_path, .{}) catch continue
432 else
433 ctx.source_base;
434 defer if (src_dir_path.len > 0) src_dir.close();
435
436 var dst_dir = if (dst_dir_path.len > 0)
437 ctx.dest_base.openDir(dst_dir_path, .{}) catch continue
438 else
439 ctx.dest_base;
440 defer if (dst_dir_path.len > 0) dst_dir.close();
441
442 ctx.linker.linkFile(src_dir, dst_dir, filename) catch {};
443 },
444 .sym_link => {
445 if (item.link_target) |target| {
446 const dst_dir_path = std.fs.path.dirname(item.dest_path) orelse "";
447 const filename = std.fs.path.basename(item.dest_path);
448
449 var dst_dir = if (dst_dir_path.len > 0)
450 ctx.dest_base.openDir(dst_dir_path, .{}) catch continue
451 else
452 ctx.dest_base;
453 defer if (dst_dir_path.len > 0) dst_dir.close();
454 createSymlinkOrCopy(dst_dir, target, filename) catch {};
455 }
456 },
457 else => {},
458 }
459 }
460 }
461
462 fn collectWorkItems(self: *Linker, source: std.fs.Dir, dest: std.fs.Dir, prefix: []const u8, work_items: *std.ArrayListUnmanaged(FileWorkItem)) !void {
463 var iter = source.iterate();
464 while (try iter.next()) |entry| {
465 const rel_path = if (prefix.len > 0)
466 try std.fmt.allocPrint(self.allocator, "{s}/{s}", .{ prefix, entry.name })
467 else
468 try self.allocator.dupe(u8, entry.name);
469 errdefer self.allocator.free(rel_path);
470
471 switch (entry.kind) {
472 .directory => {
473 dest.makePath(rel_path) catch |err| switch (err) {
474 error.PathAlreadyExists => {},
475 else => {
476 self.allocator.free(rel_path);
477 return error.IoError;
478 },
479 };
480 _ = self.stats.dirs_created.fetchAdd(1, .release);
481
482 var child_source = source.openDir(entry.name, .{ .iterate = true }) catch {
483 self.allocator.free(rel_path);
484 continue;
485 };
486 defer child_source.close();
487
488 try self.collectWorkItems(child_source, dest, rel_path, work_items);
489 self.allocator.free(rel_path);
490 },
491 .file => {
492 try work_items.append(self.allocator, .{
493 .source_path = rel_path,
494 .dest_path = try self.allocator.dupe(u8, rel_path),
495 .kind = .file,
496 .link_target = null,
497 });
498 },
499 .sym_link => {
500 var link_buf: [std.fs.max_path_bytes]u8 = undefined;
501 const target = source.readLink(entry.name, &link_buf) catch {
502 self.allocator.free(rel_path);
503 continue;
504 };
505 try work_items.append(self.allocator, .{
506 .source_path = rel_path,
507 .dest_path = try self.allocator.dupe(u8, rel_path),
508 .kind = .sym_link,
509 .link_target = try self.allocator.dupe(u8, target),
510 });
511 },
512 else => self.allocator.free(rel_path),
513 }
514 }
515 }
516
517 fn linkFile(self: *Linker, source_dir: std.fs.Dir, dest_dir: std.fs.Dir, name: []const u8) !void {
518 dest_dir.deleteFile(name) catch {};
519
520 if (comptime builtin.os.tag != .windows) {
521 if (!self.cross_device.load(.acquire)) {
522 if (linkAt(source_dir, name, dest_dir, name)) {
523 _ = self.stats.files_linked.fetchAdd(1, .release);
524 return;
525 } else |err| {
526 if (err == error.CrossDevice) {
527 self.cross_device.store(true, .release);
528 } else if (err != error.IoError) return err;
529 }
530 }
531 }
532
533 if (comptime builtin.os.tag == .macos) {
534 if (fclonefileat(source_dir.fd, name, dest_dir.fd, name)) {
535 _ = self.stats.files_cloned.fetchAdd(1, .release);
536 return;
537 } else |_| {}
538 }
539
540 try self.copyFile(source_dir, dest_dir, name);
541 _ = self.stats.files_copied.fetchAdd(1, .release);
542 }
543
544 fn linkAt(source_dir: std.fs.Dir, source_name: []const u8, dest_dir: std.fs.Dir, dest_name: []const u8) !void {
545 if (comptime builtin.os.tag == .windows) return error.IoError;
546
547 var source_buf: [256]u8 = undefined;
548 var dest_buf: [256]u8 = undefined;
549
550 if (source_name.len >= source_buf.len or dest_name.len >= dest_buf.len) {
551 return error.PathTooLong;
552 }
553
554 @memcpy(source_buf[0..source_name.len], source_name);
555 source_buf[source_name.len] = 0;
556
557 @memcpy(dest_buf[0..dest_name.len], dest_name);
558 dest_buf[dest_name.len] = 0;
559
560 const source_z: [*:0]const u8 = source_buf[0..source_name.len :0];
561 const dest_z: [*:0]const u8 = dest_buf[0..dest_name.len :0];
562
563 const result = std.c.linkat(source_dir.fd, source_z, dest_dir.fd, dest_z, 0);
564 if (result != 0) {
565 const errno = std.posix.errno(result);
566 return switch (errno) {
567 .XDEV => error.CrossDevice,
568 .PERM, .ACCES => error.PermissionDenied,
569 else => error.IoError,
570 };
571 }
572 }
573
574 fn fclonefileat(src_fd: std.posix.fd_t, src_name: []const u8, dst_fd: std.posix.fd_t, dst_name: []const u8) !void {
575 var src_buf: [256]u8 = undefined;
576 var dst_buf: [256]u8 = undefined;
577
578 if (src_name.len >= src_buf.len or dst_name.len >= dst_buf.len) {
579 return error.PathTooLong;
580 }
581
582 @memcpy(src_buf[0..src_name.len], src_name);
583 src_buf[src_name.len] = 0;
584
585 @memcpy(dst_buf[0..dst_name.len], dst_name);
586 dst_buf[dst_name.len] = 0;
587
588 const src_z: [*:0]const u8 = src_buf[0..src_name.len :0];
589 const dst_z: [*:0]const u8 = dst_buf[0..dst_name.len :0];
590
591 const fclonefileat_fn = struct {
592 extern "c" fn fclonefileat(c_int, [*:0]const u8, c_int, [*:0]const u8, c_uint) c_int;
593 }.fclonefileat;
594
595 const result = fclonefileat_fn(src_fd, src_z, dst_fd, dst_z, 0);
596 if (result != 0) {
597 return error.IoError;
598 }
599 }
600
601 fn copyFile(self: *Linker, source_dir: std.fs.Dir, dest_dir: std.fs.Dir, name: []const u8) !void {
602 _ = self;
603
604 var source = source_dir.openFile(name, .{}) catch return error.IoError;
605 defer source.close();
606
607 const stat = source.stat() catch return error.IoError;
608 var dest = dest_dir.createFile(name,
609 if (comptime builtin.os.tag != .windows) .{ .mode = stat.mode } else .{}
610 ) catch return error.IoError; defer dest.close();
611
612 var buf: [64 * 1024]u8 = undefined;
613 while (true) {
614 const bytes_read = source.read(&buf) catch return error.IoError;
615 if (bytes_read == 0) break;
616 dest.writeAll(buf[0..bytes_read]) catch return error.IoError;
617 }
618 }
619
620 pub fn getStats(self: *const Linker) StatsSnapshot {
621 return self.stats.snapshot();
622 }
623};