const std = @import("std"); const builtin = @import("builtin"); const json = @import("json.zig"); const debug = @import("debug.zig"); const PARALLEL_LINK_THRESHOLD = 500; const LINK_THREAD_COUNT = 8; pub fn createSymlinkOrCopy(dir: std.fs.Dir, target: []const u8, link_name: []const u8) !void { if (comptime builtin.os.tag == .windows) { try createSymlinkWindows(dir, target, link_name); } else try dir.symLink(target, link_name, .{}); } pub fn createSymlinkAbsolute(target: []const u8, link_path: []const u8) void { if (comptime builtin.os.tag == .windows) { createSymlinkAbsoluteWindows(target, link_path); } else std.posix.symlink(target, link_path) catch {}; } fn createSymlinkWindows(dir: std.fs.Dir, target: []const u8, link_name: []const u8) !void { if (comptime builtin.os.tag != .windows) return; var target_utf16: [std.fs.max_path_bytes]u16 = undefined; var link_utf16: [std.fs.max_path_bytes]u16 = undefined; const target_len = try std.unicode.utf8ToUtf16Le(&target_utf16, target); const link_len = try std.unicode.utf8ToUtf16Le(&link_utf16, link_name); target_utf16[target_len] = 0; _ = try std.os.windows.CreateSymbolicLink( dir.fd, link_utf16[0..link_len], target_utf16[0..target_len :0], false, ); } fn createSymlinkAbsoluteWindows(target: []const u8, link_path: []const u8) void { if (comptime builtin.os.tag != .windows) return; var target_utf16: [std.fs.max_path_bytes]u16 = undefined; var link_utf16: [std.fs.max_path_bytes]u16 = undefined; const target_len = std.unicode.utf8ToUtf16Le(&target_utf16, target) catch return; const link_len = std.unicode.utf8ToUtf16Le(&link_utf16, link_path) catch return; target_utf16[target_len] = 0; _ = std.os.windows.CreateSymbolicLink( null, link_utf16[0..link_len], target_utf16[0..target_len :0], false, ) catch {}; } pub const LinkError = error{ IoError, PathNotFound, CrossDevice, PermissionDenied, OutOfMemory, PathTooLong, }; pub const StatsSnapshot = struct { files_linked: u32, files_copied: u32, files_cloned: u32, bytes_linked: u64, bytes_copied: u64, dirs_created: u32, bins_linked: u32, packages_installed: u32, packages_skipped: u32, }; pub const LinkStats = struct { files_linked: std.atomic.Value(u32), files_copied: std.atomic.Value(u32), files_cloned: std.atomic.Value(u32), bytes_linked: std.atomic.Value(u64), bytes_copied: std.atomic.Value(u64), dirs_created: std.atomic.Value(u32), bins_linked: std.atomic.Value(u32), packages_installed: std.atomic.Value(u32), packages_skipped: std.atomic.Value(u32), pub fn init() LinkStats { return .{ .files_linked = std.atomic.Value(u32).init(0), .files_copied = std.atomic.Value(u32).init(0), .files_cloned = std.atomic.Value(u32).init(0), .bytes_linked = std.atomic.Value(u64).init(0), .bytes_copied = std.atomic.Value(u64).init(0), .dirs_created = std.atomic.Value(u32).init(0), .bins_linked = std.atomic.Value(u32).init(0), .packages_installed = std.atomic.Value(u32).init(0), .packages_skipped = std.atomic.Value(u32).init(0), }; } pub fn snapshot(self: *const LinkStats) StatsSnapshot { return .{ .files_linked = self.files_linked.load(.acquire), .files_copied = self.files_copied.load(.acquire), .files_cloned = self.files_cloned.load(.acquire), .bytes_linked = self.bytes_linked.load(.acquire), .bytes_copied = self.bytes_copied.load(.acquire), .dirs_created = self.dirs_created.load(.acquire), .bins_linked = self.bins_linked.load(.acquire), .packages_installed = self.packages_installed.load(.acquire), .packages_skipped = self.packages_skipped.load(.acquire), }; } }; pub const PackageLink = struct { cache_path: []const u8, node_modules_path: []const u8, name: []const u8, parent_path: ?[]const u8 = null, file_count: u32 = 0, has_bin: bool = true, }; pub const Linker = struct { allocator: std.mem.Allocator, stats: LinkStats, node_modules_dir: ?std.fs.Dir, bin_dir: ?std.fs.Dir, node_modules_path: []const u8, cross_device: std.atomic.Value(bool), pub fn init(allocator: std.mem.Allocator) Linker { return .{ .allocator = allocator, .stats = LinkStats.init(), .node_modules_dir = null, .bin_dir = null, .node_modules_path = "", .cross_device = std.atomic.Value(bool).init(false), }; } pub fn deinit(self: *Linker) void { if (self.node_modules_dir) |*d| d.close(); if (self.bin_dir) |*d| d.close(); if (self.node_modules_path.len > 0) self.allocator.free(self.node_modules_path); } pub fn setNodeModulesPath(self: *Linker, path: []const u8) !void { std.fs.cwd().makePath(path) catch |err| switch (err) { error.PathAlreadyExists => {}, else => return error.IoError, }; var new_nm_dir = try std.fs.cwd().openDir(path, .{}); errdefer new_nm_dir.close(); const new_path = try self.allocator.dupe(u8, path); errdefer self.allocator.free(new_path); const bin_path = try std.fmt.allocPrint(self.allocator, "{s}/.bin", .{path}); defer self.allocator.free(bin_path); std.fs.cwd().makePath(bin_path) catch |err| switch (err) { error.PathAlreadyExists => {}, else => return error.IoError, }; const new_bin_dir = try std.fs.cwd().openDir(bin_path, .{}); if (self.bin_dir) |*d| d.close(); if (self.node_modules_dir) |*d| d.close(); if (self.node_modules_path.len > 0) self.allocator.free(self.node_modules_path); self.node_modules_dir = new_nm_dir; self.node_modules_path = new_path; self.bin_dir = new_bin_dir; } pub fn linkPackage(self: *Linker, pkg: PackageLink) !void { const node_modules = self.node_modules_dir orelse return error.IoError; const install_path = if (pkg.parent_path) |parent| try std.fmt.allocPrint(self.allocator, "{s}/node_modules/{s}", .{ parent, pkg.name }) else try self.allocator.dupe(u8, pkg.name); defer self.allocator.free(install_path); var source_dir = std.fs.cwd().openDir(pkg.cache_path, .{ .iterate = true }) catch { return error.PathNotFound; }; defer source_dir.close(); const source_version = readPackageVersion(self.allocator, source_dir); defer if (source_version) |v| self.allocator.free(v); var should_skip = false; var has_existing_install = false; var has_existing_package = false; { const existing = node_modules.openDir(install_path, .{ .iterate = true }) catch null; if (existing) |dir| { has_existing_install = true; var installed_dir = dir; defer installed_dir.close(); const installed_version = readPackageVersion(self.allocator, installed_dir); defer if (installed_version) |v| self.allocator.free(v); has_existing_package = installed_version != null; should_skip = packageVersionsMatch(source_version, installed_version) and installedFileCountMatches(installed_dir, pkg.file_count); } } if (should_skip) { _ = self.stats.packages_skipped.fetchAdd(1, .release); return; } if (has_existing_install and has_existing_package) { node_modules.deleteTree(install_path) catch return error.IoError; } node_modules.makePath(install_path) catch |err| switch (err) { error.PathAlreadyExists => {}, else => return error.IoError, }; _ = self.stats.dirs_created.fetchAdd(1, .release); var dest_dir = node_modules.openDir(install_path, .{}) catch return error.IoError; defer dest_dir.close(); self.linkDirectoryWithHint(source_dir, dest_dir, pkg.file_count, pkg.name) catch |err| return err; if (pkg.parent_path == null and pkg.has_bin) try self.linkBinaries(pkg.name); _ = self.stats.packages_installed.fetchAdd(1, .release); } fn linkBinaries(self: *Linker, pkg_name: []const u8) !void { const bin_dir = self.bin_dir orelse return; const node_modules = self.node_modules_dir orelse return; var pkg_dir = node_modules.openDir(pkg_name, .{}) catch return; defer pkg_dir.close(); const pkg_json_file = pkg_dir.openFile("package.json", .{}) catch return; defer pkg_json_file.close(); const content = pkg_json_file.readToEndAlloc(self.allocator, 1024 * 1024) catch return; defer self.allocator.free(content); var doc = json.JsonDoc.parse(content) catch return; defer doc.deinit(); const root_val = doc.root(); if (root_val.getObject("bin")) |bin_obj| { var iter = bin_obj.objectIterator() orelse return; while (iter.next()) |entry| { const bin_path = entry.value.asString() orelse continue; self.createBinSymlink(pkg_name, entry.key, bin_path, bin_dir) catch continue; } } else if (root_val.getString("bin")) |bin_path| { const simple_name = if (std.mem.indexOf(u8, pkg_name, "/")) |slash| pkg_name[slash + 1 ..] else pkg_name; self.createBinSymlink(pkg_name, simple_name, bin_path, bin_dir) catch {}; } } fn readPackageVersion(allocator: std.mem.Allocator, dir: std.fs.Dir) ?[]const u8 { const pkg_json = dir.openFile("package.json", .{}) catch return null; defer pkg_json.close(); const content = pkg_json.readToEndAlloc(allocator, 256 * 1024) catch return null; defer allocator.free(content); var doc = json.JsonDoc.parse(content) catch return null; defer doc.deinit(); const version = doc.root().getString("version") orelse return null; return allocator.dupe(u8, version) catch null; } fn packageVersionsMatch(source_version: ?[]const u8, installed_version: ?[]const u8) bool { const src = source_version orelse return false; const dst = installed_version orelse return false; return std.mem.eql(u8, src, dst); } fn installedFileCountMatches(dir: std.fs.Dir, expected_files: u32) bool { if (expected_files == 0) return true; const actual_files = countFilesRecursive(dir) catch return false; return actual_files >= expected_files; } fn countFilesRecursive(dir: std.fs.Dir) !u32 { var count: u32 = 0; var iter = dir.iterate(); while (try iter.next()) |entry| { switch (entry.kind) { .file => count += 1, .directory => { var child = dir.openDir(entry.name, .{ .iterate = true }) catch continue; defer child.close(); count += try countFilesRecursive(child); }, else => {}, }} return count; } fn createBinSymlink(self: *Linker, pkg_name: []const u8, cmd_name: []const u8, bin_path: []const u8, bin_dir: std.fs.Dir) !void { var normalized_path = bin_path; if (std.mem.startsWith(u8, normalized_path, "./")) { normalized_path = normalized_path[2..]; } const target = try std.fmt.allocPrint(self.allocator, "../{s}/{s}", .{ pkg_name, normalized_path }); defer self.allocator.free(target); bin_dir.deleteFile(cmd_name) catch {}; try createSymlinkOrCopy(bin_dir, target, cmd_name); _ = self.stats.bins_linked.fetchAdd(1, .release); } const FileWorkItem = struct { source_path: []const u8, dest_path: []const u8, kind: std.fs.Dir.Entry.Kind, link_target: ?[]const u8, }; fn linkDirectory(self: *Linker, source: std.fs.Dir, dest: std.fs.Dir) !void { try self.linkDirectorySequential(source, dest); } pub fn linkDirectoryWithHint(self: *Linker, source: std.fs.Dir, dest: std.fs.Dir, file_count: u32, name: []const u8) !void { if (file_count >= PARALLEL_LINK_THRESHOLD) { debug.log(" parallel link: {s} ({d} files)", .{ name, file_count }); try self.linkDirectoryParallel(source, dest); } else try self.linkDirectorySequential(source, dest); } fn linkDirectorySequential(self: *Linker, source: std.fs.Dir, dest: std.fs.Dir) !void { var iter = source.iterate(); while (try iter.next()) |entry| { switch (entry.kind) { .directory => { dest.makePath(entry.name) catch |err| switch (err) { error.PathAlreadyExists => {}, else => return error.IoError, }; _ = self.stats.dirs_created.fetchAdd(1, .release); var child_source = source.openDir(entry.name, .{ .iterate = true }) catch continue; defer child_source.close(); var child_dest = dest.openDir(entry.name, .{}) catch continue; defer child_dest.close(); try self.linkDirectorySequential(child_source, child_dest); }, .file => try self.linkFile(source, dest, entry.name), .sym_link => { var link_buf: [std.fs.max_path_bytes]u8 = undefined; const target = source.readLink(entry.name, &link_buf) catch continue; createSymlinkOrCopy(dest, target, entry.name) catch {}; }, else => {}, } } } const ParallelThreadContext = struct { linker: *Linker, items: []const FileWorkItem, source_base: std.fs.Dir, dest_base: std.fs.Dir, }; fn linkDirectoryParallel(self: *Linker, source: std.fs.Dir, dest: std.fs.Dir) !void { var work_items = std.ArrayListUnmanaged(FileWorkItem){}; defer { for (work_items.items) |item| { self.allocator.free(item.source_path); self.allocator.free(item.dest_path); if (item.link_target) |t| self.allocator.free(t); } work_items.deinit(self.allocator); } try self.collectWorkItems(source, dest, "", &work_items); if (work_items.items.len == 0) return; const items_slice = work_items.items; const chunk_size = (items_slice.len + LINK_THREAD_COUNT - 1) / LINK_THREAD_COUNT; var threads: [LINK_THREAD_COUNT]?std.Thread = undefined; for (&threads) |*t| t.* = null; var contexts: [LINK_THREAD_COUNT]ParallelThreadContext = undefined; var offset: usize = 0; for (0..LINK_THREAD_COUNT) |i| { if (offset >= items_slice.len) break; const end = @min(offset + chunk_size, items_slice.len); contexts[i] = .{ .linker = self, .items = items_slice[offset..end], .source_base = source, .dest_base = dest, }; threads[i] = std.Thread.spawn(.{}, processWorkItems, .{&contexts[i]}) catch |err| blk: { debug.log("Thread spawn failed for chunk {d}-{d}: {s}", .{ offset, end, @errorName(err) }); processWorkItems(&contexts[i]); break :blk null; }; offset = end; } for (&threads) |*t| if (t.*) |thread| thread.join(); } fn processWorkItems(ctx: *const ParallelThreadContext) void { for (ctx.items) |item| { switch (item.kind) { .file => { const src_dir_path = std.fs.path.dirname(item.source_path) orelse ""; const dst_dir_path = std.fs.path.dirname(item.dest_path) orelse ""; const filename = std.fs.path.basename(item.source_path); var src_dir = if (src_dir_path.len > 0) ctx.source_base.openDir(src_dir_path, .{}) catch continue else ctx.source_base; defer if (src_dir_path.len > 0) src_dir.close(); var dst_dir = if (dst_dir_path.len > 0) ctx.dest_base.openDir(dst_dir_path, .{}) catch continue else ctx.dest_base; defer if (dst_dir_path.len > 0) dst_dir.close(); ctx.linker.linkFile(src_dir, dst_dir, filename) catch {}; }, .sym_link => { if (item.link_target) |target| { const dst_dir_path = std.fs.path.dirname(item.dest_path) orelse ""; const filename = std.fs.path.basename(item.dest_path); var dst_dir = if (dst_dir_path.len > 0) ctx.dest_base.openDir(dst_dir_path, .{}) catch continue else ctx.dest_base; defer if (dst_dir_path.len > 0) dst_dir.close(); createSymlinkOrCopy(dst_dir, target, filename) catch {}; } }, else => {}, } } } fn collectWorkItems(self: *Linker, source: std.fs.Dir, dest: std.fs.Dir, prefix: []const u8, work_items: *std.ArrayListUnmanaged(FileWorkItem)) !void { var iter = source.iterate(); while (try iter.next()) |entry| { const rel_path = if (prefix.len > 0) try std.fmt.allocPrint(self.allocator, "{s}/{s}", .{ prefix, entry.name }) else try self.allocator.dupe(u8, entry.name); errdefer self.allocator.free(rel_path); switch (entry.kind) { .directory => { dest.makePath(rel_path) catch |err| switch (err) { error.PathAlreadyExists => {}, else => { self.allocator.free(rel_path); return error.IoError; }, }; _ = self.stats.dirs_created.fetchAdd(1, .release); var child_source = source.openDir(entry.name, .{ .iterate = true }) catch { self.allocator.free(rel_path); continue; }; defer child_source.close(); try self.collectWorkItems(child_source, dest, rel_path, work_items); self.allocator.free(rel_path); }, .file => { try work_items.append(self.allocator, .{ .source_path = rel_path, .dest_path = try self.allocator.dupe(u8, rel_path), .kind = .file, .link_target = null, }); }, .sym_link => { var link_buf: [std.fs.max_path_bytes]u8 = undefined; const target = source.readLink(entry.name, &link_buf) catch { self.allocator.free(rel_path); continue; }; try work_items.append(self.allocator, .{ .source_path = rel_path, .dest_path = try self.allocator.dupe(u8, rel_path), .kind = .sym_link, .link_target = try self.allocator.dupe(u8, target), }); }, else => self.allocator.free(rel_path), } } } fn linkFile(self: *Linker, source_dir: std.fs.Dir, dest_dir: std.fs.Dir, name: []const u8) !void { dest_dir.deleteFile(name) catch {}; if (comptime builtin.os.tag != .windows) { if (!self.cross_device.load(.acquire)) { if (linkAt(source_dir, name, dest_dir, name)) { _ = self.stats.files_linked.fetchAdd(1, .release); return; } else |err| { if (err == error.CrossDevice) { self.cross_device.store(true, .release); } else if (err != error.IoError) return err; } } } if (comptime builtin.os.tag == .macos) { if (fclonefileat(source_dir.fd, name, dest_dir.fd, name)) { _ = self.stats.files_cloned.fetchAdd(1, .release); return; } else |_| {} } try self.copyFile(source_dir, dest_dir, name); _ = self.stats.files_copied.fetchAdd(1, .release); } fn linkAt(source_dir: std.fs.Dir, source_name: []const u8, dest_dir: std.fs.Dir, dest_name: []const u8) !void { if (comptime builtin.os.tag == .windows) return error.IoError; var source_buf: [256]u8 = undefined; var dest_buf: [256]u8 = undefined; if (source_name.len >= source_buf.len or dest_name.len >= dest_buf.len) { return error.PathTooLong; } @memcpy(source_buf[0..source_name.len], source_name); source_buf[source_name.len] = 0; @memcpy(dest_buf[0..dest_name.len], dest_name); dest_buf[dest_name.len] = 0; const source_z: [*:0]const u8 = source_buf[0..source_name.len :0]; const dest_z: [*:0]const u8 = dest_buf[0..dest_name.len :0]; const result = std.c.linkat(source_dir.fd, source_z, dest_dir.fd, dest_z, 0); if (result != 0) { const errno = std.posix.errno(result); return switch (errno) { .XDEV => error.CrossDevice, .PERM, .ACCES => error.PermissionDenied, else => error.IoError, }; } } fn fclonefileat(src_fd: std.posix.fd_t, src_name: []const u8, dst_fd: std.posix.fd_t, dst_name: []const u8) !void { var src_buf: [256]u8 = undefined; var dst_buf: [256]u8 = undefined; if (src_name.len >= src_buf.len or dst_name.len >= dst_buf.len) { return error.PathTooLong; } @memcpy(src_buf[0..src_name.len], src_name); src_buf[src_name.len] = 0; @memcpy(dst_buf[0..dst_name.len], dst_name); dst_buf[dst_name.len] = 0; const src_z: [*:0]const u8 = src_buf[0..src_name.len :0]; const dst_z: [*:0]const u8 = dst_buf[0..dst_name.len :0]; const fclonefileat_fn = struct { extern "c" fn fclonefileat(c_int, [*:0]const u8, c_int, [*:0]const u8, c_uint) c_int; }.fclonefileat; const result = fclonefileat_fn(src_fd, src_z, dst_fd, dst_z, 0); if (result != 0) { return error.IoError; } } fn copyFile(self: *Linker, source_dir: std.fs.Dir, dest_dir: std.fs.Dir, name: []const u8) !void { _ = self; var source = source_dir.openFile(name, .{}) catch return error.IoError; defer source.close(); const stat = source.stat() catch return error.IoError; var dest = dest_dir.createFile(name, if (comptime builtin.os.tag != .windows) .{ .mode = stat.mode } else .{} ) catch return error.IoError; defer dest.close(); var buf: [64 * 1024]u8 = undefined; while (true) { const bytes_read = source.read(&buf) catch return error.IoError; if (bytes_read == 0) break; dest.writeAll(buf[0..bytes_read]) catch return error.IoError; } } pub fn getStats(self: *const Linker) StatsSnapshot { return self.stats.snapshot(); } };