From 558d97d08484d3d1279ac55ac7f1e3d845b5096c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=89=E5=92=B2=E9=9B=85=20=C2=B7=20Misaki=20Masa?= Date: Tue, 8 Oct 2024 19:33:38 +0800 Subject: [PATCH] perf: merge multiple file operations into one to greatly speed up updates in large directories (#1745) --- Cargo.lock | 110 +++++++++++++------------------ Cargo.toml | 4 +- yazi-boot/Cargo.toml | 2 +- yazi-cli/Cargo.toml | 2 +- yazi-config/Cargo.toml | 2 +- yazi-core/src/manager/watcher.rs | 11 ++-- yazi-plugin/src/isolate/fetch.rs | 3 + yazi-shared/src/fs/op.rs | 22 +++++++ 8 files changed, 80 insertions(+), 76 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7176ad5d..63ac44ba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "addr2line" -version = "0.24.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5fb1d8e4442bd405fdfd1dacb42792696b0cf9cb15882e5d097b742a676d375" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" dependencies = [ "gimli", ] @@ -23,18 +23,6 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" -[[package]] -name = "ahash" -version = "0.8.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" -dependencies = [ - "cfg-if", - "once_cell", - "version_check", - "zerocopy", -] - [[package]] name = "aho-corasick" version = "1.1.3" @@ -351,9 +339,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.24" +version = "1.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "812acba72f0a070b003d3697490d2b55b837230ae7c6c6497f05cc2ddbb8d938" +checksum = "2e80e3b6a3ab07840e1cae9b0666a63970dc28e8ed5ffbcdacbfc760c281bfc1" dependencies = [ "jobserver", "libc", @@ -850,6 +838,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f81ec6369c545a7d40e4589b5597581fa1c441fe1cce96dd1de43159910a36a2" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -870,9 +864,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" dependencies = [ "futures-channel", "futures-core", @@ -885,9 +879,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", "futures-sink", @@ -895,15 +889,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-executor" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" dependencies = [ "futures-core", "futures-task", @@ -912,15 +906,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-macro" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", @@ -929,21 +923,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" [[package]] name = "futures-task" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" [[package]] name = "futures-util" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ "futures-channel", "futures-core", @@ -990,9 +984,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.31.0" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32085ea23f3234fc7846555e85283ba4de91e21016dc0455a16286d87a292d64" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "globset" @@ -1017,21 +1011,16 @@ dependencies = [ "crunchy", ] -[[package]] -name = "hashbrown" -version = "0.14.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" -dependencies = [ - "ahash", - "allocator-api2", -] - [[package]] name = "hashbrown" version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] [[package]] name = "heck" @@ -1144,7 +1133,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "equivalent", - "hashbrown 0.15.0", + "hashbrown", "serde", ] @@ -1348,11 +1337,11 @@ dependencies = [ [[package]] name = "lru" -version = "0.12.4" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37ee39891760e7d94734f6f63fedc29a2e4a152f836120753a72503f09fcf904" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" dependencies = [ - "hashbrown 0.14.5", + "hashbrown", ] [[package]] @@ -1662,21 +1651,18 @@ dependencies = [ [[package]] name = "object" -version = "0.36.4" +version = "0.36.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" +checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" dependencies = [ "memchr", ] [[package]] name = "once_cell" -version = "1.20.1" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82881c4be219ab5faaf2ad5e5e5ecdff8c66bd7402ca3160975c93b24961afd1" -dependencies = [ - "portable-atomic", -] +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "onig" @@ -1806,12 +1792,6 @@ dependencies = [ "miniz_oxide 0.8.0", ] -[[package]] -name = "portable-atomic" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2" - [[package]] name = "powerfmt" version = "0.2.0" @@ -1853,9 +1833,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "b3e4daa0dcf6feba26f985457cdf104d4b4256fc5a09547140f3631bb076b19a" dependencies = [ "unicode-ident", ] diff --git a/Cargo.toml b/Cargo.toml index f5047133..d5d6f0d1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,10 +15,10 @@ anyhow = "1.0.89" arc-swap = "1.7.1" base64 = "0.22.1" bitflags = "2.6.0" -clap = { version = "4.5.18", features = [ "derive" ] } +clap = { version = "4.5.19", features = [ "derive" ] } crossterm = { version = "0.28.1", features = [ "event-stream" ] } dirs = "5.0.1" -futures = "0.3.30" +futures = "0.3.31" globset = "0.4.15" libc = "0.2.159" md-5 = "0.10.6" diff --git a/yazi-boot/Cargo.toml b/yazi-boot/Cargo.toml index e9f9eade..1272a131 100644 --- a/yazi-boot/Cargo.toml +++ b/yazi-boot/Cargo.toml @@ -20,7 +20,7 @@ serde = { workspace = true } [build-dependencies] clap = { workspace = true } -clap_complete = "4.5.29" +clap_complete = "4.5.32" clap_complete_fig = "4.5.2" clap_complete_nushell = "4.5.3" vergen-gitcl = { version = "1.0.1", features = [ "build" ] } diff --git a/yazi-cli/Cargo.toml b/yazi-cli/Cargo.toml index cc068dd9..ba8f2abe 100644 --- a/yazi-cli/Cargo.toml +++ b/yazi-cli/Cargo.toml @@ -26,7 +26,7 @@ toml_edit = "0.22.22" # External build dependencies anyhow = { workspace = true } clap = { workspace = true } -clap_complete = "4.5.29" +clap_complete = "4.5.32" clap_complete_fig = "4.5.2" clap_complete_nushell = "4.5.3" serde_json = { workspace = true } diff --git a/yazi-config/Cargo.toml b/yazi-config/Cargo.toml index 36a6b801..f2f8eb75 100644 --- a/yazi-config/Cargo.toml +++ b/yazi-config/Cargo.toml @@ -17,7 +17,7 @@ arc-swap = { workspace = true } bitflags = { workspace = true } crossterm = { workspace = true } globset = { workspace = true } -indexmap = { version = "2.5.0", features = [ "serde" ] } +indexmap = { version = "2.6.0", features = [ "serde" ] } ratatui = { workspace = true } regex = { workspace = true } serde = { workspace = true } diff --git a/yazi-core/src/manager/watcher.rs b/yazi-core/src/manager/watcher.rs index 26e5f26c..dc094338 100644 --- a/yazi-core/src/manager/watcher.rs +++ b/yazi-core/src/manager/watcher.rs @@ -122,12 +122,13 @@ impl Watcher { let mut cached: HashMap<_, _> = HashMap::new(); let _permit = WATCHER.acquire().await.unwrap(); + let mut ops = Vec::with_capacity(urls.len()); let mut reload = Vec::with_capacity(urls.len()); for u in urls { let Some((parent, urn)) = u.pair() else { continue }; let Ok(file) = File::from(u).await else { - FilesOp::Deleting(parent, HashSet::from_iter([urn])).emit(); + ops.push(FilesOp::Deleting(parent, HashSet::from_iter([urn]))); continue; }; @@ -136,19 +137,17 @@ impl Watcher { || realname_unchecked(u, &mut cached).await.is_ok_and(|s| urn.as_urn() == s); if !eq { - FilesOp::Deleting(parent, HashSet::from_iter([urn])).emit(); + ops.push(FilesOp::Deleting(parent, HashSet::from_iter([urn]))); continue; } if !file.is_dir() { reload.push(file.clone()); } - FilesOp::Upserting(parent, HashMap::from_iter([(urn, file)])).emit(); + ops.push(FilesOp::Upserting(parent, HashMap::from_iter([(urn, file)]))); } - if reload.is_empty() { - continue; - } + FilesOp::mutate(ops); if let Err(e) = isolate::fetch("mime", reload).await { error!("Fetch `mime` failed in watcher: {e}"); } diff --git a/yazi-plugin/src/isolate/fetch.rs b/yazi-plugin/src/isolate/fetch.rs index c0e6b935..1e92b36e 100644 --- a/yazi-plugin/src/isolate/fetch.rs +++ b/yazi-plugin/src/isolate/fetch.rs @@ -6,6 +6,9 @@ use super::slim_lua; use crate::{bindings::Cast, elements::Rect, file::File, loader::LOADER}; pub async fn fetch(name: &str, files: Vec) -> mlua::Result { + if files.is_empty() { + return Ok(1); + } LOADER.ensure(name).await.into_lua_err()?; let name = name.to_owned(); diff --git a/yazi-shared/src/fs/op.rs b/yazi-shared/src/fs/op.rs index 13b1543c..7883058c 100644 --- a/yazi-shared/src/fs/op.rs +++ b/yazi-shared/src/fs/op.rs @@ -70,6 +70,28 @@ impl FilesOp { } } + pub fn mutate(ops: Vec) { + let mut parents: HashMap<_, (HashMap<_, _>, HashSet<_>)> = Default::default(); + for op in ops { + match op { + Self::Upserting(p, map) => parents.entry(p).or_default().0.extend(map), + Self::Deleting(p, urns) => parents.entry(p).or_default().1.extend(urns), + _ => unreachable!(), + } + } + for (p, (u, d)) in parents { + match (u.is_empty(), d.is_empty()) { + (true, true) => unreachable!(), + (true, false) => Self::Deleting(p, d).emit(), + (false, true) => Self::Upserting(p, u).emit(), + (false, false) => { + Self::Deleting(p.clone(), d).emit(); + Self::Upserting(p, u).emit(); + } + } + } + } + pub fn rebase(&self, new: &Url) -> Self { macro_rules! files { ($files:expr) => {{ $files.iter().map(|f| f.rebase(new)).collect() }};