From 9d9d954870ca3a87f9a4c3e4fb93ce50620641d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=89=E5=92=B2=E9=9B=85=20=C2=B7=20Misaki=20Masa?= Date: Wed, 31 Jan 2024 01:22:55 +0800 Subject: [PATCH] perf: read directory in bulk in the background at startup (#599) --- Cargo.lock | 75 ++++++++++++-------------- README.md | 2 +- yazi-core/src/folder/files.rs | 30 ++++++++++- yazi-core/src/manager/commands/open.rs | 16 +++--- yazi-core/src/manager/watcher.rs | 38 ++++++------- 5 files changed, 93 insertions(+), 68 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3186b330..195393ec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -230,9 +230,9 @@ checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" [[package]] name = "bytemuck" -version = "1.14.0" +version = "1.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "374d28ec25809ee0e23827c2ab573d729e293f281dfe393500e7ad618baa61c6" +checksum = "ed2490600f404f2b94c167e31d3ed1d5f3c225a0f3b80230053b3e0b7b962bd9" [[package]] name = "byteorder" @@ -269,14 +269,14 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.31" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +checksum = "9f13690e35a5e4ace198e7beea2895d29f3a9cc55015fcebe6336bd2010af9eb" dependencies = [ "android-tzdata", "iana-time-zone", "num-traits", - "windows-targets 0.48.5", + "windows-targets 0.52.0", ] [[package]] @@ -303,9 +303,9 @@ dependencies = [ [[package]] name = "clap_complete" -version = "4.4.8" +version = "4.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaf7dcb7c21d8ca1a2482ee0f1d341f437c9a7af6ca6da359dc5e1b164e98215" +checksum = "df631ae429f6613fcd3a7c1adbdb65f637271e561b03680adaa6573015dfb106" dependencies = [ "clap", ] @@ -918,9 +918,9 @@ checksum = "029d73f573d8e8d63e6d5020011d3255b28c3ba85d6cf870a07184ed23de9284" [[package]] name = "indexmap" -version = "2.1.0" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" +checksum = "433de089bd45971eecf4668ee0ee8f4cec17db4f8bd8f7bc3197a6ce37aa7d9b" dependencies = [ "equivalent", "hashbrown", @@ -952,15 +952,6 @@ dependencies = [ "libc", ] -[[package]] -name = "itertools" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.12.0" @@ -1085,9 +1076,9 @@ checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] name = "lru" -version = "0.12.1" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2994eeba8ed550fd9b47a0b38f0242bc3344e496483c6180b69139cc2fa5d1d7" +checksum = "db2c024b41519440580066ba82aab04092b333e09066a5eb86c7c4890df31f22" dependencies = [ "hashbrown", ] @@ -1103,9 +1094,9 @@ dependencies = [ [[package]] name = "luajit-src" -version = "210.5.4+c525bcb" +version = "210.5.5+f2336c4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a10ab4ed12d22cb50ef43ece4f6c5ca594b2d2480019e87facfd422225a9908" +checksum = "d8bcba9790f4e3b1c1467d75cdd011a63bbe6bc75da95af5d2cb4e3631f939c4" dependencies = [ "cc", "which", @@ -1166,9 +1157,9 @@ dependencies = [ [[package]] name = "mlua" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "069264935e816c85884b99e88c8b408d6d92e40ae8760f726c983526a53546b5" +checksum = "1d3561f79659ff3afad7b25e2bf2ec21507fe601ebecb7f81088669ec4bfd51e" dependencies = [ "bstr", "erased-serde", @@ -1184,9 +1175,9 @@ dependencies = [ [[package]] name = "mlua-sys" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4655631a02e3739d014951291ecfa08db49c4da3f7f8c6f3931ed236af5dd78e" +checksum = "2847b42764435201d8cbee1f517edb79c4cca4181877b90047587c89e1b7bce4" dependencies = [ "cc", "cfg-if", @@ -1197,11 +1188,11 @@ dependencies = [ [[package]] name = "mlua_derive" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15f677457e995582451fc52a5c5aa02c13d2ea1e0f44afcf87dd4dc2961f3b1c" +checksum = "aaade5f94e5829db58791664ba98f35fea6a3ffebc783becb51dc97c7a21abee" dependencies = [ - "itertools 0.11.0", + "itertools", "once_cell", "proc-macro-error", "proc-macro2", @@ -1511,7 +1502,7 @@ dependencies = [ "cassowary", "crossterm", "indoc", - "itertools 0.12.0", + "itertools", "lru", "paste", "stability", @@ -1574,9 +1565,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b7fa1134405e2ec9353fd416b17f8dacd46c473d7d3fd1cf202706a14eb792a" +checksum = "5bb987efffd3c6d0d8f5f89510bb458559eab11e4f869acb20bf845e016259cd" dependencies = [ "aho-corasick", "memchr", @@ -1655,9 +1646,9 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "serde" -version = "1.0.195" +version = "1.0.196" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02" +checksum = "870026e60fa08c69f064aa766c10f10b1d62db9ccd4d0abb206472bee0ce3b32" dependencies = [ "serde_derive", ] @@ -1674,9 +1665,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.195" +version = "1.0.196" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c" +checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67" dependencies = [ "proc-macro2", "quote", @@ -1685,9 +1676,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.111" +version = "1.0.113" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "176e46fa42316f18edd598015a5166857fc835ec732f5215eac6b7bdbf0a84f4" +checksum = "69801b70b1c3dac963ecb03a364ba0ceda9cf60c71cfe475e99864759c8b8a79" dependencies = [ "itoa", "ryu", @@ -2351,9 +2342,9 @@ checksum = "4d91413b1c31d7539ba5ef2451af3f0b833a005eb27a631cec32bc0635a8602b" [[package]] name = "weezl" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9193164d4de03a926d909d3bc7c30543cecb35400c02114792c2cae20d5e2dbb" +checksum = "53a85b86a771b1c87058196170769dd264f66c0782acf1ae6cc51bfd64b39082" [[package]] name = "which" @@ -2608,9 +2599,9 @@ checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" [[package]] name = "winnow" -version = "0.5.34" +version = "0.5.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7cf47b659b318dccbd69cc4797a39ae128f533dce7902a1096044d1967b9c16" +checksum = "1931d78a9c73861da0134f453bb1f790ce49b2e30eba8410b4b79bac72b46a2d" dependencies = [ "memchr", ] diff --git a/README.md b/README.md index 8067cf56..e327360c 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Yazi (means "duck") is a terminal file manager written in Rust, based on non-blo - 🧰 Integration with fd, rg, fzf, zoxide - 💫 Vim-like input/select component, auto-completion for cd paths - 🏷️ Multi-Tab Support, Scrollable Preview (for videos, PDFs, archives, directories, code, etc.) -- 🔄 Batch Renaming, Visual Mode, File Chooser +- 🔄 Bulk Renaming, Visual Mode, File Chooser - 🎨 Theme System, Custom Layouts, Trash Bin, CSI u - ... and more! diff --git a/yazi-core/src/folder/files.rs b/yazi-core/src/folder/files.rs index df31b365..88088bd6 100644 --- a/yazi-core/src/folder/files.rs +++ b/yazi-core/src/folder/files.rs @@ -1,7 +1,7 @@ use std::{collections::{BTreeMap, BTreeSet}, mem, ops::Deref, sync::atomic::Ordering}; use anyhow::Result; -use tokio::{fs, select, sync::mpsc::{self, UnboundedReceiver}}; +use tokio::{fs::{self, DirEntry}, select, sync::mpsc::{self, UnboundedReceiver}}; use yazi_config::{manager::SortBy, MANAGER}; use yazi_shared::fs::{File, Url, FILES_TICKET}; @@ -66,6 +66,34 @@ impl Files { }); Ok(rx) } + + pub async fn from_dir_bulk(url: &Url) -> Result> { + let mut it = fs::read_dir(url).await?; + let mut items = Vec::with_capacity(5000); + while let Ok(Some(item)) = it.next_entry().await { + items.push(item); + } + + let (first, rest) = items.split_at(items.len() / 3); + let (second, third) = rest.split_at(items.len() / 3); + async fn go(entities: &[DirEntry]) -> Vec { + let mut files = Vec::with_capacity(entities.len() / 3 + 1); + for entry in entities { + if let Ok(meta) = entry.metadata().await { + files.push(File::from_meta(Url::from(entry.path()), meta).await); + } + } + files + } + + Ok( + futures::future::join_all([go(first), go(second), go(third)]) + .await + .into_iter() + .flatten() + .collect(), + ) + } } impl Files { diff --git a/yazi-core/src/manager/commands/open.rs b/yazi-core/src/manager/commands/open.rs index e682aa10..ff467029 100644 --- a/yazi-core/src/manager/commands/open.rs +++ b/yazi-core/src/manager/commands/open.rs @@ -8,13 +8,16 @@ use yazi_shared::{emit, event::{EventQuit, Exec}, fs::{File, Url}, Layer, MIME_D use crate::{manager::Manager, select::Select, tasks::Tasks}; pub struct Opt { - targets: Option)>>, + targets: Vec<(Url, Option)>, interactive: bool, } impl From for Opt { fn from(mut e: Exec) -> Self { - Self { targets: e.take_data(), interactive: e.named.contains_key("interactive") } + Self { + targets: e.take_data().unwrap_or_default(), + interactive: e.named.contains_key("interactive"), + } } } @@ -40,7 +43,7 @@ impl Manager { let mut opt = opt.into() as Opt; if todo.is_empty() { - opt.targets = Some(done); + opt.targets = done; return self.open_do(opt, tasks); } @@ -64,11 +67,12 @@ impl Manager { pub fn open_do(&mut self, opt: impl Into, tasks: &Tasks) { let opt = opt.into() as Opt; - let Some(targets) = opt.targets else { + if opt.targets.is_empty() { return; - }; + } - let targets: Vec<_> = targets + let targets: Vec<_> = opt + .targets .into_iter() .filter_map(|(u, m)| m.or_else(|| self.mimetype.get(&u).cloned()).map(|m| (u, m))) .collect(); diff --git a/yazi-core/src/manager/watcher.rs b/yazi-core/src/manager/watcher.rs index a96d52c2..1e25317a 100644 --- a/yazi-core/src/manager/watcher.rs +++ b/yazi-core/src/manager/watcher.rs @@ -1,5 +1,6 @@ -use std::{collections::{BTreeMap, BTreeSet}, sync::Arc, time::Duration}; +use std::{collections::{BTreeMap, BTreeSet}, sync::Arc, time::{Duration, SystemTime}}; +use anyhow::Result; use notify::{event::{MetadataKind, ModifyKind}, EventKind, RecommendedWatcher, RecursiveMode, Watcher as _Watcher}; use parking_lot::RwLock; use tokio::{fs, pin, sync::mpsc::{self, UnboundedReceiver}}; @@ -90,26 +91,27 @@ impl Watcher { return; } - tokio::spawn(async move { - for (url, mtime) in todo { - let Ok(meta) = fs::metadata(&url).await else { - if let Ok(m) = fs::symlink_metadata(&url).await { - FilesOp::Full(url, vec![], m.modified().ok()).emit(); - } else if let Some(p) = url.parent_url() { - FilesOp::Deleting(p, vec![url]).emit(); - } - continue; - }; - - if meta.modified().ok() == mtime { - continue; + async fn go(url: Url, mtime: Option) { + let Ok(meta) = fs::metadata(&url).await else { + if let Ok(m) = fs::symlink_metadata(&url).await { + FilesOp::Full(url, vec![], m.modified().ok()).emit(); + } else if let Some(p) = url.parent_url() { + FilesOp::Deleting(p, vec![url]).emit(); } + return; + }; - if let Ok(rx) = Files::from_dir(&url).await { - let files: Vec<_> = UnboundedReceiverStream::new(rx).collect().await; - FilesOp::Full(url, files, meta.modified().ok()).emit(); - } + if meta.modified().ok() == mtime { + return; } + + if let Ok(files) = Files::from_dir_bulk(&url).await { + FilesOp::Full(url, files, meta.modified().ok()).emit(); + } + } + + tokio::spawn(async move { + futures::future::join_all(todo.into_iter().map(|(url, mtime)| go(url, mtime))).await; }); }