mirror of
https://github.com/sxyazi/yazi.git
synced 2026-07-21 23:01:05 +00:00
feat: prefetcher (#1061)
This commit is contained in:
parent
c2affae3a9
commit
bf1c325d00
24 changed files with 277 additions and 156 deletions
|
|
@ -12,7 +12,7 @@ Yazi (means "duck") is a terminal file manager written in Rust, based on non-blo
|
|||
- 💪 **Powerful Async Task Scheduling and Management**: Provides real-time progress updates, task cancellation, and internal task priority assignment.
|
||||
- 🖼️ **Built-in Support for Multiple Image Protocols**: Also integrated with Überzug++, covering almost all terminals.
|
||||
- 🌟 **Built-in Code Highlighting and Image Decoding**: Combined with the pre-loading mechanism, greatly accelerates image and normal file loading.
|
||||
- 🔌 **Concurrent Plugin System**: UI plugins (rewriting most of the UI), functional plugins, custom previewer, and custom preloader; Just some pieces of Lua.
|
||||
- 🔌 **Concurrent Plugin System**: UI plugins (rewriting most of the UI), functional plugins, custom previewer/preloader/prefetcher; Just some pieces of Lua.
|
||||
- 📡 **Data Distribution Service**: Built on a client-server architecture (no additional server process required), integrated with a Lua-based publish-subscribe model, achieving cross-instance communication and state persistence.
|
||||
- 📦 **Package Manager**: Install plugins and themes with one command, keeping them always up to date, or pin them to a specific version.
|
||||
- 🧰 Integration with fd, rg, fzf, zoxide
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
{"language":"en","flagWords":[],"words":["Punct","KEYMAP","splitn","crossterm","YAZI","unar","peekable","ratatui","syntect","pbpaste","pbcopy","ffmpegthumbnailer","oneshot","Posix","Lsar","XADDOS","zoxide","cands","Deque","precache","imageops","IFBLK","IFCHR","IFDIR","IFIFO","IFLNK","IFMT","IFSOCK","IRGRP","IROTH","IRUSR","ISGID","ISUID","ISVTX","IWGRP","IWOTH","IWUSR","IXGRP","IXOTH","IXUSR","libc","winsize","TIOCGWINSZ","xpixel","ypixel","ioerr","appender","Catppuccin","macchiato","gitmodules","Dotfiles","bashprofile","vimrc","flac","webp","exiftool","mediainfo","ripgrep","nvim","indexmap","indexmap","unwatch","canonicalize","serde","fsevent","Ueberzug","iterm","wezterm","sixel","chafa","ueberzugpp","️ Überzug","️ Überzug","Konsole","Alacritty","Überzug","pkgs","paru","unarchiver","pdftoppm","poppler","prebuild","singlefile","jpegopt","EXIF","rustfmt","mktemp","nanos","xclip","xsel","natord","Mintty","nixos","nixpkgs","SIGTSTP","SIGCONT","SIGCONT","mlua","nonstatic","userdata","metatable","natsort","backstack","luajit","Succ","Succ","cand","fileencoding","foldmethod","lightgreen","darkgray","lightred","lightyellow","lightcyan","nushell","msvc","aarch","linemode","sxyazi","rsplit","ZELLIJ","bitflags","bitflags","USERPROFILE","Neovim","vergen","gitcl","Renderable","preloaders","prec","imagesize","Upserting","prio","Ghostty","Catmull","Lanczos","cmds","unyank","scrolloff","headsup","unsub","uzers","scopeguard","SPDLOG","globset","filetime","magick","magick"],"version":"0.2"}
|
||||
{"words":["Punct","KEYMAP","splitn","crossterm","YAZI","unar","peekable","ratatui","syntect","pbpaste","pbcopy","ffmpegthumbnailer","oneshot","Posix","Lsar","XADDOS","zoxide","cands","Deque","precache","imageops","IFBLK","IFCHR","IFDIR","IFIFO","IFLNK","IFMT","IFSOCK","IRGRP","IROTH","IRUSR","ISGID","ISUID","ISVTX","IWGRP","IWOTH","IWUSR","IXGRP","IXOTH","IXUSR","libc","winsize","TIOCGWINSZ","xpixel","ypixel","ioerr","appender","Catppuccin","macchiato","gitmodules","Dotfiles","bashprofile","vimrc","flac","webp","exiftool","mediainfo","ripgrep","nvim","indexmap","indexmap","unwatch","canonicalize","serde","fsevent","Ueberzug","iterm","wezterm","sixel","chafa","ueberzugpp","️ Überzug","️ Überzug","Konsole","Alacritty","Überzug","pkgs","paru","unarchiver","pdftoppm","poppler","prebuild","singlefile","jpegopt","EXIF","rustfmt","mktemp","nanos","xclip","xsel","natord","Mintty","nixos","nixpkgs","SIGTSTP","SIGCONT","SIGCONT","mlua","nonstatic","userdata","metatable","natsort","backstack","luajit","Succ","Succ","cand","fileencoding","foldmethod","lightgreen","darkgray","lightred","lightyellow","lightcyan","nushell","msvc","aarch","linemode","sxyazi","rsplit","ZELLIJ","bitflags","bitflags","USERPROFILE","Neovim","vergen","gitcl","Renderable","preloaders","prec","imagesize","Upserting","prio","Ghostty","Catmull","Lanczos","cmds","unyank","scrolloff","headsup","unsub","uzers","scopeguard","SPDLOG","globset","filetime","magick","magick","prefetcher","Prework","prefetchers","PREWORKERS"],"version":"0.2","language":"en","flagWords":[]}
|
||||
|
|
@ -79,8 +79,11 @@ suppress_preload = false
|
|||
|
||||
[plugin]
|
||||
|
||||
prefetchers = [
|
||||
# Mimetype
|
||||
{ name = "*", cond = "!mime", run = "mime", prio = "high" },
|
||||
]
|
||||
preloaders = [
|
||||
{ name = "*", cond = "!mime", run = "mime", next = true, multi = true, prio = "high" },
|
||||
# Image
|
||||
{ mime = "image/svg+xml", run = "magick" },
|
||||
{ mime = "image/heic", run = "magick" },
|
||||
|
|
|
|||
|
|
@ -1,9 +1,11 @@
|
|||
mod plugin;
|
||||
mod prefetcher;
|
||||
mod preloader;
|
||||
mod previewer;
|
||||
|
||||
pub use plugin::*;
|
||||
pub use prefetcher::*;
|
||||
pub use preloader::*;
|
||||
pub use previewer::*;
|
||||
|
||||
pub const MAX_PRELOADERS: u8 = 32;
|
||||
pub const MAX_PREWORKERS: u8 = 32;
|
||||
|
|
|
|||
|
|
@ -3,13 +3,14 @@ use std::path::Path;
|
|||
use serde::Deserialize;
|
||||
use yazi_shared::MIME_DIR;
|
||||
|
||||
use super::{Preloader, Previewer};
|
||||
use crate::{plugin::MAX_PRELOADERS, Preset, MERGED_YAZI};
|
||||
use super::{Prefetcher, Preloader, Previewer};
|
||||
use crate::{plugin::MAX_PREWORKERS, Preset, MERGED_YAZI};
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct Plugin {
|
||||
pub preloaders: Vec<Preloader>,
|
||||
pub previewers: Vec<Previewer>,
|
||||
pub prefetchers: Vec<Prefetcher>,
|
||||
pub preloaders: Vec<Preloader>,
|
||||
pub previewers: Vec<Previewer>,
|
||||
}
|
||||
|
||||
impl Default for Plugin {
|
||||
|
|
@ -21,6 +22,12 @@ impl Default for Plugin {
|
|||
|
||||
#[derive(Deserialize)]
|
||||
struct Shadow {
|
||||
prefetchers: Vec<Prefetcher>,
|
||||
#[serde(default)]
|
||||
prepend_prefetchers: Vec<Prefetcher>,
|
||||
#[serde(default)]
|
||||
append_prefetchers: Vec<Prefetcher>,
|
||||
|
||||
preloaders: Vec<Preloader>,
|
||||
#[serde(default)]
|
||||
prepend_preloaders: Vec<Preloader>,
|
||||
|
|
@ -42,36 +49,53 @@ impl Default for Plugin {
|
|||
shadow.previewers.retain(|r| !r.any_dir());
|
||||
}
|
||||
|
||||
Preset::mix(&mut shadow.prefetchers, shadow.prepend_prefetchers, shadow.append_prefetchers);
|
||||
Preset::mix(&mut shadow.preloaders, shadow.prepend_preloaders, shadow.append_preloaders);
|
||||
Preset::mix(&mut shadow.previewers, shadow.prepend_previewers, shadow.append_previewers);
|
||||
|
||||
if shadow.preloaders.len() > MAX_PRELOADERS as usize {
|
||||
panic!("Too many preloaders");
|
||||
if shadow.prefetchers.len() + shadow.preloaders.len() > MAX_PREWORKERS as usize {
|
||||
panic!("Prefetchers and preloaders exceed the limit of {MAX_PREWORKERS}");
|
||||
}
|
||||
|
||||
for (i, preloader) in shadow.preloaders.iter_mut().enumerate() {
|
||||
preloader.id = i as u8;
|
||||
for (i, p) in shadow.prefetchers.iter_mut().enumerate() {
|
||||
p.id = i as u8;
|
||||
}
|
||||
for (i, p) in shadow.preloaders.iter_mut().enumerate() {
|
||||
p.id = shadow.prefetchers.len() as u8 + i as u8;
|
||||
}
|
||||
|
||||
Self { preloaders: shadow.preloaders, previewers: shadow.previewers }
|
||||
Self {
|
||||
prefetchers: shadow.prefetchers,
|
||||
preloaders: shadow.preloaders,
|
||||
previewers: shadow.previewers,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Plugin {
|
||||
pub fn preloaders(
|
||||
pub fn prefetchers(
|
||||
&self,
|
||||
path: &Path,
|
||||
mime: Option<&str>,
|
||||
f: impl Fn(&str) -> bool + Copy,
|
||||
) -> Vec<&Preloader> {
|
||||
) -> Vec<&Prefetcher> {
|
||||
let is_dir = mime == Some(MIME_DIR);
|
||||
self
|
||||
.prefetchers
|
||||
.iter()
|
||||
.filter(|&p| {
|
||||
p.cond.as_ref().and_then(|c| c.eval(f)) != Some(false)
|
||||
&& (p.mime.as_ref().zip(mime).map_or(false, |(p, m)| p.match_mime(m))
|
||||
|| p.name.as_ref().is_some_and(|p| p.match_path(path, is_dir)))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn preloaders(&self, path: &Path, mime: Option<&str>) -> Vec<&Preloader> {
|
||||
let is_dir = mime == Some(MIME_DIR);
|
||||
let mut preloaders = Vec::with_capacity(1);
|
||||
|
||||
for p in &self.preloaders {
|
||||
if p.cond.as_ref().and_then(|c| c.eval(f)) == Some(false) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if !p.mime.as_ref().zip(mime).map_or(false, |(p, m)| p.match_mime(m))
|
||||
&& !p.name.as_ref().is_some_and(|p| p.match_path(path, is_dir))
|
||||
{
|
||||
|
|
|
|||
29
yazi-config/src/plugin/prefetcher.rs
Normal file
29
yazi-config/src/plugin/prefetcher.rs
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
use serde::Deserialize;
|
||||
use yazi_shared::{event::Cmd, Condition};
|
||||
|
||||
use crate::{Pattern, Priority};
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct Prefetcher {
|
||||
#[serde(skip)]
|
||||
pub id: u8,
|
||||
pub cond: Option<Condition>,
|
||||
pub name: Option<Pattern>,
|
||||
pub mime: Option<Pattern>,
|
||||
pub run: Cmd,
|
||||
#[serde(default)]
|
||||
pub prio: Priority,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct PrefetcherProps {
|
||||
pub id: u8,
|
||||
pub name: String,
|
||||
pub prio: Priority,
|
||||
}
|
||||
|
||||
impl From<&Prefetcher> for PrefetcherProps {
|
||||
fn from(prefetcher: &Prefetcher) -> Self {
|
||||
Self { id: prefetcher.id, name: prefetcher.run.name.to_owned(), prio: prefetcher.prio }
|
||||
}
|
||||
}
|
||||
|
|
@ -1,39 +1,30 @@
|
|||
use serde::Deserialize;
|
||||
use yazi_shared::{event::Cmd, Condition};
|
||||
use yazi_shared::event::Cmd;
|
||||
|
||||
use crate::{Pattern, Priority};
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct Preloader {
|
||||
#[serde(skip)]
|
||||
pub id: u8,
|
||||
pub cond: Option<Condition>,
|
||||
pub name: Option<Pattern>,
|
||||
pub mime: Option<Pattern>,
|
||||
pub run: Cmd,
|
||||
pub id: u8,
|
||||
pub name: Option<Pattern>,
|
||||
pub mime: Option<Pattern>,
|
||||
pub run: Cmd,
|
||||
#[serde(default)]
|
||||
pub next: bool,
|
||||
pub next: bool,
|
||||
#[serde(default)]
|
||||
pub multi: bool,
|
||||
#[serde(default)]
|
||||
pub prio: Priority,
|
||||
pub prio: Priority,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct PreloaderProps {
|
||||
pub id: u8,
|
||||
pub name: String,
|
||||
pub multi: bool,
|
||||
pub prio: Priority,
|
||||
pub id: u8,
|
||||
pub name: String,
|
||||
pub prio: Priority,
|
||||
}
|
||||
|
||||
impl From<&Preloader> for PreloaderProps {
|
||||
fn from(preloader: &Preloader) -> Self {
|
||||
Self {
|
||||
id: preloader.id,
|
||||
name: preloader.run.name.to_owned(),
|
||||
multi: preloader.multi,
|
||||
prio: preloader.prio,
|
||||
}
|
||||
Self { id: preloader.id, name: preloader.run.name.to_owned(), prio: preloader.prio }
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -63,8 +63,8 @@ impl Manager {
|
|||
}
|
||||
|
||||
done.extend(files.iter().map(|f| (f.url(), String::new())));
|
||||
if let Err(e) = isolate::preload("mime", files, true).await {
|
||||
error!("preload in open failed: {e}");
|
||||
if let Err(e) = isolate::prefetch("mime", files).await {
|
||||
error!("prefetch `mime` failed in opening: {e}");
|
||||
}
|
||||
|
||||
ManagerProxy::open_do(OpenDoOpt { hovered, targets: done, interactive: opt.interactive });
|
||||
|
|
|
|||
|
|
@ -35,6 +35,6 @@ impl Manager {
|
|||
self.hover(None);
|
||||
self.update_paged((), tasks);
|
||||
|
||||
tasks.preload_sorted(&self.current().files);
|
||||
tasks.prework_sorted(&self.current().files);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -88,7 +88,7 @@ impl Manager {
|
|||
ManagerProxy::hover(None); // Re-hover in next loop
|
||||
ManagerProxy::update_paged(); // Update for paged files in next loop
|
||||
if calc {
|
||||
tasks.preload_sorted(&tab.current.files);
|
||||
tasks.prework_sorted(&tab.current.files);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -52,7 +52,7 @@ impl Manager {
|
|||
self.mimetype.extend(updates);
|
||||
self.peek(false);
|
||||
|
||||
tasks.preload_affected(&affected, &self.mimetype);
|
||||
tasks.prework_affected(&affected, &self.mimetype);
|
||||
render!();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ impl Manager {
|
|||
}
|
||||
|
||||
let targets = self.current().paginate(opt.page.unwrap_or(self.current().page));
|
||||
tasks.prefetch_paged(targets, &self.mimetype);
|
||||
tasks.preload_paged(targets, &self.mimetype);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -117,8 +117,8 @@ impl Watcher {
|
|||
if reload.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if let Err(e) = isolate::preload("mime", reload, true).await {
|
||||
error!("preload in watcher failed: {e}");
|
||||
if let Err(e) = isolate::prefetch("mime", reload).await {
|
||||
error!("prefetch `mime` failed in watcher: {e}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,6 +18,6 @@ impl Tab {
|
|||
self.apply_files_attrs();
|
||||
ManagerProxy::update_paged();
|
||||
|
||||
tasks.preload_sorted(&self.current.files);
|
||||
tasks.prework_sorted(&self.current.files);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,17 +1,15 @@
|
|||
use std::{collections::HashMap, mem};
|
||||
use std::collections::HashMap;
|
||||
|
||||
use yazi_config::{manager::SortBy, plugin::{Preloader, MAX_PRELOADERS}, PLUGIN};
|
||||
use yazi_config::{manager::SortBy, plugin::MAX_PREWORKERS, PLUGIN};
|
||||
use yazi_shared::{fs::{File, Url}, MIME_DIR};
|
||||
|
||||
use super::Tasks;
|
||||
use crate::folder::Files;
|
||||
|
||||
impl Tasks {
|
||||
pub fn preload_paged(&self, paged: &[File], mimetype: &HashMap<Url, String>) {
|
||||
let mut single_tasks = Vec::with_capacity(paged.len());
|
||||
let mut multi_tasks: [Vec<_>; MAX_PRELOADERS as usize] = Default::default();
|
||||
|
||||
let loaded = self.scheduler.preload.rule_loaded.read();
|
||||
pub fn prefetch_paged(&self, paged: &[File], mimetype: &HashMap<Url, String>) {
|
||||
let mut loaded = self.scheduler.prework.loaded.lock();
|
||||
let mut tasks: [Vec<_>; MAX_PREWORKERS as usize] = Default::default();
|
||||
for f in paged {
|
||||
let mime = if f.is_dir() { Some(MIME_DIR) } else { mimetype.get(&f.url).map(|s| &**s) };
|
||||
let factors = |s: &str| match s {
|
||||
|
|
@ -19,61 +17,58 @@ impl Tasks {
|
|||
_ => false,
|
||||
};
|
||||
|
||||
for rule in PLUGIN.preloaders(&f.url, mime, factors) {
|
||||
if loaded.get(&f.url).is_some_and(|x| x & (1 << rule.id) != 0) {
|
||||
continue;
|
||||
}
|
||||
if rule.multi {
|
||||
multi_tasks[rule.id as usize].push(f);
|
||||
} else {
|
||||
single_tasks.push((rule, f));
|
||||
for p in PLUGIN.prefetchers(&f.url, mime, factors) {
|
||||
match loaded.get_mut(&f.url) {
|
||||
Some(n) if *n & (1 << p.id) != 0 => continue,
|
||||
Some(n) => *n |= 1 << p.id,
|
||||
None => _ = loaded.insert(f.url.clone(), 1 << p.id),
|
||||
}
|
||||
tasks[p.id as usize].push(f.clone());
|
||||
}
|
||||
}
|
||||
|
||||
drop(loaded);
|
||||
let mut loaded = self.scheduler.preload.rule_loaded.write();
|
||||
|
||||
let mut go = |preloader: &Preloader, targets: Vec<&File>| {
|
||||
for &f in &targets {
|
||||
if let Some(n) = loaded.get_mut(&f.url) {
|
||||
*n |= 1 << preloader.id;
|
||||
} else {
|
||||
loaded.insert(f.url.clone(), 1 << preloader.id);
|
||||
}
|
||||
for (i, tasks) in tasks.into_iter().enumerate() {
|
||||
if !tasks.is_empty() {
|
||||
self.scheduler.prefetch_paged(&PLUGIN.prefetchers[i], tasks);
|
||||
}
|
||||
self.scheduler.preload_paged(preloader, targets);
|
||||
};
|
||||
|
||||
#[allow(clippy::needless_range_loop)]
|
||||
for i in 0..PLUGIN.preloaders.len() {
|
||||
if !multi_tasks[i].is_empty() {
|
||||
go(&PLUGIN.preloaders[i], mem::take(&mut multi_tasks[i]));
|
||||
}
|
||||
}
|
||||
for (rule, target) in single_tasks {
|
||||
go(rule, vec![target]);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn preload_affected(&self, affected: &[File], mimetype: &HashMap<Url, String>) {
|
||||
pub fn preload_paged(&self, paged: &[File], mimetype: &HashMap<Url, String>) {
|
||||
let mut loaded = self.scheduler.prework.loaded.lock();
|
||||
for f in paged {
|
||||
let mime = if f.is_dir() { Some(MIME_DIR) } else { mimetype.get(&f.url).map(|s| &**s) };
|
||||
for p in PLUGIN.preloaders(&f.url, mime) {
|
||||
match loaded.get_mut(&f.url) {
|
||||
Some(n) if *n & (1 << p.id) != 0 => continue,
|
||||
Some(n) => *n |= 1 << p.id,
|
||||
None => _ = loaded.insert(f.url.clone(), 1 << p.id),
|
||||
}
|
||||
self.scheduler.preload_paged(p, f);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn prework_affected(&self, affected: &[File], mimetype: &HashMap<Url, String>) {
|
||||
{
|
||||
let mut loaded = self.scheduler.preload.rule_loaded.write();
|
||||
let mut loaded = self.scheduler.prework.loaded.lock();
|
||||
for f in affected {
|
||||
loaded.remove(&f.url);
|
||||
}
|
||||
}
|
||||
|
||||
self.prefetch_paged(affected, mimetype);
|
||||
self.preload_paged(affected, mimetype);
|
||||
}
|
||||
|
||||
pub fn preload_sorted(&self, targets: &Files) {
|
||||
pub fn prework_sorted(&self, targets: &Files) {
|
||||
if targets.sorter().by != SortBy::Size {
|
||||
return;
|
||||
}
|
||||
|
||||
let targets: Vec<_> = {
|
||||
let loading = self.scheduler.preload.size_loading.read();
|
||||
let loading = self.scheduler.prework.size_loading.read();
|
||||
targets
|
||||
.iter()
|
||||
.filter(|f| f.is_dir() && !targets.sizes.contains_key(&f.url) && !loading.contains(&f.url))
|
||||
|
|
@ -84,11 +79,11 @@ impl Tasks {
|
|||
return;
|
||||
}
|
||||
|
||||
let mut loading = self.scheduler.preload.size_loading.write();
|
||||
let mut loading = self.scheduler.prework.size_loading.write();
|
||||
for &target in &targets {
|
||||
loading.insert(target.clone());
|
||||
}
|
||||
|
||||
self.scheduler.preload_size(targets);
|
||||
self.scheduler.prework_size(targets);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ local function match_mimetype(s)
|
|||
end
|
||||
end
|
||||
|
||||
function M:preload()
|
||||
function M:prefetch()
|
||||
local urls = {}
|
||||
for _, file in ipairs(self.files) do
|
||||
urls[#urls + 1] = tostring(file.url)
|
||||
|
|
|
|||
|
|
@ -3,11 +3,13 @@
|
|||
mod entry;
|
||||
mod isolate;
|
||||
mod peek;
|
||||
mod prefetch;
|
||||
mod preload;
|
||||
mod seek;
|
||||
|
||||
pub use entry::*;
|
||||
pub use isolate::*;
|
||||
pub use peek::*;
|
||||
pub use prefetch::*;
|
||||
pub use preload::*;
|
||||
pub use seek::*;
|
||||
|
|
|
|||
33
yazi-plugin/src/isolate/prefetch.rs
Normal file
33
yazi-plugin/src/isolate/prefetch.rs
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
use mlua::{ExternalError, ExternalResult, Table, TableExt};
|
||||
use tokio::runtime::Handle;
|
||||
use yazi_config::LAYOUT;
|
||||
|
||||
use super::slim_lua;
|
||||
use crate::{bindings::{Cast, File}, elements::Rect, loader::LOADER};
|
||||
|
||||
pub async fn prefetch(name: &str, files: Vec<yazi_shared::fs::File>) -> mlua::Result<u8> {
|
||||
LOADER.ensure(name).await.into_lua_err()?;
|
||||
|
||||
let name = name.to_owned();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let lua = slim_lua(&name)?;
|
||||
let plugin: Table = if let Some(b) = LOADER.read().get(&name) {
|
||||
lua.load(b.as_ref()).call(())?
|
||||
} else {
|
||||
return Err("unloaded plugin".into_lua_err());
|
||||
};
|
||||
|
||||
let files = files.into_iter().filter_map(|f| File::cast(&lua, f).ok()).collect::<Vec<_>>();
|
||||
if files.is_empty() {
|
||||
return Err("no files".into_lua_err());
|
||||
}
|
||||
|
||||
plugin.raw_set("skip", 0)?;
|
||||
plugin.raw_set("area", Rect::cast(&lua, LAYOUT.load().preview)?)?;
|
||||
plugin.raw_set("files", files)?;
|
||||
|
||||
Handle::current().block_on(plugin.call_async_method("prefetch", ()))
|
||||
})
|
||||
.await
|
||||
.into_lua_err()?
|
||||
}
|
||||
|
|
@ -5,11 +5,7 @@ use yazi_config::LAYOUT;
|
|||
use super::slim_lua;
|
||||
use crate::{bindings::{Cast, File}, elements::Rect, loader::LOADER};
|
||||
|
||||
pub async fn preload(
|
||||
name: &str,
|
||||
files: Vec<yazi_shared::fs::File>,
|
||||
multi: bool,
|
||||
) -> mlua::Result<u8> {
|
||||
pub async fn preload(name: &str, file: yazi_shared::fs::File) -> mlua::Result<u8> {
|
||||
LOADER.ensure(name).await.into_lua_err()?;
|
||||
|
||||
let name = name.to_owned();
|
||||
|
|
@ -21,18 +17,9 @@ pub async fn preload(
|
|||
return Err("unloaded plugin".into_lua_err());
|
||||
};
|
||||
|
||||
let mut files = files.into_iter().filter_map(|f| File::cast(&lua, f).ok()).collect::<Vec<_>>();
|
||||
if files.is_empty() {
|
||||
return Err("no files".into_lua_err());
|
||||
}
|
||||
|
||||
plugin.raw_set("skip", 0)?;
|
||||
plugin.raw_set("area", Rect::cast(&lua, LAYOUT.load().preview)?)?;
|
||||
if multi {
|
||||
plugin.raw_set("files", files)?;
|
||||
} else {
|
||||
plugin.raw_set("file", files.remove(0))?;
|
||||
}
|
||||
plugin.raw_set("file", File::cast(&lua, file)?)?;
|
||||
|
||||
Handle::current().block_on(plugin.call_async_method("preload", ()))
|
||||
})
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
use crate::{file::FileOp, plugin::PluginOp, preload::PreloadOp};
|
||||
use crate::{file::FileOp, plugin::PluginOp, preload::PreworkOp};
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum TaskOp {
|
||||
File(Box<FileOp>),
|
||||
Plugin(Box<PluginOp>),
|
||||
Preload(Box<PreloadOp>),
|
||||
Prework(Box<PreworkOp>),
|
||||
}
|
||||
|
||||
impl TaskOp {
|
||||
|
|
@ -12,7 +12,7 @@ impl TaskOp {
|
|||
match self {
|
||||
TaskOp::File(op) => op.id(),
|
||||
TaskOp::Plugin(op) => op.id(),
|
||||
TaskOp::Preload(op) => op.id(),
|
||||
TaskOp::Prework(op) => op.id(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -25,6 +25,6 @@ impl From<PluginOp> for TaskOp {
|
|||
fn from(op: PluginOp) -> Self { Self::Plugin(Box::new(op)) }
|
||||
}
|
||||
|
||||
impl From<PreloadOp> for TaskOp {
|
||||
fn from(op: PreloadOp) -> Self { Self::Preload(Box::new(op)) }
|
||||
impl From<PreworkOp> for TaskOp {
|
||||
fn from(op: PreworkOp) -> Self { Self::Prework(Box::new(op)) }
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
#![allow(clippy::module_inception)]
|
||||
|
||||
mod op;
|
||||
mod preload;
|
||||
mod prework;
|
||||
|
||||
pub use op::*;
|
||||
pub use preload::*;
|
||||
pub use prework::*;
|
||||
|
|
|
|||
|
|
@ -1,32 +1,41 @@
|
|||
use std::sync::Arc;
|
||||
|
||||
use yazi_config::plugin::PreloaderProps;
|
||||
use yazi_config::plugin::{PrefetcherProps, PreloaderProps};
|
||||
use yazi_shared::{fs::Url, Throttle};
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum PreloadOp {
|
||||
Rule(PreloadOpRule),
|
||||
Size(PreloadOpSize),
|
||||
pub enum PreworkOp {
|
||||
Fetch(PreworkOpFetch),
|
||||
Load(PreworkOpLoad),
|
||||
Size(PreworkOpSize),
|
||||
}
|
||||
|
||||
impl PreloadOp {
|
||||
impl PreworkOp {
|
||||
pub fn id(&self) -> usize {
|
||||
match self {
|
||||
Self::Rule(op) => op.id,
|
||||
Self::Fetch(op) => op.id,
|
||||
Self::Load(op) => op.id,
|
||||
Self::Size(op) => op.id,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct PreloadOpRule {
|
||||
pub struct PreworkOpFetch {
|
||||
pub id: usize,
|
||||
pub plugin: PreloaderProps,
|
||||
pub plugin: PrefetcherProps,
|
||||
pub targets: Vec<yazi_shared::fs::File>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct PreworkOpLoad {
|
||||
pub id: usize,
|
||||
pub plugin: PreloaderProps,
|
||||
pub target: yazi_shared::fs::File,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct PreloadOpSize {
|
||||
pub struct PreworkOpSize {
|
||||
pub id: usize,
|
||||
pub target: Url,
|
||||
pub throttle: Arc<Throttle<(Url, u64)>>,
|
||||
|
|
|
|||
|
|
@ -1,37 +1,57 @@
|
|||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use anyhow::{anyhow, Result};
|
||||
use parking_lot::RwLock;
|
||||
use parking_lot::{Mutex, RwLock};
|
||||
use tokio::sync::mpsc;
|
||||
use tracing::error;
|
||||
use yazi_config::Priority;
|
||||
use yazi_plugin::isolate;
|
||||
use yazi_shared::fs::{calculate_size, FilesOp, Url};
|
||||
|
||||
use super::{PreloadOp, PreloadOpRule, PreloadOpSize};
|
||||
use super::{PreworkOp, PreworkOpFetch, PreworkOpLoad, PreworkOpSize};
|
||||
use crate::{TaskOp, TaskProg, HIGH, NORMAL};
|
||||
|
||||
pub struct Preload {
|
||||
pub struct Prework {
|
||||
macro_: async_priority_channel::Sender<TaskOp, u8>,
|
||||
prog: mpsc::UnboundedSender<TaskProg>,
|
||||
|
||||
pub rule_loaded: RwLock<HashMap<Url, u32>>,
|
||||
pub loaded: Mutex<HashMap<Url, u32>>,
|
||||
pub size_loading: RwLock<HashSet<Url>>,
|
||||
}
|
||||
|
||||
impl Preload {
|
||||
impl Prework {
|
||||
pub fn new(
|
||||
macro_: async_priority_channel::Sender<TaskOp, u8>,
|
||||
prog: mpsc::UnboundedSender<TaskProg>,
|
||||
) -> Self {
|
||||
Self { macro_, prog, rule_loaded: Default::default(), size_loading: Default::default() }
|
||||
Self { macro_, prog, loaded: Default::default(), size_loading: Default::default() }
|
||||
}
|
||||
|
||||
pub async fn work(&self, op: PreloadOp) -> Result<()> {
|
||||
pub async fn work(&self, op: PreworkOp) -> Result<()> {
|
||||
match op {
|
||||
PreloadOp::Rule(task) => {
|
||||
PreworkOp::Fetch(task) => {
|
||||
let urls: Vec<_> = task.targets.iter().map(|f| f.url()).collect();
|
||||
let result = isolate::preload(&task.plugin.name, task.targets, task.plugin.multi).await;
|
||||
let result = isolate::prefetch(&task.plugin.name, task.targets).await;
|
||||
if let Err(e) = result {
|
||||
self.fail(task.id, format!("Prefetch task failed:\n{e}"))?;
|
||||
return Err(e.into());
|
||||
};
|
||||
|
||||
let code = result.unwrap();
|
||||
if code & 1 == 0 {
|
||||
error!("Prefetch task `{}` returned {code}", task.plugin.name);
|
||||
}
|
||||
if code >> 1 & 1 != 0 {
|
||||
let mut loaded = self.loaded.lock();
|
||||
for url in urls {
|
||||
loaded.get_mut(&url).map(|x| *x ^= 1 << task.plugin.id);
|
||||
}
|
||||
}
|
||||
self.prog.send(TaskProg::Adv(task.id, 1, 0))?;
|
||||
}
|
||||
PreworkOp::Load(task) => {
|
||||
let url = task.target.url();
|
||||
let result = isolate::preload(&task.plugin.name, task.target).await;
|
||||
if let Err(e) = result {
|
||||
self.fail(task.id, format!("Preload task failed:\n{e}"))?;
|
||||
return Err(e.into());
|
||||
|
|
@ -42,14 +62,12 @@ impl Preload {
|
|||
error!("Preload task `{}` returned {code}", task.plugin.name);
|
||||
}
|
||||
if code >> 1 & 1 != 0 {
|
||||
let mut loaded = self.rule_loaded.write();
|
||||
for url in urls {
|
||||
loaded.get_mut(&url).map(|x| *x ^= 1 << task.plugin.id);
|
||||
}
|
||||
let mut loaded = self.loaded.lock();
|
||||
loaded.get_mut(&url).map(|x| *x ^= 1 << task.plugin.id);
|
||||
}
|
||||
self.prog.send(TaskProg::Adv(task.id, 1, 0))?;
|
||||
}
|
||||
PreloadOp::Size(task) => {
|
||||
PreworkOp::Size(task) => {
|
||||
let length = calculate_size(&task.target).await;
|
||||
task.throttle.done((task.target, length), |buf| {
|
||||
{
|
||||
|
|
@ -68,28 +86,40 @@ impl Preload {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn rule(&self, task: PreloadOpRule) -> Result<()> {
|
||||
pub async fn fetch(&self, task: PreworkOpFetch) -> Result<()> {
|
||||
let id = task.id;
|
||||
self.prog.send(TaskProg::New(id, 0))?;
|
||||
|
||||
match task.plugin.prio {
|
||||
Priority::Low => self.queue(PreloadOp::Rule(task), NORMAL).await?,
|
||||
Priority::Normal => self.queue(PreloadOp::Rule(task), HIGH).await?,
|
||||
Priority::High => self.work(PreloadOp::Rule(task)).await?,
|
||||
Priority::Low => self.queue(PreworkOp::Fetch(task), NORMAL).await?,
|
||||
Priority::Normal => self.queue(PreworkOp::Fetch(task), HIGH).await?,
|
||||
Priority::High => self.work(PreworkOp::Fetch(task)).await?,
|
||||
}
|
||||
self.succ(id)
|
||||
}
|
||||
|
||||
pub async fn size(&self, task: PreloadOpSize) -> Result<()> {
|
||||
pub async fn load(&self, task: PreworkOpLoad) -> Result<()> {
|
||||
let id = task.id;
|
||||
self.prog.send(TaskProg::New(id, 0))?;
|
||||
|
||||
match task.plugin.prio {
|
||||
Priority::Low => self.queue(PreworkOp::Load(task), NORMAL).await?,
|
||||
Priority::Normal => self.queue(PreworkOp::Load(task), HIGH).await?,
|
||||
Priority::High => self.work(PreworkOp::Load(task)).await?,
|
||||
}
|
||||
self.succ(id)
|
||||
}
|
||||
|
||||
pub async fn size(&self, task: PreworkOpSize) -> Result<()> {
|
||||
let id = task.id;
|
||||
|
||||
self.prog.send(TaskProg::New(id, 0))?;
|
||||
self.work(PreloadOp::Size(task)).await?;
|
||||
self.work(PreworkOp::Size(task)).await?;
|
||||
self.succ(id)
|
||||
}
|
||||
}
|
||||
|
||||
impl Preload {
|
||||
impl Prework {
|
||||
#[inline]
|
||||
fn succ(&self, id: usize) -> Result<()> { Ok(self.prog.send(TaskProg::Succ(id))?) }
|
||||
|
||||
|
|
@ -4,17 +4,17 @@ use anyhow::Result;
|
|||
use futures::{future::BoxFuture, FutureExt};
|
||||
use parking_lot::Mutex;
|
||||
use tokio::{fs, select, sync::{mpsc::{self, UnboundedReceiver}, oneshot}, task::JoinHandle};
|
||||
use yazi_config::{open::Opener, plugin::Preloader, TASKS};
|
||||
use yazi_config::{open::Opener, plugin::{Prefetcher, Preloader}, TASKS};
|
||||
use yazi_dds::Pump;
|
||||
use yazi_shared::{event::Data, fs::{unique_path, Url}, Throttle};
|
||||
|
||||
use super::{Ongoing, TaskProg, TaskStage};
|
||||
use crate::{file::{File, FileOpDelete, FileOpLink, FileOpPaste, FileOpTrash}, plugin::{Plugin, PluginOpEntry}, preload::{Preload, PreloadOpRule, PreloadOpSize}, process::{Process, ProcessOpBg, ProcessOpBlock, ProcessOpOrphan}, TaskKind, TaskOp, HIGH, LOW, NORMAL};
|
||||
use crate::{file::{File, FileOpDelete, FileOpLink, FileOpPaste, FileOpTrash}, plugin::{Plugin, PluginOpEntry}, preload::{Prework, PreworkOpFetch, PreworkOpLoad, PreworkOpSize}, process::{Process, ProcessOpBg, ProcessOpBlock, ProcessOpOrphan}, TaskKind, TaskOp, HIGH, LOW, NORMAL};
|
||||
|
||||
pub struct Scheduler {
|
||||
pub file: Arc<File>,
|
||||
pub plugin: Arc<Plugin>,
|
||||
pub preload: Arc<Preload>,
|
||||
pub prework: Arc<Prework>,
|
||||
pub process: Arc<Process>,
|
||||
|
||||
micro: async_priority_channel::Sender<BoxFuture<'static, ()>, u8>,
|
||||
|
|
@ -32,7 +32,7 @@ impl Scheduler {
|
|||
let mut scheduler = Self {
|
||||
file: Arc::new(File::new(macro_tx.clone(), prog_tx.clone())),
|
||||
plugin: Arc::new(Plugin::new(macro_tx.clone(), prog_tx.clone())),
|
||||
preload: Arc::new(Preload::new(macro_tx.clone(), prog_tx.clone())),
|
||||
prework: Arc::new(Prework::new(macro_tx.clone(), prog_tx.clone())),
|
||||
process: Arc::new(Process::new(prog_tx.clone())),
|
||||
|
||||
micro: micro_tx,
|
||||
|
|
@ -218,25 +218,40 @@ impl Scheduler {
|
|||
self.plugin.macro_(PluginOpEntry { id, name, args }).ok();
|
||||
}
|
||||
|
||||
pub fn preload_paged(&self, preloader: &Preloader, targets: Vec<&yazi_shared::fs::File>) {
|
||||
pub fn prefetch_paged(&self, prefetcher: &Prefetcher, targets: Vec<yazi_shared::fs::File>) {
|
||||
let id = self.ongoing.lock().add(
|
||||
TaskKind::Preload,
|
||||
format!("Run preloader `{}` with {} target(s)", preloader.run.name, targets.len()),
|
||||
format!("Run prefetcher `{}` with {} target(s)", prefetcher.run.name, targets.len()),
|
||||
);
|
||||
|
||||
let plugin = preloader.into();
|
||||
let targets = targets.into_iter().cloned().collect();
|
||||
let preload = self.preload.clone();
|
||||
let plugin = prefetcher.into();
|
||||
let prework = self.prework.clone();
|
||||
_ = self.micro.try_send(
|
||||
async move {
|
||||
preload.rule(PreloadOpRule { id, plugin, targets }).await.ok();
|
||||
prework.fetch(PreworkOpFetch { id, plugin, targets }).await.ok();
|
||||
}
|
||||
.boxed(),
|
||||
NORMAL,
|
||||
);
|
||||
}
|
||||
|
||||
pub fn preload_size(&self, targets: Vec<&Url>) {
|
||||
pub fn preload_paged(&self, preloader: &Preloader, target: &yazi_shared::fs::File) {
|
||||
let id =
|
||||
self.ongoing.lock().add(TaskKind::Preload, format!("Run preloader `{}`", preloader.run.name));
|
||||
|
||||
let plugin = preloader.into();
|
||||
let target = target.clone();
|
||||
let prework = self.prework.clone();
|
||||
_ = self.micro.try_send(
|
||||
async move {
|
||||
prework.load(PreworkOpLoad { id, plugin, target }).await.ok();
|
||||
}
|
||||
.boxed(),
|
||||
NORMAL,
|
||||
);
|
||||
}
|
||||
|
||||
pub fn prework_size(&self, targets: Vec<&Url>) {
|
||||
let throttle = Arc::new(Throttle::new(targets.len(), Duration::from_millis(300)));
|
||||
let mut ongoing = self.ongoing.lock();
|
||||
|
||||
|
|
@ -245,10 +260,10 @@ impl Scheduler {
|
|||
let target = target.clone();
|
||||
let throttle = throttle.clone();
|
||||
|
||||
let preload = self.preload.clone();
|
||||
let prework = self.prework.clone();
|
||||
_ = self.micro.try_send(
|
||||
async move {
|
||||
preload.size(PreloadOpSize { id, target, throttle }).await.ok();
|
||||
prework.size(PreworkOpSize { id, target, throttle }).await.ok();
|
||||
}
|
||||
.boxed(),
|
||||
NORMAL,
|
||||
|
|
@ -329,7 +344,7 @@ impl Scheduler {
|
|||
) -> JoinHandle<()> {
|
||||
let file = self.file.clone();
|
||||
let plugin = self.plugin.clone();
|
||||
let preload = self.preload.clone();
|
||||
let prework = self.prework.clone();
|
||||
|
||||
let prog = self.prog.clone();
|
||||
let ongoing = self.ongoing.clone();
|
||||
|
|
@ -349,7 +364,7 @@ impl Scheduler {
|
|||
let result = match op {
|
||||
TaskOp::File(op) => file.work(*op).await,
|
||||
TaskOp::Plugin(op) => plugin.work(*op).await,
|
||||
TaskOp::Preload(op) => preload.work(*op).await,
|
||||
TaskOp::Prework(op) => prework.work(*op).await,
|
||||
};
|
||||
|
||||
if let Err(e) = result {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue