feat: prefetcher (#1061)

This commit is contained in:
三咲雅 · Misaki Masa 2024-05-20 17:25:55 +08:00 committed by GitHub
parent c2affae3a9
commit bf1c325d00
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
24 changed files with 277 additions and 156 deletions

View file

@ -12,7 +12,7 @@ Yazi (means "duck") is a terminal file manager written in Rust, based on non-blo
- 💪 **Powerful Async Task Scheduling and Management**: Provides real-time progress updates, task cancellation, and internal task priority assignment.
- 🖼️ **Built-in Support for Multiple Image Protocols**: Also integrated with Überzug++, covering almost all terminals.
- 🌟 **Built-in Code Highlighting and Image Decoding**: Combined with the pre-loading mechanism, greatly accelerates image and normal file loading.
- 🔌 **Concurrent Plugin System**: UI plugins (rewriting most of the UI), functional plugins, custom previewer, and custom preloader; Just some pieces of Lua.
- 🔌 **Concurrent Plugin System**: UI plugins (rewriting most of the UI), functional plugins, custom previewer/preloader/prefetcher; Just some pieces of Lua.
- 📡 **Data Distribution Service**: Built on a client-server architecture (no additional server process required), integrated with a Lua-based publish-subscribe model, achieving cross-instance communication and state persistence.
- 📦 **Package Manager**: Install plugins and themes with one command, keeping them always up to date, or pin them to a specific version.
- 🧰 Integration with fd, rg, fzf, zoxide

View file

@ -1 +1 @@
{"language":"en","flagWords":[],"words":["Punct","KEYMAP","splitn","crossterm","YAZI","unar","peekable","ratatui","syntect","pbpaste","pbcopy","ffmpegthumbnailer","oneshot","Posix","Lsar","XADDOS","zoxide","cands","Deque","precache","imageops","IFBLK","IFCHR","IFDIR","IFIFO","IFLNK","IFMT","IFSOCK","IRGRP","IROTH","IRUSR","ISGID","ISUID","ISVTX","IWGRP","IWOTH","IWUSR","IXGRP","IXOTH","IXUSR","libc","winsize","TIOCGWINSZ","xpixel","ypixel","ioerr","appender","Catppuccin","macchiato","gitmodules","Dotfiles","bashprofile","vimrc","flac","webp","exiftool","mediainfo","ripgrep","nvim","indexmap","indexmap","unwatch","canonicalize","serde","fsevent","Ueberzug","iterm","wezterm","sixel","chafa","ueberzugpp"," Überzug"," Überzug","Konsole","Alacritty","Überzug","pkgs","paru","unarchiver","pdftoppm","poppler","prebuild","singlefile","jpegopt","EXIF","rustfmt","mktemp","nanos","xclip","xsel","natord","Mintty","nixos","nixpkgs","SIGTSTP","SIGCONT","SIGCONT","mlua","nonstatic","userdata","metatable","natsort","backstack","luajit","Succ","Succ","cand","fileencoding","foldmethod","lightgreen","darkgray","lightred","lightyellow","lightcyan","nushell","msvc","aarch","linemode","sxyazi","rsplit","ZELLIJ","bitflags","bitflags","USERPROFILE","Neovim","vergen","gitcl","Renderable","preloaders","prec","imagesize","Upserting","prio","Ghostty","Catmull","Lanczos","cmds","unyank","scrolloff","headsup","unsub","uzers","scopeguard","SPDLOG","globset","filetime","magick","magick"],"version":"0.2"}
{"words":["Punct","KEYMAP","splitn","crossterm","YAZI","unar","peekable","ratatui","syntect","pbpaste","pbcopy","ffmpegthumbnailer","oneshot","Posix","Lsar","XADDOS","zoxide","cands","Deque","precache","imageops","IFBLK","IFCHR","IFDIR","IFIFO","IFLNK","IFMT","IFSOCK","IRGRP","IROTH","IRUSR","ISGID","ISUID","ISVTX","IWGRP","IWOTH","IWUSR","IXGRP","IXOTH","IXUSR","libc","winsize","TIOCGWINSZ","xpixel","ypixel","ioerr","appender","Catppuccin","macchiato","gitmodules","Dotfiles","bashprofile","vimrc","flac","webp","exiftool","mediainfo","ripgrep","nvim","indexmap","indexmap","unwatch","canonicalize","serde","fsevent","Ueberzug","iterm","wezterm","sixel","chafa","ueberzugpp"," Überzug"," Überzug","Konsole","Alacritty","Überzug","pkgs","paru","unarchiver","pdftoppm","poppler","prebuild","singlefile","jpegopt","EXIF","rustfmt","mktemp","nanos","xclip","xsel","natord","Mintty","nixos","nixpkgs","SIGTSTP","SIGCONT","SIGCONT","mlua","nonstatic","userdata","metatable","natsort","backstack","luajit","Succ","Succ","cand","fileencoding","foldmethod","lightgreen","darkgray","lightred","lightyellow","lightcyan","nushell","msvc","aarch","linemode","sxyazi","rsplit","ZELLIJ","bitflags","bitflags","USERPROFILE","Neovim","vergen","gitcl","Renderable","preloaders","prec","imagesize","Upserting","prio","Ghostty","Catmull","Lanczos","cmds","unyank","scrolloff","headsup","unsub","uzers","scopeguard","SPDLOG","globset","filetime","magick","magick","prefetcher","Prework","prefetchers","PREWORKERS"],"version":"0.2","language":"en","flagWords":[]}

View file

@ -79,8 +79,11 @@ suppress_preload = false
[plugin]
prefetchers = [
# Mimetype
{ name = "*", cond = "!mime", run = "mime", prio = "high" },
]
preloaders = [
{ name = "*", cond = "!mime", run = "mime", next = true, multi = true, prio = "high" },
# Image
{ mime = "image/svg+xml", run = "magick" },
{ mime = "image/heic", run = "magick" },

View file

@ -1,9 +1,11 @@
mod plugin;
mod prefetcher;
mod preloader;
mod previewer;
pub use plugin::*;
pub use prefetcher::*;
pub use preloader::*;
pub use previewer::*;
pub const MAX_PRELOADERS: u8 = 32;
pub const MAX_PREWORKERS: u8 = 32;

View file

@ -3,13 +3,14 @@ use std::path::Path;
use serde::Deserialize;
use yazi_shared::MIME_DIR;
use super::{Preloader, Previewer};
use crate::{plugin::MAX_PRELOADERS, Preset, MERGED_YAZI};
use super::{Prefetcher, Preloader, Previewer};
use crate::{plugin::MAX_PREWORKERS, Preset, MERGED_YAZI};
#[derive(Deserialize)]
pub struct Plugin {
pub preloaders: Vec<Preloader>,
pub previewers: Vec<Previewer>,
pub prefetchers: Vec<Prefetcher>,
pub preloaders: Vec<Preloader>,
pub previewers: Vec<Previewer>,
}
impl Default for Plugin {
@ -21,6 +22,12 @@ impl Default for Plugin {
#[derive(Deserialize)]
struct Shadow {
prefetchers: Vec<Prefetcher>,
#[serde(default)]
prepend_prefetchers: Vec<Prefetcher>,
#[serde(default)]
append_prefetchers: Vec<Prefetcher>,
preloaders: Vec<Preloader>,
#[serde(default)]
prepend_preloaders: Vec<Preloader>,
@ -42,36 +49,53 @@ impl Default for Plugin {
shadow.previewers.retain(|r| !r.any_dir());
}
Preset::mix(&mut shadow.prefetchers, shadow.prepend_prefetchers, shadow.append_prefetchers);
Preset::mix(&mut shadow.preloaders, shadow.prepend_preloaders, shadow.append_preloaders);
Preset::mix(&mut shadow.previewers, shadow.prepend_previewers, shadow.append_previewers);
if shadow.preloaders.len() > MAX_PRELOADERS as usize {
panic!("Too many preloaders");
if shadow.prefetchers.len() + shadow.preloaders.len() > MAX_PREWORKERS as usize {
panic!("Prefetchers and preloaders exceed the limit of {MAX_PREWORKERS}");
}
for (i, preloader) in shadow.preloaders.iter_mut().enumerate() {
preloader.id = i as u8;
for (i, p) in shadow.prefetchers.iter_mut().enumerate() {
p.id = i as u8;
}
for (i, p) in shadow.preloaders.iter_mut().enumerate() {
p.id = shadow.prefetchers.len() as u8 + i as u8;
}
Self { preloaders: shadow.preloaders, previewers: shadow.previewers }
Self {
prefetchers: shadow.prefetchers,
preloaders: shadow.preloaders,
previewers: shadow.previewers,
}
}
}
impl Plugin {
pub fn preloaders(
pub fn prefetchers(
&self,
path: &Path,
mime: Option<&str>,
f: impl Fn(&str) -> bool + Copy,
) -> Vec<&Preloader> {
) -> Vec<&Prefetcher> {
let is_dir = mime == Some(MIME_DIR);
self
.prefetchers
.iter()
.filter(|&p| {
p.cond.as_ref().and_then(|c| c.eval(f)) != Some(false)
&& (p.mime.as_ref().zip(mime).map_or(false, |(p, m)| p.match_mime(m))
|| p.name.as_ref().is_some_and(|p| p.match_path(path, is_dir)))
})
.collect()
}
pub fn preloaders(&self, path: &Path, mime: Option<&str>) -> Vec<&Preloader> {
let is_dir = mime == Some(MIME_DIR);
let mut preloaders = Vec::with_capacity(1);
for p in &self.preloaders {
if p.cond.as_ref().and_then(|c| c.eval(f)) == Some(false) {
continue;
}
if !p.mime.as_ref().zip(mime).map_or(false, |(p, m)| p.match_mime(m))
&& !p.name.as_ref().is_some_and(|p| p.match_path(path, is_dir))
{

View file

@ -0,0 +1,29 @@
use serde::Deserialize;
use yazi_shared::{event::Cmd, Condition};
use crate::{Pattern, Priority};
#[derive(Debug, Deserialize)]
pub struct Prefetcher {
#[serde(skip)]
pub id: u8,
pub cond: Option<Condition>,
pub name: Option<Pattern>,
pub mime: Option<Pattern>,
pub run: Cmd,
#[serde(default)]
pub prio: Priority,
}
#[derive(Debug, Clone)]
pub struct PrefetcherProps {
pub id: u8,
pub name: String,
pub prio: Priority,
}
impl From<&Prefetcher> for PrefetcherProps {
fn from(prefetcher: &Prefetcher) -> Self {
Self { id: prefetcher.id, name: prefetcher.run.name.to_owned(), prio: prefetcher.prio }
}
}

View file

@ -1,39 +1,30 @@
use serde::Deserialize;
use yazi_shared::{event::Cmd, Condition};
use yazi_shared::event::Cmd;
use crate::{Pattern, Priority};
#[derive(Debug, Deserialize)]
pub struct Preloader {
#[serde(skip)]
pub id: u8,
pub cond: Option<Condition>,
pub name: Option<Pattern>,
pub mime: Option<Pattern>,
pub run: Cmd,
pub id: u8,
pub name: Option<Pattern>,
pub mime: Option<Pattern>,
pub run: Cmd,
#[serde(default)]
pub next: bool,
pub next: bool,
#[serde(default)]
pub multi: bool,
#[serde(default)]
pub prio: Priority,
pub prio: Priority,
}
#[derive(Debug, Clone)]
pub struct PreloaderProps {
pub id: u8,
pub name: String,
pub multi: bool,
pub prio: Priority,
pub id: u8,
pub name: String,
pub prio: Priority,
}
impl From<&Preloader> for PreloaderProps {
fn from(preloader: &Preloader) -> Self {
Self {
id: preloader.id,
name: preloader.run.name.to_owned(),
multi: preloader.multi,
prio: preloader.prio,
}
Self { id: preloader.id, name: preloader.run.name.to_owned(), prio: preloader.prio }
}
}

View file

@ -63,8 +63,8 @@ impl Manager {
}
done.extend(files.iter().map(|f| (f.url(), String::new())));
if let Err(e) = isolate::preload("mime", files, true).await {
error!("preload in open failed: {e}");
if let Err(e) = isolate::prefetch("mime", files).await {
error!("prefetch `mime` failed in opening: {e}");
}
ManagerProxy::open_do(OpenDoOpt { hovered, targets: done, interactive: opt.interactive });

View file

@ -35,6 +35,6 @@ impl Manager {
self.hover(None);
self.update_paged((), tasks);
tasks.preload_sorted(&self.current().files);
tasks.prework_sorted(&self.current().files);
}
}

View file

@ -88,7 +88,7 @@ impl Manager {
ManagerProxy::hover(None); // Re-hover in next loop
ManagerProxy::update_paged(); // Update for paged files in next loop
if calc {
tasks.preload_sorted(&tab.current.files);
tasks.prework_sorted(&tab.current.files);
}
}

View file

@ -52,7 +52,7 @@ impl Manager {
self.mimetype.extend(updates);
self.peek(false);
tasks.preload_affected(&affected, &self.mimetype);
tasks.prework_affected(&affected, &self.mimetype);
render!();
}
}

View file

@ -32,6 +32,7 @@ impl Manager {
}
let targets = self.current().paginate(opt.page.unwrap_or(self.current().page));
tasks.prefetch_paged(targets, &self.mimetype);
tasks.preload_paged(targets, &self.mimetype);
}
}

View file

@ -117,8 +117,8 @@ impl Watcher {
if reload.is_empty() {
continue;
}
if let Err(e) = isolate::preload("mime", reload, true).await {
error!("preload in watcher failed: {e}");
if let Err(e) = isolate::prefetch("mime", reload).await {
error!("prefetch `mime` failed in watcher: {e}");
}
}
}

View file

@ -18,6 +18,6 @@ impl Tab {
self.apply_files_attrs();
ManagerProxy::update_paged();
tasks.preload_sorted(&self.current.files);
tasks.prework_sorted(&self.current.files);
}
}

View file

@ -1,17 +1,15 @@
use std::{collections::HashMap, mem};
use std::collections::HashMap;
use yazi_config::{manager::SortBy, plugin::{Preloader, MAX_PRELOADERS}, PLUGIN};
use yazi_config::{manager::SortBy, plugin::MAX_PREWORKERS, PLUGIN};
use yazi_shared::{fs::{File, Url}, MIME_DIR};
use super::Tasks;
use crate::folder::Files;
impl Tasks {
pub fn preload_paged(&self, paged: &[File], mimetype: &HashMap<Url, String>) {
let mut single_tasks = Vec::with_capacity(paged.len());
let mut multi_tasks: [Vec<_>; MAX_PRELOADERS as usize] = Default::default();
let loaded = self.scheduler.preload.rule_loaded.read();
pub fn prefetch_paged(&self, paged: &[File], mimetype: &HashMap<Url, String>) {
let mut loaded = self.scheduler.prework.loaded.lock();
let mut tasks: [Vec<_>; MAX_PREWORKERS as usize] = Default::default();
for f in paged {
let mime = if f.is_dir() { Some(MIME_DIR) } else { mimetype.get(&f.url).map(|s| &**s) };
let factors = |s: &str| match s {
@ -19,61 +17,58 @@ impl Tasks {
_ => false,
};
for rule in PLUGIN.preloaders(&f.url, mime, factors) {
if loaded.get(&f.url).is_some_and(|x| x & (1 << rule.id) != 0) {
continue;
}
if rule.multi {
multi_tasks[rule.id as usize].push(f);
} else {
single_tasks.push((rule, f));
for p in PLUGIN.prefetchers(&f.url, mime, factors) {
match loaded.get_mut(&f.url) {
Some(n) if *n & (1 << p.id) != 0 => continue,
Some(n) => *n |= 1 << p.id,
None => _ = loaded.insert(f.url.clone(), 1 << p.id),
}
tasks[p.id as usize].push(f.clone());
}
}
drop(loaded);
let mut loaded = self.scheduler.preload.rule_loaded.write();
let mut go = |preloader: &Preloader, targets: Vec<&File>| {
for &f in &targets {
if let Some(n) = loaded.get_mut(&f.url) {
*n |= 1 << preloader.id;
} else {
loaded.insert(f.url.clone(), 1 << preloader.id);
}
for (i, tasks) in tasks.into_iter().enumerate() {
if !tasks.is_empty() {
self.scheduler.prefetch_paged(&PLUGIN.prefetchers[i], tasks);
}
self.scheduler.preload_paged(preloader, targets);
};
#[allow(clippy::needless_range_loop)]
for i in 0..PLUGIN.preloaders.len() {
if !multi_tasks[i].is_empty() {
go(&PLUGIN.preloaders[i], mem::take(&mut multi_tasks[i]));
}
}
for (rule, target) in single_tasks {
go(rule, vec![target]);
}
}
pub fn preload_affected(&self, affected: &[File], mimetype: &HashMap<Url, String>) {
pub fn preload_paged(&self, paged: &[File], mimetype: &HashMap<Url, String>) {
let mut loaded = self.scheduler.prework.loaded.lock();
for f in paged {
let mime = if f.is_dir() { Some(MIME_DIR) } else { mimetype.get(&f.url).map(|s| &**s) };
for p in PLUGIN.preloaders(&f.url, mime) {
match loaded.get_mut(&f.url) {
Some(n) if *n & (1 << p.id) != 0 => continue,
Some(n) => *n |= 1 << p.id,
None => _ = loaded.insert(f.url.clone(), 1 << p.id),
}
self.scheduler.preload_paged(p, f);
}
}
}
pub fn prework_affected(&self, affected: &[File], mimetype: &HashMap<Url, String>) {
{
let mut loaded = self.scheduler.preload.rule_loaded.write();
let mut loaded = self.scheduler.prework.loaded.lock();
for f in affected {
loaded.remove(&f.url);
}
}
self.prefetch_paged(affected, mimetype);
self.preload_paged(affected, mimetype);
}
pub fn preload_sorted(&self, targets: &Files) {
pub fn prework_sorted(&self, targets: &Files) {
if targets.sorter().by != SortBy::Size {
return;
}
let targets: Vec<_> = {
let loading = self.scheduler.preload.size_loading.read();
let loading = self.scheduler.prework.size_loading.read();
targets
.iter()
.filter(|f| f.is_dir() && !targets.sizes.contains_key(&f.url) && !loading.contains(&f.url))
@ -84,11 +79,11 @@ impl Tasks {
return;
}
let mut loading = self.scheduler.preload.size_loading.write();
let mut loading = self.scheduler.prework.size_loading.write();
for &target in &targets {
loading.insert(target.clone());
}
self.scheduler.preload_size(targets);
self.scheduler.prework_size(targets);
}
}

View file

@ -9,7 +9,7 @@ local function match_mimetype(s)
end
end
function M:preload()
function M:prefetch()
local urls = {}
for _, file in ipairs(self.files) do
urls[#urls + 1] = tostring(file.url)

View file

@ -3,11 +3,13 @@
mod entry;
mod isolate;
mod peek;
mod prefetch;
mod preload;
mod seek;
pub use entry::*;
pub use isolate::*;
pub use peek::*;
pub use prefetch::*;
pub use preload::*;
pub use seek::*;

View file

@ -0,0 +1,33 @@
use mlua::{ExternalError, ExternalResult, Table, TableExt};
use tokio::runtime::Handle;
use yazi_config::LAYOUT;
use super::slim_lua;
use crate::{bindings::{Cast, File}, elements::Rect, loader::LOADER};
pub async fn prefetch(name: &str, files: Vec<yazi_shared::fs::File>) -> mlua::Result<u8> {
LOADER.ensure(name).await.into_lua_err()?;
let name = name.to_owned();
tokio::task::spawn_blocking(move || {
let lua = slim_lua(&name)?;
let plugin: Table = if let Some(b) = LOADER.read().get(&name) {
lua.load(b.as_ref()).call(())?
} else {
return Err("unloaded plugin".into_lua_err());
};
let files = files.into_iter().filter_map(|f| File::cast(&lua, f).ok()).collect::<Vec<_>>();
if files.is_empty() {
return Err("no files".into_lua_err());
}
plugin.raw_set("skip", 0)?;
plugin.raw_set("area", Rect::cast(&lua, LAYOUT.load().preview)?)?;
plugin.raw_set("files", files)?;
Handle::current().block_on(plugin.call_async_method("prefetch", ()))
})
.await
.into_lua_err()?
}

View file

@ -5,11 +5,7 @@ use yazi_config::LAYOUT;
use super::slim_lua;
use crate::{bindings::{Cast, File}, elements::Rect, loader::LOADER};
pub async fn preload(
name: &str,
files: Vec<yazi_shared::fs::File>,
multi: bool,
) -> mlua::Result<u8> {
pub async fn preload(name: &str, file: yazi_shared::fs::File) -> mlua::Result<u8> {
LOADER.ensure(name).await.into_lua_err()?;
let name = name.to_owned();
@ -21,18 +17,9 @@ pub async fn preload(
return Err("unloaded plugin".into_lua_err());
};
let mut files = files.into_iter().filter_map(|f| File::cast(&lua, f).ok()).collect::<Vec<_>>();
if files.is_empty() {
return Err("no files".into_lua_err());
}
plugin.raw_set("skip", 0)?;
plugin.raw_set("area", Rect::cast(&lua, LAYOUT.load().preview)?)?;
if multi {
plugin.raw_set("files", files)?;
} else {
plugin.raw_set("file", files.remove(0))?;
}
plugin.raw_set("file", File::cast(&lua, file)?)?;
Handle::current().block_on(plugin.call_async_method("preload", ()))
})

View file

@ -1,10 +1,10 @@
use crate::{file::FileOp, plugin::PluginOp, preload::PreloadOp};
use crate::{file::FileOp, plugin::PluginOp, preload::PreworkOp};
#[derive(Debug)]
pub enum TaskOp {
File(Box<FileOp>),
Plugin(Box<PluginOp>),
Preload(Box<PreloadOp>),
Prework(Box<PreworkOp>),
}
impl TaskOp {
@ -12,7 +12,7 @@ impl TaskOp {
match self {
TaskOp::File(op) => op.id(),
TaskOp::Plugin(op) => op.id(),
TaskOp::Preload(op) => op.id(),
TaskOp::Prework(op) => op.id(),
}
}
}
@ -25,6 +25,6 @@ impl From<PluginOp> for TaskOp {
fn from(op: PluginOp) -> Self { Self::Plugin(Box::new(op)) }
}
impl From<PreloadOp> for TaskOp {
fn from(op: PreloadOp) -> Self { Self::Preload(Box::new(op)) }
impl From<PreworkOp> for TaskOp {
fn from(op: PreworkOp) -> Self { Self::Prework(Box::new(op)) }
}

View file

@ -1,7 +1,7 @@
#![allow(clippy::module_inception)]
mod op;
mod preload;
mod prework;
pub use op::*;
pub use preload::*;
pub use prework::*;

View file

@ -1,32 +1,41 @@
use std::sync::Arc;
use yazi_config::plugin::PreloaderProps;
use yazi_config::plugin::{PrefetcherProps, PreloaderProps};
use yazi_shared::{fs::Url, Throttle};
#[derive(Debug)]
pub enum PreloadOp {
Rule(PreloadOpRule),
Size(PreloadOpSize),
pub enum PreworkOp {
Fetch(PreworkOpFetch),
Load(PreworkOpLoad),
Size(PreworkOpSize),
}
impl PreloadOp {
impl PreworkOp {
pub fn id(&self) -> usize {
match self {
Self::Rule(op) => op.id,
Self::Fetch(op) => op.id,
Self::Load(op) => op.id,
Self::Size(op) => op.id,
}
}
}
#[derive(Clone, Debug)]
pub struct PreloadOpRule {
pub struct PreworkOpFetch {
pub id: usize,
pub plugin: PreloaderProps,
pub plugin: PrefetcherProps,
pub targets: Vec<yazi_shared::fs::File>,
}
#[derive(Clone, Debug)]
pub struct PreworkOpLoad {
pub id: usize,
pub plugin: PreloaderProps,
pub target: yazi_shared::fs::File,
}
#[derive(Debug)]
pub struct PreloadOpSize {
pub struct PreworkOpSize {
pub id: usize,
pub target: Url,
pub throttle: Arc<Throttle<(Url, u64)>>,

View file

@ -1,37 +1,57 @@
use std::collections::{HashMap, HashSet};
use anyhow::{anyhow, Result};
use parking_lot::RwLock;
use parking_lot::{Mutex, RwLock};
use tokio::sync::mpsc;
use tracing::error;
use yazi_config::Priority;
use yazi_plugin::isolate;
use yazi_shared::fs::{calculate_size, FilesOp, Url};
use super::{PreloadOp, PreloadOpRule, PreloadOpSize};
use super::{PreworkOp, PreworkOpFetch, PreworkOpLoad, PreworkOpSize};
use crate::{TaskOp, TaskProg, HIGH, NORMAL};
pub struct Preload {
pub struct Prework {
macro_: async_priority_channel::Sender<TaskOp, u8>,
prog: mpsc::UnboundedSender<TaskProg>,
pub rule_loaded: RwLock<HashMap<Url, u32>>,
pub loaded: Mutex<HashMap<Url, u32>>,
pub size_loading: RwLock<HashSet<Url>>,
}
impl Preload {
impl Prework {
pub fn new(
macro_: async_priority_channel::Sender<TaskOp, u8>,
prog: mpsc::UnboundedSender<TaskProg>,
) -> Self {
Self { macro_, prog, rule_loaded: Default::default(), size_loading: Default::default() }
Self { macro_, prog, loaded: Default::default(), size_loading: Default::default() }
}
pub async fn work(&self, op: PreloadOp) -> Result<()> {
pub async fn work(&self, op: PreworkOp) -> Result<()> {
match op {
PreloadOp::Rule(task) => {
PreworkOp::Fetch(task) => {
let urls: Vec<_> = task.targets.iter().map(|f| f.url()).collect();
let result = isolate::preload(&task.plugin.name, task.targets, task.plugin.multi).await;
let result = isolate::prefetch(&task.plugin.name, task.targets).await;
if let Err(e) = result {
self.fail(task.id, format!("Prefetch task failed:\n{e}"))?;
return Err(e.into());
};
let code = result.unwrap();
if code & 1 == 0 {
error!("Prefetch task `{}` returned {code}", task.plugin.name);
}
if code >> 1 & 1 != 0 {
let mut loaded = self.loaded.lock();
for url in urls {
loaded.get_mut(&url).map(|x| *x ^= 1 << task.plugin.id);
}
}
self.prog.send(TaskProg::Adv(task.id, 1, 0))?;
}
PreworkOp::Load(task) => {
let url = task.target.url();
let result = isolate::preload(&task.plugin.name, task.target).await;
if let Err(e) = result {
self.fail(task.id, format!("Preload task failed:\n{e}"))?;
return Err(e.into());
@ -42,14 +62,12 @@ impl Preload {
error!("Preload task `{}` returned {code}", task.plugin.name);
}
if code >> 1 & 1 != 0 {
let mut loaded = self.rule_loaded.write();
for url in urls {
loaded.get_mut(&url).map(|x| *x ^= 1 << task.plugin.id);
}
let mut loaded = self.loaded.lock();
loaded.get_mut(&url).map(|x| *x ^= 1 << task.plugin.id);
}
self.prog.send(TaskProg::Adv(task.id, 1, 0))?;
}
PreloadOp::Size(task) => {
PreworkOp::Size(task) => {
let length = calculate_size(&task.target).await;
task.throttle.done((task.target, length), |buf| {
{
@ -68,28 +86,40 @@ impl Preload {
Ok(())
}
pub async fn rule(&self, task: PreloadOpRule) -> Result<()> {
pub async fn fetch(&self, task: PreworkOpFetch) -> Result<()> {
let id = task.id;
self.prog.send(TaskProg::New(id, 0))?;
match task.plugin.prio {
Priority::Low => self.queue(PreloadOp::Rule(task), NORMAL).await?,
Priority::Normal => self.queue(PreloadOp::Rule(task), HIGH).await?,
Priority::High => self.work(PreloadOp::Rule(task)).await?,
Priority::Low => self.queue(PreworkOp::Fetch(task), NORMAL).await?,
Priority::Normal => self.queue(PreworkOp::Fetch(task), HIGH).await?,
Priority::High => self.work(PreworkOp::Fetch(task)).await?,
}
self.succ(id)
}
pub async fn size(&self, task: PreloadOpSize) -> Result<()> {
pub async fn load(&self, task: PreworkOpLoad) -> Result<()> {
let id = task.id;
self.prog.send(TaskProg::New(id, 0))?;
match task.plugin.prio {
Priority::Low => self.queue(PreworkOp::Load(task), NORMAL).await?,
Priority::Normal => self.queue(PreworkOp::Load(task), HIGH).await?,
Priority::High => self.work(PreworkOp::Load(task)).await?,
}
self.succ(id)
}
pub async fn size(&self, task: PreworkOpSize) -> Result<()> {
let id = task.id;
self.prog.send(TaskProg::New(id, 0))?;
self.work(PreloadOp::Size(task)).await?;
self.work(PreworkOp::Size(task)).await?;
self.succ(id)
}
}
impl Preload {
impl Prework {
#[inline]
fn succ(&self, id: usize) -> Result<()> { Ok(self.prog.send(TaskProg::Succ(id))?) }

View file

@ -4,17 +4,17 @@ use anyhow::Result;
use futures::{future::BoxFuture, FutureExt};
use parking_lot::Mutex;
use tokio::{fs, select, sync::{mpsc::{self, UnboundedReceiver}, oneshot}, task::JoinHandle};
use yazi_config::{open::Opener, plugin::Preloader, TASKS};
use yazi_config::{open::Opener, plugin::{Prefetcher, Preloader}, TASKS};
use yazi_dds::Pump;
use yazi_shared::{event::Data, fs::{unique_path, Url}, Throttle};
use super::{Ongoing, TaskProg, TaskStage};
use crate::{file::{File, FileOpDelete, FileOpLink, FileOpPaste, FileOpTrash}, plugin::{Plugin, PluginOpEntry}, preload::{Preload, PreloadOpRule, PreloadOpSize}, process::{Process, ProcessOpBg, ProcessOpBlock, ProcessOpOrphan}, TaskKind, TaskOp, HIGH, LOW, NORMAL};
use crate::{file::{File, FileOpDelete, FileOpLink, FileOpPaste, FileOpTrash}, plugin::{Plugin, PluginOpEntry}, preload::{Prework, PreworkOpFetch, PreworkOpLoad, PreworkOpSize}, process::{Process, ProcessOpBg, ProcessOpBlock, ProcessOpOrphan}, TaskKind, TaskOp, HIGH, LOW, NORMAL};
pub struct Scheduler {
pub file: Arc<File>,
pub plugin: Arc<Plugin>,
pub preload: Arc<Preload>,
pub prework: Arc<Prework>,
pub process: Arc<Process>,
micro: async_priority_channel::Sender<BoxFuture<'static, ()>, u8>,
@ -32,7 +32,7 @@ impl Scheduler {
let mut scheduler = Self {
file: Arc::new(File::new(macro_tx.clone(), prog_tx.clone())),
plugin: Arc::new(Plugin::new(macro_tx.clone(), prog_tx.clone())),
preload: Arc::new(Preload::new(macro_tx.clone(), prog_tx.clone())),
prework: Arc::new(Prework::new(macro_tx.clone(), prog_tx.clone())),
process: Arc::new(Process::new(prog_tx.clone())),
micro: micro_tx,
@ -218,25 +218,40 @@ impl Scheduler {
self.plugin.macro_(PluginOpEntry { id, name, args }).ok();
}
pub fn preload_paged(&self, preloader: &Preloader, targets: Vec<&yazi_shared::fs::File>) {
pub fn prefetch_paged(&self, prefetcher: &Prefetcher, targets: Vec<yazi_shared::fs::File>) {
let id = self.ongoing.lock().add(
TaskKind::Preload,
format!("Run preloader `{}` with {} target(s)", preloader.run.name, targets.len()),
format!("Run prefetcher `{}` with {} target(s)", prefetcher.run.name, targets.len()),
);
let plugin = preloader.into();
let targets = targets.into_iter().cloned().collect();
let preload = self.preload.clone();
let plugin = prefetcher.into();
let prework = self.prework.clone();
_ = self.micro.try_send(
async move {
preload.rule(PreloadOpRule { id, plugin, targets }).await.ok();
prework.fetch(PreworkOpFetch { id, plugin, targets }).await.ok();
}
.boxed(),
NORMAL,
);
}
pub fn preload_size(&self, targets: Vec<&Url>) {
pub fn preload_paged(&self, preloader: &Preloader, target: &yazi_shared::fs::File) {
let id =
self.ongoing.lock().add(TaskKind::Preload, format!("Run preloader `{}`", preloader.run.name));
let plugin = preloader.into();
let target = target.clone();
let prework = self.prework.clone();
_ = self.micro.try_send(
async move {
prework.load(PreworkOpLoad { id, plugin, target }).await.ok();
}
.boxed(),
NORMAL,
);
}
pub fn prework_size(&self, targets: Vec<&Url>) {
let throttle = Arc::new(Throttle::new(targets.len(), Duration::from_millis(300)));
let mut ongoing = self.ongoing.lock();
@ -245,10 +260,10 @@ impl Scheduler {
let target = target.clone();
let throttle = throttle.clone();
let preload = self.preload.clone();
let prework = self.prework.clone();
_ = self.micro.try_send(
async move {
preload.size(PreloadOpSize { id, target, throttle }).await.ok();
prework.size(PreworkOpSize { id, target, throttle }).await.ok();
}
.boxed(),
NORMAL,
@ -329,7 +344,7 @@ impl Scheduler {
) -> JoinHandle<()> {
let file = self.file.clone();
let plugin = self.plugin.clone();
let preload = self.preload.clone();
let prework = self.prework.clone();
let prog = self.prog.clone();
let ongoing = self.ongoing.clone();
@ -349,7 +364,7 @@ impl Scheduler {
let result = match op {
TaskOp::File(op) => file.work(*op).await,
TaskOp::Plugin(op) => plugin.work(*op).await,
TaskOp::Preload(op) => preload.work(*op).await,
TaskOp::Prework(op) => prework.work(*op).await,
};
if let Err(e) = result {