perf: avoid unnecessary allocations in code highlighting (#3804)

.github/workflows/cachix.yml

@@ -21,7 +21,7 @@ jobs:
         uses: cachix/install-nix-action@v31
 
       - name: Authenticate with Cachix
-        uses: cachix/cachix-action@v16
+        uses: cachix/cachix-action@v17
         with:
           name: yazi
           authToken: "${{ secrets.CACHIX_AUTH_TOKEN }}"

CHANGELOG.md

@@ -59,6 +59,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/):
 
 - Reduce memory allocations by using Lua 5.5 external strings ([#3634])
 - Reuse previewed and spotted widgets when possible ([#3765])
+- Avoid unnecessary allocations in code highlighting ([#3804])
 
 ## [v26.1.22]
 
@@ -1690,3 +1691,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/):
 [#3780]: https://github.com/sxyazi/yazi/pull/3780
 [#3781]: https://github.com/sxyazi/yazi/pull/3781
 [#3792]: https://github.com/sxyazi/yazi/pull/3792
+[#3804]: https://github.com/sxyazi/yazi/pull/3804

Cargo.lock

@@ -171,9 +171,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1"
 
 [[package]]
 name = "arc-swap"
-version = "1.8.2"
+version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f9f3647c145568cec02c42054e07bdf9a5a698e15b466fb2341bfc393cd24aa5"
+checksum = "a07d1f37ff60921c83bdfc7407723bdefe89b44b98a9b772f225c8f9d67141a6"
 dependencies = [
  "rustversion",
 ]
@@ -6131,6 +6131,8 @@ dependencies = [
  "crossterm 0.29.0",
  "ratatui",
  "twox-hash",
+ "unicode-segmentation",
+ "unicode-width",
  "yazi-macro",
 ]
 
@@ -6289,9 +6291,9 @@ dependencies = [
 
 [[package]]
 name = "zune-jpeg"
-version = "0.5.13"
+version = "0.5.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec5f41c76397b7da451efd19915684f727d7e1d516384ca6bd0ec43ec94de23c"
+checksum = "0b7a1c0af6e5d8d1363f4994b7a091ccf963d8b694f7da5b0b9cceb82da2c0a6"
 dependencies = [
  "zune-core",
 ]

cspell.json

@@ -1 +1 @@
-{"version":"0.2","flagWords":[],"words":["Punct","KEYMAP","splitn","crossterm","YAZI","peekable","ratatui","syntect","pbpaste","pbcopy","oneshot","Posix","Lsar","XADDOS","zoxide","cands","Deque","precache","imageops","IFBLK","IFCHR","IFDIR","IFIFO","IFLNK","IFMT","IFSOCK","IRGRP","IROTH","IRUSR","ISGID","ISUID","ISVTX","IWGRP","IWOTH","IWUSR","IXGRP","IXOTH","IXUSR","libc","winsize","TIOCGWINSZ","xpixel","ypixel","ioerr","appender","Catppuccin","macchiato","gitmodules","Dotfiles","bashprofile","vimrc","flac","webp","exiftool","mediainfo","ripgrep","indexmap","indexmap","unwatch","canonicalize","serde","fsevent","Ueberzug","iterm","wezterm","sixel","chafa","ueberzugpp","Konsole","Überzug","pkgs","pdftoppm","poppler","singlefile","jpegopt","EXIF","rustfmt","mktemp","nanos","xclip","xsel","natord","Mintty","nixos","nixpkgs","SIGTSTP","SIGCONT","SIGCONT","mlua","nonstatic","userdata","metatable","natsort","backstack","luajit","Succ","Succ","cand","fileencoding","foldmethod","lightgreen","darkgray","lightred","lightyellow","lightcyan","nushell","msvc","aarch","linemode","sxyazi","rsplit","ZELLIJ","bitflags","bitflags","USERPROFILE","Neovim","vergen","gitcl","Renderable","preloaders","prec","Upserting","prio","Ghostty","Catmull","Lanczos","cmds","unyank","scrolloff","headsup","unsub","uzers","scopeguard","SPDLOG","globset","filetime","magick","magick","prefetcher","Prework","prefetchers","PREWORKERS","conds","translit","rxvt","Urxvt","realpath","realname","REPARSE","hardlink","hardlinking","nlink","nlink","linemodes","SIGSTOP","sevenzip","rsplitn","replacen","DECSET","DECRQM","repeek","cwds","tcsi","Hyprland","Wayfire","SWAYSOCK","btime","nsec","codegen","gethostname","fchmod","fdfind","Rustc","rustc","ffprobe","vframes","luma","obase","outln","errln","tmtheme","twox","cfgs","fstype","objc","rdev","runloop","exfat","rclone","DECRQSS","DECSCUSR","libvterm","Uninit","lockin","rposition","resvg","foldhash","tilded","futs","chdir","hashbrown","JEMALLOC","RUSTFLAGS","RDONLY","GETPATH","fcntl","casefold","inodes","Splatable","casefied","thiserror","memchr","memmem","russh","deadpool","keepalive","nodelay","publickey","deadpool","initing","treelize","TOCTOU","fellback","watchee"],"language":"en"}
+{"language":"en","words":["Punct","KEYMAP","splitn","crossterm","YAZI","peekable","ratatui","syntect","pbpaste","pbcopy","oneshot","Posix","Lsar","XADDOS","zoxide","cands","Deque","precache","imageops","IFBLK","IFCHR","IFDIR","IFIFO","IFLNK","IFMT","IFSOCK","IRGRP","IROTH","IRUSR","ISGID","ISUID","ISVTX","IWGRP","IWOTH","IWUSR","IXGRP","IXOTH","IXUSR","libc","winsize","TIOCGWINSZ","xpixel","ypixel","ioerr","appender","Catppuccin","macchiato","gitmodules","Dotfiles","bashprofile","vimrc","flac","webp","exiftool","mediainfo","ripgrep","indexmap","indexmap","unwatch","canonicalize","serde","fsevent","Ueberzug","iterm","wezterm","sixel","chafa","ueberzugpp","Konsole","Überzug","pkgs","pdftoppm","poppler","singlefile","jpegopt","EXIF","rustfmt","mktemp","nanos","xclip","xsel","natord","Mintty","nixos","nixpkgs","SIGTSTP","SIGCONT","SIGCONT","mlua","nonstatic","userdata","metatable","natsort","backstack","luajit","Succ","Succ","cand","fileencoding","foldmethod","lightgreen","darkgray","lightred","lightyellow","lightcyan","nushell","msvc","aarch","linemode","sxyazi","rsplit","ZELLIJ","bitflags","bitflags","USERPROFILE","Neovim","vergen","gitcl","Renderable","preloaders","prec","Upserting","prio","Ghostty","Catmull","Lanczos","cmds","unyank","scrolloff","headsup","unsub","uzers","scopeguard","SPDLOG","globset","filetime","magick","magick","prefetcher","Prework","prefetchers","PREWORKERS","conds","translit","rxvt","Urxvt","realpath","realname","REPARSE","hardlink","hardlinking","nlink","nlink","linemodes","SIGSTOP","sevenzip","rsplitn","replacen","DECSET","DECRQM","repeek","cwds","tcsi","Hyprland","Wayfire","SWAYSOCK","btime","nsec","codegen","gethostname","fchmod","fdfind","Rustc","rustc","ffprobe","vframes","luma","obase","outln","errln","tmtheme","twox","cfgs","fstype","objc","rdev","runloop","exfat","rclone","DECRQSS","DECSCUSR","libvterm","Uninit","lockin","rposition","resvg","foldhash","tilded","futs","chdir","hashbrown","JEMALLOC","RUSTFLAGS","RDONLY","GETPATH","fcntl","casefold","inodes","Splatable","casefied","thiserror","memchr","memmem","russh","deadpool","keepalive","nodelay","publickey","deadpool","initing","treelize","TOCTOU","fellback","watchee","Textlike"],"flagWords":[],"version":"0.2"}

yazi-binding/src/elements/line.rs

@@ -51,6 +51,12 @@ impl Line {
 	}
 }
 
+impl From<ratatui::text::Line<'static>> for Line {
+	fn from(value: ratatui::text::Line<'static>) -> Self {
+		Self { inner: value, ..Default::default() }
+	}
+}
+
 impl TryFrom<Table> for Line {
 	type Error = mlua::Error;
 

yazi-fs/Cargo.toml

@@ -20,7 +20,7 @@ yazi-shim   = { path = "../yazi-shim", version = "26.2.2" }
 
 # External dependencies
 anyhow           = { workspace = true }
-arc-swap         = "1.8.2"
+arc-swap         = "1.9.0"
 bitflags         = { workspace = true }
 dirs             = { workspace = true }
 either           = { workspace = true }

yazi-plugin/preset/plugins/json.lua

@@ -11,9 +11,9 @@ function M:peek(job)
 		return require("code"):peek(job)
 	end
 
-	local wrap = rt.preview.wrap
+	local opt = { ansi = true, tab_size = rt.preview.tab_size, wrap = rt.preview.wrap, width = job.area.w }
 	local limit = job.area.h
-	local i, lines = 0, ""
+	local i, lines = 0, {}
 	repeat
 		local next, event = child:read_line()
 		if event == 1 then
@@ -22,9 +22,13 @@ function M:peek(job)
 			break
 		end
 
-		i = i + ui.height(next, { width = job.area.w, ansi = true, wrap = wrap })
+		local wrapped = ui.lines(next, opt)
-		if i > job.skip then
+		local from = math.max(1, job.skip - i + 1)
-			lines = lines .. next
+		local to = math.min(#wrapped, job.skip + limit - i)
+
+		i = i + #wrapped
+		for j = from, to do
+			lines[#lines + 1] = wrapped[j]
 		end
 	until i >= job.skip + limit
 
@@ -32,8 +36,7 @@ function M:peek(job)
 	if job.skip > 0 and i < job.skip + limit then
 		ya.emit("peek", { math.max(0, i - limit), only_if = job.file.url, upper_bound = true })
 	else
-		lines = lines:gsub("\t", string.rep(" ", rt.preview.tab_size))
+		ya.preview_widget(job, ui.Text(lines):area(job.area))
-		ya.preview_widget(job, ui.Text.parse(lines):area(job.area):wrap(wrap))
 	end
 end
 

yazi-plugin/src/elements/elements.rs

@@ -1,22 +1,22 @@
-use std::borrow::Cow;
+use std::{borrow::Cow, iter};
 
 use ansi_to_tui::IntoText;
 use mlua::{AnyUserData, ExternalError, ExternalResult, IntoLua, Lua, ObjectLike, Table, Value};
 use tracing::error;
 use unicode_width::{UnicodeWidthChar, UnicodeWidthStr};
 use yazi_binding::{Composer, ComposerGet, ComposerSet, Permit, PermitRef, elements::{Line, Rect, Span, Wrap}, runtime};
-use yazi_config::{LAYOUT, YAZI};
+use yazi_config::LAYOUT;
 use yazi_proxy::AppProxy;
 use yazi_shared::replace_to_printable;
-use yazi_shim::ratatui::line_count;
+use yazi_shim::ratatui::LineIter;
 use yazi_term::YIELD_TO_SUBPROCESS;
 
 pub fn compose() -> Composer<ComposerGet, ComposerSet> {
 	fn get(lua: &Lua, key: &[u8]) -> mlua::Result<Value> {
 		match key {
 			b"area" => area(lua)?,
-			b"height" => height(lua)?,
 			b"hide" => hide(lua)?,
+			b"lines" => lines(lua)?,
 			b"printable" => printable(lua)?,
 			b"redraw" => redraw(lua)?,
 			b"render" => render(lua)?,
@@ -46,21 +46,6 @@ pub(super) fn area(lua: &Lua) -> mlua::Result<Value> {
 	f.into_lua(lua)
 }
 
-pub(super) fn height(lua: &Lua) -> mlua::Result<Value> {
-	let f = lua.create_function(|_, (s, opts): (mlua::String, Table)| {
-		let width = opts.raw_get("width")?;
-		let wrap: Wrap = opts.raw_get("wrap")?;
-
-		Ok(if opts.raw_get("ansi")? {
-			line_count(s.to_string_lossy().to_text().into_lua_err()?, width, YAZI.preview.indent(), wrap)
-		} else {
-			line_count(s.to_string_lossy(), width, YAZI.preview.indent(), wrap)
-		})
-	})?;
-
-	f.into_lua(lua)
-}
-
 pub(super) fn hide(lua: &Lua) -> mlua::Result<Value> {
 	let f = lua.create_async_function(|lua, ()| async move {
 		if runtime!(lua)?.blocking {
@@ -81,6 +66,31 @@ pub(super) fn hide(lua: &Lua) -> mlua::Result<Value> {
 	f.into_lua(lua)
 }
 
+pub(super) fn lines(lua: &Lua) -> mlua::Result<Value> {
+	let f = lua.create_function(|lua, (s, opts): (mlua::String, Table)| {
+		let b = s.as_bytes();
+		let s = &*String::from_utf8_lossy(&b);
+
+		let tab_size = opts.raw_get("tab_size")?;
+		let mut it = if opts.raw_get("ansi")? {
+			LineIter::parsed(s.to_text().into_lua_err()?.lines, tab_size)
+		} else {
+			LineIter::source(&s, tab_size)
+		};
+
+		if let Some(wrap) = *opts.raw_get::<Wrap>("wrap")? {
+			it = it.wrapped(wrap, opts.raw_get("width")?);
+		}
+
+		lua.create_sequence_from(iter::from_fn(|| {
+			let (spans, align) = it.next()?;
+			Some(Line::from(spans.into_static_line().alignment(align)))
+		}))
+	})?;
+
+	f.into_lua(lua)
+}
+
 pub(super) fn printable(lua: &Lua) -> mlua::Result<Value> {
 	let f = lua.create_function(|lua, s: mlua::String| {
 		Ok(match replace_to_printable(&s.as_bytes(), false, 1, true) {

yazi-plugin/src/external/highlighter.rs

@@ -1,30 +1,147 @@
-use std::{io::Cursor, path::{Path, PathBuf}, sync::OnceLock};
+use std::{io::{BufRead, BufReader, Cursor, Seek}, path::PathBuf, sync::OnceLock};
 
-use anyhow::{Result, anyhow};
+use anyhow::{Result, bail};
 use ratatui::{layout::Size, text::{Line, Span, Text}};
 use syntect::{LoadingError, dumps, easy::HighlightLines, highlighting::{self, Theme, ThemeSet}, parsing::{SyntaxReference, SyntaxSet}};
-use tokio::io::{AsyncBufReadExt, AsyncSeekExt, BufReader};
+use yazi_config::{THEME, YAZI};
-use yazi_config::{THEME, YAZI, preview::PreviewWrap};
+use yazi_shared::{Id, Ids, errors::PeekError, replace_to_printable};
-use yazi_fs::provider::{Provider, local::Local};
+use yazi_shim::ratatui::LineIter;
-use yazi_shared::{Ids, errors::PeekError, push_printable_char};
-use yazi_shim::ratatui::line_count;
 
 static INCR: Ids = Ids::new();
 static SYNTECT: OnceLock<(Theme, SyntaxSet)> = OnceLock::new();
 
 pub struct Highlighter {
-	path: PathBuf,
+	path:   PathBuf,
+	reader: BufReader<std::fs::File>,
+
+	skip:   usize,
+	size:   Size,
+	ticket: Id,
+
+	theme:    &'static Theme,
+	syntaxes: &'static SyntaxSet,
+	inner:    Option<HighlightLines<'static>>,
+	syntax:   Option<&'static SyntaxReference>,
 }
 
 impl Highlighter {
-	pub fn new<P>(path: P) -> Self
+	pub async fn oneshot<P>(path: P, skip: usize, size: Size) -> Result<Text<'static>, PeekError>
 	where
 		P: Into<PathBuf>,
 	{
-		Self { path: path.into() }
+		let path = path.into();
+		tokio::task::spawn_blocking(move || Self::make(path, skip, size)?.highlight()).await?
 	}
 
-	pub fn init() -> (&'static Theme, &'static SyntaxSet) {
+	fn make<P>(path: P, skip: usize, size: Size) -> Result<Self>
+	where
+		P: Into<PathBuf>,
+	{
+		let path = path.into();
+		let (theme, syntaxes) = Self::load();
+
+		Ok(Self {
+			reader: BufReader::new(std::fs::File::open(&path)?),
+			path,
+
+			skip,
+			size,
+			ticket: INCR.current(),
+
+			theme,
+			syntaxes,
+			inner: None,
+			syntax: None,
+		})
+	}
+
+	pub fn abort() { INCR.next(); }
+
+	fn highlight(mut self) -> Result<Text<'static>, PeekError> {
+		self.load_syntax()?;
+		let mut plain = self.syntax.is_none();
+
+		let mut i = 0;
+		let mut buf = vec![];
+		let mut lines = Vec::with_capacity(self.size.height as usize);
+		let mut inspected = 0u16;
+		while self.reader.read_until(b'\n', &mut buf).is_ok_and(|n| n > 0) {
+			if Self::is_binary(&buf, &mut inspected) {
+				return Err("Binary file".into());
+			}
+
+			let remaining = Self::normalize_control_chars(&mut buf);
+			if remaining || buf.len() > 5000 {
+				plain = true;
+			}
+
+			self.ensure_not_cancelled()?;
+			if plain && !self.process_plain(&buf, &mut i, &mut lines)? {
+				break;
+			} else if !plain && !self.process_hyper(&buf, &mut i, &mut lines)? {
+				break;
+			}
+			buf.clear();
+		}
+
+		if self.skip > 0 && i < self.skip + self.size.height as usize {
+			return Err(PeekError::Exceed(i.saturating_sub(self.size.height as _)));
+		}
+
+		Ok(Text::from(lines))
+	}
+
+	fn process_plain(&mut self, buf: &[u8], i: &mut usize, lines: &mut Vec<Line>) -> Result<bool> {
+		let b = replace_to_printable(buf, true, YAZI.preview.tab_size, false);
+		let s = String::from_utf8_lossy(&b);
+
+		let mut it = LineIter::source(&s, YAZI.preview.tab_size);
+		if let Some(wrap) = YAZI.preview.wrap.into() {
+			it = it.wrapped(wrap, self.size.width);
+		}
+
+		while let Some((spans, _)) = it.next() {
+			*i += 1;
+			if *i > self.skip + self.size.height as usize {
+				return Ok(false);
+			} else if *i > self.skip {
+				lines.push(spans.into_static_line());
+			}
+			self.ensure_not_cancelled()?;
+		}
+		Ok(true)
+	}
+
+	fn process_hyper(&mut self, buf: &[u8], i: &mut usize, lines: &mut Vec<Line>) -> Result<bool> {
+		let Some(syntax) = self.syntax else { bail!("No syntax") };
+		let h = self.inner.get_or_insert_with(|| HighlightLines::new(syntax, self.theme));
+
+		let s = String::from_utf8_lossy(buf);
+		let line = Self::to_line_widget(h.highlight_line(&s, self.syntaxes)?);
+
+		let mut it = LineIter::parsed(vec![line], YAZI.preview.tab_size);
+		if let Some(wrap) = YAZI.preview.wrap.into() {
+			it = it.wrapped(wrap, self.size.width);
+		}
+
+		while let Some((spans, _)) = it.next() {
+			*i += 1;
+			if *i > self.skip + self.size.height as usize {
+				return Ok(false);
+			} else if *i > self.skip {
+				lines.push(spans.into_static_line());
+			}
+			self.ensure_not_cancelled()?;
+		}
+		Ok(true)
+	}
+
+	#[inline]
+	fn ensure_not_cancelled(&self) -> Result<(), PeekError> {
+		if self.ticket != INCR.current() { Err("Highlighting cancelled".into()) } else { Ok(()) }
+	}
+
+	fn load() -> (&'static Theme, &'static SyntaxSet) {
 		let f = || {
 			let theme = std::fs::File::open(&THEME.mgr.syntect_theme)
 				.map_err(LoadingError::Io)
@@ -40,114 +157,25 @@ impl Highlighter {
 		(theme, syntaxes)
 	}
 
-	pub fn abort() { INCR.next(); }
+	fn load_syntax(&mut self) -> Result<()> {
+		let name = self.path.file_name().map(|n| n.to_string_lossy()).unwrap_or_default();
+		if let Some(s) = self.syntaxes.find_syntax_by_extension(&name) {
+			self.syntax = Some(s);
+			return Ok(());
+		}
 
-	pub async fn highlight(&self, skip: usize, size: Size) -> Result<Text<'static>, PeekError> {
+		let ext = self.path.extension().map(|e| e.to_string_lossy()).unwrap_or_default();
-		let indent = YAZI.preview.indent();
+		if let Some(s) = self.syntaxes.find_syntax_by_extension(&ext) {
-		let mut reader = BufReader::new(Local::regular(&self.path).open().await?);
+			self.syntax = Some(s);
+			return Ok(());
+		}
 
-		let syntax = Self::find_syntax(&self.path, &mut reader).await;
-		let mut plain = syntax.is_err();
-
-		let mut before = Vec::with_capacity(if plain { 0 } else { skip });
-		let mut after = Vec::with_capacity(size.height as _);
-
-		let mut i = 0;
 		let mut buf = vec![];
-		let mut inspected = 0u16;
+		if self.reader.read_until(b'\n', &mut buf).is_ok_and(|n| n > 0) {
-		while reader.read_until(b'\n', &mut buf).await.is_ok_and(|n| n > 0) {
+			self.reader.rewind()?;
-			if Self::is_binary(&buf, &mut inspected) {
+			self.syntax = self.syntaxes.find_syntax_by_first_line(&String::from_utf8_lossy(&buf));
-				return Err("Binary file".into());
-			}
-
-			let remaining = Self::normalize_control_chars(&mut buf);
-			if !plain && (remaining || buf.len() > 5000) {
-				plain = true;
-				before.clear();
-			}
-
-			i += if i >= skip {
-				after.push(String::from_utf8_lossy(&buf).into_owned());
-				line_count(&**after.last().unwrap(), size.width, &indent, YAZI.preview.wrap)
-			} else if !plain {
-				before.push(String::from_utf8_lossy(&buf).into_owned());
-				line_count(&**before.last().unwrap(), size.width, &indent, YAZI.preview.wrap)
-			} else if YAZI.preview.wrap != PreviewWrap::No {
-				line_count(String::from_utf8_lossy(&buf), size.width, &indent, YAZI.preview.wrap)
-			} else {
-				1
-			};
-
-			buf.clear();
-			if i > skip + size.height as usize {
-				break;
-			}
 		}
-
+		Ok(())
-		if skip > 0 && i < skip + size.height as usize {
-			return Err(PeekError::Exceed(i.saturating_sub(size.height as _)));
-		}
-
-		Ok(if plain {
-			Text::from(Self::merge_highlight_lines(&after, YAZI.preview.tab_size))
-		} else {
-			Self::highlight_with(before, after, syntax.unwrap()).await?
-		})
-	}
-
-	async fn highlight_with(
-		before: Vec<String>,
-		after: Vec<String>,
-		syntax: &'static SyntaxReference,
-	) -> Result<Text<'static>, PeekError> {
-		let ticket = INCR.current();
-
-		tokio::task::spawn_blocking(move || {
-			let (theme, syntaxes) = Self::init();
-			let mut h = HighlightLines::new(syntax, theme);
-
-			for line in before {
-				if ticket != INCR.current() {
-					return Err("Highlighting cancelled".into());
-				}
-				h.highlight_line(&line, syntaxes).map_err(|e| anyhow!(e))?;
-			}
-
-			let indent = YAZI.preview.indent();
-			let mut lines = Vec::with_capacity(after.len());
-			for line in after {
-				if ticket != INCR.current() {
-					return Err("Highlighting cancelled".into());
-				}
-
-				let regions = h.highlight_line(&line, syntaxes).map_err(|e| anyhow!(e))?;
-				lines.push(Self::to_line_widget(regions, &indent));
-			}
-
-			Ok(Text::from(lines))
-		})
-		.await?
-	}
-
-	async fn find_syntax(
-		path: &Path,
-		reader: &mut BufReader<tokio::fs::File>,
-	) -> Result<&'static SyntaxReference> {
-		let (_, syntaxes) = Self::init();
-		let name = path.file_name().map(|n| n.to_string_lossy()).unwrap_or_default();
-		if let Some(s) = syntaxes.find_syntax_by_extension(&name) {
-			return Ok(s);
-		}
-
-		let ext = path.extension().map(|e| e.to_string_lossy()).unwrap_or_default();
-		if let Some(s) = syntaxes.find_syntax_by_extension(&ext) {
-			return Ok(s);
-		}
-
-		let mut line = String::new();
-		reader.read_line(&mut line).await?;
-		reader.rewind().await?;
-		syntaxes.find_syntax_by_first_line(&line).ok_or_else(|| anyhow!("No syntax found"))
 	}
 
 	#[inline(always)]
@@ -177,50 +205,35 @@ impl Highlighter {
 		}
 		remaining
 	}
-
-	fn merge_highlight_lines(s: &[String], tab_size: u8) -> String {
-		let mut buf = Vec::new();
-		buf.reserve_exact(s.iter().map(|s| s.len()).sum::<usize>() | 15);
-
-		for &b in s.iter().flat_map(|s| s.as_bytes()) {
-			push_printable_char(&mut buf, b, true, tab_size, false);
-		}
-		unsafe { String::from_utf8_unchecked(buf) }
-	}
 }
 
 impl Highlighter {
-	pub fn to_line_widget(regions: Vec<(highlighting::Style, &str)>, indent: &str) -> Line<'static> {
+	fn to_line_widget<'a>(regions: Vec<(highlighting::Style, &'a str)>) -> Line<'a> {
-		let spans: Vec<_> = regions
+		Line::from_iter(regions.into_iter().map(|(style, s)| {
-			.into_iter()
+			let mut modifier = ratatui::style::Modifier::empty();
-			.map(|(style, s)| {
+			if style.font_style.contains(highlighting::FontStyle::BOLD) {
-				let mut modifier = ratatui::style::Modifier::empty();
+				modifier |= ratatui::style::Modifier::BOLD;
-				if style.font_style.contains(highlighting::FontStyle::BOLD) {
+			}
-					modifier |= ratatui::style::Modifier::BOLD;
+			if style.font_style.contains(highlighting::FontStyle::ITALIC) {
-				}
+				modifier |= ratatui::style::Modifier::ITALIC;
-				if style.font_style.contains(highlighting::FontStyle::ITALIC) {
+			}
-					modifier |= ratatui::style::Modifier::ITALIC;
+			if style.font_style.contains(highlighting::FontStyle::UNDERLINE) {
-				}
+				modifier |= ratatui::style::Modifier::UNDERLINED;
-				if style.font_style.contains(highlighting::FontStyle::UNDERLINE) {
+			}
-					modifier |= ratatui::style::Modifier::UNDERLINED;
-				}
 
-				Span {
+			Span {
-					content: s.replace('\t', indent).into(),
+				content: s.into(),
-					style:   ratatui::style::Style {
+				style:   ratatui::style::Style {
-						fg: Self::to_ansi_color(style.foreground),
+					fg: Self::to_ansi_color(style.foreground),
-						// bg: Self::to_ansi_color(style.background),
+					// bg: Self::to_ansi_color(style.background),
-						add_modifier: modifier,
+					add_modifier: modifier,
-						..Default::default()
+					..Default::default()
-					},
+				},
-				}
+			}
-			})
+		}))
-			.collect();
-
-		Line::from(spans)
 	}
 
-	// Copy from https://github.com/sharkdp/bat/blob/master/src/terminal.rs
+	// Copied from https://github.com/sharkdp/bat/blob/master/src/terminal.rs
 	fn to_ansi_color(color: highlighting::Color) -> Option<ratatui::style::Color> {
 		if color.a == 0 {
 			// Themes can specify one of the user-configurable terminal colors by

yazi-plugin/src/utils/preview.rs

@@ -16,7 +16,7 @@ impl Utils {
 			let mut lock = PreviewLock::try_from(t)?;
 
 			let path = lock.url.as_url().unified_path();
-			let inner = match Highlighter::new(path).highlight(lock.skip, area.size()).await {
+			let inner = match Highlighter::oneshot(path, lock.skip, area.size()).await {
 				Ok(text) => text,
 				Err(e @ PeekError::Exceed(max)) => return (e.to_string(), max).into_lua_multi(&lua),
 				Err(e @ PeekError::Unexpected(_)) => {

yazi-shim/Cargo.toml

@@ -19,6 +19,10 @@ yazi-macro = { path = "../yazi-macro", version = "26.2.2" }
 crossterm = { workspace = true }
 ratatui   = { workspace = true }
 twox-hash = { workspace = true }
+unicode-width = { workspace = true }
+
+[dependencies.unicode-segmentation]
+version = "1"
 
 [target.'cfg(target_os = "macos")'.dependencies]
 crossterm = { workspace = true, features = [ "use-dev-tty", "libc" ] }

yazi-shim/src/ratatui/grapheme.rs

@@ -0,0 +1,47 @@
+// Copied from https://github.com/ratatui/ratatui/blob/main/ratatui-core/src/text/grapheme.rs
+use std::borrow::Cow;
+
+use ratatui::style::{Style, Styled};
+
+const NBSP: &str = "\u{00a0}";
+const ZWSP: &str = "\u{200b}";
+
+/// A grapheme associated to a style.
+/// Note that, although `StyledGrapheme` is the smallest divisible unit of text,
+/// it actually is not a member of the text type hierarchy (`Text` -> `Line` ->
+/// `Span`). It is a separate type used mostly for rendering purposes. A `Span`
+/// consists of components that can be split into `StyledGrapheme`s, but it does
+/// not contain a collection of `StyledGrapheme`s.
+#[derive(Debug, Default, Clone, Eq, PartialEq, Hash)]
+pub struct StyledGrapheme<'a> {
+	pub symbol: Cow<'a, str>,
+	pub style:  Style,
+}
+
+impl<'a> StyledGrapheme<'a> {
+	/// Creates a new `StyledGrapheme` with the given symbol and style.
+	///
+	/// `style` accepts any type that is convertible to [`Style`] (e.g. [`Style`],
+	/// [`Color`], or your own type that implements [`Into<Style>`]).
+	///
+	/// [`Color`]: crate::style::Color
+	pub fn new<S: Into<Cow<'a, str>>>(symbol: S, style: Style) -> Self {
+		Self { symbol: symbol.into(), style }
+	}
+
+	pub fn is_whitespace(&self) -> bool {
+		let symbol = &*self.symbol;
+		symbol == ZWSP || symbol.chars().all(char::is_whitespace) && symbol != NBSP
+	}
+}
+
+impl Styled for StyledGrapheme<'_> {
+	type Item = Self;
+
+	fn style(&self) -> Style { self.style }
+
+	fn set_style<S: Into<Style>>(mut self, style: S) -> Self::Item {
+		self.style = style.into();
+		self
+	}
+}

yazi-shim/src/ratatui/line.rs

@@ -0,0 +1,83 @@
+use std::{boxed::Box, iter, mem, str::{self, Lines}, vec::IntoIter};
+
+use ratatui::{layout::Alignment, text::Line, widgets::Wrap};
+
+use super::wrapper::{LineComposer, WordWrapper};
+use crate::ratatui::SpanIter;
+
+type WrappedLines<'text> = Box<dyn Iterator<Item = (SpanIter<'text, 'text>, Alignment)> + 'text>;
+
+pub struct LineIter<'text> {
+	inner:    LineIterInner<'text>,
+	tab_size: u8,
+}
+
+enum LineIterInner<'text> {
+	Source { empty: bool, lines: Lines<'text> },
+	Parsed(IntoIter<Line<'text>>),
+	Wrapped(WordWrapper<'text, WrappedLines<'text>, SpanIter<'text, 'text>>),
+	Unlimited(WrappedLines<'text>),
+}
+
+impl<'text> LineIter<'text> {
+	pub fn source(source: &'text str, tab_size: u8) -> Self {
+		Self {
+			inner: LineIterInner::Source { empty: source.is_empty(), lines: source.lines() },
+			tab_size,
+		}
+	}
+
+	pub fn parsed(mut text: Vec<Line<'text>>, tab_size: u8) -> Self {
+		if text.is_empty() {
+			text.push(Line::from(""));
+		}
+		Self { inner: LineIterInner::Parsed(text.into_iter()), tab_size }
+	}
+
+	pub fn wrapped(mut self, wrap: Wrap, width: u16) -> Self {
+		let lines = Box::new(iter::from_fn(move || self.next_owned()));
+		Self {
+			inner:    if width == 0 {
+				LineIterInner::Unlimited(lines)
+			} else {
+				LineIterInner::Wrapped(WordWrapper::new(lines, width, wrap.trim))
+			},
+			tab_size: 0,
+		}
+	}
+
+	pub fn next<'lend>(&'lend mut self) -> Option<(SpanIter<'lend, 'text>, Alignment)> {
+		match self.inner {
+			LineIterInner::Source { .. } | LineIterInner::Parsed(_) => self.next_owned(),
+			LineIterInner::Wrapped(ref mut wrapper) => {
+				let wrapped = wrapper.next_line()?;
+				Some((SpanIter::Wrapped(wrapped.graphemes.iter()), wrapped.alignment))
+			}
+			LineIterInner::Unlimited(ref mut lines) => lines.next(),
+		}
+	}
+
+	fn next_owned(&mut self) -> Option<(SpanIter<'text, 'text>, Alignment)> {
+		match &mut self.inner {
+			LineIterInner::Source { empty, lines } => {
+				let line = if mem::replace(empty, false) {
+					Some(Line::from(""))
+				} else {
+					lines.next().map(Line::from)
+				}?;
+				let alignment = line.alignment.unwrap_or(Alignment::Left);
+
+				Some((SpanIter::new(line, self.tab_size), alignment))
+			}
+			LineIterInner::Parsed(lines) => {
+				let line = lines.next()?;
+				let alignment = line.alignment.unwrap_or(Alignment::Left);
+
+				Some((SpanIter::new(line, self.tab_size), alignment))
+			}
+			LineIterInner::Wrapped(_) | LineIterInner::Unlimited(_) => {
+				unreachable!(); // This branch is handled by next() and should never call next_owned()
+			}
+		}
+	}
+}

yazi-shim/src/ratatui/mod.rs

@@ -1 +1 @@
-yazi_macro::mod_flat!(paragraph);
+yazi_macro::mod_flat!(grapheme line span wrapper);

yazi-shim/src/ratatui/paragraph.rs

@@ -1,52 +0,0 @@
-use ratatui::{text::Text, widgets::{Paragraph, Wrap}};
-
-pub fn line_count<'a, T, W, I>(text: T, width: u16, indent: I, wrap: W) -> usize
-where
-	T: Into<Text<'a>>,
-	I: AsRef<str>,
-	W: Into<Option<Wrap>>,
-{
-	line_count_impl(text.into(), width, indent.as_ref(), wrap.into())
-}
-
-fn line_count_impl(mut text: Text<'_>, mut width: u16, indent: &str, wrap: Option<Wrap>) -> usize {
-	width = width.max(1);
-
-	let Some(wrap) = wrap else {
-		return Paragraph::new(text).line_count(width);
-	};
-
-	if indent.len() == 1 {
-		return Paragraph::new(text).wrap(wrap).line_count(width);
-	}
-
-	let extra = indent.len().saturating_sub(1);
-	for line in &mut text.lines {
-		for span in &mut line.spans {
-			let mut out = None::<String>;
-			let mut start = 0;
-			for (idx, b) in span.content.bytes().enumerate() {
-				if b != b'\t' {
-					continue;
-				}
-
-				let out = out.get_or_insert_with(|| String::with_capacity(span.content.len() + extra));
-				if start < idx {
-					out.push_str(unsafe { span.content.get_unchecked(start..idx) });
-				}
-
-				out.push_str(indent);
-				start = idx + 1;
-			}
-
-			if let Some(mut out) = out {
-				if start < span.content.len() {
-					out.push_str(unsafe { span.content.get_unchecked(start..) });
-				}
-				span.content = out.into();
-			}
-		}
-	}
-
-	Paragraph::new(text).wrap(wrap).line_count(width)
-}

yazi-shim/src/ratatui/span.rs

@@ -0,0 +1,101 @@
+use std::{borrow::Cow, slice::Iter, vec::IntoIter};
+
+use ratatui::{style::Style, text::{Line, Span}};
+use unicode_segmentation::{Graphemes, UnicodeSegmentation};
+
+use crate::ratatui::StyledGrapheme;
+
+#[allow(private_interfaces)]
+pub enum SpanIter<'lend, 'text> {
+	Line {
+		spans:         IntoIter<Span<'text>>,
+		line_style:    Style,
+		current:       Option<CurrentSpan<'text>>,
+		tab_size:      u8,
+		pending_tabs:  u8,
+		pending_style: Style,
+	},
+	Wrapped(Iter<'lend, StyledGrapheme<'text>>),
+}
+
+impl<'lend, 'text> SpanIter<'lend, 'text> {
+	pub(super) fn new(line: Line<'text>, tab_size: u8) -> Self {
+		Self::Line {
+			spans: line.spans.into_iter(),
+			line_style: line.style,
+			current: None,
+			tab_size,
+			pending_tabs: 0,
+			pending_style: Style::default(),
+		}
+	}
+
+	pub fn into_static_line(self) -> Line<'static> {
+		Line::from_iter(self.map(|g| Span { style: g.style, content: g.symbol.into_owned().into() }))
+	}
+}
+
+impl<'lend, 'text> Iterator for SpanIter<'lend, 'text> {
+	type Item = StyledGrapheme<'text>;
+
+	fn next(&mut self) -> Option<Self::Item> {
+		match self {
+			Self::Wrapped(inner) => inner.next().cloned(),
+			Self::Line { spans, line_style, current, tab_size, pending_tabs, pending_style } => loop {
+				if *pending_tabs > 0 {
+					*pending_tabs -= 1;
+					return Some(StyledGrapheme::new(" ", *pending_style));
+				}
+
+				if let Some(span) = current
+					&& let Some(symbol) = span.next_symbol()
+				{
+					if symbol == "\t" {
+						if *tab_size == 0 {
+							continue;
+						}
+
+						*pending_tabs = tab_size.saturating_sub(1);
+						*pending_style = span.style();
+						return Some(StyledGrapheme::new(" ", span.style()));
+					}
+
+					return Some(StyledGrapheme::new(symbol, span.style()));
+				}
+
+				let span = spans.next()?;
+				*current = Some(match span.content {
+					Cow::Borrowed(content) => CurrentSpan::Borrowed {
+						style:     line_style.patch(span.style),
+						graphemes: content.graphemes(true),
+					},
+					Cow::Owned(content) => CurrentSpan::Owned {
+						style:     line_style.patch(span.style),
+						graphemes: content.graphemes(true).map(str::to_owned).collect::<Vec<_>>().into_iter(),
+					},
+				});
+			},
+		}
+	}
+}
+
+// --- CurrentSpan
+enum CurrentSpan<'text> {
+	Borrowed { style: Style, graphemes: Graphemes<'text> },
+	Owned { style: Style, graphemes: IntoIter<String> },
+}
+
+impl<'text> CurrentSpan<'text> {
+	fn style(&self) -> Style {
+		match self {
+			Self::Borrowed { style, .. } | Self::Owned { style, .. } => *style,
+		}
+	}
+
+	fn next_symbol(&mut self) -> Option<Cow<'text, str>> {
+		match self {
+			Self::Borrowed { graphemes, .. } => graphemes.next().map(Cow::Borrowed),
+			Self::Owned { graphemes, .. } => graphemes.next().map(Cow::Owned),
+		}
+	}
+}

yazi-shim/src/ratatui/wrapper.rs

@@ -0,0 +1,196 @@
+// Copied from https://github.com/ratatui/ratatui/blob/main/ratatui-widgets/src/reflow.rs
+use std::{collections::VecDeque, mem};
+
+use ratatui::layout::Alignment;
+use unicode_width::UnicodeWidthStr;
+
+use crate::ratatui::StyledGrapheme;
+
+pub trait LineComposer<'a> {
+	fn next_line<'lend>(&'lend mut self) -> Option<WrappedLine<'lend, 'a>>;
+}
+
+pub struct WrappedLine<'lend, 'text> {
+	pub graphemes: &'lend [StyledGrapheme<'text>],
+	pub width:     u16,
+	pub alignment: Alignment,
+}
+
+#[derive(Debug, Default, Clone)]
+pub struct WordWrapper<'a, O, I>
+where
+	O: Iterator<Item = (I, Alignment)>,
+	I: Iterator<Item = StyledGrapheme<'a>>,
+{
+	input_lines:       O,
+	max_line_width:    u16,
+	wrapped_lines:     VecDeque<Vec<StyledGrapheme<'a>>>,
+	current_alignment: Alignment,
+	current_line:      Vec<StyledGrapheme<'a>>,
+	trim:              bool,
+
+	pending_word:       Vec<StyledGrapheme<'a>>,
+	pending_whitespace: VecDeque<StyledGrapheme<'a>>,
+	pending_line_pool:  Vec<Vec<StyledGrapheme<'a>>>,
+}
+
+impl<'a, O, I> WordWrapper<'a, O, I>
+where
+	O: Iterator<Item = (I, Alignment)>,
+	I: Iterator<Item = StyledGrapheme<'a>>,
+{
+	pub const fn new(lines: O, max_line_width: u16, trim: bool) -> Self {
+		Self {
+			input_lines: lines,
+			max_line_width,
+			wrapped_lines: VecDeque::new(),
+			current_alignment: Alignment::Left,
+			current_line: vec![],
+			trim,
+
+			pending_word: Vec::new(),
+			pending_whitespace: VecDeque::new(),
+			pending_line_pool: Vec::new(),
+		}
+	}
+
+	fn process_input(&mut self, line_symbols: impl IntoIterator<Item = StyledGrapheme<'a>>) {
+		let mut pending_line = self.pending_line_pool.pop().unwrap_or_default();
+		let mut line_width = 0;
+		let mut word_width = 0;
+		let mut whitespace_width = 0;
+		let mut non_whitespace_previous = false;
+
+		self.pending_word.clear();
+		self.pending_whitespace.clear();
+		pending_line.clear();
+
+		for grapheme in line_symbols {
+			let is_whitespace = grapheme.is_whitespace();
+			let symbol_width = grapheme.symbol.width() as u16;
+
+			if symbol_width > self.max_line_width {
+				continue;
+			}
+
+			let word_found = non_whitespace_previous && is_whitespace;
+			let trimmed_overflow =
+				pending_line.is_empty() && self.trim && word_width + symbol_width > self.max_line_width;
+			let whitespace_overflow = pending_line.is_empty()
+				&& self.trim
+				&& whitespace_width + symbol_width > self.max_line_width;
+			let untrimmed_overflow = pending_line.is_empty()
+				&& !self.trim
+				&& word_width + whitespace_width + symbol_width > self.max_line_width;
+
+			if word_found || trimmed_overflow || whitespace_overflow || untrimmed_overflow {
+				if !pending_line.is_empty() || !self.trim {
+					pending_line.extend(self.pending_whitespace.drain(..));
+					line_width += whitespace_width;
+				}
+
+				pending_line.append(&mut self.pending_word);
+				line_width += word_width;
+
+				self.pending_whitespace.clear();
+				whitespace_width = 0;
+				word_width = 0;
+			}
+
+			let line_full = line_width >= self.max_line_width;
+			let pending_word_overflow =
+				symbol_width > 0 && line_width + whitespace_width + word_width >= self.max_line_width;
+
+			if line_full || pending_word_overflow {
+				let mut remaining_width = u16::saturating_sub(self.max_line_width, line_width);
+
+				self.wrapped_lines.push_back(mem::take(&mut pending_line));
+				line_width = 0;
+
+				while let Some(grapheme) = self.pending_whitespace.front() {
+					let width = grapheme.symbol.width() as u16;
+
+					if width > remaining_width {
+						break;
+					}
+
+					whitespace_width -= width;
+					remaining_width -= width;
+					self.pending_whitespace.pop_front();
+				}
+
+				if is_whitespace && self.pending_whitespace.is_empty() {
+					continue;
+				}
+			}
+
+			if is_whitespace {
+				whitespace_width += symbol_width;
+				self.pending_whitespace.push_back(grapheme);
+			} else {
+				word_width += symbol_width;
+				self.pending_word.push(grapheme);
+			}
+
+			non_whitespace_previous = !is_whitespace;
+		}
+
+		if pending_line.is_empty()
+			&& self.pending_word.is_empty()
+			&& !self.pending_whitespace.is_empty()
+			&& self.trim
+		{
+			self.wrapped_lines.push_back(vec![]);
+		}
+		if !pending_line.is_empty() || !self.trim {
+			pending_line.extend(self.pending_whitespace.drain(..));
+		}
+		pending_line.append(&mut self.pending_word);
+
+		#[expect(clippy::else_if_without_else)]
+		if !pending_line.is_empty() {
+			self.wrapped_lines.push_back(pending_line);
+		} else if pending_line.capacity() > 0 {
+			self.pending_line_pool.push(pending_line);
+		}
+		if self.wrapped_lines.is_empty() {
+			self.wrapped_lines.push_back(vec![]);
+		}
+	}
+
+	fn replace_current_line(&mut self, line: Vec<StyledGrapheme<'a>>) {
+		let cache = mem::replace(&mut self.current_line, line);
+		if cache.capacity() > 0 {
+			self.pending_line_pool.push(cache);
+		}
+	}
+}
+
+impl<'a, O, I> LineComposer<'a> for WordWrapper<'a, O, I>
+where
+	O: Iterator<Item = (I, Alignment)>,
+	I: Iterator<Item = StyledGrapheme<'a>>,
+{
+	fn next_line<'lend>(&'lend mut self) -> Option<WrappedLine<'lend, 'a>> {
+		if self.max_line_width == 0 {
+			return None;
+		}
+
+		loop {
+			if let Some(line) = self.wrapped_lines.pop_front() {
+				let line_width = line.iter().map(|grapheme| grapheme.symbol.width() as u16).sum();
+
+				self.replace_current_line(line);
+				return Some(WrappedLine {
+					graphemes: &self.current_line,
+					width:     line_width,
+					alignment: self.current_alignment,
+				});
+			}
+
+			let (line_symbols, line_alignment) = self.input_lines.next()?;
+			self.current_alignment = line_alignment;
+			self.process_input(line_symbols);
+		}
+	}
+}

yazi-tty/src/windows.rs

@@ -1,5 +1,4 @@
 // Copied from https://github.com/rust-lang/rust/blob/master/library/std/src/sys/pal/windows/stdio.rs
-
 use std::{mem::MaybeUninit, os::windows::io::RawHandle, str};
 
 use windows_sys::Win32::{Globalization::{CP_UTF8, MB_ERR_INVALID_CHARS, MultiByteToWideChar}, System::Console::WriteConsoleW};