perf: string interning to reduce memory usage of mimetype and URL domain (#3084)

yazi-actor/src/mgr/update_mimes.rs

@@ -3,7 +3,7 @@ use hashbrown::HashMap;
 use yazi_core::mgr::LINKED;
 use yazi_macro::{act, render, succ};
 use yazi_parser::mgr::UpdateMimesOpt;
-use yazi_shared::{event::Data, url::UrlCov};
+use yazi_shared::{event::Data, pool::InternStr, url::UrlCov};
 
 use crate::{Actor, Ctx};
 
@@ -23,9 +23,9 @@ impl Actor for UpdateMimes {
 			.filter(|(url, mime)| cx.mgr.mimetype.by_url(url) != Some(mime))
 			.fold(HashMap::new(), |mut map, (u, m)| {
 				for u in linked.from_file(u.as_url()) {
-					map.insert(u.into(), m.to_string());
+					map.insert(u.into(), m.intern());
 				}
-				map.insert(u.into(), m.into_owned());
+				map.insert(u.into(), m.intern());
 				map
 			});
 

yazi-core/src/mgr/mimetype.rs

@@ -1,21 +1,19 @@
-use std::borrow::Cow;
-
 use hashbrown::HashMap;
 use yazi_fs::File;
-use yazi_shared::{MIME_DIR, SStr, url::{Url, UrlBufCov, UrlCov}};
+use yazi_shared::{MIME_DIR, pool::{InternStr, Symbol}, url::{Url, UrlBufCov, UrlCov}};
 
 #[derive(Default)]
-pub struct Mimetype(HashMap<UrlBufCov, String>);
+pub struct Mimetype(HashMap<UrlBufCov, Symbol<str>>);
 
 impl Mimetype {
 	#[inline]
 	pub fn by_url<'a>(&self, url: impl Into<Url<'a>>) -> Option<&str> {
-		self.0.get(&UrlCov::new(url)).map(|s| s.as_str())
+		self.0.get(&UrlCov::new(url)).map(|s| s.as_ref())
 	}
 
 	#[inline]
-	pub fn by_url_owned<'a>(&self, url: impl Into<Url<'a>>) -> Option<SStr> {
+	pub fn by_url_owned<'a>(&self, url: impl Into<Url<'a>>) -> Option<Symbol<str>> {
-		self.by_url(url).map(|s| Cow::Owned(s.to_owned()))
+		self.0.get(&UrlCov::new(url)).cloned()
 	}
 
 	#[inline]
@@ -24,8 +22,8 @@ impl Mimetype {
 	}
 
 	#[inline]
-	pub fn by_file_owned(&self, file: &File) -> Option<SStr> {
+	pub fn by_file_owned(&self, file: &File) -> Option<Symbol<str>> {
-		if file.is_dir() { Some(Cow::Borrowed(MIME_DIR)) } else { self.by_url_owned(&file.url) }
+		if file.is_dir() { Some(MIME_DIR.intern()) } else { self.by_url_owned(&file.url) }
 	}
 
 	#[inline]
@@ -34,7 +32,7 @@ impl Mimetype {
 	}
 
 	#[inline]
-	pub fn extend(&mut self, iter: impl IntoIterator<Item = (UrlBufCov, String)>) {
+	pub fn extend(&mut self, iter: impl IntoIterator<Item = (UrlBufCov, Symbol<str>)>) {
 		self.0.extend(iter);
 	}
 }

yazi-core/src/spot/spot.rs

@@ -4,7 +4,7 @@ use yazi_fs::File;
 use yazi_macro::render;
 use yazi_parser::mgr::SpotLock;
 use yazi_plugin::isolate;
-use yazi_shared::{SStr, url::UrlBuf};
+use yazi_shared::{pool::Symbol, url::UrlBuf};
 
 #[derive(Default)]
 pub struct Spot {
@@ -15,7 +15,7 @@ pub struct Spot {
 }
 
 impl Spot {
-	pub fn go(&mut self, file: File, mime: SStr) {
+	pub fn go(&mut self, file: File, mime: Symbol<str>) {
 		if mime.is_empty() {
 			return; // Wait till mimetype is resolved to avoid flickering
 		} else if self.same_lock(&file, &mime) {

yazi-core/src/tab/preview.rs

@@ -1,4 +1,4 @@
-use std::{borrow::Cow, time::Duration};
+use std::time::Duration;
 
 use tokio::{pin, task::JoinHandle};
 use tokio_stream::{StreamExt, wrappers::UnboundedReceiverStream};
@@ -9,7 +9,7 @@ use yazi_fs::{File, Files, FilesOp, cha::Cha};
 use yazi_macro::render;
 use yazi_parser::mgr::PreviewLock;
 use yazi_plugin::{external::Highlighter, isolate};
-use yazi_shared::{MIME_DIR, SStr, url::UrlBuf};
+use yazi_shared::{MIME_DIR, pool::{InternStr, Symbol}, url::UrlBuf};
 
 #[derive(Default)]
 pub struct Preview {
@@ -21,7 +21,7 @@ pub struct Preview {
 }
 
 impl Preview {
-	pub fn go(&mut self, file: File, mime: SStr, force: bool) {
+	pub fn go(&mut self, file: File, mime: Symbol<str>, force: bool) {
 		if mime.is_empty() {
 			return; // Wait till mimetype is resolved to avoid flickering
 		} else if !force && self.same_lock(&file, &mime) {
@@ -40,7 +40,7 @@ impl Preview {
 		let same = self.same_file(&file, MIME_DIR);
 		let (wd, cha, internal) = (file.url_owned(), file.cha, file.url.is_internal());
 
-		self.go(file, Cow::Borrowed(MIME_DIR), force);
+		self.go(file, MIME_DIR.intern(), force);
 		if same || !internal {
 			return;
 		}

yazi-fs/src/path/clean.rs

@@ -54,6 +54,7 @@ mod tests {
 
 	#[test]
 	fn test_clean_url() -> anyhow::Result<()> {
+		yazi_shared::init_tests();
 		let cases = [
 			// CurDir
 			("archive://:3//./tmp/test.zip/foo/bar", "archive://:3//tmp/test.zip/foo/bar"),

yazi-fs/src/path/expand.rs

@@ -106,6 +106,7 @@ mod tests {
 	#[cfg(unix)]
 	#[test]
 	fn test_expand_url() -> Result<()> {
+		yazi_shared::init_tests();
 		unsafe {
 			std::env::set_var("FOO", "foo");
 			std::env::set_var("BAR_BAZ", "bar/baz");

yazi-fs/src/path/path.rs

@@ -134,6 +134,8 @@ mod tests {
 
 	#[test]
 	fn test_path_relative_to() {
+		yazi_shared::init_tests();
+
 		fn assert(from: &str, to: &str, ret: &str) {
 			assert_eq!(
 				url_relative_to(&from.parse().unwrap(), &to.parse().unwrap()).unwrap(),

yazi-plugin/src/isolate/peek.rs

@@ -7,7 +7,7 @@ use yazi_config::LAYOUT;
 use yazi_dds::Sendable;
 use yazi_parser::app::{PluginCallback, PluginOpt};
 use yazi_proxy::AppProxy;
-use yazi_shared::{SStr, event::Cmd};
+use yazi_shared::{event::Cmd, pool::Symbol};
 
 use super::slim_lua;
 use crate::loader::LOADER;
@@ -15,7 +15,7 @@ use crate::loader::LOADER;
 pub fn peek(
 	cmd: &'static Cmd,
 	file: yazi_fs::File,
-	mime: SStr,
+	mime: Symbol<str>,
 	skip: usize,
 ) -> Option<CancellationToken> {
 	let ct = CancellationToken::new();
@@ -46,7 +46,7 @@ pub fn peek(
 	Some(ct)
 }
 
-fn peek_sync(cmd: &'static Cmd, file: yazi_fs::File, mime: SStr, skip: usize) {
+fn peek_sync(cmd: &'static Cmd, file: yazi_fs::File, mime: Symbol<str>, skip: usize) {
 	let cb: PluginCallback = Box::new(move |lua, plugin| {
 		let job = lua.create_table_from([
 			("area", Rect::from(LAYOUT.get().preview).into_lua(lua)?),
@@ -65,7 +65,7 @@ fn peek_sync(cmd: &'static Cmd, file: yazi_fs::File, mime: SStr, skip: usize) {
 fn peek_async(
 	cmd: &'static Cmd,
 	file: yazi_fs::File,
-	mime: SStr,
+	mime: Symbol<str>,
 	skip: usize,
 	ct: CancellationToken,
 ) {

yazi-plugin/src/isolate/spot.rs

@@ -4,14 +4,19 @@ use tokio_util::sync::CancellationToken;
 use tracing::error;
 use yazi_binding::{File, Id};
 use yazi_dds::Sendable;
-use yazi_shared::{Ids, SStr, event::Cmd};
+use yazi_shared::{Ids, event::Cmd, pool::Symbol};
 
 use super::slim_lua;
 use crate::loader::LOADER;
 
 static IDS: Ids = Ids::new();
 
-pub fn spot(cmd: &'static Cmd, file: yazi_fs::File, mime: SStr, skip: usize) -> CancellationToken {
+pub fn spot(
+	cmd: &'static Cmd,
+	file: yazi_fs::File,
+	mime: Symbol<str>,
+	skip: usize,
+) -> CancellationToken {
 	let ct = CancellationToken::new();
 	let (ct1, ct2) = (ct.clone(), ct.clone());
 

yazi-shared/src/lib.rs

@@ -1,10 +1,12 @@
 #![allow(clippy::option_map_unit_fn)]
 
-yazi_macro::mod_pub!(errors event loc shell translit url);
+yazi_macro::mod_pub!(errors event loc pool shell translit url);
 
-yazi_macro::mod_flat!(alias bytes chars condition debounce either env id layer natsort os osstr rand ro_cell source string sync_cell terminal throttle time utf8);
+yazi_macro::mod_flat!(alias bytes chars condition debounce either env id layer natsort os osstr rand ro_cell source string sync_cell terminal tests throttle time utf8);
 
 pub fn init() {
+	pool::init();
+
 	LOG_LEVEL.replace(<_>::from(std::env::var("YAZI_LOG").unwrap_or_default()));
 
 	#[cfg(unix)]

yazi-shared/src/loc/buf.rs

@@ -253,6 +253,7 @@ mod tests {
 
 	#[test]
 	fn test_set_name() -> Result<()> {
+		crate::init_tests();
 		let cases = [
 			// Regular
 			("/", "a", "regular:///a"),

yazi-shared/src/pool/mod.rs

@@ -0,0 +1,6 @@
+yazi_macro::mod_flat!(pool ptr symbol traits);
+
+static SYMBOLS: crate::RoCell<parking_lot::Mutex<hashbrown::HashMap<SymbolPtr, u64>>> =
+	crate::RoCell::new();
+
+pub(super) fn init() { SYMBOLS.with(<_>::default); }

yazi-shared/src/pool/pool.rs

@@ -0,0 +1,31 @@
+use std::marker::PhantomData;
+
+use crate::pool::{SYMBOLS, Symbol, SymbolPtr};
+
+pub struct Pool<T: ?Sized> {
+	_phantom: PhantomData<T>,
+}
+
+impl Pool<[u8]> {
+	pub fn intern(value: &[u8]) -> Symbol<[u8]> {
+		let mut lock = SYMBOLS.lock();
+
+		if let Some((ptr, count)) = lock.get_key_value_mut(value) {
+			*count += 1;
+			return Symbol::new(ptr.clone());
+		}
+
+		let boxed = value.to_vec().into_boxed_slice();
+		let ptr = SymbolPtr::leaked(Box::leak(boxed));
+
+		lock.insert(ptr.clone(), 1);
+		Symbol::new(ptr)
+	}
+}
+
+impl Pool<str> {
+	pub fn intern(value: impl AsRef<str>) -> Symbol<str> {
+		let symbol = Pool::<[u8]>::intern(value.as_ref().as_bytes());
+		Symbol::new(symbol.into_ptr())
+	}
+}

yazi-shared/src/pool/ptr.rs

@@ -0,0 +1,36 @@
+use std::{borrow::Borrow, hash::{Hash, Hasher}, ops::Deref, ptr::NonNull};
+
+use hashbrown::Equivalent;
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub(super) struct SymbolPtr(NonNull<[u8]>);
+
+unsafe impl Send for SymbolPtr {}
+
+unsafe impl Sync for SymbolPtr {}
+
+impl Deref for SymbolPtr {
+	type Target = NonNull<[u8]>;
+
+	fn deref(&self) -> &Self::Target { &self.0 }
+}
+
+impl Borrow<[u8]> for SymbolPtr {
+	fn borrow(&self) -> &[u8] { self.bytes() }
+}
+
+impl Hash for SymbolPtr {
+	fn hash<H: Hasher>(&self, state: &mut H) { self.bytes().hash(state); }
+}
+
+impl Equivalent<[u8]> for SymbolPtr {
+	fn equivalent(&self, key: &[u8]) -> bool { self.bytes() == key }
+}
+
+impl SymbolPtr {
+	#[inline]
+	pub(super) fn leaked(leaked: &'static mut [u8]) -> Self { Self(NonNull::from(leaked)) }
+
+	#[inline]
+	pub(super) fn bytes(&self) -> &[u8] { unsafe { self.0.as_ref() } }
+}

yazi-shared/src/pool/symbol.rs

@@ -0,0 +1,118 @@
+use std::{hash::{Hash, Hasher}, marker::PhantomData, mem::ManuallyDrop, ops::Deref, str};
+
+use crate::pool::{Pool, SYMBOLS, SymbolPtr};
+
+pub struct Symbol<T: ?Sized> {
+	ptr:      SymbolPtr,
+	_phantom: PhantomData<T>,
+}
+
+unsafe impl<T: ?Sized> Send for Symbol<T> {}
+
+unsafe impl<T: ?Sized> Sync for Symbol<T> {}
+
+impl<T: ?Sized> Clone for Symbol<T> {
+	fn clone(&self) -> Self {
+		*SYMBOLS.lock().get_mut(&self.ptr).unwrap() += 1;
+		Symbol::new(self.ptr.clone())
+	}
+}
+
+impl<T: ?Sized> Drop for Symbol<T> {
+	fn drop(&mut self) {
+		let mut lock = SYMBOLS.lock();
+		let count = lock.get_mut(&self.ptr).unwrap();
+
+		*count -= 1;
+		if *count == 0 {
+			lock.remove(&self.ptr);
+			unsafe {
+				drop(Box::from_raw(self.ptr.as_ptr()));
+			}
+		}
+	}
+}
+
+impl AsRef<[u8]> for Symbol<[u8]> {
+	fn as_ref(&self) -> &[u8] { self.ptr.bytes() }
+}
+
+impl AsRef<str> for Symbol<str> {
+	fn as_ref(&self) -> &str { unsafe { str::from_utf8_unchecked(self.ptr.as_ref()) } }
+}
+
+impl Deref for Symbol<[u8]> {
+	type Target = [u8];
+
+	fn deref(&self) -> &Self::Target { self.as_ref() }
+}
+
+impl Deref for Symbol<str> {
+	type Target = str;
+
+	fn deref(&self) -> &Self::Target { self.as_ref() }
+}
+
+// --- Default
+impl Default for Symbol<[u8]> {
+	fn default() -> Self { Pool::<[u8]>::intern(b"") }
+}
+
+impl Default for Symbol<str> {
+	fn default() -> Self { Pool::<str>::intern("") }
+}
+
+// --- Eq
+impl<T: ?Sized> PartialEq for Symbol<T> {
+	fn eq(&self, other: &Self) -> bool { self.ptr == other.ptr }
+}
+
+impl<T: ?Sized> Eq for Symbol<T> {}
+
+// --- Hash
+impl<T: ?Sized> Hash for Symbol<T> {
+	fn hash<H: Hasher>(&self, state: &mut H) { self.ptr.as_ptr().hash(state); }
+}
+
+// --- PartialOrd
+impl PartialOrd for Symbol<[u8]> {
+	fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+		self.as_ref().partial_cmp(other.as_ref())
+	}
+}
+
+impl PartialOrd for Symbol<str> {
+	fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+		self.as_ref().partial_cmp(other.as_ref())
+	}
+}
+
+// --- Ord
+impl Ord for Symbol<[u8]> {
+	fn cmp(&self, other: &Self) -> std::cmp::Ordering { self.as_ref().cmp(other.as_ref()) }
+}
+
+impl Ord for Symbol<str> {
+	fn cmp(&self, other: &Self) -> std::cmp::Ordering { self.as_ref().cmp(other.as_ref()) }
+}
+
+// --- Debug
+impl std::fmt::Debug for Symbol<[u8]> {
+	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+		write!(f, "Symbol<[u8]>({:?})", self.as_ref())
+	}
+}
+
+impl std::fmt::Debug for Symbol<str> {
+	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+		write!(f, "Symbol<str>({:?})", self.as_ref())
+	}
+}
+
+impl<T: ?Sized> Symbol<T> {
+	#[inline]
+	pub(super) fn new(ptr: SymbolPtr) -> Self { Self { ptr, _phantom: PhantomData } }
+
+	#[inline]
+	pub(super) fn into_ptr(self) -> SymbolPtr { ManuallyDrop::new(self).ptr.clone() }
+}

yazi-shared/src/pool/traits.rs

@@ -0,0 +1,9 @@
+use crate::pool::{Pool, Symbol};
+
+pub trait InternStr {
+	fn intern(&self) -> Symbol<str>;
+}
+
+impl<T: AsRef<str>> InternStr for T {
+	fn intern(&self) -> Symbol<str> { Pool::<str>::intern(self) }
+}

yazi-shared/src/tests.rs

@@ -0,0 +1,7 @@
+pub fn init_tests() {
+	static INIT: std::sync::OnceLock<()> = std::sync::OnceLock::new();
+
+	INIT.get_or_init(|| {
+		crate::init();
+	});
+}

yazi-shared/src/url/buf.rs

@@ -4,7 +4,7 @@ use anyhow::Result;
 use serde::{Deserialize, Serialize};
 
 use super::UrnBuf;
-use crate::{loc::LocBuf, url::{Components, Display, Encode, EncodeTilded, Scheme, Url, UrlCow, Urn}};
+use crate::{loc::LocBuf, pool::Pool, url::{Components, Display, Encode, EncodeTilded, Scheme, Url, UrlCow, Urn}};
 
 #[derive(Clone, Default, Eq, Ord, PartialOrd, PartialEq, Hash)]
 pub struct UrlBuf {
@@ -212,14 +212,14 @@ impl UrlBuf {
 	pub fn to_search(&self, domain: impl AsRef<str>) -> Self {
 		Self {
 			loc:    LocBuf::zeroed(self.loc.to_path()),
-			scheme: Scheme::Search(domain.as_ref().to_owned()),
+			scheme: Scheme::Search(Pool::<str>::intern(domain)),
 		}
 	}
 
 	#[inline]
 	pub fn into_search(mut self, domain: impl AsRef<str>) -> Self {
 		self.loc = LocBuf::zeroed(self.loc.into_path());
-		self.scheme = Scheme::Search(domain.as_ref().to_owned());
+		self.scheme = Scheme::Search(Pool::<str>::intern(domain));
 		self
 	}
 
@@ -278,6 +278,7 @@ mod tests {
 
 	#[test]
 	fn test_join() -> anyhow::Result<()> {
+		crate::init_tests();
 		let cases = [
 			// Regular
 			("/a", "b/c", "regular:///a/b/c"),
@@ -309,6 +310,7 @@ mod tests {
 
 	#[test]
 	fn test_parent_url() -> anyhow::Result<()> {
+		crate::init_tests();
 		let cases = [
 			// Regular
 			("/a", Some("regular:///")),
@@ -342,6 +344,7 @@ mod tests {
 
 	#[test]
 	fn test_into_search() -> Result<()> {
+		crate::init_tests();
 		const S: char = std::path::MAIN_SEPARATOR;
 
 		let u: UrlBuf = "/root".parse()?;

yazi-shared/src/url/component.rs

@@ -168,19 +168,22 @@ mod tests {
 	use std::path::Path;
 
 	use super::*;
+	use crate::pool::InternStr;
 
 	#[test]
 	fn test_collect() {
+		crate::init_tests();
+
 		let search: UrlBuf = "search://keyword//root/projects/yazi".parse().unwrap();
 		assert_eq!(search.loc.uri().as_os_str(), OsStr::new(""));
-		assert_eq!(search.scheme, Scheme::Search("keyword".to_owned()));
+		assert_eq!(search.scheme, Scheme::Search("keyword".intern()));
 
 		let item = search.join("main.rs");
 		assert_eq!(item.loc.uri().as_os_str(), OsStr::new("main.rs"));
-		assert_eq!(item.scheme, Scheme::Search("keyword".to_owned()));
+		assert_eq!(item.scheme, Scheme::Search("keyword".intern()));
 
 		let u: UrlBuf = item.components().take(4).collect();
-		assert_eq!(u.scheme, Scheme::Search("keyword".to_owned()));
+		assert_eq!(u.scheme, Scheme::Search("keyword".intern()));
 		assert_eq!(u.loc.as_path(), Path::new("/root/projects"));
 
 		let u: UrlBuf = item
@@ -188,7 +191,7 @@ mod tests {
 			.take(5)
 			.chain([Component::Normal(OsStr::new("target/release/yazi"))])
 			.collect();
-		assert_eq!(u.scheme, Scheme::Search("keyword".to_owned()));
+		assert_eq!(u.scheme, Scheme::Search("keyword".intern()));
 		assert_eq!(u.loc.as_path(), Path::new("/root/projects/yazi/target/release/yazi"));
 	}
 }

yazi-shared/src/url/scheme.rs

@@ -3,18 +3,18 @@ use std::{borrow::Cow, ops::Not, path::Path};
 use anyhow::{Result, bail, ensure};
 use percent_encoding::percent_decode;
 
-use crate::BytesExt;
+use crate::{BytesExt, pool::{InternStr, Symbol}};
 
 #[derive(Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)]
 pub enum Scheme {
 	#[default]
 	Regular,
 
-	Search(String),
+	Search(Symbol<str>),
 
-	Archive(String),
+	Archive(Symbol<str>),
 
-	Sftp(String),
+	Sftp(Symbol<str>),
 }
 
 impl Scheme {
@@ -100,15 +100,15 @@ impl Scheme {
 	fn decode_param(
 		bytes: &[u8],
 		skip: &mut usize,
-	) -> Result<(String, Option<usize>, Option<usize>)> {
+	) -> Result<(Symbol<str>, Option<usize>, Option<usize>)> {
 		let mut len = bytes.iter().copied().take_while(|&b| b != b'/').count();
 		let slash = bytes.get(len).is_some_and(|&b| b == b'/');
 		*skip += len + slash as usize;
 
 		let (uri, urn) = Self::decode_ports(&bytes[..len], &mut len)?;
 		let domain = match Cow::from(percent_decode(&bytes[..len])) {
-			Cow::Borrowed(b) => str::from_utf8(b)?.to_owned(),
+			Cow::Borrowed(b) => str::from_utf8(b)?.intern(),
-			Cow::Owned(b) => String::from_utf8(b)?,
+			Cow::Owned(b) => String::from_utf8(b)?.intern(),
 		};
 
 		Ok((domain, uri, urn))