feat: add deunicoded, natural sorting

Cargo.lock

@@ -550,6 +550,12 @@ dependencies = [
  "powerfmt",
 ]
 
+[[package]]
+name = "deunicode"
+version = "1.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "339544cc9e2c4dc3fc7149fd630c5f22263a4fdf18a98afd0075784968b5cf00"
+
 [[package]]
 name = "digest"
 version = "0.10.7"
@@ -2915,6 +2921,7 @@ dependencies = [
  "anyhow",
  "bitflags 2.5.0",
  "crossterm",
+ "deunicode",
  "dirs",
  "filetime",
  "futures",

yazi-config/preset/keymap.toml

@@ -116,6 +116,8 @@ keymap = [
 	{ on = [ ",", "A" ], run = "sort alphabetical --reverse --dir-first", 			desc = "Sort alphabetically (reverse)" },
 	{ on = [ ",", "n" ], run = "sort natural --dir-first",                			desc = "Sort naturally" },
 	{ on = [ ",", "N" ], run = "sort natural --reverse --dir-first",      			desc = "Sort naturally (reverse)" },
+	{ on = [ ",", "d" ], run = "sort deunicode_natural --dir-first",                desc = "Sort naturally (deunicode)" },
+	{ on = [ ",", "D" ], run = "sort deunicode_natural --reverse --dir-first",      desc = "Sort naturally (deunicode + reverse)" },
 	{ on = [ ",", "s" ], run = "sort size --dir-first",                   			desc = "Sort by size" },
 	{ on = [ ",", "S" ], run = "sort size --reverse --dir-first",         			desc = "Sort by size (reverse)" },
 

yazi-config/src/manager/sorting.rs

@@ -13,6 +13,7 @@ pub enum SortBy {
 	Extension,
 	Alphabetical,
 	Natural,
+	DeunicodeNatural,
 	Size,
 }
 
@@ -27,6 +28,7 @@ impl FromStr for SortBy {
 			"extension" => Self::Extension,
 			"alphabetical" => Self::Alphabetical,
 			"natural" => Self::Natural,
+			"deunicode_natural" => Self::DeunicodeNatural,
 			"size" => Self::Size,
 			_ => bail!("invalid sort_by value: {s}"),
 		})
@@ -48,6 +50,7 @@ impl Display for SortBy {
 			Self::Extension => "extension",
 			Self::Alphabetical => "alphabetical",
 			Self::Natural => "natural",
+			Self::DeunicodeNatural => "deunicode_natural",
 			Self::Size => "size",
 		})
 	}

yazi-core/src/folder/sorter.rs

@@ -1,7 +1,7 @@
 use std::{cmp::Ordering, collections::HashMap, mem};
 
 use yazi_config::manager::SortBy;
-use yazi_shared::{fs::{File, Url}, natsort};
+use yazi_shared::{deunicode_natsort, fs::{File, Url}, natsort};
 
 #[derive(Clone, Copy, Default, PartialEq)]
 pub struct FilesSorter {
@@ -52,7 +52,8 @@ impl FilesSorter {
 				if ord == Ordering::Equal { by_alphabetical(a, b) } else { ord }
 			}),
 			SortBy::Alphabetical => items.sort_unstable_by(by_alphabetical),
-			SortBy::Natural => self.sort_naturally(items),
+			SortBy::Natural => self.sort_naturally(items, false),
+			SortBy::DeunicodeNatural => self.sort_naturally(items, true),
 			SortBy::Size => items.sort_unstable_by(|a, b| {
 				let aa = if a.is_dir() { sizes.get(&a.url).copied() } else { None };
 				let bb = if b.is_dir() { sizes.get(&b.url).copied() } else { None };
@@ -62,7 +63,7 @@ impl FilesSorter {
 		}
 	}
 
-	fn sort_naturally(&self, items: &mut Vec<File>) {
+	fn sort_naturally(&self, items: &mut Vec<File>, deunicoded: bool) {
 		let mut indices = Vec::with_capacity(items.len());
 		let mut entities = Vec::with_capacity(items.len());
 		for (i, file) in items.iter().enumerate() {
@@ -76,7 +77,11 @@ impl FilesSorter {
 				return promote;
 			}
 
-			let ordering = natsort(entities[a], entities[b], !self.sensitive);
+			let ordering = match deunicoded {
+				true => deunicode_natsort(entities[a], entities[b], !self.sensitive),
+				false => natsort(entities[a], entities[b], !self.sensitive)
+			};
+
 			if self.reverse { ordering.reverse() } else { ordering }
 		});
 

yazi-shared/Cargo.toml

@@ -12,6 +12,7 @@ repository  = "https://github.com/sxyazi/yazi"
 anyhow           = "1.0.83"
 bitflags         = "2.5.0"
 crossterm        = "0.27.0"
+deunicode        = "1.6.0"
 dirs             = "5.0.1"
 filetime         = "0.2.23"
 futures          = "0.3.30"

yazi-shared/src/natsort.rs

@@ -6,6 +6,8 @@
 
 use std::cmp::Ordering;
 
+use deunicode::deunicode;
+
 macro_rules! return_unless_equal {
 	($ord:expr) => {
 		match $ord {
@@ -123,6 +125,17 @@ pub fn natsort(left: &[u8], right: &[u8], insensitive: bool) -> Ordering {
 	}
 }
 
+pub fn deunicode_natsort(left: &[u8], right: &[u8], insensitive: bool) -> Ordering {
+	let utf8_left = std::str::from_utf8(left);
+	let utf8_right = std::str::from_utf8(right);
+	match (utf8_left, utf8_right) {
+		(Ok(l), Ok(r)) => natsort(deunicode(l).as_bytes(), deunicode(r).as_bytes(), insensitive),
+		(Ok(l), Err(_)) => natsort(deunicode(l).as_bytes(), right, insensitive),
+		(Err(_), Ok(r)) => natsort(left, deunicode(r).as_bytes(), insensitive),
+		_ => natsort(left, right, insensitive),
+	}
+}
+
 #[cfg(test)]
 mod tests {
 	use super::*;
@@ -133,6 +146,12 @@ mod tests {
 		assert_eq!(left, right);
 	}
 
+	fn cmp_deun(left: &[&str]) {
+		let mut right = left.to_vec();
+		right.sort_by(|a, b| deunicode_natsort(a.as_bytes(), b.as_bytes(), true));
+		assert_eq!(left, right);
+	}
+
 	#[test]
 	fn test_natsort() {
 		let dates = vec!["1999-3-3", "1999-12-25", "2000-1-2", "2000-1-10", "2000-3-23"];
@@ -176,4 +195,33 @@ mod tests {
 		cmp(&fractions);
 		cmp(&words);
 	}
+
+	#[test]
+	fn test_deunicode_natsort() {
+		let deun = vec![
+			"Acorn",
+			"Ætt",
+			"Alpha",
+			"Átriu",
+			"Attention",
+			"Beta",
+			"Elemér",
+			"Érvényes",
+			"Große",
+			"Grotto",
+			"Hedvig",
+			"Ilona",
+			"Írott",
+			"Olga",
+			"Órmotlan",
+			"Öveges",
+			"Őzike",
+			"Ubul",
+			"Űr",
+			"Útvonal",
+			"Üveg",
+		];
+
+		cmp_deun(&deun)
+	}
 }