From c8bf2c507a1ff4b4051d018833b659edc3ddf637 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=89=E5=92=B2=E9=9B=85=20=C2=B7=20Misaki=20Masa?= Date: Mon, 10 Mar 2025 14:41:02 +0800 Subject: [PATCH] refactor: prefer `WriteConsoleW` for Windows console output (#2464) --- Cargo.lock | 31 ++--- Cargo.toml | 2 +- cspell.json | 2 +- yazi-adapter/src/drivers/chafa.rs | 12 +- yazi-adapter/src/drivers/iip.rs | 10 +- yazi-adapter/src/drivers/kgp.rs | 14 +- yazi-adapter/src/drivers/kgp_old.rs | 4 +- yazi-adapter/src/drivers/sixel.rs | 10 +- yazi-codegen/Cargo.toml | 2 +- yazi-core/src/mgr/commands/bulk_rename.rs | 4 +- yazi-shared/src/lib.rs | 2 +- yazi-shared/src/tty/handle.rs | 33 +++-- yazi-shared/src/tty/mod.rs | 5 +- yazi-shared/src/tty/tty.rs | 10 -- yazi-shared/src/tty/windows.rs | 160 ++++++++++++++++++++++ yazi-shared/src/utf8.rs | 53 +++++++ 16 files changed, 283 insertions(+), 71 deletions(-) create mode 100644 yazi-shared/src/tty/windows.rs create mode 100644 yazi-shared/src/utf8.rs diff --git a/Cargo.lock b/Cargo.lock index ea6e69ef..61caa028 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -299,12 +299,6 @@ version = "1.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6b1fc10dbac614ebc03540c9dbd60e83887fda27794998c6528f1782047d540" -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - [[package]] name = "byteorder-lite" version = "0.1.0" @@ -1913,9 +1907,9 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "ppv-lite86" -version = "0.2.20" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" dependencies = [ "zerocopy", ] @@ -2252,9 +2246,9 @@ checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" [[package]] name = "serde" -version = "1.0.218" +version = "1.0.219" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8dfc9d19bdbf6d17e22319da49161d5d0108e4188e8b680aef6299eed22df60" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" dependencies = [ "serde_derive", ] @@ -2271,9 +2265,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.218" +version = "1.0.219" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f09503e191f4e797cb8aac08e9a4a4695c5edf6a2e70e376d961ddd5c969f82b" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", @@ -2452,9 +2446,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.99" +version = "2.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e02e925281e18ffd9d640e234264753c43edc62d64b2d4cf898f1bc5e75f3fc2" +checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0" dependencies = [ "proc-macro2", "quote", @@ -3661,19 +3655,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.7.35" +version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +checksum = "fd97444d05a4328b90e75e503a34bad781f14e28a823ad3557f0750df1ebcbc6" dependencies = [ - "byteorder", "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.35" +version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +checksum = "6352c01d0edd5db859a63e2605f4ea3183ddbd15e2c4a9e7d32184df75e4f154" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 2ea36e1a..357b80be 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,7 +35,7 @@ parking_lot = "0.12.3" ratatui = { version = "0.29.0", features = [ "unstable-rendered-line-info" ] } regex = "1.11.1" scopeguard = "1.2.0" -serde = { version = "1.0.218", features = [ "derive" ] } +serde = { version = "1.0.219", features = [ "derive" ] } serde_json = "1.0.140" tokio = { version = "1.44.0", features = [ "full" ] } tokio-stream = "0.1.17" diff --git a/cspell.json b/cspell.json index 41b40792..3216a2e4 100644 --- a/cspell.json +++ b/cspell.json @@ -1 +1 @@ -{"version":"0.2","language":"en","flagWords":[],"words":["Punct","KEYMAP","splitn","crossterm","YAZI","peekable","ratatui","syntect","pbpaste","pbcopy","oneshot","Posix","Lsar","XADDOS","zoxide","cands","Deque","precache","imageops","IFBLK","IFCHR","IFDIR","IFIFO","IFLNK","IFMT","IFSOCK","IRGRP","IROTH","IRUSR","ISGID","ISUID","ISVTX","IWGRP","IWOTH","IWUSR","IXGRP","IXOTH","IXUSR","libc","winsize","TIOCGWINSZ","xpixel","ypixel","ioerr","appender","Catppuccin","macchiato","gitmodules","Dotfiles","bashprofile","vimrc","flac","webp","exiftool","mediainfo","ripgrep","indexmap","indexmap","unwatch","canonicalize","serde","fsevent","Ueberzug","iterm","wezterm","sixel","chafa","ueberzugpp","Konsole","Überzug","pkgs","pdftoppm","poppler","prebuild","singlefile","jpegopt","EXIF","rustfmt","mktemp","nanos","xclip","xsel","natord","Mintty","nixos","nixpkgs","SIGTSTP","SIGCONT","SIGCONT","mlua","nonstatic","userdata","metatable","natsort","backstack","luajit","Succ","Succ","cand","fileencoding","foldmethod","lightgreen","darkgray","lightred","lightyellow","lightcyan","nushell","msvc","aarch","linemode","sxyazi","rsplit","ZELLIJ","bitflags","bitflags","USERPROFILE","Neovim","vergen","gitcl","Renderable","preloaders","prec","Upserting","prio","Ghostty","Catmull","Lanczos","cmds","unyank","scrolloff","headsup","unsub","uzers","scopeguard","SPDLOG","globset","filetime","magick","magick","prefetcher","Prework","prefetchers","PREWORKERS","conds","translit","rxvt","Urxvt","realpath","realname","REPARSE","hardlink","hardlinking","nlink","nlink","linemodes","SIGSTOP","sevenzip","rsplitn","replacen","DECSET","DECRQM","repeek","cwds","tcsi","Hyprland","Wayfire","SWAYSOCK","btime","nsec","codegen","gethostname","fchmod","fdfind","Rustc","rustc","ffprobe","vframes","luma","obase","outln","errln","tmtheme","twox","cfgs","fstype","objc","rdev","runloop","exfat","rclone","DECRQSS","DECSCUSR","libvterm","Uninit","lockin"]} \ No newline at end of file +{"version":"0.2","language":"en","words":["Punct","KEYMAP","splitn","crossterm","YAZI","peekable","ratatui","syntect","pbpaste","pbcopy","oneshot","Posix","Lsar","XADDOS","zoxide","cands","Deque","precache","imageops","IFBLK","IFCHR","IFDIR","IFIFO","IFLNK","IFMT","IFSOCK","IRGRP","IROTH","IRUSR","ISGID","ISUID","ISVTX","IWGRP","IWOTH","IWUSR","IXGRP","IXOTH","IXUSR","libc","winsize","TIOCGWINSZ","xpixel","ypixel","ioerr","appender","Catppuccin","macchiato","gitmodules","Dotfiles","bashprofile","vimrc","flac","webp","exiftool","mediainfo","ripgrep","indexmap","indexmap","unwatch","canonicalize","serde","fsevent","Ueberzug","iterm","wezterm","sixel","chafa","ueberzugpp","Konsole","Überzug","pkgs","pdftoppm","poppler","prebuild","singlefile","jpegopt","EXIF","rustfmt","mktemp","nanos","xclip","xsel","natord","Mintty","nixos","nixpkgs","SIGTSTP","SIGCONT","SIGCONT","mlua","nonstatic","userdata","metatable","natsort","backstack","luajit","Succ","Succ","cand","fileencoding","foldmethod","lightgreen","darkgray","lightred","lightyellow","lightcyan","nushell","msvc","aarch","linemode","sxyazi","rsplit","ZELLIJ","bitflags","bitflags","USERPROFILE","Neovim","vergen","gitcl","Renderable","preloaders","prec","Upserting","prio","Ghostty","Catmull","Lanczos","cmds","unyank","scrolloff","headsup","unsub","uzers","scopeguard","SPDLOG","globset","filetime","magick","magick","prefetcher","Prework","prefetchers","PREWORKERS","conds","translit","rxvt","Urxvt","realpath","realname","REPARSE","hardlink","hardlinking","nlink","nlink","linemodes","SIGSTOP","sevenzip","rsplitn","replacen","DECSET","DECRQM","repeek","cwds","tcsi","Hyprland","Wayfire","SWAYSOCK","btime","nsec","codegen","gethostname","fchmod","fdfind","Rustc","rustc","ffprobe","vframes","luma","obase","outln","errln","tmtheme","twox","cfgs","fstype","objc","rdev","runloop","exfat","rclone","DECRQSS","DECSCUSR","libvterm","Uninit","lockin","rposition"],"flagWords":[]} \ No newline at end of file diff --git a/yazi-adapter/src/drivers/chafa.rs b/yazi-adapter/src/drivers/chafa.rs index 5c9131d4..b8c18b56 100644 --- a/yazi-adapter/src/drivers/chafa.rs +++ b/yazi-adapter/src/drivers/chafa.rs @@ -55,10 +55,10 @@ impl Chafa { Adapter::Chafa.image_hide()?; Adapter::shown_store(area); - Emulator::move_lock((max.x, max.y), |stderr| { + Emulator::move_lock((max.x, max.y), |w| { for (i, line) in lines.into_iter().enumerate() { - stderr.write_all(line)?; - queue!(stderr, MoveTo(max.x, max.y + i as u16 + 1))?; + w.write_all(line)?; + queue!(w, MoveTo(max.x, max.y + i as u16 + 1))?; } Ok(area) }) @@ -66,10 +66,10 @@ impl Chafa { pub(crate) fn image_erase(area: Rect) -> Result<()> { let s = " ".repeat(area.width as usize); - Emulator::move_lock((0, 0), |stderr| { + Emulator::move_lock((0, 0), |w| { for y in area.top()..area.bottom() { - queue!(stderr, MoveTo(area.x, y))?; - write!(stderr, "{s}")?; + queue!(w, MoveTo(area.x, y))?; + write!(w, "{s}")?; } Ok(()) }) diff --git a/yazi-adapter/src/drivers/iip.rs b/yazi-adapter/src/drivers/iip.rs index 967adc7c..1f8d6528 100644 --- a/yazi-adapter/src/drivers/iip.rs +++ b/yazi-adapter/src/drivers/iip.rs @@ -19,18 +19,18 @@ impl Iip { Adapter::Iip.image_hide()?; Adapter::shown_store(area); - Emulator::move_lock((max.x, max.y), |stderr| { - stderr.write_all(&b)?; + Emulator::move_lock((max.x, max.y), |w| { + w.write_all(&b)?; Ok(area) }) } pub(crate) fn image_erase(area: Rect) -> Result<()> { let s = " ".repeat(area.width as usize); - Emulator::move_lock((0, 0), |stderr| { + Emulator::move_lock((0, 0), |w| { for y in area.top()..area.bottom() { - queue!(stderr, MoveTo(area.x, y))?; - write!(stderr, "{s}")?; + queue!(w, MoveTo(area.x, y))?; + write!(w, "{s}")?; } Ok(()) }) diff --git a/yazi-adapter/src/drivers/kgp.rs b/yazi-adapter/src/drivers/kgp.rs index 051ca123..1290cf35 100644 --- a/yazi-adapter/src/drivers/kgp.rs +++ b/yazi-adapter/src/drivers/kgp.rs @@ -321,22 +321,22 @@ impl Kgp { Adapter::Kgp.image_hide()?; Adapter::shown_store(area); - Emulator::move_lock((area.x, area.y), |stderr| { - stderr.write_all(&b1)?; - stderr.write_all(&b2)?; + Emulator::move_lock((area.x, area.y), |w| { + w.write_all(&b1)?; + w.write_all(&b2)?; Ok(area) }) } pub(crate) fn image_erase(area: Rect) -> Result<()> { let s = " ".repeat(area.width as usize); - Emulator::move_lock((0, 0), |stderr| { + Emulator::move_lock((0, 0), |w| { for y in area.top()..area.bottom() { - queue!(stderr, MoveTo(area.x, y))?; - write!(stderr, "{s}")?; + queue!(w, MoveTo(area.x, y))?; + write!(w, "{s}")?; } - write!(stderr, "{START}_Gq=2,a=d,d=A{ESCAPE}\\{CLOSE}")?; + write!(w, "{START}_Gq=2,a=d,d=A{ESCAPE}\\{CLOSE}")?; Ok(()) }) } diff --git a/yazi-adapter/src/drivers/kgp_old.rs b/yazi-adapter/src/drivers/kgp_old.rs index f149bf8a..e21bc6fa 100644 --- a/yazi-adapter/src/drivers/kgp_old.rs +++ b/yazi-adapter/src/drivers/kgp_old.rs @@ -19,8 +19,8 @@ impl KgpOld { Adapter::KgpOld.image_hide()?; Adapter::shown_store(area); - Emulator::move_lock((area.x, area.y), |stderr| { - stderr.write_all(&b)?; + Emulator::move_lock((area.x, area.y), |w| { + w.write_all(&b)?; Ok(area) }) } diff --git a/yazi-adapter/src/drivers/sixel.rs b/yazi-adapter/src/drivers/sixel.rs index 56a30993..526341ca 100644 --- a/yazi-adapter/src/drivers/sixel.rs +++ b/yazi-adapter/src/drivers/sixel.rs @@ -19,18 +19,18 @@ impl Sixel { Adapter::Sixel.image_hide()?; Adapter::shown_store(area); - Emulator::move_lock((area.x, area.y), |stderr| { - stderr.write_all(&b)?; + Emulator::move_lock((area.x, area.y), |w| { + w.write_all(&b)?; Ok(area) }) } pub(crate) fn image_erase(area: Rect) -> Result<()> { let s = " ".repeat(area.width as usize); - Emulator::move_lock((0, 0), |stderr| { + Emulator::move_lock((0, 0), |w| { for y in area.top()..area.bottom() { - queue!(stderr, MoveTo(area.x, y))?; - write!(stderr, "{s}")?; + queue!(w, MoveTo(area.x, y))?; + write!(w, "{s}")?; } Ok(()) }) diff --git a/yazi-codegen/Cargo.toml b/yazi-codegen/Cargo.toml index 8cadb9bf..0e632caf 100644 --- a/yazi-codegen/Cargo.toml +++ b/yazi-codegen/Cargo.toml @@ -13,5 +13,5 @@ proc-macro = true [dependencies] # External dependencies -syn = { version = "2.0.99", features = [ "full" ] } +syn = { version = "2.0.100", features = [ "full" ] } quote = "1.0.39" diff --git a/yazi-core/src/mgr/commands/bulk_rename.rs b/yazi-core/src/mgr/commands/bulk_rename.rs index 1376a361..c30a8302 100644 --- a/yazi-core/src/mgr/commands/bulk_rename.rs +++ b/yazi-core/src/mgr/commands/bulk_rename.rs @@ -66,8 +66,8 @@ impl Mgr { { let mut w = TTY.lockout(); - for (o, n) in &todo { - writeln!(w, "{} -> {}", o.display(), n.display())?; + for (old, new) in &todo { + writeln!(w, "{} -> {}", old.display(), new.display())?; } write!(w, "Continue to rename? (y/N): ")?; w.flush()?; diff --git a/yazi-shared/src/lib.rs b/yazi-shared/src/lib.rs index ca4260ab..abc04594 100644 --- a/yazi-shared/src/lib.rs +++ b/yazi-shared/src/lib.rs @@ -2,7 +2,7 @@ yazi_macro::mod_pub!(errors event shell theme translit tty url); -yazi_macro::mod_flat!(chars condition debounce either env id layer natsort number os rand ro_cell sync_cell terminal throttle time); +yazi_macro::mod_flat!(chars condition debounce either env id layer natsort number os rand ro_cell sync_cell terminal throttle time utf8); pub fn init() { tty::init(); diff --git a/yazi-shared/src/tty/handle.rs b/yazi-shared/src/tty/handle.rs index 23544fd1..7a54936f 100644 --- a/yazi-shared/src/tty/handle.rs +++ b/yazi-shared/src/tty/handle.rs @@ -2,10 +2,14 @@ use std::{io::{Error, ErrorKind, Read, Write}, time::Duration}; pub struct Handle { #[cfg(unix)] - inner: std::os::fd::RawFd, + inner: std::os::fd::RawFd, #[cfg(windows)] - inner: std::os::windows::io::RawHandle, - close: bool, + inner: std::os::windows::io::RawHandle, + close: bool, + #[cfg(windows)] + out_utf8: bool, + #[cfg(windows)] + incomplete_utf8: super::IncompleteUtf8, } impl Drop for Handle { @@ -49,16 +53,20 @@ impl Write for Handle { use std::os::{fd::IntoRawFd, unix::io::FromRawFd}; let mut f = unsafe { std::fs::File::from_raw_fd(self.inner) }; let result = f.write(buf); - self.inner = f.into_raw_fd(); + _ = f.into_raw_fd(); result } #[cfg(windows)] { use std::os::windows::io::{FromRawHandle, IntoRawHandle}; - let mut f = unsafe { std::fs::File::from_raw_handle(self.inner) }; - let result = f.write(buf); - self.inner = f.into_raw_handle(); - result + if self.out_utf8 { + let mut f = unsafe { std::fs::File::from_raw_handle(self.inner) }; + let result = f.write(buf); + _ = f.into_raw_handle(); + result + } else { + super::write_console_utf16(buf, &mut self.incomplete_utf8, self.inner) + } } } @@ -114,7 +122,7 @@ impl Handle { #[cfg(windows)] impl Handle { pub(super) fn new(out: bool) -> std::io::Result { - use windows_sys::Win32::{Foundation::{GENERIC_READ, GENERIC_WRITE, INVALID_HANDLE_VALUE}, Storage::FileSystem::{CreateFileW, FILE_SHARE_READ, FILE_SHARE_WRITE, OPEN_EXISTING}}; + use windows_sys::Win32::{Foundation::{GENERIC_READ, GENERIC_WRITE, INVALID_HANDLE_VALUE}, Globalization::CP_UTF8, Storage::FileSystem::{CreateFileW, FILE_SHARE_READ, FILE_SHARE_WRITE, OPEN_EXISTING}, System::Console::GetConsoleOutputCP}; let name: Vec = if out { "CONOUT$\0" } else { "CONIN$\0" }.encode_utf16().collect(); let result = unsafe { @@ -132,7 +140,12 @@ impl Handle { if result == INVALID_HANDLE_VALUE { Err(std::io::Error::last_os_error()) } else { - Ok(Self { inner: result, close: true }) + Ok(Self { + inner: result, + close: true, + out_utf8: unsafe { GetConsoleOutputCP() } == CP_UTF8, + incomplete_utf8: Default::default(), + }) } } diff --git a/yazi-shared/src/tty/mod.rs b/yazi-shared/src/tty/mod.rs index 9a54aebc..faef2c9c 100644 --- a/yazi-shared/src/tty/mod.rs +++ b/yazi-shared/src/tty/mod.rs @@ -1,4 +1,7 @@ -yazi_macro::mod_flat!(tty handle); +yazi_macro::mod_flat!(handle tty); + +#[cfg(windows)] +yazi_macro::mod_flat!(windows); pub static TTY: crate::RoCell = crate::RoCell::new(); diff --git a/yazi-shared/src/tty/tty.rs b/yazi-shared/src/tty/tty.rs index 733877a5..74690348 100644 --- a/yazi-shared/src/tty/tty.rs +++ b/yazi-shared/src/tty/tty.rs @@ -11,9 +11,6 @@ pub struct Tty { impl Default for Tty { fn default() -> Self { - #[cfg(windows)] - Self::set_code_page().expect("failed to set terminal code page"); - let stdin = Handle::new(false).expect("failed to open stdin"); let stdout = Handle::new(true).expect("failed to open stdout"); Self { stdin: Mutex::new(stdin), stdout: Mutex::new(BufWriter::new(stdout)) } @@ -58,13 +55,6 @@ impl Tty { let result = read(); (buf, result) } - - #[cfg(windows)] - fn set_code_page() -> std::io::Result<()> { - use windows_sys::Win32::{Globalization::CP_UTF8, System::Console::SetConsoleOutputCP}; - - if unsafe { SetConsoleOutputCP(CP_UTF8) } == 0 { Err(Error::last_os_error()) } else { Ok(()) } - } } // --- Reader diff --git a/yazi-shared/src/tty/windows.rs b/yazi-shared/src/tty/windows.rs new file mode 100644 index 00000000..72d8c2bd --- /dev/null +++ b/yazi-shared/src/tty/windows.rs @@ -0,0 +1,160 @@ +// Copied from https://github.com/rust-lang/rust/blob/master/library/std/src/sys/pal/windows/stdio.rs + +use std::{mem::MaybeUninit, os::windows::io::RawHandle, str}; + +use windows_sys::Win32::{Globalization::{CP_UTF8, MB_ERR_INVALID_CHARS, MultiByteToWideChar}, System::Console::WriteConsoleW}; + +use crate::{floor_char_boundary, utf8_char_width}; + +// Apparently Windows doesn't handle large reads on stdin or writes to +// stdout/stderr well (see #13304 for details). +// +// From MSDN (2011): "The storage for this buffer is allocated from a shared +// heap for the process that is 64 KB in size. The maximum size of the buffer +// will depend on heap usage." +// +// We choose the cap at 8 KiB because libuv does the same, and it seems to be +// acceptable so far. +const MAX_BUFFER_SIZE: usize = 8192; + +#[derive(Default)] +pub(super) struct IncompleteUtf8 { + bytes: [u8; 4], + len: u8, +} + +pub(super) fn write_console_utf16( + data: &[u8], + incomplete_utf8: &mut IncompleteUtf8, + handle: RawHandle, +) -> std::io::Result { + if incomplete_utf8.len > 0 { + assert!(incomplete_utf8.len < 4, "Unexpected number of bytes for incomplete UTF-8 codepoint."); + if data[0] >> 6 != 0b10 { + // not a continuation byte - reject + incomplete_utf8.len = 0; + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "Windows stdio in console mode does not support writing non-UTF-8 byte sequences", + )); + } + incomplete_utf8.bytes[incomplete_utf8.len as usize] = data[0]; + incomplete_utf8.len += 1; + let char_width = utf8_char_width(incomplete_utf8.bytes[0]); + if (incomplete_utf8.len as usize) < char_width { + // more bytes needed + return Ok(1); + } + let s = str::from_utf8(&incomplete_utf8.bytes[0..incomplete_utf8.len as usize]); + incomplete_utf8.len = 0; + match s { + Ok(s) => { + assert_eq!(char_width, s.len()); + let written = write_valid_utf8_to_console(handle, s)?; + assert_eq!(written, s.len()); // guaranteed by write_valid_utf8_to_console() for single codepoint writes + return Ok(1); + } + Err(_) => { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "Windows stdio in console mode does not support writing non-UTF-8 byte sequences", + )); + } + } + } + + // As the console is meant for presenting text, we assume bytes of `data` are + // encoded as UTF-8, which needs to be encoded as UTF-16. + // + // If the data is not valid UTF-8 we write out as many bytes as are valid. + // If the first byte is invalid it is either first byte of a multi-byte sequence + // but the provided byte slice is too short or it is the first byte of an + // invalid multi-byte sequence. + let len = std::cmp::min(data.len(), MAX_BUFFER_SIZE / 2); + let utf8 = match str::from_utf8(&data[..len]) { + Ok(s) => s, + Err(ref e) if e.valid_up_to() == 0 => { + let first_byte_char_width = utf8_char_width(data[0]); + if first_byte_char_width > 1 && data.len() < first_byte_char_width { + incomplete_utf8.bytes[0] = data[0]; + incomplete_utf8.len = 1; + return Ok(1); + } else { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "Windows stdio in console mode does not support writing non-UTF-8 byte sequences", + )); + } + } + Err(e) => str::from_utf8(&data[..e.valid_up_to()]).unwrap(), + }; + + write_valid_utf8_to_console(handle, utf8) +} + +fn write_valid_utf8_to_console(handle: RawHandle, utf8: &str) -> std::io::Result { + debug_assert!(!utf8.is_empty()); + + let mut utf16 = [MaybeUninit::::uninit(); MAX_BUFFER_SIZE / 2]; + let utf8 = &utf8[..floor_char_boundary(utf8, utf16.len())]; + + let utf16: &[u16] = unsafe { + // Note that this theoretically checks validity twice in the (most common) case + // where the underlying byte sequence is valid utf-8 (given the check in + // `write()`). + let result = MultiByteToWideChar( + CP_UTF8, + MB_ERR_INVALID_CHARS, + utf8.as_ptr(), + utf8.len() as i32, + utf16.as_mut_ptr() as *mut _, + utf16.len() as i32, + ); + assert!(result != 0, "Unexpected error in MultiByteToWideChar"); + + // Safety: MultiByteToWideChar initializes `result` values. + &*(&utf16[..result as usize] as *const [MaybeUninit] as *const [u16]) + }; + + let mut written = write_u16s(handle, utf16)?; + + // Figure out how many bytes of as UTF-8 were written away as UTF-16. + if written == utf16.len() { + Ok(utf8.len()) + } else { + // Make sure we didn't end up writing only half of a surrogate pair (even though + // the chance is tiny). Because it is not possible for user code to re-slice + // `data` in such a way that a missing surrogate can be produced (and also + // because of the UTF-8 validation above), write the missing surrogate out + // now. Buffering it would mean we have to lie about the number of bytes + // written. + let first_code_unit_remaining = utf16[written]; + if matches!(first_code_unit_remaining, 0xdcee..=0xdfff) { + // low surrogate + // We just hope this works, and give up otherwise + let _ = write_u16s(handle, &utf16[written..written + 1]); + written += 1; + } + // Calculate the number of bytes of `utf8` that were actually written. + let mut count = 0; + for ch in utf16[..written].iter() { + count += match ch { + 0x0000..=0x007f => 1, + 0x0080..=0x07ff => 2, + 0xdcee..=0xdfff => 1, // Low surrogate. We already counted 3 bytes for the other. + _ => 3, + }; + } + debug_assert!(String::from_utf16(&utf16[..written]).unwrap() == utf8[..count]); + Ok(count) + } +} + +fn write_u16s(handle: RawHandle, data: &[u16]) -> std::io::Result { + debug_assert!(data.len() < u32::MAX as usize); + let mut written = 0; + let result = unsafe { + WriteConsoleW(handle, data.as_ptr(), data.len() as u32, &mut written, std::ptr::null_mut()) + }; + if result == 0 { Err(std::io::Error::last_os_error()) } else { Ok(written as usize) } +} diff --git a/yazi-shared/src/utf8.rs b/yazi-shared/src/utf8.rs new file mode 100644 index 00000000..6bf021d7 --- /dev/null +++ b/yazi-shared/src/utf8.rs @@ -0,0 +1,53 @@ +// https://tools.ietf.org/html/rfc3629 +const UTF8_CHAR_WIDTH: &[u8; 256] = &[ + // 1 2 3 4 5 6 7 8 9 A B C D E F + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 0 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 1 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 2 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 3 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 4 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 5 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 6 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 7 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 8 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 9 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // A + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // B + 0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // C + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // D + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, // E + 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // F +]; + +/// Given a first byte, determines how many bytes are in this UTF-8 character. +#[must_use] +#[inline] +pub const fn utf8_char_width(b: u8) -> usize { UTF8_CHAR_WIDTH[b as usize] as usize } + +/// Finds the closest `x` not exceeding `index` where [`is_char_boundary(x)`] is +/// `true`. +/// +/// This method can help you truncate a string so that it's still valid UTF-8, +/// but doesn't exceed a given number of bytes. Note that this is done purely at +/// the character level and can still visually split graphemes, even though the +/// underlying characters aren't split. For example, the emoji 🧑‍🔬 (scientist) +/// could be split so that the string only includes 🧑 (person) instead. +#[inline] +pub fn floor_char_boundary(s: &str, index: usize) -> usize { + if index >= s.len() { + s.len() + } else { + let lower_bound = index.saturating_sub(3); + let new_index = + s.as_bytes()[lower_bound..=index].iter().rposition(|&b| is_utf8_char_boundary(b)); + + // SAFETY: we know that the character boundary will be within four bytes + unsafe { lower_bound + new_index.unwrap_unchecked() } + } +} + +#[inline] +const fn is_utf8_char_boundary(b: u8) -> bool { + // This is bit magic equivalent to: b < 128 || b >= 192 + (b as i8) >= -0x40 +}