From 42a0fcd5cfba17641c0ef04bc997fb04b53a2f67 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E4=B8=89=E5=92=B2=E9=9B=85=20=C2=B7=20Misaki=20Masa?=
 <sxyazi@gmail.com>
Date: Fri, 26 Apr 2024 20:42:39 +0800
Subject: [PATCH] feat: support previewing files containing non-UTF-8
 characters (#958)

---
 yazi-plugin/src/external/highlighter.rs | 26 +++++++++++++++----------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/yazi-plugin/src/external/highlighter.rs b/yazi-plugin/src/external/highlighter.rs
index 584e614f..b4fd6b39 100644
--- a/yazi-plugin/src/external/highlighter.rs
+++ b/yazi-plugin/src/external/highlighter.rs
@@ -57,7 +57,7 @@ impl Highlighter {
 	}
 
 	pub async fn highlight(&self, skip: usize, limit: usize) -> Result<Text<'static>, PeekError> {
-		let mut reader = BufReader::new(File::open(&self.path).await?).lines();
+		let mut reader = BufReader::new(File::open(&self.path).await?);
 
 		let syntax = Self::find_syntax(&self.path).await;
 		let mut plain = syntax.is_err();
@@ -66,24 +66,30 @@ impl Highlighter {
 		let mut after = Vec::with_capacity(limit);
 
 		let mut i = 0;
-		while let Some(mut line) = reader.next_line().await? {
+		let mut buf = vec![];
+		while reader.read_until(b'\n', &mut buf).await.is_ok() {
 			i += 1;
-			if i > skip + limit {
+			if buf.is_empty() || i > skip + limit {
 				break;
 			}
 
-			if !plain && line.len() > 6000 {
+			if !plain && buf.len() > 6000 {
 				plain = true;
 				drop(mem::take(&mut before));
 			}
 
-			if i > skip {
-				line.push('\n');
-				after.push(line);
-			} else if !plain {
-				line.push('\n');
-				before.push(line);
+			if buf.ends_with(b"\r\n") {
+				buf.pop();
+				buf.pop();
+				buf.push(b'\n');
 			}
+
+			if i > skip {
+				after.push(String::from_utf8_lossy(&buf).into_owned());
+			} else if !plain {
+				before.push(String::from_utf8_lossy(&buf).into_owned());
+			}
+			buf.clear();
 		}
 
 		if skip > 0 && i < skip + limit {