From 6753ade6ff6788908a50ff123acaeb7e88ae5074 Mon Sep 17 00:00:00 2001 From: Oded Falik Date: Tue, 7 Jul 2026 11:18:39 -0700 Subject: [PATCH] feat(rclone): add read-only rclone VFS provider MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds an `rclone:///` scheme backed by the rclone CLI, giving yazi native browsing of any of rclone's 100+ remotes (S3, GCS, Azure, SFTP, WebDAV, local, …) without a FUSE mount. - New SchemeKind::Rclone threaded through the scheme/URL type system, mirroring Sftp (remote, virtual, Unix-path, byte-encoded path semantics). - yazi-vfs rclone provider: - read_dir via `rclone lsjson` - metadata via `rclone lsjson --stat` (synthesized POSIX mode) - seekable read-only File streaming `rclone cat --offset`; seek respawns the child at the new offset. Detects premature EOF and unknown sizes. - all write ops return Unsupported (read-only for now) - `remote:/path` targets preserve the leading slash, so filesystem-like backends (local, sftp) work alongside object stores. - Config: [services.] type = "rclone", remote = "..." - Tests: - tests/rclone_local.rs: hermetic, points an rclone `local` remote at a temp fixture dir, so it needs no cloud/credentials and skips if `rclone` isn't installed. Covers list, read, seek, missing-object, and a >8 MiB parallel multi-chunk copy with position-varying bytes. - tests/rclone_read.rs: env-driven live tests against a real remote (ignored by default; paths supplied via env vars, nothing hard-coded). --- .github/workflows/test.yml | 12 ++ Cargo.lock | 1 + RCLONE_VFS.md | 94 ++++++++++++ RCLONE_VFS_PROPOSAL.md | 96 ++++++++++++ yazi-config/src/pattern.rs | 3 + yazi-fs/src/path/expand.rs | 4 + yazi-fs/src/provider/local/absolute.rs | 2 +- yazi-fs/src/provider/local/local.rs | 4 +- yazi-fs/src/scheme.rs | 3 + yazi-fs/src/url.rs | 6 +- yazi-shared/src/loc/loc.rs | 2 +- yazi-shared/src/path/kind.rs | 2 +- yazi-shared/src/scheme/cow.rs | 15 +- yazi-shared/src/scheme/encode.rs | 3 +- yazi-shared/src/scheme/kind.rs | 9 +- yazi-shared/src/scheme/ref.rs | 13 +- yazi-shared/src/scheme/scheme.rs | 9 +- yazi-shared/src/scheme/traits.rs | 1 + yazi-shared/src/strand/kind.rs | 2 +- yazi-shared/src/url/buf.rs | 12 +- yazi-shared/src/url/components.rs | 4 + yazi-shared/src/url/cow.rs | 22 ++- yazi-shared/src/url/encode.rs | 3 + yazi-shared/src/url/traits.rs | 2 + yazi-shared/src/url/url.rs | 65 +++++--- yazi-vfs/Cargo.toml | 1 + yazi-vfs/src/config/mod.rs | 2 +- yazi-vfs/src/config/rclone.rs | 28 ++++ yazi-vfs/src/config/service.rs | 15 +- yazi-vfs/src/provider/dir_entry.rs | 6 + yazi-vfs/src/provider/gate.rs | 1 + yazi-vfs/src/provider/mod.rs | 2 +- yazi-vfs/src/provider/provider.rs | 1 + yazi-vfs/src/provider/providers.rs | 29 ++++ yazi-vfs/src/provider/rclone/absolute.rs | 21 +++ yazi-vfs/src/provider/rclone/file.rs | 145 ++++++++++++++++++ yazi-vfs/src/provider/rclone/gate.rs | 73 +++++++++ yazi-vfs/src/provider/rclone/metadata.rs | 47 ++++++ yazi-vfs/src/provider/rclone/mod.rs | 1 + yazi-vfs/src/provider/rclone/rclone.rs | 180 +++++++++++++++++++++++ yazi-vfs/src/provider/rclone/read_dir.rs | 44 ++++++ yazi-vfs/src/provider/read_dir.rs | 2 + yazi-vfs/src/provider/rw_file.rs | 16 ++ yazi-vfs/src/provider/sftp/sftp.rs | 2 +- yazi-vfs/tests/rclone_local.rs | 158 ++++++++++++++++++++ yazi-vfs/tests/rclone_read.rs | 179 ++++++++++++++++++++++ yazi-watcher/src/reporter.rs | 2 +- yazi-watcher/src/watched.rs | 20 ++- 48 files changed, 1308 insertions(+), 56 deletions(-) create mode 100644 RCLONE_VFS.md create mode 100644 RCLONE_VFS_PROPOSAL.md create mode 100644 yazi-vfs/src/config/rclone.rs create mode 100644 yazi-vfs/src/provider/rclone/absolute.rs create mode 100644 yazi-vfs/src/provider/rclone/file.rs create mode 100644 yazi-vfs/src/provider/rclone/gate.rs create mode 100644 yazi-vfs/src/provider/rclone/metadata.rs create mode 100644 yazi-vfs/src/provider/rclone/mod.rs create mode 100644 yazi-vfs/src/provider/rclone/rclone.rs create mode 100644 yazi-vfs/src/provider/rclone/read_dir.rs create mode 100644 yazi-vfs/tests/rclone_local.rs create mode 100644 yazi-vfs/tests/rclone_read.rs diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7ea29bc1..e4693685 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -26,6 +26,18 @@ jobs: - name: Setup Rust toolchain run: rustup toolchain install stable --profile minimal + # For the rclone VFS provider's hermetic tests (Unix only; they drive an + # rclone `local` remote, so no cloud account or credentials are needed). + - name: Install rclone + if: runner.os != 'Windows' + shell: bash + run: | + if [ "$RUNNER_OS" = "macOS" ]; then + brew install rclone + else + sudo apt-get update && sudo apt-get install -y rclone + fi + - name: Setup sccache uses: mozilla-actions/sccache-action@v0.0.10 diff --git a/Cargo.lock b/Cargo.lock index cec37e11..99403e10 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5357,6 +5357,7 @@ dependencies = [ "parking_lot", "russh", "serde", + "serde_json", "tokio", "toml", "tracing", diff --git a/RCLONE_VFS.md b/RCLONE_VFS.md new file mode 100644 index 00000000..773335b3 --- /dev/null +++ b/RCLONE_VFS.md @@ -0,0 +1,94 @@ +# rclone VFS provider for yazi + +Browse **any of rclone's 100+ remotes** — S3, GCS, Azure, Backblaze, SFTP, +WebDAV, Google Drive, … — natively inside [yazi](https://github.com/sxyazi/yazi), +with no FUSE/NFS mount. It rides on yazi's existing VFS layer (the same +infrastructure behind the built-in SFTP provider) and shells out to the `rclone` +binary you already have. + +Addresses the "Integrate rclone" item on #3395. Read-only for now. + +--- + +## Usage + +Point a VFS service at an existing rclone remote in `~/.config/yazi/vfs.toml`: + +```toml +[services.gcs] +type = "rclone" +remote = "gcs" # any remote from `rclone listremotes` + +[services.s3] +type = "rclone" +remote = "s3" +# binary = "/opt/homebrew/bin/rclone" # optional: path to rclone +# config_file = "/path/to/rclone.conf" # optional: non-default config +# flags = ["--fast-list"] # optional: extra rclone args +``` + +Then browse with a `rclone:////` URL: + +``` +rclone://gcs//my-bucket/some/dir +rclone://s3//my-bucket +``` + +`rclone://gcs//` (empty path) lists the remote's buckets. Everything else is +normal yazi navigation, preview, and copy-out. + +## What works + +- **Directory listing** — `rclone lsjson` +- **Metadata** — `rclone lsjson --stat` (size + mtime; POSIX mode is synthesized + since object stores have no permissions) +- **Reading / preview / copy-out** — a seekable file handle that streams + `rclone cat --offset` +- **Object stores and filesystem backends** — `remote:/path` keeps its leading + slash, so `local`, `sftp`, etc. work alongside S3/GCS/… + +## What doesn't (yet) + +- **Writes** — every mutating op returns `Unsupported`. Read-only by design for a + first cut; uploads (`rclone rcat`/`copyto`) and mutations are a clean follow-up. +- **Remote search** — still disabled globally for remote schemes (per #3395). + +## How it works + +yazi's provider dispatch is a set of static enums, so a new `rclone://` scheme +threads a `SchemeKind::Rclone` variant through the scheme/URL type system — +mostly folded into the existing `Sftp` arms, since rclone remotes share the same +traits (remote, virtual, Unix-style paths). + +The interesting part is the file handle. rclone has no persistent open-file +concept, so `Provider::File` (which must be `AsyncRead + AsyncSeek`) is +implemented by streaming `rclone cat` from the current position; a **seek kills +the child process and the next read respawns `rclone cat --offset `**. This +is enough for yazi's previewer and for its parallel chunked copier (which opens +several handles and seeks each to its 8 MiB slice). + +## Testing + +Two layers, no credentials required to run CI: + +- **`tests/rclone_local.rs` — hermetic.** Points an rclone `local` remote at a + generated temp directory, so `lsjson`/`cat` run against the filesystem. Needs + only the `rclone` binary; **skips itself if rclone isn't installed** (so CI + without rclone stays green). Covers listing, read, seek, missing-object, and a + >8 MiB parallel multi-chunk copy with position-varying bytes so any offset slip + is caught. +- **`tests/rclone_read.rs` — live, `#[ignore]`d.** Runs against a real remote you + point it at via env vars (`YAZI_RCLONE_TEST_SMALL` / `_BIG` / `_DIR`) — nothing + is hard-coded. + +Verified end-to-end against live **GCS and S3** buckets: byte-exact reads, +mid-file seeks, and both simple and progressive (parallel multi-chunk) copy-out. + +## Notes / open questions for upstream + +- **CLI vs daemon.** This spawns an `rclone` process per operation — simple, no + lifecycle to manage. A long-lived `rclone rcd` (remote-control daemon) would + cut per-op latency and enable connection pooling like the SFTP provider does. + Happy to go either way. +- **Read-only scope.** Matches the earlier S3 PR (#3843). Writes can follow once + the direction is settled. diff --git a/RCLONE_VFS_PROPOSAL.md b/RCLONE_VFS_PROPOSAL.md new file mode 100644 index 00000000..af5d5bcf --- /dev/null +++ b/RCLONE_VFS_PROPOSAL.md @@ -0,0 +1,96 @@ +# Proposal: rclone-backed VFS provider (re: #3395) + +> **STATUS: LOCAL DRAFT — not posted anywhere.** This is the comment we'd +> put on issue #3395 once we decide to engage upstream. Nothing has been +> pushed or shared. + +## Summary + +A read-only `rclone://` VFS provider that shells out to the `rclone` CLI, +giving yazi native browsing of any of rclone's 100+ remotes (S3, GCS, Azure, +Backblaze, WebDAV, …) through a single provider — no FUSE/NFS mount. This is +the "integrate rclone" item on the #3395 roadmap, and reuses the VFS +infrastructure from #3821 / the (closed) S3 PR #3843. + +A working proof-of-concept is implemented and verified against live GCS and +S3 buckets (directory browsing, metadata, byte-exact streaming reads + seeks). + +## URL / scheme shape + +``` +rclone://// +``` + +`` is a name defined in `vfs.toml`; it maps to an rclone remote. +One scheme covers every backend — the backend choice lives in rclone's own +config, not in yazi. + +```toml +# ~/.config/yazi/vfs.toml +[services.gcs] +type = "rclone" +remote = "gcs" # an existing rclone remote (`rclone listremotes`) + +[services.s3] +type = "rclone" +remote = "s3" +# binary = "/opt/homebrew/bin/rclone" # optional +# config_file = "/path/to/rclone.conf" # optional +# flags = ["--fast-list"] # optional extra args +``` + +Then: `rclone://gcs//my-bucket/some/dir`. + +`SchemeKind::Rclone` mirrors `Sftp` — remote, virtual, Unix-path, +byte-encoded filenames — so it threads through the existing scheme/URL enums +with minimal new surface (mostly folded into existing `Sftp` match arms). + +## How each operation maps + +| Provider op | rclone | +|--------------------|------------------------------------------------| +| `read_dir` | `rclone lsjson :` | +| `metadata` | `rclone lsjson --stat :` | +| `open`/read | `rclone cat --offset :` | +| write/create/… | `Unsupported` (read-only v1) | + +### The `AsyncSeek`-over-a-CLI problem + +`Provider::File` must be `AsyncRead + AsyncSeek + AsyncWrite`. rclone has no +persistent file handle, so the `File` type streams `rclone cat` from the +current offset; a `seek` kills the child process and the next read respawns +`rclone cat --offset `. This makes the file seekable enough for yazi's +previewer and copy paths without any random-access protocol. Verified +byte-exact vs `rclone cat` for both full reads and post-seek tail reads. + +## Scope decisions (seeking maintainer input) + +1. **Read-only first.** Matches PR #3843's scope. Writes (`rcat`/`copyto`) + and mutations (`deletefile`, `mkdir`, `moveto`) are a clean follow-up but + raise consistency/UX questions worth deciding separately. +2. **CLI vs `rclone rcd` daemon.** The PoC spawns a subprocess per op + (simple, no lifecycle). A long-lived `rclone rcd` + HTTP API would cut + per-op latency and enable a connection-pool model like the SFTP provider. + Happy to go either way — which do you prefer? +3. **Metadata.** rclone gives `Size`/`ModTime`/`IsDir` but no + uid/gid/perm/nlink; the PoC synthesizes `0755`/`0644`. `Capabilities` + only models `symlink` today, so read-only-ness surfaces as per-op + `Unsupported`. Want a `writable`/`read_only` capability flag? +4. **Pagination.** `lsjson` returns a whole directory as one JSON array + (simple, but memory-heavy for very large dirs). Incremental listing would + need `--files-only`/chunking or the rc daemon. Acceptable for v1? +5. **`search` on remote schemes** is currently disabled (per #3395) — this + provider doesn't change that. + +## Rough edit surface + +~40 files, but the vast majority are one-line enum/match additions folded +into existing `Sftp` arms. The real code is the ~6-file provider module in +`yazi-vfs/src/provider/rclone/` plus `config/rclone.rs`. Diffstat of the +PoC: 44 files, ~790 insertions. + +## Open question for the maintainer + +Before we polish this into a PR: does this scheme shape and the CLI-first +(vs rcd-daemon) approach match what you discussed on Telegram? If the daemon +model is preferred, we'll refactor the transport before submitting. diff --git a/yazi-config/src/pattern.rs b/yazi-config/src/pattern.rs index 37fd6dc3..9e35dd55 100644 --- a/yazi-config/src/pattern.rs +++ b/yazi-config/src/pattern.rs @@ -123,6 +123,7 @@ enum PatternScheme { Search, Archive, Sftp, + Rclone, } impl PatternScheme { @@ -141,6 +142,7 @@ impl PatternScheme { "search" => Self::Search, "archive" => Self::Archive, "sftp" => Self::Sftp, + "rclone" => Self::Rclone, "" => bail!("Invalid URL pattern: protocol is empty"), _ => bail!("Unknown protocol in URL pattern: {protocol}"), @@ -163,6 +165,7 @@ impl PatternScheme { (Self::Search, K::Search) => true, (Self::Archive, K::Archive) => true, (Self::Sftp, K::Sftp) => true, + (Self::Rclone, K::Rclone) => true, _ => false, } diff --git a/yazi-fs/src/path/expand.rs b/yazi-fs/src/path/expand.rs index b5a49027..caa40554 100644 --- a/yazi-fs/src/path/expand.rs +++ b/yazi-fs/src/path/expand.rs @@ -44,6 +44,10 @@ fn expand_url_impl(url: UrlCow) -> UrlCow { loc: LocBuf::::with(path.into_unix().unwrap(), uri, urn).unwrap(), domain: domain.intern(), }, + Url::Rclone { domain, .. } => UrlBuf::Rclone { + loc: LocBuf::::with(path.into_unix().unwrap(), uri, urn).unwrap(), + domain: domain.intern(), + }, } .into() } diff --git a/yazi-fs/src/provider/local/absolute.rs b/yazi-fs/src/provider/local/absolute.rs index d22cceda..004f1498 100644 --- a/yazi-fs/src/provider/local/absolute.rs +++ b/yazi-fs/src/provider/local/absolute.rs @@ -51,6 +51,6 @@ fn try_absolute_impl<'a>(url: UrlCow<'a>) -> Option> { Some(match url.as_url() { Url::Regular(_) => UrlBuf::Regular(loc).into(), Url::Search { domain, .. } => UrlBuf::Search { loc, domain: domain.intern() }.into(), - Url::Archive { .. } | Url::Sftp { .. } => None?, + Url::Archive { .. } | Url::Sftp { .. } | Url::Rclone { .. } => None?, }) } diff --git a/yazi-fs/src/provider/local/local.rs b/yazi-fs/src/provider/local/local.rs index 6ec0557d..87f4b12b 100644 --- a/yazi-fs/src/provider/local/local.rs +++ b/yazi-fs/src/provider/local/local.rs @@ -80,7 +80,7 @@ impl<'a> Provider for Local<'a> { async fn new<'b>(url: Url<'b>) -> io::Result> { match url { Url::Regular(loc) | Url::Search { loc, .. } => Ok(Self::Me { url, path: loc.as_inner() }), - Url::Archive { .. } | Url::Sftp { .. } => { + Url::Archive { .. } | Url::Sftp { .. } | Url::Rclone { .. } => { Err(io::Error::new(io::ErrorKind::InvalidInput, format!("Not a local URL: {url:?}"))) } } @@ -94,7 +94,7 @@ impl<'a> Provider for Local<'a> { reader: tokio::fs::read_dir(self.path).await?, dir: Arc::new(self.url.to_owned()), }, - SchemeKind::Archive | SchemeKind::Sftp => Err(io::Error::new( + SchemeKind::Archive | SchemeKind::Sftp | SchemeKind::Rclone => Err(io::Error::new( io::ErrorKind::InvalidInput, format!("Not a local URL: {:?}", self.url), ))?, diff --git a/yazi-fs/src/scheme.rs b/yazi-fs/src/scheme.rs index 415a0252..1c74d3cd 100644 --- a/yazi-fs/src/scheme.rs +++ b/yazi-fs/src/scheme.rs @@ -19,6 +19,9 @@ impl FsScheme for SchemeRef<'_> { Self::Sftp { domain, .. } => { Some(Xdg::temp_dir().join(format!("sftp-{}", yazi_shared::scheme::Encode::domain(domain)))) } + Self::Rclone { domain, .. } => Some( + Xdg::temp_dir().join(format!("rclone-{}", yazi_shared::scheme::Encode::domain(domain))), + ), } } } diff --git a/yazi-fs/src/url.rs b/yazi-fs/src/url.rs index 80d17a9a..5049a785 100644 --- a/yazi-fs/src/url.rs +++ b/yazi-fs/src/url.rs @@ -50,7 +50,7 @@ impl<'a> FsUrl<'a> for Url<'a> { fn unified_path(self) -> Cow<'a, Path> { match self { Self::Regular(loc) | Self::Search { loc, .. } => loc.as_inner().into(), - Self::Archive { .. } | Self::Sftp { .. } => { + Self::Archive { .. } | Self::Sftp { .. } | Self::Rclone { .. } => { self.cache().expect("non-local URL should have a cache path").into() } } @@ -65,7 +65,7 @@ impl FsUrl<'_> for UrlBuf { fn unified_path(self) -> Cow<'static, Path> { match self { Self::Regular(loc) | Self::Search { loc, .. } => loc.into_inner().into(), - Self::Archive { .. } | Self::Sftp { .. } => { + Self::Archive { .. } | Self::Sftp { .. } | Self::Rclone { .. } => { self.cache().expect("non-local URL should have a cache path").into() } } @@ -80,7 +80,7 @@ impl<'a> FsUrl<'a> for UrlCow<'a> { fn unified_path(self) -> Cow<'a, Path> { match self { Self::Regular(loc) | Self::Search { loc, .. } => loc.into_inner(), - Self::Archive { .. } | Self::Sftp { .. } => { + Self::Archive { .. } | Self::Sftp { .. } | Self::Rclone { .. } => { self.cache().expect("non-local URL should have a cache path").into() } } diff --git a/yazi-shared/src/loc/loc.rs b/yazi-shared/src/loc/loc.rs index a1824ee8..10ba960e 100644 --- a/yazi-shared/src/loc/loc.rs +++ b/yazi-shared/src/loc/loc.rs @@ -155,7 +155,7 @@ where SchemeKind::Regular => Self::bare(path), SchemeKind::Search => Self::zeroed(path), SchemeKind::Archive => Self::zeroed(path), - SchemeKind::Sftp => Self::bare(path), + SchemeKind::Sftp | SchemeKind::Rclone => Self::bare(path), } } diff --git a/yazi-shared/src/path/kind.rs b/yazi-shared/src/path/kind.rs index fb2c9a1f..a0d37659 100644 --- a/yazi-shared/src/path/kind.rs +++ b/yazi-shared/src/path/kind.rs @@ -15,7 +15,7 @@ impl From for PathKind { SchemeKind::Regular => Self::Os, SchemeKind::Search => Self::Os, SchemeKind::Archive => Self::Os, - SchemeKind::Sftp => Self::Unix, + SchemeKind::Sftp | SchemeKind::Rclone => Self::Unix, } } } diff --git a/yazi-shared/src/scheme/cow.rs b/yazi-shared/src/scheme/cow.rs index 30a4eea0..a941f13f 100644 --- a/yazi-shared/src/scheme/cow.rs +++ b/yazi-shared/src/scheme/cow.rs @@ -67,6 +67,16 @@ impl<'a> SchemeCow<'a> { } } + pub fn rclone(domain: T, uri: usize, urn: usize) -> Self + where + T: Into>, + { + match domain.into() { + Cow::Borrowed(domain) => SchemeRef::Rclone { domain, uri, urn }.into(), + Cow::Owned(domain) => Scheme::Rclone { domain: domain.intern(), uri, urn }.into(), + } + } + pub fn parse(bytes: &'a [u8]) -> Result<(Self, PathCow<'a>)> { let Some((kind, tilde)) = SchemeKind::parse(bytes)? else { let path = Self::decode_path(SchemeKind::Regular, false, bytes)?; @@ -81,6 +91,7 @@ impl<'a> SchemeCow<'a> { SchemeKind::Search => Self::decode_param(&bytes[skip..], &mut skip)?, SchemeKind::Archive => Self::decode_param(&bytes[skip..], &mut skip)?, SchemeKind::Sftp => Self::decode_param(&bytes[skip..], &mut skip)?, + SchemeKind::Rclone => Self::decode_param(&bytes[skip..], &mut skip)?, }; // Decode path @@ -93,6 +104,7 @@ impl<'a> SchemeCow<'a> { SchemeKind::Search => Self::search(domain, uri, urn), SchemeKind::Archive => Self::archive(domain, uri, urn), SchemeKind::Sftp => Self::sftp(domain, uri, urn), + SchemeKind::Rclone => Self::rclone(domain, uri, urn), }; Ok((scheme, path)) @@ -154,7 +166,7 @@ impl<'a> SchemeCow<'a> { (uri, urn) } SchemeKind::Archive => (uri.unwrap_or(0), urn.unwrap_or(0)), - SchemeKind::Sftp => { + SchemeKind::Sftp | SchemeKind::Rclone => { let uri = uri.unwrap_or(path.name().is_some() as usize); let urn = urn.unwrap_or(path.name().is_some() as usize); (uri, urn) @@ -168,6 +180,7 @@ impl<'a> SchemeCow<'a> { Url::Search { loc, .. } => (loc.uri().components().count(), loc.urn().components().count()), Url::Archive { loc, .. } => (loc.uri().components().count(), loc.urn().components().count()), Url::Sftp { loc, .. } => (loc.uri().components().count(), loc.urn().components().count()), + Url::Rclone { loc, .. } => (loc.uri().components().count(), loc.urn().components().count()), } } } diff --git a/yazi-shared/src/scheme/encode.rs b/yazi-shared/src/scheme/encode.rs index c51e28ea..40166959 100644 --- a/yazi-shared/src/scheme/encode.rs +++ b/yazi-shared/src/scheme/encode.rs @@ -38,7 +38,7 @@ impl<'a> Encode<'a> { match self.0.0.kind() { SchemeKind::Regular => Ok(()), SchemeKind::Search | SchemeKind::Archive => w!(0, 0), - SchemeKind::Sftp => { + SchemeKind::Sftp | SchemeKind::Rclone => { w!(self.0.0.loc().name().is_some() as usize, self.0.0.loc().name().is_some() as usize) } } @@ -58,6 +58,7 @@ impl Display for Encode<'_> { write!(f, "archive://{}{}/", Self::domain(domain), self.ports()) } Url::Sftp { domain, .. } => write!(f, "sftp://{}{}/", Self::domain(domain), self.ports()), + Url::Rclone { domain, .. } => write!(f, "rclone://{}{}/", Self::domain(domain), self.ports()), } } } diff --git a/yazi-shared/src/scheme/kind.rs b/yazi-shared/src/scheme/kind.rs index c5ddc3dc..948805f0 100644 --- a/yazi-shared/src/scheme/kind.rs +++ b/yazi-shared/src/scheme/kind.rs @@ -10,6 +10,7 @@ pub enum SchemeKind { Search, Archive, Sftp, + Rclone, } impl From for SchemeKind @@ -22,6 +23,7 @@ where SchemeRef::Search { .. } => Self::Search, SchemeRef::Archive { .. } => Self::Archive, SchemeRef::Sftp { .. } => Self::Sftp, + SchemeRef::Rclone { .. } => Self::Rclone, } } } @@ -35,6 +37,7 @@ impl TryFrom<&[u8]> for SchemeKind { b"search" => Ok(Self::Search), b"archive" => Ok(Self::Archive), b"sftp" => Ok(Self::Sftp), + b"rclone" => Ok(Self::Rclone), _ => bail!("invalid scheme kind: {}", String::from_utf8_lossy(value)), } } @@ -45,7 +48,7 @@ impl SchemeKind { pub fn is_local(self) -> bool { match self { Self::Regular | Self::Search => true, - Self::Archive | Self::Sftp => false, + Self::Archive | Self::Sftp | Self::Rclone => false, } } @@ -53,7 +56,7 @@ impl SchemeKind { pub fn is_remote(self) -> bool { match self { Self::Regular | Self::Search | Self::Archive => false, - Self::Sftp => true, + Self::Sftp | Self::Rclone => true, } } @@ -61,7 +64,7 @@ impl SchemeKind { pub fn is_virtual(self) -> bool { match self { Self::Regular | Self::Search => false, - Self::Archive | Self::Sftp => true, + Self::Archive | Self::Sftp | Self::Rclone => true, } } diff --git a/yazi-shared/src/scheme/ref.rs b/yazi-shared/src/scheme/ref.rs index 6259f699..3c09575d 100644 --- a/yazi-shared/src/scheme/ref.rs +++ b/yazi-shared/src/scheme/ref.rs @@ -8,6 +8,7 @@ pub enum SchemeRef<'a> { Search { domain: &'a str, uri: usize, urn: usize }, Archive { domain: &'a str, uri: usize, urn: usize }, Sftp { domain: &'a str, uri: usize, urn: usize }, + Rclone { domain: &'a str, uri: usize, urn: usize }, } impl Deref for SchemeRef<'_> { @@ -19,6 +20,7 @@ impl Deref for SchemeRef<'_> { Self::Search { .. } => &SchemeKind::Search, Self::Archive { .. } => &SchemeKind::Archive, Self::Sftp { .. } => &SchemeKind::Sftp, + Self::Rclone { .. } => &SchemeKind::Rclone, } } } @@ -49,9 +51,10 @@ impl<'a> SchemeRef<'a> { pub const fn domain(self) -> Option<&'a str> { match self { Self::Regular { .. } => None, - Self::Search { domain, .. } | Self::Archive { domain, .. } | Self::Sftp { domain, .. } => { - Some(domain) - } + Self::Search { domain, .. } + | Self::Archive { domain, .. } + | Self::Sftp { domain, .. } + | Self::Rclone { domain, .. } => Some(domain), } } @@ -61,6 +64,7 @@ impl<'a> SchemeRef<'a> { Self::Search { .. } => SchemeKind::Search, Self::Archive { .. } => SchemeKind::Archive, Self::Sftp { .. } => SchemeKind::Sftp, + Self::Rclone { .. } => SchemeKind::Rclone, } } @@ -70,6 +74,7 @@ impl<'a> SchemeRef<'a> { Self::Search { uri, urn, .. } => (uri, urn), Self::Archive { uri, urn, .. } => (uri, urn), Self::Sftp { uri, urn, .. } => (uri, urn), + Self::Rclone { uri, urn, .. } => (uri, urn), } } @@ -79,6 +84,7 @@ impl<'a> SchemeRef<'a> { Self::Search { domain, uri, urn } => Scheme::Search { domain: domain.intern(), uri, urn }, Self::Archive { domain, uri, urn } => Scheme::Archive { domain: domain.intern(), uri, urn }, Self::Sftp { domain, uri, urn } => Scheme::Sftp { domain: domain.intern(), uri, urn }, + Self::Rclone { domain, uri, urn } => Scheme::Rclone { domain: domain.intern(), uri, urn }, } } @@ -88,6 +94,7 @@ impl<'a> SchemeRef<'a> { Self::Search { domain, .. } => Self::Search { domain, uri, urn }, Self::Archive { domain, .. } => Self::Archive { domain, uri, urn }, Self::Sftp { domain, .. } => Self::Sftp { domain, uri, urn }, + Self::Rclone { domain, .. } => Self::Rclone { domain, uri, urn }, } } diff --git a/yazi-shared/src/scheme/scheme.rs b/yazi-shared/src/scheme/scheme.rs index 8148a431..d9186475 100644 --- a/yazi-shared/src/scheme/scheme.rs +++ b/yazi-shared/src/scheme/scheme.rs @@ -11,6 +11,7 @@ pub enum Scheme { Search { domain: Symbol, uri: usize, urn: usize }, Archive { domain: Symbol, uri: usize, urn: usize }, Sftp { domain: Symbol, uri: usize, urn: usize }, + Rclone { domain: Symbol, uri: usize, urn: usize }, } impl Hash for Scheme { @@ -26,9 +27,10 @@ impl Scheme { pub fn into_domain(self) -> Option> { match self { Self::Regular { .. } => None, - Self::Search { domain, .. } | Self::Archive { domain, .. } | Self::Sftp { domain, .. } => { - Some(domain) - } + Self::Search { domain, .. } + | Self::Archive { domain, .. } + | Self::Sftp { domain, .. } + | Self::Rclone { domain, .. } => Some(domain), } } @@ -39,6 +41,7 @@ impl Scheme { Self::Search { domain, .. } => Self::Search { domain, uri, urn }, Self::Archive { domain, .. } => Self::Archive { domain, uri, urn }, Self::Sftp { domain, .. } => Self::Sftp { domain, uri, urn }, + Self::Rclone { domain, .. } => Self::Rclone { domain, uri, urn }, } } diff --git a/yazi-shared/src/scheme/traits.rs b/yazi-shared/src/scheme/traits.rs index b36cbc56..8ec49ffa 100644 --- a/yazi-shared/src/scheme/traits.rs +++ b/yazi-shared/src/scheme/traits.rs @@ -17,6 +17,7 @@ impl AsScheme for Scheme { Self::Search { ref domain, uri, urn } => SchemeRef::Search { domain, uri, urn }, Self::Archive { ref domain, uri, urn } => SchemeRef::Archive { domain, uri, urn }, Self::Sftp { ref domain, uri, urn } => SchemeRef::Sftp { domain, uri, urn }, + Self::Rclone { ref domain, uri, urn } => SchemeRef::Rclone { domain, uri, urn }, } } } diff --git a/yazi-shared/src/strand/kind.rs b/yazi-shared/src/strand/kind.rs index 103afc20..1fcdd130 100644 --- a/yazi-shared/src/strand/kind.rs +++ b/yazi-shared/src/strand/kind.rs @@ -22,7 +22,7 @@ impl From for StrandKind { SchemeKind::Regular => Self::Os, SchemeKind::Search => Self::Os, SchemeKind::Archive => Self::Os, - SchemeKind::Sftp => Self::Bytes, + SchemeKind::Sftp | SchemeKind::Rclone => Self::Bytes, } } } diff --git a/yazi-shared/src/url/buf.rs b/yazi-shared/src/url/buf.rs index ef9b2e05..6085658e 100644 --- a/yazi-shared/src/url/buf.rs +++ b/yazi-shared/src/url/buf.rs @@ -13,6 +13,7 @@ pub enum UrlBuf { Search { loc: LocBuf, domain: Symbol }, Archive { loc: LocBuf, domain: Symbol }, Sftp { loc: LocBuf, domain: Symbol }, + Rclone { loc: LocBuf, domain: Symbol }, } impl_data_any!(UrlBuf); @@ -33,6 +34,7 @@ impl From> for UrlBuf { Url::Search { loc, domain } => Self::Search { loc: loc.into(), domain: domain.intern() }, Url::Archive { loc, domain } => Self::Archive { loc: loc.into(), domain: domain.intern() }, Url::Sftp { loc, domain } => Self::Sftp { loc: loc.into(), domain: domain.intern() }, + Url::Rclone { loc, domain } => Self::Rclone { loc: loc.into(), domain: domain.intern() }, } } } @@ -138,7 +140,7 @@ impl UrlBuf { Self::Regular(loc) => loc.into_inner().into(), Self::Search { loc, .. } => loc.into_inner().into(), Self::Archive { loc, .. } => loc.into_inner().into(), - Self::Sftp { loc, .. } => loc.into_inner().into(), + Self::Sftp { loc, .. } | Self::Rclone { loc, .. } => loc.into_inner().into(), } } @@ -153,7 +155,9 @@ impl UrlBuf { Self::Regular(loc) => loc.try_set_name(name.as_os()?)?, Self::Search { loc, .. } => loc.try_set_name(name.as_os()?)?, Self::Archive { loc, .. } => loc.try_set_name(name.as_os()?)?, - Self::Sftp { loc, .. } => loc.try_set_name(name.encoded_bytes())?, + Self::Sftp { loc, .. } | Self::Rclone { loc, .. } => { + loc.try_set_name(name.encoded_bytes())? + } }) } @@ -170,6 +174,10 @@ impl UrlBuf { todo!(); // Self::Sftp { loc: loc.rebase(base), domain: domain.clone() } } + Self::Rclone { loc, domain } => { + todo!(); + // Self::Rclone { loc: loc.rebase(base), domain: domain.clone() } + } } } } diff --git a/yazi-shared/src/url/components.rs b/yazi-shared/src/url/components.rs index 50778f81..7199777b 100644 --- a/yazi-shared/src/url/components.rs +++ b/yazi-shared/src/url/components.rs @@ -60,6 +60,7 @@ impl<'a> Components<'a> { Url::Search { domain, .. } => SchemeRef::Search { domain, uri, urn }, Url::Archive { domain, .. } => SchemeRef::Archive { domain, uri, urn }, Url::Sftp { domain, .. } => SchemeRef::Sftp { domain, uri, urn }, + Url::Rclone { domain, .. } => SchemeRef::Rclone { domain, uri, urn }, } } @@ -89,6 +90,9 @@ impl<'a> Components<'a> { Url::Sftp { domain, .. } => { Url::Sftp { loc: Loc::with(path.as_unix().unwrap(), uri, urn).unwrap(), domain } } + Url::Rclone { domain, .. } => { + Url::Rclone { loc: Loc::with(path.as_unix().unwrap(), uri, urn).unwrap(), domain } + } } } } diff --git a/yazi-shared/src/url/cow.rs b/yazi-shared/src/url/cow.rs index 323e34fb..02fea695 100644 --- a/yazi-shared/src/url/cow.rs +++ b/yazi-shared/src/url/cow.rs @@ -12,6 +12,7 @@ pub enum UrlCow<'a> { Search { loc: LocCow<'a>, domain: SymbolCow<'a, str> }, Archive { loc: LocCow<'a>, domain: SymbolCow<'a, str> }, Sftp { loc: LocCow<'a, &'a UnixPath, UnixPathBuf>, domain: SymbolCow<'a, str> }, + Rclone { loc: LocCow<'a, &'a UnixPath, UnixPathBuf>, domain: SymbolCow<'a, str> }, } // FIXME: remove @@ -26,6 +27,7 @@ impl<'a> From> for UrlCow<'a> { Url::Search { loc, domain } => Self::Search { loc: loc.into(), domain: domain.into() }, Url::Archive { loc, domain } => Self::Archive { loc: loc.into(), domain: domain.into() }, Url::Sftp { loc, domain } => Self::Sftp { loc: loc.into(), domain: domain.into() }, + Url::Rclone { loc, domain } => Self::Rclone { loc: loc.into(), domain: domain.into() }, } } } @@ -46,6 +48,7 @@ impl From for UrlCow<'_> { Self::Archive { loc: loc.into(), domain: domain.into() } } UrlBuf::Sftp { loc, domain } => Self::Sftp { loc: loc.into(), domain: domain.into() }, + UrlBuf::Rclone { loc, domain } => Self::Rclone { loc: loc.into(), domain: domain.into() }, } } } @@ -157,6 +160,10 @@ impl<'a> TryFrom<(SchemeCow<'a>, PathDyn<'a>)> for UrlCow<'a> { loc: Loc::with(path.as_unix()?, uri, urn)?.into(), domain: domain.ok_or_else(|| anyhow!("missing domain for sftp scheme"))?, }, + SchemeKind::Rclone => Self::Rclone { + loc: Loc::with(path.as_unix()?, uri, urn)?.into(), + domain: domain.ok_or_else(|| anyhow!("missing domain for rclone scheme"))?, + }, }) } } @@ -184,6 +191,10 @@ impl<'a> TryFrom<(SchemeCow<'a>, PathBufDyn)> for UrlCow<'a> { loc: LocBuf::::with(path.try_into()?, uri, urn)?.into(), domain: domain.ok_or_else(|| anyhow!("missing domain for sftp scheme"))?, }, + SchemeKind::Rclone => Self::Rclone { + loc: LocBuf::::with(path.try_into()?, uri, urn)?.into(), + domain: domain.ok_or_else(|| anyhow!("missing domain for rclone scheme"))?, + }, }) } } @@ -210,7 +221,7 @@ impl<'a> UrlCow<'a> { Self::Regular(loc) => loc.is_owned(), Self::Search { loc, .. } => loc.is_owned(), Self::Archive { loc, .. } => loc.is_owned(), - Self::Sftp { loc, .. } => loc.is_owned(), + Self::Sftp { loc, .. } | Self::Rclone { loc, .. } => loc.is_owned(), } } @@ -226,6 +237,9 @@ impl<'a> UrlCow<'a> { Self::Sftp { loc, domain } => { UrlBuf::Sftp { loc: loc.into_owned(), domain: domain.into() } } + Self::Rclone { loc, domain } => { + UrlBuf::Rclone { loc: loc.into_owned(), domain: domain.into() } + } } } @@ -254,6 +268,12 @@ impl<'a> UrlCow<'a> { } SymbolCow::Owned(domain) => (Scheme::Sftp { domain, uri, urn }.into(), loc.into_path()), }, + Self::Rclone { loc, domain } => match domain { + SymbolCow::Borrowed(domain) => { + (SchemeRef::Rclone { domain, uri, urn }.into(), loc.into_path()) + } + SymbolCow::Owned(domain) => (Scheme::Rclone { domain, uri, urn }.into(), loc.into_path()), + }, } } diff --git a/yazi-shared/src/url/encode.rs b/yazi-shared/src/url/encode.rs index 1695e50e..ba8a4e62 100644 --- a/yazi-shared/src/url/encode.rs +++ b/yazi-shared/src/url/encode.rs @@ -23,6 +23,9 @@ impl Display for Encode<'_> { Url::Sftp { domain, .. } => { write!(f, "sftp~://{}{}/{loc}", E::domain(domain), E::ports((*self).into())) } + Url::Rclone { domain, .. } => { + write!(f, "rclone~://{}{}/{loc}", E::domain(domain), E::ports((*self).into())) + } } } } diff --git a/yazi-shared/src/url/traits.rs b/yazi-shared/src/url/traits.rs index c1ea5387..c3c4a3cc 100644 --- a/yazi-shared/src/url/traits.rs +++ b/yazi-shared/src/url/traits.rs @@ -43,6 +43,7 @@ impl AsUrl for UrlBuf { Self::Search { loc, domain } => Url::Search { loc: loc.as_loc(), domain }, Self::Archive { loc, domain } => Url::Archive { loc: loc.as_loc(), domain }, Self::Sftp { loc, domain } => Url::Sftp { loc: loc.as_loc(), domain }, + Self::Rclone { loc, domain } => Url::Rclone { loc: loc.as_loc(), domain }, } } } @@ -64,6 +65,7 @@ impl AsUrl for UrlCow<'_> { Self::Search { loc, domain } => Url::Search { loc: loc.as_loc(), domain }, Self::Archive { loc, domain } => Url::Archive { loc: loc.as_loc(), domain }, Self::Sftp { loc, domain } => Url::Sftp { loc: loc.as_loc(), domain }, + Self::Rclone { loc, domain } => Url::Rclone { loc: loc.as_loc(), domain }, } } } diff --git a/yazi-shared/src/url/url.rs b/yazi-shared/src/url/url.rs index a17abc9a..175af4a4 100644 --- a/yazi-shared/src/url/url.rs +++ b/yazi-shared/src/url/url.rs @@ -13,6 +13,7 @@ pub enum Url<'a> { Search { loc: Loc<'a>, domain: &'a str }, Archive { loc: Loc<'a>, domain: &'a str }, Sftp { loc: Loc<'a, &'a typed_path::UnixPath>, domain: &'a str }, + Rclone { loc: Loc<'a, &'a typed_path::UnixPath>, domain: &'a str }, } // --- Eq @@ -64,6 +65,7 @@ impl<'a> Url<'a> { Self::Search { loc, domain } => Self::Search { loc: Loc::zeroed(loc.base()), domain }, Self::Archive { loc, domain } => Self::Archive { loc: Loc::zeroed(loc.base()), domain }, Self::Sftp { loc, domain } => Self::Sftp { loc: Loc::bare(loc.base()), domain }, + Self::Rclone { loc, domain } => Self::Rclone { loc: Loc::bare(loc.base()), domain }, } } @@ -82,7 +84,7 @@ impl<'a> Url<'a> { Self::Regular(loc) => loc.extension()?.as_strand(), Self::Search { loc, .. } => loc.extension()?.as_strand(), Self::Archive { loc, .. } => loc.extension()?.as_strand(), - Self::Sftp { loc, .. } => loc.extension()?.as_strand(), + Self::Sftp { loc, .. } | Self::Rclone { loc, .. } => loc.extension()?.as_strand(), }) } @@ -92,7 +94,7 @@ impl<'a> Url<'a> { Self::Regular(loc) => loc.has_base(), Self::Search { loc, .. } => loc.has_base(), Self::Archive { loc, .. } => loc.has_base(), - Self::Sftp { loc, .. } => loc.has_base(), + Self::Sftp { loc, .. } | Self::Rclone { loc, .. } => loc.has_base(), } } @@ -105,7 +107,7 @@ impl<'a> Url<'a> { Self::Regular(loc) => loc.has_trail(), Self::Search { loc, .. } => loc.has_trail(), Self::Archive { loc, .. } => loc.has_trail(), - Self::Sftp { loc, .. } => loc.has_trail(), + Self::Sftp { loc, .. } | Self::Rclone { loc, .. } => loc.has_trail(), } } @@ -118,7 +120,7 @@ impl<'a> Url<'a> { #[inline] pub fn is_internal(self) -> bool { match self { - Self::Regular(_) | Self::Sftp { .. } => true, + Self::Regular(_) | Self::Sftp { .. } | Self::Rclone { .. } => true, Self::Search { .. } => !self.uri().is_empty(), Self::Archive { .. } => false, } @@ -137,6 +139,7 @@ impl<'a> Url<'a> { Self::Search { .. } => SchemeKind::Search, Self::Archive { .. } => SchemeKind::Archive, Self::Sftp { .. } => SchemeKind::Sftp, + Self::Rclone { .. } => SchemeKind::Rclone, } } @@ -146,7 +149,7 @@ impl<'a> Url<'a> { Self::Regular(loc) => loc.as_path(), Self::Search { loc, .. } => loc.as_path(), Self::Archive { loc, .. } => loc.as_path(), - Self::Sftp { loc, .. } => loc.as_path(), + Self::Sftp { loc, .. } | Self::Rclone { loc, .. } => loc.as_path(), } } @@ -156,7 +159,7 @@ impl<'a> Url<'a> { Self::Regular(loc) => loc.file_name()?.as_strand(), Self::Search { loc, .. } => loc.file_name()?.as_strand(), Self::Archive { loc, .. } => loc.file_name()?.as_strand(), - Self::Sftp { loc, .. } => loc.file_name()?.as_strand(), + Self::Sftp { loc, .. } | Self::Rclone { loc, .. } => loc.file_name()?.as_strand(), }) } @@ -190,6 +193,9 @@ impl<'a> Url<'a> { // SFTP Self::Sftp { loc, domain } => Self::Sftp { loc: Loc::bare(loc.parent()?), domain }, + + // Rclone + Self::Rclone { loc, domain } => Self::Rclone { loc: Loc::bare(loc.parent()?), domain }, }) } @@ -206,6 +212,7 @@ impl<'a> Url<'a> { Self::Search { domain, .. } => SchemeRef::Search { domain, uri, urn }, Self::Archive { domain, .. } => SchemeRef::Archive { domain, uri, urn }, Self::Sftp { domain, .. } => SchemeRef::Sftp { domain, uri, urn }, + Self::Rclone { domain, .. } => SchemeRef::Rclone { domain, uri, urn }, } } @@ -215,7 +222,7 @@ impl<'a> Url<'a> { Self::Regular(loc) => loc.file_stem()?.as_strand(), Self::Search { loc, .. } => loc.file_stem()?.as_strand(), Self::Archive { loc, .. } => loc.file_stem()?.as_strand(), - Self::Sftp { loc, .. } => loc.file_stem()?.as_strand(), + Self::Sftp { loc, .. } | Self::Rclone { loc, .. } => loc.file_stem()?.as_strand(), }) } @@ -249,6 +256,8 @@ impl<'a> Url<'a> { } Self::Sftp { loc, domain } => Self::Sftp { loc: Loc::bare(loc.trail()), domain }, + + Self::Rclone { loc, domain } => Self::Rclone { loc: Loc::bare(loc.trail()), domain }, } } @@ -258,7 +267,7 @@ impl<'a> Url<'a> { let (base, rest, urn) = loc.triple(); (base.as_path(), rest.as_path(), urn.as_path()) } - Self::Sftp { loc, .. } => { + Self::Sftp { loc, .. } | Self::Rclone { loc, .. } => { let (base, rest, urn) = loc.triple(); (base.as_path(), rest.as_path(), urn.as_path()) } @@ -298,6 +307,10 @@ impl<'a> Url<'a> { Self::Sftp { domain, .. } => { UrlBuf::Sftp { loc: joined.into_unix()?.into(), domain: domain.intern() } } + + Self::Rclone { domain, .. } => { + UrlBuf::Rclone { loc: joined.into_unix()?.into(), domain: domain.intern() } + } }) } @@ -332,6 +345,10 @@ impl<'a> Url<'a> { loc: LocBuf::::new(path.into_unix()?, loc.base(), loc.trail()), domain: domain.intern(), }, + Self::Rclone { loc, domain } if path.try_starts_with(loc.trail())? => UrlBuf::Rclone { + loc: LocBuf::::new(path.into_unix()?, loc.base(), loc.trail()), + domain: domain.intern(), + }, Self::Search { domain, .. } => UrlBuf::Search { loc: LocBuf::::saturated(path.into_os()?, self.kind()), @@ -345,6 +362,10 @@ impl<'a> Url<'a> { loc: LocBuf::::saturated(path.into_unix()?, self.kind()), domain: domain.intern(), }, + Self::Rclone { domain, .. } => UrlBuf::Rclone { + loc: LocBuf::::saturated(path.into_unix()?, self.kind()), + domain: domain.intern(), + }, }; Ok(url.into()) @@ -373,6 +394,9 @@ impl<'a> Url<'a> { (U::Sftp { domain: a, .. }, U::Sftp { domain: b, .. }) => { (a == b).then_some(prefix).ok_or(Exotic) } + (U::Rclone { domain: a, .. }, U::Rclone { domain: b, .. }) => { + (a == b).then_some(prefix).ok_or(Exotic) + } // Both are local files (U::Regular(_), U::Search { .. }) => Ok(prefix), @@ -392,13 +416,8 @@ impl<'a> Url<'a> { self.uri().is_empty().then_some(prefix).ok_or(NotPrefix) } - // Independent virtual file space - (U::Regular(_), U::Sftp { .. }) => Err(Exotic), - (U::Search { .. }, U::Sftp { .. }) => Err(Exotic), - (U::Archive { .. }, U::Sftp { .. }) => Err(Exotic), - (U::Sftp { .. }, U::Regular(_)) => Err(Exotic), - (U::Sftp { .. }, U::Search { .. }) => Err(Exotic), - (U::Sftp { .. }, U::Archive { .. }) => Err(Exotic), + // Independent virtual file space (Sftp/Rclone crossed with anything else) + _ => Err(Exotic), } } @@ -419,6 +438,9 @@ impl<'a> Url<'a> { (U::Sftp { domain: a, .. }, U::Sftp { domain: b, .. }) => { (a == b).then_some(suffix).ok_or(Exotic) } + (U::Rclone { domain: a, .. }, U::Rclone { domain: b, .. }) => { + (a == b).then_some(suffix).ok_or(Exotic) + } // Both are local files (U::Regular(_), U::Search { .. }) => Ok(suffix), @@ -438,13 +460,8 @@ impl<'a> Url<'a> { self.uri().is_empty().then_some(suffix).ok_or(NotSuffix) } - // Independent virtual file space - (U::Regular(_), U::Sftp { .. }) => Err(Exotic), - (U::Search { .. }, U::Sftp { .. }) => Err(Exotic), - (U::Archive { .. }, U::Sftp { .. }) => Err(Exotic), - (U::Sftp { .. }, U::Regular(_)) => Err(Exotic), - (U::Sftp { .. }, U::Search { .. }) => Err(Exotic), - (U::Sftp { .. }, U::Archive { .. }) => Err(Exotic), + // Independent virtual file space (Sftp/Rclone crossed with anything else) + _ => Err(Exotic), } } @@ -454,7 +471,7 @@ impl<'a> Url<'a> { Self::Regular(loc) => loc.uri().as_path(), Self::Search { loc, .. } => loc.uri().as_path(), Self::Archive { loc, .. } => loc.uri().as_path(), - Self::Sftp { loc, .. } => loc.uri().as_path(), + Self::Sftp { loc, .. } | Self::Rclone { loc, .. } => loc.uri().as_path(), } } @@ -464,7 +481,7 @@ impl<'a> Url<'a> { Self::Regular(loc) => loc.urn().as_path(), Self::Search { loc, .. } => loc.urn().as_path(), Self::Archive { loc, .. } => loc.urn().as_path(), - Self::Sftp { loc, .. } => loc.urn().as_path(), + Self::Sftp { loc, .. } | Self::Rclone { loc, .. } => loc.urn().as_path(), } } } diff --git a/yazi-vfs/Cargo.toml b/yazi-vfs/Cargo.toml index 4bbbe40c..c3eaf028 100644 --- a/yazi-vfs/Cargo.toml +++ b/yazi-vfs/Cargo.toml @@ -32,6 +32,7 @@ mlua = { workspace = true } parking_lot = { workspace = true } russh = { workspace = true } serde = { workspace = true } +serde_json = { workspace = true } tokio = { workspace = true } toml = { workspace = true } tracing = { workspace = true } diff --git a/yazi-vfs/src/config/mod.rs b/yazi-vfs/src/config/mod.rs index e35a3ea7..0df85aee 100644 --- a/yazi-vfs/src/config/mod.rs +++ b/yazi-vfs/src/config/mod.rs @@ -1 +1 @@ -yazi_macro::mod_flat!(service services sftp vfs); +yazi_macro::mod_flat!(rclone service services sftp vfs); diff --git a/yazi-vfs/src/config/rclone.rs b/yazi-vfs/src/config/rclone.rs new file mode 100644 index 00000000..f7b6bd86 --- /dev/null +++ b/yazi-vfs/src/config/rclone.rs @@ -0,0 +1,28 @@ +use std::path::PathBuf; + +use serde::{Deserialize, Deserializer, Serialize, de}; +use yazi_fs::path::sanitize_path; + +#[derive(Deserialize, Eq, Hash, PartialEq, Serialize)] +pub struct ServiceRclone { + pub remote: String, + #[serde(default)] + pub binary: PathBuf, + #[serde(default, deserialize_with = "deserialize_path")] + pub config_file: PathBuf, + #[serde(default)] + pub flags: Vec, +} + +fn deserialize_path<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + let mut path = PathBuf::deserialize(deserializer)?; + if !path.as_os_str().is_empty() { + path = sanitize_path(path) + .ok_or_else(|| de::Error::custom("path must be either empty or an absolute path"))?; + } + + Ok(path) +} diff --git a/yazi-vfs/src/config/service.rs b/yazi-vfs/src/config/service.rs index 283f7db7..522be8a5 100644 --- a/yazi-vfs/src/config/service.rs +++ b/yazi-vfs/src/config/service.rs @@ -1,11 +1,12 @@ use serde::{Deserialize, Serialize}; -use crate::config::ServiceSftp; +use crate::config::{ServiceRclone, ServiceSftp}; #[derive(Deserialize, Serialize)] #[serde(tag = "type", rename_all = "kebab-case")] pub enum Service { Sftp(ServiceSftp), + Rclone(ServiceRclone), } impl TryFrom<&'static Service> for &'static ServiceSftp { @@ -14,6 +15,18 @@ impl TryFrom<&'static Service> for &'static ServiceSftp { fn try_from(value: &'static Service) -> Result { match value { Service::Sftp(p) => Ok(p), + _ => Err("expected an SFTP service"), + } + } +} + +impl TryFrom<&'static Service> for &'static ServiceRclone { + type Error = &'static str; + + fn try_from(value: &'static Service) -> Result { + match value { + Service::Rclone(p) => Ok(p), + _ => Err("expected an rclone service"), } } } diff --git a/yazi-vfs/src/provider/dir_entry.rs b/yazi-vfs/src/provider/dir_entry.rs index 40b75e64..0262c91f 100644 --- a/yazi-vfs/src/provider/dir_entry.rs +++ b/yazi-vfs/src/provider/dir_entry.rs @@ -6,6 +6,7 @@ use yazi_shared::{path::PathBufDyn, strand::StrandCow, url::UrlBuf}; pub enum DirEntry { Local(yazi_fs::provider::local::DirEntry), Sftp(super::sftp::DirEntry), + Rclone(super::rclone::DirEntry), } impl FileHolder for DirEntry { @@ -13,6 +14,7 @@ impl FileHolder for DirEntry { match self { Self::Local(entry) => entry.file_type().await, Self::Sftp(entry) => entry.file_type().await, + Self::Rclone(entry) => entry.file_type().await, } } @@ -20,6 +22,7 @@ impl FileHolder for DirEntry { match self { Self::Local(entry) => entry.metadata().await, Self::Sftp(entry) => entry.metadata().await, + Self::Rclone(entry) => entry.metadata().await, } } @@ -27,6 +30,7 @@ impl FileHolder for DirEntry { match self { Self::Local(entry) => entry.name(), Self::Sftp(entry) => entry.name(), + Self::Rclone(entry) => entry.name(), } } @@ -34,6 +38,7 @@ impl FileHolder for DirEntry { match self { Self::Local(entry) => entry.path(), Self::Sftp(entry) => entry.path(), + Self::Rclone(entry) => entry.path(), } } @@ -41,6 +46,7 @@ impl FileHolder for DirEntry { match self { Self::Local(entry) => entry.url(), Self::Sftp(entry) => entry.url(), + Self::Rclone(entry) => entry.url(), } } } diff --git a/yazi-vfs/src/provider/gate.rs b/yazi-vfs/src/provider/gate.rs index 3210dd16..2deaf0e4 100644 --- a/yazi-vfs/src/provider/gate.rs +++ b/yazi-vfs/src/provider/gate.rs @@ -52,6 +52,7 @@ impl FileBuilder for Gate { Err(io::Error::new(io::ErrorKind::Unsupported, "Unsupported filesystem: archive"))? } SchemeKind::Sftp => self.build::().open(url).await?.into(), + SchemeKind::Rclone => self.build::().open(url).await?.into(), }) } diff --git a/yazi-vfs/src/provider/mod.rs b/yazi-vfs/src/provider/mod.rs index b8afef8f..a63a9271 100644 --- a/yazi-vfs/src/provider/mod.rs +++ b/yazi-vfs/src/provider/mod.rs @@ -1,4 +1,4 @@ -yazi_macro::mod_pub!(sftp); +yazi_macro::mod_pub!(rclone sftp); yazi_macro::mod_flat!(calculator copier dir_entry gate lua provider providers read_dir rw_file); diff --git a/yazi-vfs/src/provider/provider.rs b/yazi-vfs/src/provider/provider.rs index 3535bcb0..ec5bb66f 100644 --- a/yazi-vfs/src/provider/provider.rs +++ b/yazi-vfs/src/provider/provider.rs @@ -270,6 +270,7 @@ where Url::Regular(_) | Url::Search { .. } => yazi_fs::provider::local::try_absolute(url), Url::Archive { .. } => None, // TODO Url::Sftp { .. } => crate::provider::sftp::try_absolute(url), + Url::Rclone { .. } => crate::provider::rclone::try_absolute(url), } } diff --git a/yazi-vfs/src/provider/providers.rs b/yazi-vfs/src/provider/providers.rs index 5c1d30f5..743c0b5d 100644 --- a/yazi-vfs/src/provider/providers.rs +++ b/yazi-vfs/src/provider/providers.rs @@ -8,6 +8,7 @@ use yazi_shared::{path::{AsPath, PathBufDyn}, strand::AsStrand, url::{Url, UrlBu pub(super) enum Providers<'a> { Local(yazi_fs::provider::local::Local<'a>), Sftp(super::sftp::Sftp<'a>), + Rclone(super::rclone::Rclone<'a>), } impl<'a> Provider for Providers<'a> { @@ -21,6 +22,7 @@ impl<'a> Provider for Providers<'a> { match self { Self::Local(p) => p.absolute().await, Self::Sftp(p) => p.absolute().await, + Self::Rclone(p) => p.absolute().await, } } @@ -28,6 +30,7 @@ impl<'a> Provider for Providers<'a> { match self { Self::Local(p) => p.canonicalize().await, Self::Sftp(p) => p.canonicalize().await, + Self::Rclone(p) => p.canonicalize().await, } } @@ -35,6 +38,7 @@ impl<'a> Provider for Providers<'a> { match self { Self::Local(p) => p.capabilities(), Self::Sftp(p) => p.capabilities(), + Self::Rclone(p) => p.capabilities(), } } @@ -42,6 +46,7 @@ impl<'a> Provider for Providers<'a> { match self { Self::Local(p) => p.casefold().await, Self::Sftp(p) => p.casefold().await, + Self::Rclone(p) => p.casefold().await, } } @@ -52,6 +57,7 @@ impl<'a> Provider for Providers<'a> { match self { Self::Local(p) => p.copy(to, attrs).await, Self::Sftp(p) => p.copy(to, attrs).await, + Self::Rclone(p) => p.copy(to, attrs).await, } } @@ -63,6 +69,7 @@ impl<'a> Provider for Providers<'a> { match self { Self::Local(p) => p.copy_with_progress(to, attrs), Self::Sftp(p) => p.copy_with_progress(to, attrs), + Self::Rclone(p) => p.copy_with_progress(to, attrs), } } @@ -70,6 +77,7 @@ impl<'a> Provider for Providers<'a> { Ok(match self { Self::Local(p) => p.create().await?.into(), Self::Sftp(p) => p.create().await?.into(), + Self::Rclone(p) => p.create().await?.into(), }) } @@ -77,6 +85,7 @@ impl<'a> Provider for Providers<'a> { match self { Self::Local(p) => p.create_dir().await, Self::Sftp(p) => p.create_dir().await, + Self::Rclone(p) => p.create_dir().await, } } @@ -84,6 +93,7 @@ impl<'a> Provider for Providers<'a> { match self { Self::Local(p) => p.create_dir_all().await, Self::Sftp(p) => p.create_dir_all().await, + Self::Rclone(p) => p.create_dir_all().await, } } @@ -91,6 +101,7 @@ impl<'a> Provider for Providers<'a> { Ok(match self { Self::Local(p) => p.create_new().await?.into(), Self::Sftp(p) => p.create_new().await?.into(), + Self::Rclone(p) => p.create_new().await?.into(), }) } @@ -101,6 +112,7 @@ impl<'a> Provider for Providers<'a> { match self { Self::Local(p) => p.hard_link(to).await, Self::Sftp(p) => p.hard_link(to).await, + Self::Rclone(p) => p.hard_link(to).await, } } @@ -108,6 +120,7 @@ impl<'a> Provider for Providers<'a> { match self { Self::Local(p) => p.metadata().await, Self::Sftp(p) => p.metadata().await, + Self::Rclone(p) => p.metadata().await, } } @@ -120,6 +133,7 @@ impl<'a> Provider for Providers<'a> { Err(io::Error::new(io::ErrorKind::Unsupported, "Unsupported filesystem: archive"))? } K::Sftp => Self::Me::Sftp(super::sftp::Sftp::new(url).await?), + K::Rclone => Self::Me::Rclone(super::rclone::Rclone::new(url).await?), }) } @@ -127,6 +141,7 @@ impl<'a> Provider for Providers<'a> { Ok(match self { Self::Local(p) => p.open().await?.into(), Self::Sftp(p) => p.open().await?.into(), + Self::Rclone(p) => p.open().await?.into(), }) } @@ -134,6 +149,7 @@ impl<'a> Provider for Providers<'a> { Ok(match self { Self::Local(p) => Self::ReadDir::Local(p.read_dir().await?), Self::Sftp(p) => Self::ReadDir::Sftp(p.read_dir().await?), + Self::Rclone(p) => Self::ReadDir::Rclone(p.read_dir().await?), }) } @@ -141,6 +157,7 @@ impl<'a> Provider for Providers<'a> { match self { Self::Local(p) => p.read_link().await, Self::Sftp(p) => p.read_link().await, + Self::Rclone(p) => p.read_link().await, } } @@ -148,6 +165,7 @@ impl<'a> Provider for Providers<'a> { match self { Self::Local(p) => p.remove_dir().await, Self::Sftp(p) => p.remove_dir().await, + Self::Rclone(p) => p.remove_dir().await, } } @@ -155,6 +173,7 @@ impl<'a> Provider for Providers<'a> { match self { Self::Local(p) => p.remove_dir_all().await, Self::Sftp(p) => p.remove_dir_all().await, + Self::Rclone(p) => p.remove_dir_all().await, } } @@ -162,6 +181,7 @@ impl<'a> Provider for Providers<'a> { match self { Self::Local(p) => p.remove_file().await, Self::Sftp(p) => p.remove_file().await, + Self::Rclone(p) => p.remove_file().await, } } @@ -172,6 +192,7 @@ impl<'a> Provider for Providers<'a> { match self { Self::Local(p) => p.rename(to).await, Self::Sftp(p) => p.rename(to).await, + Self::Rclone(p) => p.rename(to).await, } } @@ -179,6 +200,7 @@ impl<'a> Provider for Providers<'a> { match self { Self::Local(p) => p.set_mode(mode).await, Self::Sftp(p) => p.set_mode(mode).await, + Self::Rclone(p) => p.set_mode(mode).await, } } @@ -190,6 +212,7 @@ impl<'a> Provider for Providers<'a> { match self { Self::Local(p) => p.symlink(original, is_dir).await, Self::Sftp(p) => p.symlink(original, is_dir).await, + Self::Rclone(p) => p.symlink(original, is_dir).await, } } @@ -200,6 +223,7 @@ impl<'a> Provider for Providers<'a> { match self { Self::Local(p) => p.symlink_dir(original).await, Self::Sftp(p) => p.symlink_dir(original).await, + Self::Rclone(p) => p.symlink_dir(original).await, } } @@ -210,6 +234,7 @@ impl<'a> Provider for Providers<'a> { match self { Self::Local(p) => p.symlink_file(original).await, Self::Sftp(p) => p.symlink_file(original).await, + Self::Rclone(p) => p.symlink_file(original).await, } } @@ -217,6 +242,7 @@ impl<'a> Provider for Providers<'a> { match self { Self::Local(p) => p.symlink_metadata().await, Self::Sftp(p) => p.symlink_metadata().await, + Self::Rclone(p) => p.symlink_metadata().await, } } @@ -224,6 +250,7 @@ impl<'a> Provider for Providers<'a> { match self { Self::Local(p) => p.trash().await, Self::Sftp(p) => p.trash().await, + Self::Rclone(p) => p.trash().await, } } @@ -231,6 +258,7 @@ impl<'a> Provider for Providers<'a> { match self { Self::Local(p) => p.url(), Self::Sftp(p) => p.url(), + Self::Rclone(p) => p.url(), } } @@ -241,6 +269,7 @@ impl<'a> Provider for Providers<'a> { match self { Self::Local(p) => p.write(contents).await, Self::Sftp(p) => p.write(contents).await, + Self::Rclone(p) => p.write(contents).await, } } } diff --git a/yazi-vfs/src/provider/rclone/absolute.rs b/yazi-vfs/src/provider/rclone/absolute.rs new file mode 100644 index 00000000..567d804b --- /dev/null +++ b/yazi-vfs/src/provider/rclone/absolute.rs @@ -0,0 +1,21 @@ +use yazi_fs::CWD; +use yazi_shared::url::{UrlCow, UrlLike}; + +pub fn try_absolute<'a, U>(url: U) -> Option> +where + U: Into>, +{ + try_absolute_impl(url.into()) +} + +fn try_absolute_impl<'a>(url: UrlCow<'a>) -> Option> { + if url.is_absolute() { + Some(url) + } else if let cwd = CWD.load() + && cwd.scheme().covariant(url.scheme()) + { + Some(cwd.try_join(url.loc()).ok()?.into()) + } else { + None + } +} diff --git a/yazi-vfs/src/provider/rclone/file.rs b/yazi-vfs/src/provider/rclone/file.rs new file mode 100644 index 00000000..a801be13 --- /dev/null +++ b/yazi-vfs/src/provider/rclone/file.rs @@ -0,0 +1,145 @@ +use std::{io, pin::Pin, process::Stdio, task::{Context, Poll}}; + +use tokio::{io::{AsyncRead, AsyncSeek, AsyncWrite, ReadBuf}, process::{Child, ChildStdout}}; +use yazi_fs::cha::{Cha, ChaMode}; + +use crate::config::ServiceRclone; + +/// A read-only, seekable handle to a remote object. +/// +/// rclone has no notion of a persistent file handle, so reads are served by +/// streaming `rclone cat` from the current position; seeking kills the child +/// process and the next read respawns it with `--offset`. +pub struct File { + config: &'static ServiceRclone, + target: String, + /// Object size when known (`rclone lsjson --stat` may report `-1` for + /// backends that don't expose it, in which case we stream to EOF blindly). + size: Option, + pos: u64, + child: Option<(Child, ChildStdout)>, +} + +impl File { + pub(super) fn new(config: &'static ServiceRclone, target: String, size: i64) -> Self { + let size = if size >= 0 { Some(size as u64) } else { None }; + Self { config, target, size, pos: 0, child: None } + } + + pub(crate) fn cha(&self) -> io::Result { + Ok(Cha { + mode: ChaMode::try_from(0o100644u16) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?, + len: self.size.unwrap_or(0), + ..Default::default() + }) + } + + fn spawn(&mut self) -> io::Result<()> { + let mut cmd = super::command(self.config); + cmd.arg("cat"); + if self.pos > 0 { + cmd.arg("--offset").arg(self.pos.to_string()); + } + cmd.arg(&self.target).stderr(Stdio::null()); + + let mut child = cmd.spawn()?; + let stdout = + child.stdout.take().ok_or_else(|| io::Error::other("No stdout from `rclone cat`"))?; + + self.child = Some((child, stdout)); + Ok(()) + } +} + +impl AsyncRead for File { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + let me = self.get_mut(); + + // Fast path: we've read everything the object claims to hold. + if me.size == Some(me.pos) { + return Poll::Ready(Ok(())); + } + if me.child.is_none() { + me.spawn()?; + } + + let before = buf.filled().len(); + let (_, stdout) = me.child.as_mut().unwrap(); + match Pin::new(stdout).poll_read(cx, buf) { + Poll::Ready(Ok(())) => { + let n = (buf.filled().len() - before) as u64; + me.pos += n; + + // Premature EOF: `rclone cat` closed stdout before delivering the + // full object. Surface it instead of silently truncating. + if n == 0 + && let Some(size) = me.size + && me.pos < size + { + return Poll::Ready(Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + format!("`rclone cat` ended early: got {} of {} bytes", me.pos, size), + ))); + } + Poll::Ready(Ok(())) + } + polled => polled, + } + } +} + +impl AsyncSeek for File { + fn start_seek(self: Pin<&mut Self>, position: io::SeekFrom) -> io::Result<()> { + let me = self.get_mut(); + let base = match position { + io::SeekFrom::Start(n) => Some(n as i128), + io::SeekFrom::Current(n) => Some(me.pos as i128 + n as i128), + io::SeekFrom::End(n) => match me.size { + Some(size) => Some(size as i128 + n as i128), + None => { + return Err(io::Error::new( + io::ErrorKind::Unsupported, + "Cannot seek from the end of an object with unknown size", + )); + } + }, + }; + + let pos = base + .and_then(|p| u64::try_from(p).ok()) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "Invalid seek offset"))?; + + if pos != me.pos { + me.child = None; // Invalidate the stream; the next read respawns at the new offset + me.pos = pos; + } + Ok(()) + } + + fn poll_complete(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(self.pos)) + } +} + +impl AsyncWrite for File { + fn poll_write( + self: Pin<&mut Self>, + _cx: &mut Context<'_>, + _buf: &[u8], + ) -> Poll> { + Poll::Ready(Err(super::read_only())) + } + + fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } +} diff --git a/yazi-vfs/src/provider/rclone/gate.rs b/yazi-vfs/src/provider/rclone/gate.rs new file mode 100644 index 00000000..708a6d83 --- /dev/null +++ b/yazi-vfs/src/provider/rclone/gate.rs @@ -0,0 +1,73 @@ +use std::io; + +use yazi_fs::provider::{Attrs, FileBuilder}; +use yazi_shared::url::{AsUrl, Url}; + +use crate::config::{ServiceRclone, Vfs}; + +#[derive(Clone, Copy, Default)] +pub struct Gate(crate::provider::Gate); + +impl FileBuilder for Gate { + type File = super::File; + + fn append(&mut self, append: bool) -> &mut Self { + self.0.append(append); + self + } + + fn attrs(&mut self, attrs: Attrs) -> &mut Self { + self.0.attrs(attrs); + self + } + + fn create(&mut self, create: bool) -> &mut Self { + self.0.create(create); + self + } + + fn create_new(&mut self, create_new: bool) -> &mut Self { + self.0.create_new(create_new); + self + } + + async fn open(&self, url: U) -> io::Result + where + U: AsUrl, + { + let url = url.as_url(); + let (path, (_, config)) = match url { + Url::Rclone { loc, domain } => { + (loc.as_inner(), Vfs::service::<&ServiceRclone>(domain).await?) + } + _ => Err(io::Error::new(io::ErrorKind::InvalidInput, format!("Not an rclone URL: {url:?}")))?, + }; + + if self.0.write || self.0.append || self.0.truncate || self.0.create || self.0.create_new { + return Err(super::read_only()); + } + + let target = super::target(config, path)?; + let item = super::stat(config, &target).await?; + if item.is_dir { + return Err(io::Error::new(io::ErrorKind::InvalidInput, "Is a directory")); + } + + Ok(super::File::new(config, target, item.size)) + } + + fn read(&mut self, read: bool) -> &mut Self { + self.0.read(read); + self + } + + fn truncate(&mut self, truncate: bool) -> &mut Self { + self.0.truncate(truncate); + self + } + + fn write(&mut self, write: bool) -> &mut Self { + self.0.write(write); + self + } +} diff --git a/yazi-vfs/src/provider/rclone/metadata.rs b/yazi-vfs/src/provider/rclone/metadata.rs new file mode 100644 index 00000000..366b1b90 --- /dev/null +++ b/yazi-vfs/src/provider/rclone/metadata.rs @@ -0,0 +1,47 @@ +use std::{io, time::SystemTime}; + +use serde::Deserialize; +use yazi_fs::cha::{Cha, ChaKind, ChaMode}; + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "PascalCase")] +pub(super) struct Item { + #[serde(default)] + pub name: String, + #[serde(default)] + pub size: i64, + #[serde(default)] + pub mod_time: Option, + #[serde(default)] + pub is_dir: bool, +} + +impl Item { + pub(super) fn cha(&self) -> io::Result { + let kind = if self.name.starts_with('.') { ChaKind::HIDDEN } else { ChaKind::empty() }; + + // Object stores have no POSIX permissions; synthesize a sensible mode + let mode = ChaMode::try_from(if self.is_dir { 0o040755u16 } else { 0o100644u16 }) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + + let mtime = self + .mod_time + .as_deref() + .and_then(|s| chrono::DateTime::parse_from_rfc3339(s).ok()) + .map(SystemTime::from); + + Ok(Cha { + kind, + mode, + len: self.size.max(0) as u64, + atime: None, + btime: None, + ctime: None, + mtime, + dev: 0, + uid: 0, + gid: 0, + nlink: 0, + }) + } +} diff --git a/yazi-vfs/src/provider/rclone/mod.rs b/yazi-vfs/src/provider/rclone/mod.rs new file mode 100644 index 00000000..b926af74 --- /dev/null +++ b/yazi-vfs/src/provider/rclone/mod.rs @@ -0,0 +1 @@ +yazi_macro::mod_flat!(absolute file gate metadata rclone read_dir); diff --git a/yazi-vfs/src/provider/rclone/rclone.rs b/yazi-vfs/src/provider/rclone/rclone.rs new file mode 100644 index 00000000..d435d1cc --- /dev/null +++ b/yazi-vfs/src/provider/rclone/rclone.rs @@ -0,0 +1,180 @@ +use std::{io, path::Path, process::Stdio, sync::Arc}; + +use tokio::sync::mpsc::Receiver; +use yazi_fs::{cha::ChaMode, provider::{Capabilities, Provider}}; +use yazi_shared::{loc::LocBuf, path::{AsPath, PathBufDyn}, pool::InternStr, scheme::SchemeKind, strand::AsStrand, url::{Url, UrlBuf, UrlCow}}; + +use super::Item; +use crate::config::{ServiceRclone, Vfs}; + +#[derive(Clone)] +pub struct Rclone<'a> { + url: Url<'a>, + path: &'a typed_path::UnixPath, + + name: &'static str, + config: &'static ServiceRclone, +} + +impl<'a> Provider for Rclone<'a> { + type File = super::File; + type Gate = super::Gate; + type Me<'b> = Rclone<'b>; + type ReadDir = super::ReadDir; + type UrlCow = UrlCow<'a>; + + async fn absolute(&self) -> io::Result { + Ok(if let Some(u) = super::try_absolute(self.url) { u } else { self.url.to_owned().into() }) + } + + async fn canonicalize(&self) -> io::Result { Ok(self.url.to_owned()) } + + fn capabilities(&self) -> Capabilities { Capabilities { symlink: false } } + + async fn casefold(&self) -> io::Result { Ok(self.url.to_owned()) } + + async fn copy

(&self, _to: P, _attrs: yazi_fs::provider::Attrs) -> io::Result + where + P: AsPath, + { + Err(read_only()) + } + + fn copy_with_progress(&self, to: P, attrs: A) -> io::Result>> + where + P: AsPath, + A: Into, + { + let to = UrlBuf::Rclone { + loc: LocBuf::::saturated( + to.as_path().to_unix_owned()?, + SchemeKind::Rclone, + ), + domain: self.name.intern(), + }; + let from = self.url.to_owned(); + + Ok(crate::provider::copy_with_progress_impl(from, to, attrs.into())) + } + + async fn create_dir(&self) -> io::Result<()> { Err(read_only()) } + + async fn hard_link

(&self, _to: P) -> io::Result<()> + where + P: AsPath, + { + Err(io::Error::new(io::ErrorKind::Unsupported, "Hard links not supported")) + } + + async fn metadata(&self) -> io::Result { + stat(self.config, &target(self.config, self.path)?).await?.cha() + } + + async fn new<'b>(url: Url<'b>) -> io::Result> { + match url { + Url::Rclone { loc, domain } => { + let (name, config) = Vfs::service::<&ServiceRclone>(domain).await?; + Ok(Self::Me { url, path: loc.as_inner(), name, config }) + } + _ => Err(io::Error::new(io::ErrorKind::InvalidInput, format!("Not an rclone URL: {url:?}"))), + } + } + + async fn read_dir(self) -> io::Result { + let mut cmd = command(self.config); + cmd.arg("lsjson").arg(target(self.config, self.path)?); + + let items: Vec = serde_json::from_slice(&output(cmd).await?) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + + Ok(Self::ReadDir { dir: Arc::new(self.url.to_owned()), entries: items.into() }) + } + + async fn read_link(&self) -> io::Result { + Err(io::Error::new(io::ErrorKind::Unsupported, "Symlinks not supported")) + } + + async fn remove_dir(&self) -> io::Result<()> { Err(read_only()) } + + async fn remove_file(&self) -> io::Result<()> { Err(read_only()) } + + async fn rename

(&self, _to: P) -> io::Result<()> + where + P: AsPath, + { + Err(read_only()) + } + + async fn set_mode(&self, _mode: ChaMode) -> io::Result<()> { Err(read_only()) } + + async fn symlink(&self, _original: S, _is_dir: F) -> io::Result<()> + where + S: AsStrand, + F: AsyncFnOnce() -> io::Result, + { + Err(io::Error::new(io::ErrorKind::Unsupported, "Symlinks not supported")) + } + + async fn symlink_metadata(&self) -> io::Result { self.metadata().await } + + async fn trash(&self) -> io::Result<()> { + Err(io::Error::new(io::ErrorKind::Unsupported, "Trash not supported")) + } + + #[inline] + fn url(&self) -> Url<'_> { self.url } +} + +// --- Helpers +#[inline] +pub(super) fn read_only() -> io::Error { + io::Error::new(io::ErrorKind::Unsupported, "The rclone provider is currently read-only") +} + +pub(super) fn command(config: &ServiceRclone) -> tokio::process::Command { + let binary = + if config.binary.as_os_str().is_empty() { Path::new("rclone") } else { &config.binary }; + + let mut cmd = tokio::process::Command::new(binary); + if !config.config_file.as_os_str().is_empty() { + cmd.arg("--config").arg(&config.config_file); + } + + cmd.args(&config.flags); + cmd.stdin(Stdio::null()).stdout(Stdio::piped()).stderr(Stdio::piped()); + cmd.kill_on_drop(true); + cmd +} + +pub(super) fn target(config: &ServiceRclone, path: &typed_path::UnixPath) -> io::Result { + // Keep the leading slash: object stores treat `remote:/bucket/key` and + // `remote:bucket/key` identically, while rclone's filesystem-like backends + // (local, sftp, …) need the absolute path preserved (`remote:/abs/path`). + let s = str::from_utf8(path.as_bytes()) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "Non-UTF-8 path"))?; + + Ok(format!("{}:{s}", config.remote)) +} + +pub(super) async fn output(mut cmd: tokio::process::Command) -> io::Result> { + let output = cmd.output().await?; + if output.status.success() { + return Ok(output.stdout); + } + + // https://rclone.org/docs/#exit-code — 3: directory not found, 4: file not found + let kind = match output.status.code() { + Some(3 | 4) => io::ErrorKind::NotFound, + _ => io::ErrorKind::Other, + }; + let stderr = String::from_utf8_lossy(&output.stderr); + Err(io::Error::new(kind, format!("rclone failed: {}", stderr.trim()))) +} + +pub(super) async fn stat(config: &ServiceRclone, target: &str) -> io::Result { + let mut cmd = command(config); + cmd.arg("lsjson").arg("--stat").arg(target); + + serde_json::from_slice(&output(cmd).await?) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e)) +} diff --git a/yazi-vfs/src/provider/rclone/read_dir.rs b/yazi-vfs/src/provider/rclone/read_dir.rs new file mode 100644 index 00000000..9473c89d --- /dev/null +++ b/yazi-vfs/src/provider/rclone/read_dir.rs @@ -0,0 +1,44 @@ +use std::{collections::VecDeque, io, sync::Arc}; + +use yazi_fs::provider::{DirReader, FileHolder}; +use yazi_shared::{path::PathBufDyn, strand::StrandCow, url::{UrlBuf, UrlLike}}; + +use super::Item; + +pub struct ReadDir { + pub(super) dir: Arc, + pub(super) entries: VecDeque, +} + +impl DirReader for ReadDir { + type Entry = DirEntry; + + async fn next(&mut self) -> io::Result> { + Ok(self.entries.pop_front().map(|item| DirEntry { dir: self.dir.clone(), item })) + } +} + +// --- Entry +pub struct DirEntry { + dir: Arc, + item: Item, +} + +impl FileHolder for DirEntry { + async fn file_type(&self) -> io::Result { + Ok(self.item.cha()?.mode.into()) + } + + async fn metadata(&self) -> io::Result { self.item.cha() } + + fn name(&self) -> StrandCow<'_> { self.item.name.as_str().into() } + + fn path(&self) -> PathBufDyn { self.url().into_loc() } + + fn url(&self) -> UrlBuf { + self + .dir + .try_join(self.item.name.as_str()) + .expect("entry name is a valid component of the rclone URL") + } +} diff --git a/yazi-vfs/src/provider/read_dir.rs b/yazi-vfs/src/provider/read_dir.rs index 55e63159..b51e8832 100644 --- a/yazi-vfs/src/provider/read_dir.rs +++ b/yazi-vfs/src/provider/read_dir.rs @@ -5,6 +5,7 @@ use yazi_fs::provider::DirReader; pub enum ReadDir { Local(yazi_fs::provider::local::ReadDir), Sftp(super::sftp::ReadDir), + Rclone(super::rclone::ReadDir), } impl DirReader for ReadDir { @@ -14,6 +15,7 @@ impl DirReader for ReadDir { Ok(match self { Self::Local(reader) => reader.next().await?.map(Self::Entry::Local), Self::Sftp(reader) => reader.next().await?.map(Self::Entry::Sftp), + Self::Rclone(reader) => reader.next().await?.map(Self::Entry::Rclone), }) } } diff --git a/yazi-vfs/src/provider/rw_file.rs b/yazi-vfs/src/provider/rw_file.rs index 0b4e6e0d..402b63af 100644 --- a/yazi-vfs/src/provider/rw_file.rs +++ b/yazi-vfs/src/provider/rw_file.rs @@ -6,6 +6,7 @@ use yazi_fs::provider::Attrs; pub enum RwFile { Tokio(tokio::fs::File), Sftp(Box), + Rclone(Box), } impl From for RwFile { @@ -16,12 +17,17 @@ impl From for RwFile { fn from(f: yazi_sftp::fs::File) -> Self { Self::Sftp(Box::new(f)) } } +impl From for RwFile { + fn from(f: super::rclone::File) -> Self { Self::Rclone(Box::new(f)) } +} + impl RwFile { // FIXME: path pub async fn metadata(&self) -> io::Result { Ok(match self { Self::Tokio(f) => yazi_fs::cha::Cha::new("// FIXME", f.metadata().await?), Self::Sftp(f) => super::sftp::Cha::try_from(("// FIXME".as_bytes(), &f.fstat().await?))?.0, + Self::Rclone(f) => f.cha()?, }) } @@ -45,6 +51,7 @@ impl RwFile { f.fsetstat(&attrs).await?; } } + Self::Rclone(_) => {} } Ok(()) @@ -56,6 +63,7 @@ impl RwFile { Self::Sftp(f) => { f.fsetstat(&yazi_sftp::fs::Attrs { size: Some(size), ..Default::default() }).await? } + Self::Rclone(_) => Err(io::Error::new(io::ErrorKind::Unsupported, "Read-only file"))?, }) } } @@ -70,6 +78,7 @@ impl AsyncRead for RwFile { match &mut *self { Self::Tokio(f) => Pin::new(f).poll_read(cx, buf), Self::Sftp(f) => Pin::new(f).poll_read(cx, buf), + Self::Rclone(f) => Pin::new(f).poll_read(cx, buf), } } } @@ -80,6 +89,7 @@ impl AsyncSeek for RwFile { match &mut *self { Self::Tokio(f) => Pin::new(f).start_seek(position), Self::Sftp(f) => Pin::new(f).start_seek(position), + Self::Rclone(f) => Pin::new(f).start_seek(position), } } @@ -91,6 +101,7 @@ impl AsyncSeek for RwFile { match &mut *self { Self::Tokio(f) => Pin::new(f).poll_complete(cx), Self::Sftp(f) => Pin::new(f).poll_complete(cx), + Self::Rclone(f) => Pin::new(f).poll_complete(cx), } } } @@ -105,6 +116,7 @@ impl AsyncWrite for RwFile { match &mut *self { Self::Tokio(f) => Pin::new(f).poll_write(cx, buf), Self::Sftp(f) => Pin::new(f).poll_write(cx, buf), + Self::Rclone(f) => Pin::new(f).poll_write(cx, buf), } } @@ -116,6 +128,7 @@ impl AsyncWrite for RwFile { match &mut *self { Self::Tokio(f) => Pin::new(f).poll_flush(cx), Self::Sftp(f) => Pin::new(f).poll_flush(cx), + Self::Rclone(f) => Pin::new(f).poll_flush(cx), } } @@ -127,6 +140,7 @@ impl AsyncWrite for RwFile { match &mut *self { Self::Tokio(f) => Pin::new(f).poll_shutdown(cx), Self::Sftp(f) => Pin::new(f).poll_shutdown(cx), + Self::Rclone(f) => Pin::new(f).poll_shutdown(cx), } } @@ -139,6 +153,7 @@ impl AsyncWrite for RwFile { match &mut *self { Self::Tokio(f) => Pin::new(f).poll_write_vectored(cx, bufs), Self::Sftp(f) => Pin::new(f).poll_write_vectored(cx, bufs), + Self::Rclone(f) => Pin::new(f).poll_write_vectored(cx, bufs), } } @@ -147,6 +162,7 @@ impl AsyncWrite for RwFile { match self { Self::Tokio(f) => f.is_write_vectored(), Self::Sftp(f) => f.is_write_vectored(), + Self::Rclone(_) => false, } } } diff --git a/yazi-vfs/src/provider/sftp/sftp.rs b/yazi-vfs/src/provider/sftp/sftp.rs index a12690b2..194e1950 100644 --- a/yazi-vfs/src/provider/sftp/sftp.rs +++ b/yazi-vfs/src/provider/sftp/sftp.rs @@ -145,7 +145,7 @@ impl<'a> Provider for Sftp<'a> { async fn new<'b>(url: Url<'b>) -> io::Result> { match url { - Url::Regular(_) | Url::Search { .. } | Url::Archive { .. } => { + Url::Regular(_) | Url::Search { .. } | Url::Archive { .. } | Url::Rclone { .. } => { Err(io::Error::new(io::ErrorKind::InvalidInput, format!("Not a SFTP URL: {url:?}"))) } Url::Sftp { loc, domain } => { diff --git a/yazi-vfs/tests/rclone_local.rs b/yazi-vfs/tests/rclone_local.rs new file mode 100644 index 00000000..b1ba4a58 --- /dev/null +++ b/yazi-vfs/tests/rclone_local.rs @@ -0,0 +1,158 @@ +//! Hermetic tests for the rclone provider. +//! +//! Unix-only: they drive an rclone `local` remote with absolute filesystem +//! paths, which the (Unix-style) rclone scheme can't represent on Windows. +//! Cloud remotes work fine from any OS — this is a limitation of the test's +//! local backend, not the provider. +//! +//! These need no cloud account and no credentials: they point an rclone `local` +//! remote at a generated temp directory, so `rclone lsjson` / `rclone cat` run +//! entirely against the filesystem. The only requirement is the `rclone` binary +//! on PATH; if it's missing the tests skip themselves (so CI without rclone +//! stays green, and CI that installs rclone gets real coverage). + +#![cfg(unix)] + +use std::{path::{Path, PathBuf}, process::Command, sync::OnceLock}; + +use tokio::io::{AsyncReadExt, AsyncSeekExt}; +use yazi_fs::provider::{DirReader, FileHolder}; +use yazi_shared::url::UrlBuf; + +fn have_rclone() -> bool { + Command::new("rclone").arg("version").output().map(|o| o.status.success()).unwrap_or(false) +} + +/// Byte at position `i` of the big fixture — varies per offset so that a wrong +/// `--offset` or a botched seek produces detectably wrong bytes. +fn big_byte(i: usize) -> u8 { (i % 251) as u8 } + +const SMALL: &[u8] = b"hello rclone vfs\nsecond line\nthird line\n"; +const BIG_LEN: usize = 9 * 1024 * 1024 + 777; // > 8 MiB, non-aligned + +/// One-time fixture + config setup for the whole test binary. Returns the data +/// directory, or `None` if rclone is unavailable. Must run before any provider +/// call, because it sets `YAZI_CONFIG_HOME` before yazi caches it. +fn setup() -> Option<&'static Path> { + static SETUP: OnceLock> = OnceLock::new(); + SETUP + .get_or_init(|| { + if !have_rclone() { + return None; + } + + let root = std::env::temp_dir().join(format!("yazi_rclone_local_{}", std::process::id())); + let data = root.join("data"); + std::fs::create_dir_all(data.join("sub")).unwrap(); + std::fs::write(data.join("small.txt"), SMALL).unwrap(); + std::fs::write(data.join("sub").join("nested.txt"), b"nested\n").unwrap(); + let big: Vec = (0..BIG_LEN).map(big_byte).collect(); + std::fs::write(data.join("big.bin"), &big).unwrap(); + + let conf = root.join("rclone.conf"); + std::fs::write(&conf, "[local]\ntype = local\n").unwrap(); + + let cfg = root.join("config"); + std::fs::create_dir_all(&cfg).unwrap(); + std::fs::write( + cfg.join("vfs.toml"), + format!( + "[services.local]\ntype = \"rclone\"\nremote = \"local\"\nconfig_file = {:?}\n", + conf.to_str().unwrap() + ), + ) + .unwrap(); + + // SAFETY: set before any threads touch the environment or yazi init. + unsafe { std::env::set_var("YAZI_CONFIG_HOME", &cfg) }; + yazi_shared::init(); + yazi_fs::init(); + yazi_vfs::init(); + + Some(data) + }) + .as_deref() +} + +macro_rules! data_or_skip { + () => { + match setup() { + Some(d) => d, + None => { + eprintln!("skipping: `rclone` not found on PATH"); + return; + } + } + }; +} + +fn url(abs: &Path) -> UrlBuf { format!("rclone://local/{}", abs.display()).parse().unwrap() } + +#[tokio::test(flavor = "multi_thread")] +async fn reads_and_seeks() { + let data = data_or_skip!(); + let u = url(&data.join("small.txt")); + + let mut f = yazi_vfs::provider::open(&u).await.expect("open"); + let mut buf = Vec::new(); + f.read_to_end(&mut buf).await.expect("read"); + assert_eq!(buf, SMALL); + + let mid = SMALL.len() / 2; + let mut f = yazi_vfs::provider::open(&u).await.expect("reopen"); + f.seek(std::io::SeekFrom::Start(mid as u64)).await.expect("seek"); + let mut tail = Vec::new(); + f.read_to_end(&mut tail).await.expect("read tail"); + assert_eq!(tail, &SMALL[mid..]); + println!("OK reads_and_seeks"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn lists_dir() { + let data = data_or_skip!(); + let mut rd = yazi_vfs::provider::read_dir(&url(data)).await.expect("read_dir"); + let mut names = Vec::new(); + while let Some(e) = rd.next().await.expect("next") { + names.push(e.name().into_string_lossy()); + } + names.sort(); + assert_eq!(names, vec!["big.bin", "small.txt", "sub"]); + println!("OK lists_dir: {names:?}"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn missing_object_errors() { + let data = data_or_skip!(); + let u = url(&data.join("does_not_exist.bin")); + match yazi_vfs::provider::open(&u).await { + Ok(_) => panic!("expected an error"), + Err(e) => println!("OK missing_object_errors: {} ({:?})", e, e.kind()), + } +} + +/// Progressive chunked copy (parallel 8 MiB chunks + per-chunk seek) of the +/// >8 MiB fixture — the byte-varying content makes any offset/seek slip +/// visible. +#[tokio::test(flavor = "multi_thread")] +async fn copy_out_progressive() { + let data = data_or_skip!(); + let from = url(&data.join("big.bin")); + + // Write outside the fixture `data/` dir so it doesn't perturb `lists_dir`. + let out = data.parent().unwrap().join("copied.bin"); + let to: UrlBuf = out.to_str().unwrap().parse().unwrap(); + + let mut rx = + yazi_vfs::provider::copy_with_progress(&from, &to, yazi_fs::provider::Attrs::default()) + .await + .expect("copy_with_progress"); + let mut total = 0u64; + while let Some(msg) = rx.recv().await { + total += msg.expect("no copy error"); + } + + let got = std::fs::read(&out).unwrap(); + assert_eq!(got.len(), BIG_LEN, "size mismatch"); + assert!(got.iter().enumerate().all(|(i, &b)| b == big_byte(i)), "byte mismatch after copy"); + println!("OK copy_out_progressive: {BIG_LEN} bytes (progress sum {total})"); +} diff --git a/yazi-vfs/tests/rclone_read.rs b/yazi-vfs/tests/rclone_read.rs new file mode 100644 index 00000000..511f4fbd --- /dev/null +++ b/yazi-vfs/tests/rclone_read.rs @@ -0,0 +1,179 @@ +//! Live read/copy-path tests for the rclone provider. +//! +//! These hit a real rclone remote, so they're `#[ignore]` by default and are +//! driven entirely by environment variables — nothing is hard-coded. Set: +//! +//! YAZI_RCLONE_TEST_SMALL a small (<1 MiB) text file as `remote:path` +//! YAZI_RCLONE_TEST_BIG a >8 MiB file as `remote:path` (spans copy chunks) +//! YAZI_RCLONE_TEST_DIR a directory as `remote:path` (has children) +//! +//! Then run: +//! cargo test -p yazi-vfs --test rclone_read -- --ignored --nocapture + +use tokio::io::{AsyncReadExt, AsyncSeekExt}; +use yazi_fs::provider::{DirReader, FileHolder}; +use yazi_shared::url::UrlBuf; + +fn env(key: &str) -> Option { std::env::var(key).ok().filter(|s| !s.is_empty()) } + +/// `remote:path/to/obj` → `rclone://remote//path/to/obj` +fn to_url(remote_path: &str) -> String { + let (remote, path) = remote_path.split_once(':').expect("expected `remote:path`"); + format!("rclone://{remote}//{path}") +} + +fn cat(remote_path: &str) -> Vec { + std::process::Command::new("rclone").arg("cat").arg(remote_path).output().unwrap().stdout +} + +fn init() { + static ONCE: std::sync::Once = std::sync::Once::new(); + ONCE.call_once(|| { + yazi_shared::init(); + yazi_fs::init(); + yazi_vfs::init(); + }); +} + +macro_rules! require { + ($key:literal) => { + match env($key) { + Some(v) => v, + None => { + eprintln!("skipping: set {} to run this test", $key); + return; + } + } + }; +} + +#[tokio::test(flavor = "multi_thread")] +#[ignore] +async fn reads_and_seeks() { + init(); + let small = require!("YAZI_RCLONE_TEST_SMALL"); + let truth = cat(&small); + assert!(truth.len() > 100, "expected a non-trivial file"); + + let url: UrlBuf = to_url(&small).parse().expect("parse rclone url"); + + // Full sequential read + let mut f = yazi_vfs::provider::open(&url).await.expect("open"); + let mut buf = Vec::new(); + f.read_to_end(&mut buf).await.expect("read_to_end"); + assert_eq!(buf, truth, "streamed bytes must match `rclone cat`"); + + // Seek to the middle, read the tail, compare + let mid = (truth.len() / 2) as u64; + let mut f = yazi_vfs::provider::open(&url).await.expect("reopen"); + f.seek(std::io::SeekFrom::Start(mid)).await.expect("seek"); + let mut tail = Vec::new(); + f.read_to_end(&mut tail).await.expect("read tail"); + assert_eq!(tail, &truth[mid as usize..], "post-seek bytes must match"); + + println!("OK reads_and_seeks: {} bytes, seek-tail {} bytes", buf.len(), tail.len()); +} + +/// The remote root lists buckets; a directory path lists its children. +#[tokio::test(flavor = "multi_thread")] +#[ignore] +async fn lists_root_and_dir() { + init(); + let dir = require!("YAZI_RCLONE_TEST_DIR"); + + async fn names(url: &str) -> Vec { + let url: UrlBuf = url.parse().unwrap(); + let mut rd = yazi_vfs::provider::read_dir(&url).await.expect("read_dir"); + let mut out = Vec::new(); + while let Some(e) = rd.next().await.expect("next") { + out.push(e.name().into_string_lossy()); + } + out + } + + let remote = dir.split_once(':').unwrap().0; + let buckets = names(&format!("rclone://{remote}//")).await; + assert!(!buckets.is_empty(), "remote root should list at least one bucket"); + + let children = names(&to_url(&dir)).await; + assert!(!children.is_empty(), "directory should list its children"); + + println!("OK lists_root_and_dir: {} buckets, {} children", buckets.len(), children.len()); +} + +/// Opening a nonexistent object errors. Note: object stores cannot distinguish +/// a missing key from an empty directory prefix, so `rclone lsjson --stat` +/// reports the path as a directory — hence `InvalidInput` ("Is a directory") +/// rather than `NotFound`. Either way, the open fails cleanly instead of +/// streaming garbage. +#[tokio::test(flavor = "multi_thread")] +#[ignore] +async fn missing_object_errors() { + init(); + let dir = require!("YAZI_RCLONE_TEST_DIR"); + let remote = dir.split_once(':').unwrap().0; + + let url: UrlBuf = + to_url(&format!("{remote}:this/object/does/not/exist_xyz.bin")).parse().unwrap(); + match yazi_vfs::provider::open(&url).await { + Ok(_) => panic!("expected an error opening a nonexistent object"), + Err(e) => println!("OK missing_object_errors: {} ({:?})", e, e.kind()), + } +} + +/// Simple (non-progress) download path: `copy_impl` → open(rclone) + +/// create(local). +#[tokio::test(flavor = "multi_thread")] +#[ignore] +async fn copy_out_simple() { + init(); + let small = require!("YAZI_RCLONE_TEST_SMALL"); + let truth = cat(&small); + let from: UrlBuf = to_url(&small).parse().unwrap(); + + let dir = std::env::temp_dir().join("yazi_rclone_test_simple"); + std::fs::create_dir_all(&dir).unwrap(); + let dest = dir.join("out.bin"); + let to: UrlBuf = dest.to_str().unwrap().parse().unwrap(); + + let n = yazi_vfs::provider::copy(&from, &to, Default::default()).await.expect("copy"); + let got = std::fs::read(&dest).unwrap(); + assert_eq!(got, truth, "downloaded bytes must match"); + assert_eq!(n as usize, truth.len(), "reported byte count must match"); + std::fs::remove_dir_all(&dir).ok(); + println!("OK copy_out_simple: {n} bytes"); +} + +/// Progressive chunked download path: `ProgressiveCopier` (parallel 8 MiB +/// chunks +/// + per-chunk seek). A >8 MiB file spans multiple chunks. +#[tokio::test(flavor = "multi_thread")] +#[ignore] +async fn copy_out_progressive() { + init(); + let big = require!("YAZI_RCLONE_TEST_BIG"); + let truth = cat(&big); + assert!(truth.len() > 8 * 1024 * 1024, "need a >8 MiB file to span chunks"); + let from: UrlBuf = to_url(&big).parse().unwrap(); + + let dir = std::env::temp_dir().join("yazi_rclone_test_prog"); + std::fs::create_dir_all(&dir).unwrap(); + let dest = dir.join("out.bin"); + let to: UrlBuf = dest.to_str().unwrap().parse().unwrap(); + + let mut rx = + yazi_vfs::provider::copy_with_progress(&from, &to, yazi_fs::provider::Attrs::default()) + .await + .expect("copy_with_progress"); + + let mut total = 0u64; + while let Some(msg) = rx.recv().await { + total += msg.expect("no copy error"); + } + + let got = std::fs::read(&dest).unwrap(); + assert_eq!(got.len(), truth.len(), "downloaded size must match"); + assert_eq!(got, truth, "downloaded bytes must match byte-for-byte"); + std::fs::remove_dir_all(&dir).ok(); + println!("OK copy_out_progressive: {} bytes (progress sum {total})", got.len()); +} diff --git a/yazi-watcher/src/reporter.rs b/yazi-watcher/src/reporter.rs index 7d74b1a3..6506a68e 100644 --- a/yazi-watcher/src/reporter.rs +++ b/yazi-watcher/src/reporter.rs @@ -22,7 +22,7 @@ impl Reporter { match url.as_url().kind() { SchemeKind::Regular | SchemeKind::Search => self.report_local(url), SchemeKind::Archive => {} - SchemeKind::Sftp => self.report_remote(url), + SchemeKind::Sftp | SchemeKind::Rclone => self.report_remote(url), } } } diff --git a/yazi-watcher/src/watched.rs b/yazi-watcher/src/watched.rs index f74f7a47..2c24965e 100644 --- a/yazi-watcher/src/watched.rs +++ b/yazi-watcher/src/watched.rs @@ -44,19 +44,31 @@ impl Watched { // Parse domain let domain = it.next()?.as_normal()?.to_str().ok()?; - let domain = percent_decode_str(domain.strip_prefix("sftp-")?).decode_utf8().ok()?.intern(); + let (kind, domain) = if let Some(d) = domain.strip_prefix("sftp-") { + (SchemeKind::Sftp, d) + } else if let Some(d) = domain.strip_prefix("rclone-") { + (SchemeKind::Rclone, d) + } else { + return None; + }; + let domain = percent_decode_str(domain).decode_utf8().ok()?.intern(); // Parse path let (path, abs) = if let Ok(p) = it.path().try_strip_prefix(".%2F") { (p, false) } else { (it.path(), true) }; - let path = path.percent_decode(SchemeKind::Sftp).ok()?; + let path = path.percent_decode(kind).ok()?; let path = PathBufDyn::from_components( - SchemeKind::Sftp, + kind, abs.then_some(Component::RootDir).into_iter().chain(path.components()), ) .ok()?; - let url = UrlBuf::Sftp { loc: path.into_unix().ok()?.into(), domain }; + let loc = path.into_unix().ok()?.into(); + let url = match kind { + SchemeKind::Sftp => UrlBuf::Sftp { loc, domain }, + SchemeKind::Rclone => UrlBuf::Rclone { loc, domain }, + _ => return None, + }; if self.contains_url(&url) { Some(url) } else { None } } }