From 558622e512c755c1c4b710eec33b8d50f87dea5c Mon Sep 17 00:00:00 2001 From: Xie Zejian Date: Wed, 1 Apr 2026 14:07:03 +0800 Subject: [PATCH] feat(s3): add S3 bucket browsing as a VFS provider Implement a read-only S3 provider built on the `object_store` crate, allowing users to browse S3 buckets natively within yazi. - Add `ServiceS3` config (region, endpoint, credentials, force_path_style, allow_http) - Add `S3` variant to `Url` enum and `SchemeKind` - Implement read_dir with paginated listing, metadata, and streaming read - Write operations return `Unsupported` for now --- Cargo.lock | 804 +++++++++++++++++++++++++ yazi-config/src/vfs/service.rs | 50 ++ yazi-fs/src/path/expand.rs | 4 + yazi-fs/src/provider/local/absolute.rs | 2 +- yazi-fs/src/provider/local/local.rs | 4 +- yazi-fs/src/scheme.rs | 3 + yazi-fs/src/url.rs | 6 +- yazi-shared/src/loc/loc.rs | 1 + yazi-shared/src/path/kind.rs | 1 + yazi-shared/src/scheme/cow.rs | 18 + yazi-shared/src/scheme/encode.rs | 3 +- yazi-shared/src/scheme/kind.rs | 9 +- yazi-shared/src/scheme/ref.rs | 13 +- yazi-shared/src/scheme/scheme.rs | 9 +- yazi-shared/src/scheme/traits.rs | 1 + yazi-shared/src/strand/kind.rs | 1 + yazi-shared/src/url/buf.rs | 7 +- yazi-shared/src/url/components.rs | 4 + yazi-shared/src/url/cow.rs | 25 + yazi-shared/src/url/encode.rs | 3 + yazi-shared/src/url/traits.rs | 2 + yazi-shared/src/url/url.rs | 51 +- yazi-vfs/Cargo.toml | 1 + yazi-vfs/src/provider/dir_entry.rs | 6 + yazi-vfs/src/provider/gate.rs | 3 + yazi-vfs/src/provider/mod.rs | 7 +- yazi-vfs/src/provider/provider.rs | 7 + yazi-vfs/src/provider/providers.rs | 35 ++ yazi-vfs/src/provider/read_dir.rs | 2 + yazi-vfs/src/provider/s3/absolute.rs | 29 + yazi-vfs/src/provider/s3/metadata.rs | 97 +++ yazi-vfs/src/provider/s3/mod.rs | 10 + yazi-vfs/src/provider/s3/read_dir.rs | 125 ++++ yazi-vfs/src/provider/s3/s3.rs | 407 +++++++++++++ yazi-vfs/src/provider/sftp/sftp.rs | 2 +- yazi-watcher/src/reporter.rs | 1 + 36 files changed, 1731 insertions(+), 22 deletions(-) create mode 100644 yazi-vfs/src/provider/s3/absolute.rs create mode 100644 yazi-vfs/src/provider/s3/metadata.rs create mode 100644 yazi-vfs/src/provider/s3/mod.rs create mode 100644 yazi-vfs/src/provider/s3/read_dir.rs create mode 100644 yazi-vfs/src/provider/s3/s3.rs diff --git a/Cargo.lock b/Cargo.lock index 5f246022..13945ac5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -225,6 +225,17 @@ dependencies = [ "event-listener", ] +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "atomic" version = "0.6.1" @@ -234,6 +245,12 @@ dependencies = [ "bytemuck", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.5.0" @@ -751,6 +768,16 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -1232,6 +1259,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "document-features" version = "0.2.12" @@ -1546,6 +1584,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + [[package]] name = "fsevent-sys" version = "4.1.0" @@ -1691,9 +1738,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 5.3.0", "wasip2", + "wasm-bindgen", ] [[package]] @@ -1759,6 +1808,25 @@ dependencies = [ "subtle", ] +[[package]] +name = "h2" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap 2.13.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "half" version = "2.7.1" @@ -1840,6 +1908,51 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "http" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "humantime" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" + [[package]] name = "hybrid-array" version = "0.2.3" @@ -1858,6 +1971,67 @@ dependencies = [ "typenum", ] +[[package]] +name = "hyper" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + [[package]] name = "iana-time-zone" version = "0.1.65" @@ -1882,6 +2056,88 @@ dependencies = [ "cc", ] +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + [[package]] name = "id-arena" version = "2.3.0" @@ -1894,6 +2150,27 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + [[package]] name = "image" version = "0.25.10" @@ -2050,6 +2327,22 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "iri-string" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25e659a4bb38e810ebc252e53b5814ff908a8c58c2a9ce2fae1bbec24cbf4e20" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -2222,6 +2515,12 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + [[package]] name = "litrs" version = "1.0.0" @@ -2261,6 +2560,12 @@ dependencies = [ "hashbrown 0.16.1", ] +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + [[package]] name = "lua-src" version = "550.0.0" @@ -2308,6 +2613,16 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest 0.10.7", +] + [[package]] name = "md5" version = "0.7.0" @@ -2672,6 +2987,41 @@ dependencies = [ "memchr", ] +[[package]] +name = "object_store" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbfbfff40aeccab00ec8a910b57ca8ecf4319b335c542f2edcd19dd25a1e2a00" +dependencies = [ + "async-trait", + "base64", + "bytes", + "chrono", + "form_urlencoded", + "futures", + "http", + "http-body-util", + "humantime", + "hyper", + "itertools", + "md-5", + "parking_lot", + "percent-encoding", + "quick-xml", + "rand 0.9.2", + "reqwest", + "ring", + "serde", + "serde_json", + "serde_urlencoded", + "thiserror 2.0.18", + "tokio", + "tracing", + "url", + "wasm-bindgen-futures", + "web-time", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -2712,6 +3062,12 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + [[package]] name = "option-ext" version = "0.2.0" @@ -3123,6 +3479,15 @@ version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + [[package]] name = "powerfmt" version = "0.2.0" @@ -3251,6 +3616,62 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" dependencies = [ "memchr", + "serde", +] + +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +dependencies = [ + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand 0.9.2", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.60.2", ] [[package]] @@ -3586,6 +4007,48 @@ version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", +] + [[package]] name = "rfc6979" version = "0.4.0" @@ -3766,6 +4229,53 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "rustls" +version = "0.23.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.22" @@ -3805,6 +4315,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "schemars" version = "0.9.0" @@ -3860,6 +4379,29 @@ dependencies = [ "zeroize", ] +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags 2.11.0", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "1.0.27" @@ -3928,6 +4470,18 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "serde_with" version = "3.18.0" @@ -4303,6 +4857,26 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "syntect" version = "5.3.0" @@ -4520,6 +5094,31 @@ dependencies = [ "time-core", ] +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" version = "1.50.0" @@ -4548,6 +5147,16 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.18" @@ -4611,6 +5220,51 @@ version = "1.1.1+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db" +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +dependencies = [ + "bitflags 2.11.0", + "bytes", + "futures-util", + "http", + "http-body", + "iri-string", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + [[package]] name = "tracing" version = "0.1.44" @@ -4702,6 +5356,12 @@ dependencies = [ "windows 0.56.0", ] +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "twox-hash" version = "2.1.2" @@ -4786,12 +5446,30 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + [[package]] name = "urlencoding" version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "utf8parse" version = "0.2.2" @@ -4902,6 +5580,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -5003,6 +5690,19 @@ dependencies = [ "wasmparser", ] +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasmparser" version = "0.244.0" @@ -5015,6 +5715,26 @@ dependencies = [ "semver", ] +[[package]] +name = "web-sys" +version = "0.3.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd70027e39b12f0849461e08ffc50b9cd7688d942c1c8e3c7b22273236b4dd0a" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "weezl" version = "0.1.12" @@ -5567,6 +6287,12 @@ dependencies = [ "wasmparser", ] +[[package]] +name = "writeable" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" + [[package]] name = "wyz" version = "0.5.1" @@ -6150,6 +6876,7 @@ dependencies = [ "either", "futures", "hashbrown 0.16.1", + "object_store", "parking_lot", "russh", "tokio", @@ -6203,6 +6930,29 @@ dependencies = [ "yazi-tty", ] +[[package]] +name = "yoke" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.8.48" @@ -6223,12 +6973,66 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "zerofrom" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", + "synstructure", +] + [[package]] name = "zeroize" version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "zmij" version = "1.0.21" diff --git a/yazi-config/src/vfs/service.rs b/yazi-config/src/vfs/service.rs index 57578f67..77cc439c 100644 --- a/yazi-config/src/vfs/service.rs +++ b/yazi-config/src/vfs/service.rs @@ -7,15 +7,28 @@ use crate::normalize_path; #[derive(Deserialize, Serialize)] #[serde(tag = "type", rename_all = "kebab-case")] pub enum Service { + S3(ServiceS3), Sftp(ServiceSftp), } +impl TryFrom<&'static Service> for &'static ServiceS3 { + type Error = &'static str; + + fn try_from(value: &'static Service) -> Result { + match value { + Service::S3(p) => Ok(p), + _ => Err("expected s3 service"), + } + } +} + impl TryFrom<&'static Service> for &'static ServiceSftp { type Error = &'static str; fn try_from(value: &'static Service) -> Result { match value { Service::Sftp(p) => Ok(p), + _ => Err("expected sftp service"), } } } @@ -23,11 +36,48 @@ impl TryFrom<&'static Service> for &'static ServiceSftp { impl Service { pub(super) fn reshape(&mut self) -> io::Result<()> { match self { + Self::S3(p) => p.reshape(), Self::Sftp(p) => p.reshape(), } } } +// --- S3 +#[derive(Deserialize, Eq, Hash, PartialEq, Serialize)] +pub struct ServiceS3 { + pub region: Option, + pub endpoint: Option, + pub access_key_id: Option, + pub secret_access_key: Option, + pub session_token: Option, + #[serde(default)] + pub force_path_style: bool, + #[serde(default)] + pub allow_http: bool, +} + +impl ServiceS3 { + fn reshape(&mut self) -> io::Result<()> { + self.region = trim_option(self.region.take()); + self.endpoint = self.endpoint.take().and_then(|s| { + let s = s.trim().trim_end_matches('/').to_owned(); + (!s.is_empty()).then_some(s) + }); + self.access_key_id = trim_option(self.access_key_id.take()); + self.secret_access_key = trim_option(self.secret_access_key.take()); + self.session_token = trim_option(self.session_token.take()); + + Ok(()) + } +} + +fn trim_option(value: Option) -> Option { + value.and_then(|s| { + let s = s.trim().to_owned(); + (!s.is_empty()).then_some(s) + }) +} + // --- SFTP #[derive(Deserialize, Eq, Hash, PartialEq, Serialize)] pub struct ServiceSftp { diff --git a/yazi-fs/src/path/expand.rs b/yazi-fs/src/path/expand.rs index d1061ba5..8a6725b5 100644 --- a/yazi-fs/src/path/expand.rs +++ b/yazi-fs/src/path/expand.rs @@ -39,6 +39,10 @@ fn expand_url_impl(url: UrlCow) -> UrlCow { loc: LocBuf::::with(path.into_os().unwrap(), uri, urn).unwrap(), domain: domain.intern(), }, + Url::S3 { domain, .. } => UrlBuf::S3 { + loc: LocBuf::::with(path.into_unix().unwrap(), uri, urn).unwrap(), + domain: domain.intern(), + }, Url::Sftp { domain, .. } => UrlBuf::Sftp { loc: LocBuf::::with(path.into_unix().unwrap(), uri, urn).unwrap(), domain: domain.intern(), diff --git a/yazi-fs/src/provider/local/absolute.rs b/yazi-fs/src/provider/local/absolute.rs index d22cceda..a9cceb9a 100644 --- a/yazi-fs/src/provider/local/absolute.rs +++ b/yazi-fs/src/provider/local/absolute.rs @@ -51,6 +51,6 @@ fn try_absolute_impl<'a>(url: UrlCow<'a>) -> Option> { Some(match url.as_url() { Url::Regular(_) => UrlBuf::Regular(loc).into(), Url::Search { domain, .. } => UrlBuf::Search { loc, domain: domain.intern() }.into(), - Url::Archive { .. } | Url::Sftp { .. } => None?, + Url::Archive { .. } | Url::S3 { .. } | Url::Sftp { .. } => None?, }) } diff --git a/yazi-fs/src/provider/local/local.rs b/yazi-fs/src/provider/local/local.rs index a9abe8fc..791faf8e 100644 --- a/yazi-fs/src/provider/local/local.rs +++ b/yazi-fs/src/provider/local/local.rs @@ -80,7 +80,7 @@ impl<'a> Provider for Local<'a> { async fn new<'b>(url: Url<'b>) -> io::Result> { match url { Url::Regular(loc) | Url::Search { loc, .. } => Ok(Self::Me { url, path: loc.as_inner() }), - Url::Archive { .. } | Url::Sftp { .. } => { + Url::Archive { .. } | Url::S3 { .. } | Url::Sftp { .. } => { Err(io::Error::new(io::ErrorKind::InvalidInput, format!("Not a local URL: {url:?}"))) } } @@ -94,7 +94,7 @@ impl<'a> Provider for Local<'a> { reader: tokio::fs::read_dir(self.path).await?, dir: Arc::new(self.url.to_owned()), }, - SchemeKind::Archive | SchemeKind::Sftp => Err(io::Error::new( + SchemeKind::Archive | SchemeKind::S3 | SchemeKind::Sftp => Err(io::Error::new( io::ErrorKind::InvalidInput, format!("Not a local URL: {:?}", self.url), ))?, diff --git a/yazi-fs/src/scheme.rs b/yazi-fs/src/scheme.rs index 4eb65431..915c0adc 100644 --- a/yazi-fs/src/scheme.rs +++ b/yazi-fs/src/scheme.rs @@ -15,6 +15,9 @@ impl FsScheme for SchemeRef<'_> { Self::Archive { domain, .. } => Some( Xdg::cache_dir().join(format!("archive-{}", yazi_shared::scheme::Encode::domain(domain))), ), + Self::S3 { domain, .. } => { + Some(Xdg::cache_dir().join(format!("s3-{}", yazi_shared::scheme::Encode::domain(domain)))) + } Self::Sftp { domain, .. } => { Some(Xdg::cache_dir().join(format!("sftp-{}", yazi_shared::scheme::Encode::domain(domain)))) } diff --git a/yazi-fs/src/url.rs b/yazi-fs/src/url.rs index 9755c7ba..ceaa293c 100644 --- a/yazi-fs/src/url.rs +++ b/yazi-fs/src/url.rs @@ -49,7 +49,7 @@ impl<'a> FsUrl<'a> for Url<'a> { fn unified_path(self) -> Cow<'a, Path> { match self { Self::Regular(loc) | Self::Search { loc, .. } => loc.as_inner().into(), - Self::Archive { .. } | Self::Sftp { .. } => { + Self::Archive { .. } | Self::S3 { .. } | Self::Sftp { .. } => { self.cache().expect("non-local URL should have a cache path").into() } } @@ -64,7 +64,7 @@ impl FsUrl<'_> for UrlBuf { fn unified_path(self) -> Cow<'static, Path> { match self { Self::Regular(loc) | Self::Search { loc, .. } => loc.into_inner().into(), - Self::Archive { .. } | Self::Sftp { .. } => { + Self::Archive { .. } | Self::S3 { .. } | Self::Sftp { .. } => { self.cache().expect("non-local URL should have a cache path").into() } } @@ -79,7 +79,7 @@ impl<'a> FsUrl<'a> for UrlCow<'a> { fn unified_path(self) -> Cow<'a, Path> { match self { Self::Regular(loc) | Self::Search { loc, .. } => loc.into_inner(), - Self::Archive { .. } | Self::Sftp { .. } => { + Self::Archive { .. } | Self::S3 { .. } | Self::Sftp { .. } => { self.cache().expect("non-local URL should have a cache path").into() } } diff --git a/yazi-shared/src/loc/loc.rs b/yazi-shared/src/loc/loc.rs index a1824ee8..581d2461 100644 --- a/yazi-shared/src/loc/loc.rs +++ b/yazi-shared/src/loc/loc.rs @@ -155,6 +155,7 @@ where SchemeKind::Regular => Self::bare(path), SchemeKind::Search => Self::zeroed(path), SchemeKind::Archive => Self::zeroed(path), + SchemeKind::S3 => Self::bare(path), SchemeKind::Sftp => Self::bare(path), } } diff --git a/yazi-shared/src/path/kind.rs b/yazi-shared/src/path/kind.rs index c66781cd..b1e0f7b0 100644 --- a/yazi-shared/src/path/kind.rs +++ b/yazi-shared/src/path/kind.rs @@ -11,6 +11,7 @@ impl From for PathKind { SchemeKind::Regular => Self::Os, SchemeKind::Search => Self::Os, SchemeKind::Archive => Self::Os, + SchemeKind::S3 => Self::Unix, SchemeKind::Sftp => Self::Unix, } } diff --git a/yazi-shared/src/scheme/cow.rs b/yazi-shared/src/scheme/cow.rs index 30a4eea0..e7780579 100644 --- a/yazi-shared/src/scheme/cow.rs +++ b/yazi-shared/src/scheme/cow.rs @@ -57,6 +57,16 @@ impl<'a> SchemeCow<'a> { } } + pub fn s3(domain: T, uri: usize, urn: usize) -> Self + where + T: Into>, + { + match domain.into() { + Cow::Borrowed(domain) => SchemeRef::S3 { domain, uri, urn }.into(), + Cow::Owned(domain) => Scheme::S3 { domain: domain.intern(), uri, urn }.into(), + } + } + pub fn sftp(domain: T, uri: usize, urn: usize) -> Self where T: Into>, @@ -80,6 +90,7 @@ impl<'a> SchemeCow<'a> { SchemeKind::Regular => ("".into(), None, None), SchemeKind::Search => Self::decode_param(&bytes[skip..], &mut skip)?, SchemeKind::Archive => Self::decode_param(&bytes[skip..], &mut skip)?, + SchemeKind::S3 => Self::decode_param(&bytes[skip..], &mut skip)?, SchemeKind::Sftp => Self::decode_param(&bytes[skip..], &mut skip)?, }; @@ -92,6 +103,7 @@ impl<'a> SchemeCow<'a> { SchemeKind::Regular => Self::regular(uri, urn), SchemeKind::Search => Self::search(domain, uri, urn), SchemeKind::Archive => Self::archive(domain, uri, urn), + SchemeKind::S3 => Self::s3(domain, uri, urn), SchemeKind::Sftp => Self::sftp(domain, uri, urn), }; @@ -154,6 +166,11 @@ impl<'a> SchemeCow<'a> { (uri, urn) } SchemeKind::Archive => (uri.unwrap_or(0), urn.unwrap_or(0)), + SchemeKind::S3 => { + let uri = uri.unwrap_or(path.name().is_some() as usize); + let urn = urn.unwrap_or(path.name().is_some() as usize); + (uri, urn) + } SchemeKind::Sftp => { let uri = uri.unwrap_or(path.name().is_some() as usize); let urn = urn.unwrap_or(path.name().is_some() as usize); @@ -167,6 +184,7 @@ impl<'a> SchemeCow<'a> { Url::Regular(loc) => (loc.file_name().is_some() as usize, loc.file_name().is_some() as usize), Url::Search { loc, .. } => (loc.uri().components().count(), loc.urn().components().count()), Url::Archive { loc, .. } => (loc.uri().components().count(), loc.urn().components().count()), + Url::S3 { loc, .. } => (loc.uri().components().count(), loc.urn().components().count()), Url::Sftp { loc, .. } => (loc.uri().components().count(), loc.urn().components().count()), } } diff --git a/yazi-shared/src/scheme/encode.rs b/yazi-shared/src/scheme/encode.rs index c51e28ea..694e7f5c 100644 --- a/yazi-shared/src/scheme/encode.rs +++ b/yazi-shared/src/scheme/encode.rs @@ -38,7 +38,7 @@ impl<'a> Encode<'a> { match self.0.0.kind() { SchemeKind::Regular => Ok(()), SchemeKind::Search | SchemeKind::Archive => w!(0, 0), - SchemeKind::Sftp => { + SchemeKind::S3 | SchemeKind::Sftp => { w!(self.0.0.loc().name().is_some() as usize, self.0.0.loc().name().is_some() as usize) } } @@ -57,6 +57,7 @@ impl Display for Encode<'_> { Url::Archive { domain, .. } => { write!(f, "archive://{}{}/", Self::domain(domain), self.ports()) } + Url::S3 { domain, .. } => write!(f, "s3://{}{}/", Self::domain(domain), self.ports()), Url::Sftp { domain, .. } => write!(f, "sftp://{}{}/", Self::domain(domain), self.ports()), } } diff --git a/yazi-shared/src/scheme/kind.rs b/yazi-shared/src/scheme/kind.rs index c5ddc3dc..5ec5175b 100644 --- a/yazi-shared/src/scheme/kind.rs +++ b/yazi-shared/src/scheme/kind.rs @@ -9,6 +9,7 @@ pub enum SchemeKind { Regular, Search, Archive, + S3, Sftp, } @@ -21,6 +22,7 @@ where SchemeRef::Regular { .. } => Self::Regular, SchemeRef::Search { .. } => Self::Search, SchemeRef::Archive { .. } => Self::Archive, + SchemeRef::S3 { .. } => Self::S3, SchemeRef::Sftp { .. } => Self::Sftp, } } @@ -34,6 +36,7 @@ impl TryFrom<&[u8]> for SchemeKind { b"regular" => Ok(Self::Regular), b"search" => Ok(Self::Search), b"archive" => Ok(Self::Archive), + b"s3" => Ok(Self::S3), b"sftp" => Ok(Self::Sftp), _ => bail!("invalid scheme kind: {}", String::from_utf8_lossy(value)), } @@ -45,7 +48,7 @@ impl SchemeKind { pub fn is_local(self) -> bool { match self { Self::Regular | Self::Search => true, - Self::Archive | Self::Sftp => false, + Self::Archive | Self::S3 | Self::Sftp => false, } } @@ -53,7 +56,7 @@ impl SchemeKind { pub fn is_remote(self) -> bool { match self { Self::Regular | Self::Search | Self::Archive => false, - Self::Sftp => true, + Self::S3 | Self::Sftp => true, } } @@ -61,7 +64,7 @@ impl SchemeKind { pub fn is_virtual(self) -> bool { match self { Self::Regular | Self::Search => false, - Self::Archive | Self::Sftp => true, + Self::Archive | Self::S3 | Self::Sftp => true, } } diff --git a/yazi-shared/src/scheme/ref.rs b/yazi-shared/src/scheme/ref.rs index 6259f699..d95833e3 100644 --- a/yazi-shared/src/scheme/ref.rs +++ b/yazi-shared/src/scheme/ref.rs @@ -7,6 +7,7 @@ pub enum SchemeRef<'a> { Regular { uri: usize, urn: usize }, Search { domain: &'a str, uri: usize, urn: usize }, Archive { domain: &'a str, uri: usize, urn: usize }, + S3 { domain: &'a str, uri: usize, urn: usize }, Sftp { domain: &'a str, uri: usize, urn: usize }, } @@ -18,6 +19,7 @@ impl Deref for SchemeRef<'_> { Self::Regular { .. } => &SchemeKind::Regular, Self::Search { .. } => &SchemeKind::Search, Self::Archive { .. } => &SchemeKind::Archive, + Self::S3 { .. } => &SchemeKind::S3, Self::Sftp { .. } => &SchemeKind::Sftp, } } @@ -49,9 +51,10 @@ impl<'a> SchemeRef<'a> { pub const fn domain(self) -> Option<&'a str> { match self { Self::Regular { .. } => None, - Self::Search { domain, .. } | Self::Archive { domain, .. } | Self::Sftp { domain, .. } => { - Some(domain) - } + Self::Search { domain, .. } + | Self::Archive { domain, .. } + | Self::S3 { domain, .. } + | Self::Sftp { domain, .. } => Some(domain), } } @@ -60,6 +63,7 @@ impl<'a> SchemeRef<'a> { Self::Regular { .. } => SchemeKind::Regular, Self::Search { .. } => SchemeKind::Search, Self::Archive { .. } => SchemeKind::Archive, + Self::S3 { .. } => SchemeKind::S3, Self::Sftp { .. } => SchemeKind::Sftp, } } @@ -69,6 +73,7 @@ impl<'a> SchemeRef<'a> { Self::Regular { uri, urn } => (uri, urn), Self::Search { uri, urn, .. } => (uri, urn), Self::Archive { uri, urn, .. } => (uri, urn), + Self::S3 { uri, urn, .. } => (uri, urn), Self::Sftp { uri, urn, .. } => (uri, urn), } } @@ -78,6 +83,7 @@ impl<'a> SchemeRef<'a> { Self::Regular { uri, urn } => Scheme::Regular { uri, urn }, Self::Search { domain, uri, urn } => Scheme::Search { domain: domain.intern(), uri, urn }, Self::Archive { domain, uri, urn } => Scheme::Archive { domain: domain.intern(), uri, urn }, + Self::S3 { domain, uri, urn } => Scheme::S3 { domain: domain.intern(), uri, urn }, Self::Sftp { domain, uri, urn } => Scheme::Sftp { domain: domain.intern(), uri, urn }, } } @@ -87,6 +93,7 @@ impl<'a> SchemeRef<'a> { Self::Regular { .. } => Self::Regular { uri, urn }, Self::Search { domain, .. } => Self::Search { domain, uri, urn }, Self::Archive { domain, .. } => Self::Archive { domain, uri, urn }, + Self::S3 { domain, .. } => Self::S3 { domain, uri, urn }, Self::Sftp { domain, .. } => Self::Sftp { domain, uri, urn }, } } diff --git a/yazi-shared/src/scheme/scheme.rs b/yazi-shared/src/scheme/scheme.rs index 8148a431..3b06c41a 100644 --- a/yazi-shared/src/scheme/scheme.rs +++ b/yazi-shared/src/scheme/scheme.rs @@ -10,6 +10,7 @@ pub enum Scheme { Regular { uri: usize, urn: usize }, Search { domain: Symbol, uri: usize, urn: usize }, Archive { domain: Symbol, uri: usize, urn: usize }, + S3 { domain: Symbol, uri: usize, urn: usize }, Sftp { domain: Symbol, uri: usize, urn: usize }, } @@ -26,9 +27,10 @@ impl Scheme { pub fn into_domain(self) -> Option> { match self { Self::Regular { .. } => None, - Self::Search { domain, .. } | Self::Archive { domain, .. } | Self::Sftp { domain, .. } => { - Some(domain) - } + Self::Search { domain, .. } + | Self::Archive { domain, .. } + | Self::S3 { domain, .. } + | Self::Sftp { domain, .. } => Some(domain), } } @@ -38,6 +40,7 @@ impl Scheme { Self::Regular { .. } => Self::Regular { uri, urn }, Self::Search { domain, .. } => Self::Search { domain, uri, urn }, Self::Archive { domain, .. } => Self::Archive { domain, uri, urn }, + Self::S3 { domain, .. } => Self::S3 { domain, uri, urn }, Self::Sftp { domain, .. } => Self::Sftp { domain, uri, urn }, } } diff --git a/yazi-shared/src/scheme/traits.rs b/yazi-shared/src/scheme/traits.rs index b36cbc56..99401a4b 100644 --- a/yazi-shared/src/scheme/traits.rs +++ b/yazi-shared/src/scheme/traits.rs @@ -16,6 +16,7 @@ impl AsScheme for Scheme { Self::Regular { uri, urn } => SchemeRef::Regular { uri, urn }, Self::Search { ref domain, uri, urn } => SchemeRef::Search { domain, uri, urn }, Self::Archive { ref domain, uri, urn } => SchemeRef::Archive { domain, uri, urn }, + Self::S3 { ref domain, uri, urn } => SchemeRef::S3 { domain, uri, urn }, Self::Sftp { ref domain, uri, urn } => SchemeRef::Sftp { domain, uri, urn }, } } diff --git a/yazi-shared/src/strand/kind.rs b/yazi-shared/src/strand/kind.rs index 103afc20..08dcaa27 100644 --- a/yazi-shared/src/strand/kind.rs +++ b/yazi-shared/src/strand/kind.rs @@ -22,6 +22,7 @@ impl From for StrandKind { SchemeKind::Regular => Self::Os, SchemeKind::Search => Self::Os, SchemeKind::Archive => Self::Os, + SchemeKind::S3 => Self::Bytes, SchemeKind::Sftp => Self::Bytes, } } diff --git a/yazi-shared/src/url/buf.rs b/yazi-shared/src/url/buf.rs index cf3bbc25..f89d523b 100644 --- a/yazi-shared/src/url/buf.rs +++ b/yazi-shared/src/url/buf.rs @@ -10,6 +10,7 @@ pub enum UrlBuf { Regular(LocBuf), Search { loc: LocBuf, domain: Symbol }, Archive { loc: LocBuf, domain: Symbol }, + S3 { loc: LocBuf, domain: Symbol }, Sftp { loc: LocBuf, domain: Symbol }, } @@ -28,6 +29,7 @@ impl From> for UrlBuf { Url::Regular(loc) => Self::Regular(loc.into()), Url::Search { loc, domain } => Self::Search { loc: loc.into(), domain: domain.intern() }, Url::Archive { loc, domain } => Self::Archive { loc: loc.into(), domain: domain.intern() }, + Url::S3 { loc, domain } => Self::S3 { loc: loc.into(), domain: domain.intern() }, Url::Sftp { loc, domain } => Self::Sftp { loc: loc.into(), domain: domain.intern() }, } } @@ -134,6 +136,7 @@ impl UrlBuf { Self::Regular(loc) => loc.into_inner().into(), Self::Search { loc, .. } => loc.into_inner().into(), Self::Archive { loc, .. } => loc.into_inner().into(), + Self::S3 { loc, .. } => loc.into_inner().into(), Self::Sftp { loc, .. } => loc.into_inner().into(), } } @@ -149,6 +152,7 @@ impl UrlBuf { Self::Regular(loc) => loc.try_set_name(name.as_os()?)?, Self::Search { loc, .. } => loc.try_set_name(name.as_os()?)?, Self::Archive { loc, .. } => loc.try_set_name(name.as_os()?)?, + Self::S3 { loc, .. } => loc.try_set_name(name.encoded_bytes())?, Self::Sftp { loc, .. } => loc.try_set_name(name.encoded_bytes())?, }) } @@ -162,7 +166,8 @@ impl UrlBuf { Self::Archive { loc, domain } => { Self::Archive { loc: loc.rebase(base), domain: domain.clone() } } - Self::Sftp { loc, domain } => { + Self::S3 { .. } => self.clone(), + Self::Sftp { loc: _, domain: _ } => { todo!(); // Self::Sftp { loc: loc.rebase(base), domain: domain.clone() } } diff --git a/yazi-shared/src/url/components.rs b/yazi-shared/src/url/components.rs index 50778f81..5d455812 100644 --- a/yazi-shared/src/url/components.rs +++ b/yazi-shared/src/url/components.rs @@ -59,6 +59,7 @@ impl<'a> Components<'a> { Url::Regular(_) => SchemeRef::Regular { uri, urn }, Url::Search { domain, .. } => SchemeRef::Search { domain, uri, urn }, Url::Archive { domain, .. } => SchemeRef::Archive { domain, uri, urn }, + Url::S3 { domain, .. } => SchemeRef::S3 { domain, uri, urn }, Url::Sftp { domain, .. } => SchemeRef::Sftp { domain, uri, urn }, } } @@ -86,6 +87,9 @@ impl<'a> Components<'a> { Url::Archive { domain, .. } => { Url::Archive { loc: Loc::with(path.as_os().unwrap(), uri, urn).unwrap(), domain } } + Url::S3 { domain, .. } => { + Url::S3 { loc: Loc::with(path.as_unix().unwrap(), uri, urn).unwrap(), domain } + } Url::Sftp { domain, .. } => { Url::Sftp { loc: Loc::with(path.as_unix().unwrap(), uri, urn).unwrap(), domain } } diff --git a/yazi-shared/src/url/cow.rs b/yazi-shared/src/url/cow.rs index b1de5301..5a41b6f5 100644 --- a/yazi-shared/src/url/cow.rs +++ b/yazi-shared/src/url/cow.rs @@ -11,6 +11,7 @@ pub enum UrlCow<'a> { Regular(LocCow<'a>), Search { loc: LocCow<'a>, domain: SymbolCow<'a, str> }, Archive { loc: LocCow<'a>, domain: SymbolCow<'a, str> }, + S3 { loc: LocCow<'a, &'a UnixPath, UnixPathBuf>, domain: SymbolCow<'a, str> }, Sftp { loc: LocCow<'a, &'a UnixPath, UnixPathBuf>, domain: SymbolCow<'a, str> }, } @@ -25,6 +26,7 @@ impl<'a> From> for UrlCow<'a> { Url::Regular(loc) => Self::Regular(loc.into()), Url::Search { loc, domain } => Self::Search { loc: loc.into(), domain: domain.into() }, Url::Archive { loc, domain } => Self::Archive { loc: loc.into(), domain: domain.into() }, + Url::S3 { loc, domain } => Self::S3 { loc: loc.into(), domain: domain.into() }, Url::Sftp { loc, domain } => Self::Sftp { loc: loc.into(), domain: domain.into() }, } } @@ -45,6 +47,7 @@ impl From for UrlCow<'_> { UrlBuf::Archive { loc, domain } => { Self::Archive { loc: loc.into(), domain: domain.into() } } + UrlBuf::S3 { loc, domain } => Self::S3 { loc: loc.into(), domain: domain.into() }, UrlBuf::Sftp { loc, domain } => Self::Sftp { loc: loc.into(), domain: domain.into() }, } } @@ -153,6 +156,10 @@ impl<'a> TryFrom<(SchemeCow<'a>, PathDyn<'a>)> for UrlCow<'a> { loc: Loc::with(path.as_os()?, uri, urn)?.into(), domain: domain.ok_or_else(|| anyhow!("missing domain for archive scheme"))?, }, + SchemeKind::S3 => Self::S3 { + loc: Loc::with(path.as_unix()?, uri, urn)?.into(), + domain: domain.ok_or_else(|| anyhow!("missing domain for s3 scheme"))?, + }, SchemeKind::Sftp => Self::Sftp { loc: Loc::with(path.as_unix()?, uri, urn)?.into(), domain: domain.ok_or_else(|| anyhow!("missing domain for sftp scheme"))?, @@ -180,6 +187,10 @@ impl<'a> TryFrom<(SchemeCow<'a>, PathBufDyn)> for UrlCow<'a> { loc: LocBuf::::with(path.try_into()?, uri, urn)?.into(), domain: domain.ok_or_else(|| anyhow!("missing domain for archive scheme"))?, }, + SchemeKind::S3 => Self::S3 { + loc: LocBuf::::with(path.try_into()?, uri, urn)?.into(), + domain: domain.ok_or_else(|| anyhow!("missing domain for s3 scheme"))?, + }, SchemeKind::Sftp => Self::Sftp { loc: LocBuf::::with(path.try_into()?, uri, urn)?.into(), domain: domain.ok_or_else(|| anyhow!("missing domain for sftp scheme"))?, @@ -210,6 +221,7 @@ impl<'a> UrlCow<'a> { Self::Regular(loc) => loc.is_owned(), Self::Search { loc, .. } => loc.is_owned(), Self::Archive { loc, .. } => loc.is_owned(), + Self::S3 { loc, .. } => loc.is_owned(), Self::Sftp { loc, .. } => loc.is_owned(), } } @@ -223,9 +235,13 @@ impl<'a> UrlCow<'a> { Self::Archive { loc, domain } => { UrlBuf::Archive { loc: loc.into_owned(), domain: domain.into() } } + Self::S3 { loc, domain } => { + UrlBuf::S3 { loc: loc.into_owned(), domain: domain.into() } + } Self::Sftp { loc, domain } => { UrlBuf::Sftp { loc: loc.into_owned(), domain: domain.into() } } + } } @@ -248,6 +264,12 @@ impl<'a> UrlCow<'a> { } SymbolCow::Owned(domain) => (Scheme::Archive { domain, uri, urn }.into(), loc.into_path()), }, + Self::S3 { loc, domain } => match domain { + SymbolCow::Borrowed(domain) => { + (SchemeRef::S3 { domain, uri, urn }.into(), loc.into_path()) + } + SymbolCow::Owned(domain) => (Scheme::S3 { domain, uri, urn }.into(), loc.into_path()), + }, Self::Sftp { loc, domain } => match domain { SymbolCow::Borrowed(domain) => { (SchemeRef::Sftp { domain, uri, urn }.into(), loc.into_path()) @@ -270,6 +292,9 @@ impl<'a> UrlCow<'a> { UrlCow::Archive { loc, domain } => { UrlCow::Archive { loc: loc.into_owned().into(), domain: domain.into_owned().into() } } + UrlCow::S3 { loc, domain } => { + UrlCow::S3 { loc: loc.into_owned().into(), domain: domain.into_owned().into() } + } UrlCow::Sftp { loc, domain } => { UrlCow::Sftp { loc: loc.into_owned().into(), domain: domain.into_owned().into() } } diff --git a/yazi-shared/src/url/encode.rs b/yazi-shared/src/url/encode.rs index 1695e50e..9db09b48 100644 --- a/yazi-shared/src/url/encode.rs +++ b/yazi-shared/src/url/encode.rs @@ -20,6 +20,9 @@ impl Display for Encode<'_> { Url::Archive { domain, .. } => { write!(f, "archive~://{}{}/{loc}", E::domain(domain), E::ports((*self).into())) } + Url::S3 { domain, .. } => { + write!(f, "s3~://{}{}/{loc}", E::domain(domain), E::ports((*self).into())) + } Url::Sftp { domain, .. } => { write!(f, "sftp~://{}{}/{loc}", E::domain(domain), E::ports((*self).into())) } diff --git a/yazi-shared/src/url/traits.rs b/yazi-shared/src/url/traits.rs index 70ab0c29..cede7cc8 100644 --- a/yazi-shared/src/url/traits.rs +++ b/yazi-shared/src/url/traits.rs @@ -39,6 +39,7 @@ impl AsUrl for UrlBuf { Self::Regular(loc) => Url::Regular(loc.as_loc()), Self::Search { loc, domain } => Url::Search { loc: loc.as_loc(), domain }, Self::Archive { loc, domain } => Url::Archive { loc: loc.as_loc(), domain }, + Self::S3 { loc, domain } => Url::S3 { loc: loc.as_loc(), domain }, Self::Sftp { loc, domain } => Url::Sftp { loc: loc.as_loc(), domain }, } } @@ -60,6 +61,7 @@ impl AsUrl for UrlCow<'_> { Self::Regular(loc) => Url::Regular(loc.as_loc()), Self::Search { loc, domain } => Url::Search { loc: loc.as_loc(), domain }, Self::Archive { loc, domain } => Url::Archive { loc: loc.as_loc(), domain }, + Self::S3 { loc, domain } => Url::S3 { loc: loc.as_loc(), domain }, Self::Sftp { loc, domain } => Url::Sftp { loc: loc.as_loc(), domain }, } } diff --git a/yazi-shared/src/url/url.rs b/yazi-shared/src/url/url.rs index 44afc8da..5a82e4f9 100644 --- a/yazi-shared/src/url/url.rs +++ b/yazi-shared/src/url/url.rs @@ -12,6 +12,7 @@ pub enum Url<'a> { Regular(Loc<'a>), Search { loc: Loc<'a>, domain: &'a str }, Archive { loc: Loc<'a>, domain: &'a str }, + S3 { loc: Loc<'a, &'a typed_path::UnixPath>, domain: &'a str }, Sftp { loc: Loc<'a, &'a typed_path::UnixPath>, domain: &'a str }, } @@ -63,6 +64,7 @@ impl<'a> Url<'a> { Self::Regular(loc) => Self::Regular(Loc::bare(loc.base())), Self::Search { loc, domain } => Self::Search { loc: Loc::zeroed(loc.base()), domain }, Self::Archive { loc, domain } => Self::Archive { loc: Loc::zeroed(loc.base()), domain }, + Self::S3 { loc, domain } => Self::S3 { loc: Loc::bare(loc.base()), domain }, Self::Sftp { loc, domain } => Self::Sftp { loc: Loc::bare(loc.base()), domain }, } } @@ -82,6 +84,7 @@ impl<'a> Url<'a> { Self::Regular(loc) => loc.extension()?.as_strand(), Self::Search { loc, .. } => loc.extension()?.as_strand(), Self::Archive { loc, .. } => loc.extension()?.as_strand(), + Self::S3 { loc, .. } => loc.extension()?.as_strand(), Self::Sftp { loc, .. } => loc.extension()?.as_strand(), }) } @@ -92,6 +95,7 @@ impl<'a> Url<'a> { Self::Regular(loc) => loc.has_base(), Self::Search { loc, .. } => loc.has_base(), Self::Archive { loc, .. } => loc.has_base(), + Self::S3 { loc, .. } => loc.has_base(), Self::Sftp { loc, .. } => loc.has_base(), } } @@ -105,6 +109,7 @@ impl<'a> Url<'a> { Self::Regular(loc) => loc.has_trail(), Self::Search { loc, .. } => loc.has_trail(), Self::Archive { loc, .. } => loc.has_trail(), + Self::S3 { loc, .. } => loc.has_trail(), Self::Sftp { loc, .. } => loc.has_trail(), } } @@ -118,7 +123,7 @@ impl<'a> Url<'a> { #[inline] pub fn is_internal(self) -> bool { match self { - Self::Regular(_) | Self::Sftp { .. } => true, + Self::Regular(_) | Self::S3 { .. } | Self::Sftp { .. } => true, Self::Search { .. } => !self.uri().is_empty(), Self::Archive { .. } => false, } @@ -136,6 +141,7 @@ impl<'a> Url<'a> { Self::Regular(_) => SchemeKind::Regular, Self::Search { .. } => SchemeKind::Search, Self::Archive { .. } => SchemeKind::Archive, + Self::S3 { .. } => SchemeKind::S3, Self::Sftp { .. } => SchemeKind::Sftp, } } @@ -146,6 +152,7 @@ impl<'a> Url<'a> { Self::Regular(loc) => loc.as_path(), Self::Search { loc, .. } => loc.as_path(), Self::Archive { loc, .. } => loc.as_path(), + Self::S3 { loc, .. } => loc.as_path(), Self::Sftp { loc, .. } => loc.as_path(), } } @@ -156,6 +163,7 @@ impl<'a> Url<'a> { Self::Regular(loc) => loc.file_name()?.as_strand(), Self::Search { loc, .. } => loc.file_name()?.as_strand(), Self::Archive { loc, .. } => loc.file_name()?.as_strand(), + Self::S3 { loc, .. } => loc.file_name()?.as_strand(), Self::Sftp { loc, .. } => loc.file_name()?.as_strand(), }) } @@ -189,6 +197,7 @@ impl<'a> Url<'a> { } // SFTP + Self::S3 { loc, domain } => Self::S3 { loc: Loc::bare(loc.parent()?), domain }, Self::Sftp { loc, domain } => Self::Sftp { loc: Loc::bare(loc.parent()?), domain }, }) } @@ -205,6 +214,7 @@ impl<'a> Url<'a> { Self::Regular(_) => SchemeRef::Regular { uri, urn }, Self::Search { domain, .. } => SchemeRef::Search { domain, uri, urn }, Self::Archive { domain, .. } => SchemeRef::Archive { domain, uri, urn }, + Self::S3 { domain, .. } => SchemeRef::S3 { domain, uri, urn }, Self::Sftp { domain, .. } => SchemeRef::Sftp { domain, uri, urn }, } } @@ -215,6 +225,7 @@ impl<'a> Url<'a> { Self::Regular(loc) => loc.file_stem()?.as_strand(), Self::Search { loc, .. } => loc.file_stem()?.as_strand(), Self::Archive { loc, .. } => loc.file_stem()?.as_strand(), + Self::S3 { loc, .. } => loc.file_stem()?.as_strand(), Self::Sftp { loc, .. } => loc.file_stem()?.as_strand(), }) } @@ -248,6 +259,7 @@ impl<'a> Url<'a> { Self::Archive { loc: Loc::new(loc.trail(), loc.base(), loc.base()), domain } } + Self::S3 { loc, domain } => Self::S3 { loc: Loc::bare(loc.trail()), domain }, Self::Sftp { loc, domain } => Self::Sftp { loc: Loc::bare(loc.trail()), domain }, } } @@ -258,7 +270,7 @@ impl<'a> Url<'a> { let (base, rest, urn) = loc.triple(); (base.as_path(), rest.as_path(), urn.as_path()) } - Self::Sftp { loc, .. } => { + Self::S3 { loc, .. } | Self::Sftp { loc, .. } => { let (base, rest, urn) = loc.triple(); (base.as_path(), rest.as_path(), urn.as_path()) } @@ -295,6 +307,9 @@ impl<'a> Url<'a> { domain: domain.intern(), }, + Self::S3 { domain, .. } => { + UrlBuf::S3 { loc: joined.into_unix()?.into(), domain: domain.intern() } + } Self::Sftp { domain, .. } => { UrlBuf::Sftp { loc: joined.into_unix()?.into(), domain: domain.intern() } } @@ -328,6 +343,10 @@ impl<'a> Url<'a> { loc: LocBuf::::new(path.into_os()?, loc.base(), loc.trail()), domain: domain.intern(), }, + Self::S3 { loc, domain } if path.try_starts_with(loc.trail())? => UrlBuf::S3 { + loc: LocBuf::::new(path.into_unix()?, loc.base(), loc.trail()), + domain: domain.intern(), + }, Self::Sftp { loc, domain } if path.try_starts_with(loc.trail())? => UrlBuf::Sftp { loc: LocBuf::::new(path.into_unix()?, loc.base(), loc.trail()), domain: domain.intern(), @@ -341,6 +360,10 @@ impl<'a> Url<'a> { loc: LocBuf::::saturated(path.into_os()?, self.kind()), domain: domain.intern(), }, + Self::S3 { domain, .. } => UrlBuf::S3 { + loc: LocBuf::::saturated(path.into_unix()?, self.kind()), + domain: domain.intern(), + }, Self::Sftp { domain, .. } => UrlBuf::Sftp { loc: LocBuf::::saturated(path.into_unix()?, self.kind()), domain: domain.intern(), @@ -370,6 +393,9 @@ impl<'a> Url<'a> { (U::Archive { domain: a, .. }, U::Archive { domain: b, .. }) => { Some(prefix).filter(|_| a == b).ok_or(Exotic) } + (U::S3 { domain: a, .. }, U::S3 { domain: b, .. }) => { + Some(prefix).filter(|_| a == b).ok_or(Exotic) + } (U::Sftp { domain: a, .. }, U::Sftp { domain: b, .. }) => { Some(prefix).filter(|_| a == b).ok_or(Exotic) } @@ -393,12 +419,20 @@ impl<'a> Url<'a> { } // Independent virtual file space + (U::Regular(_), U::S3 { .. }) => Err(Exotic), + (U::Search { .. }, U::S3 { .. }) => Err(Exotic), + (U::Archive { .. }, U::S3 { .. }) => Err(Exotic), + (U::S3 { .. }, U::Regular(_)) => Err(Exotic), + (U::S3 { .. }, U::Search { .. }) => Err(Exotic), + (U::S3 { .. }, U::Archive { .. }) => Err(Exotic), + (U::S3 { .. }, U::Sftp { .. }) => Err(Exotic), (U::Regular(_), U::Sftp { .. }) => Err(Exotic), (U::Search { .. }, U::Sftp { .. }) => Err(Exotic), (U::Archive { .. }, U::Sftp { .. }) => Err(Exotic), (U::Sftp { .. }, U::Regular(_)) => Err(Exotic), (U::Sftp { .. }, U::Search { .. }) => Err(Exotic), (U::Sftp { .. }, U::Archive { .. }) => Err(Exotic), + (U::Sftp { .. }, U::S3 { .. }) => Err(Exotic), } } @@ -416,6 +450,9 @@ impl<'a> Url<'a> { (U::Archive { domain: a, .. }, U::Archive { domain: b, .. }) => { Some(suffix).filter(|_| a == b).ok_or(Exotic) } + (U::S3 { domain: a, .. }, U::S3 { domain: b, .. }) => { + Some(suffix).filter(|_| a == b).ok_or(Exotic) + } (U::Sftp { domain: a, .. }, U::Sftp { domain: b, .. }) => { Some(suffix).filter(|_| a == b).ok_or(Exotic) } @@ -439,12 +476,20 @@ impl<'a> Url<'a> { } // Independent virtual file space + (U::Regular(_), U::S3 { .. }) => Err(Exotic), + (U::Search { .. }, U::S3 { .. }) => Err(Exotic), + (U::Archive { .. }, U::S3 { .. }) => Err(Exotic), + (U::S3 { .. }, U::Regular(_)) => Err(Exotic), + (U::S3 { .. }, U::Search { .. }) => Err(Exotic), + (U::S3 { .. }, U::Archive { .. }) => Err(Exotic), + (U::S3 { .. }, U::Sftp { .. }) => Err(Exotic), (U::Regular(_), U::Sftp { .. }) => Err(Exotic), (U::Search { .. }, U::Sftp { .. }) => Err(Exotic), (U::Archive { .. }, U::Sftp { .. }) => Err(Exotic), (U::Sftp { .. }, U::Regular(_)) => Err(Exotic), (U::Sftp { .. }, U::Search { .. }) => Err(Exotic), (U::Sftp { .. }, U::Archive { .. }) => Err(Exotic), + (U::Sftp { .. }, U::S3 { .. }) => Err(Exotic), } } @@ -454,6 +499,7 @@ impl<'a> Url<'a> { Self::Regular(loc) => loc.uri().as_path(), Self::Search { loc, .. } => loc.uri().as_path(), Self::Archive { loc, .. } => loc.uri().as_path(), + Self::S3 { loc, .. } => loc.uri().as_path(), Self::Sftp { loc, .. } => loc.uri().as_path(), } } @@ -464,6 +510,7 @@ impl<'a> Url<'a> { Self::Regular(loc) => loc.urn().as_path(), Self::Search { loc, .. } => loc.urn().as_path(), Self::Archive { loc, .. } => loc.urn().as_path(), + Self::S3 { loc, .. } => loc.urn().as_path(), Self::Sftp { loc, .. } => loc.urn().as_path(), } } diff --git a/yazi-vfs/Cargo.toml b/yazi-vfs/Cargo.toml index 4e209fa2..4d56915c 100644 --- a/yazi-vfs/Cargo.toml +++ b/yazi-vfs/Cargo.toml @@ -26,6 +26,7 @@ deadpool = { version = "0.13.0", default-features = false, features = [ "mana either = { workspace = true } futures = { workspace = true } hashbrown = { workspace = true } +object_store = { version = "0.12.5", default-features = false, features = [ "aws" ] } parking_lot = { workspace = true } russh = { workspace = true } tokio = { workspace = true } diff --git a/yazi-vfs/src/provider/dir_entry.rs b/yazi-vfs/src/provider/dir_entry.rs index 40b75e64..9e3f18e9 100644 --- a/yazi-vfs/src/provider/dir_entry.rs +++ b/yazi-vfs/src/provider/dir_entry.rs @@ -5,6 +5,7 @@ use yazi_shared::{path::PathBufDyn, strand::StrandCow, url::UrlBuf}; pub enum DirEntry { Local(yazi_fs::provider::local::DirEntry), + S3(super::s3::DirEntry), Sftp(super::sftp::DirEntry), } @@ -12,6 +13,7 @@ impl FileHolder for DirEntry { async fn file_type(&self) -> io::Result { match self { Self::Local(entry) => entry.file_type().await, + Self::S3(entry) => entry.file_type().await, Self::Sftp(entry) => entry.file_type().await, } } @@ -19,6 +21,7 @@ impl FileHolder for DirEntry { async fn metadata(&self) -> io::Result { match self { Self::Local(entry) => entry.metadata().await, + Self::S3(entry) => entry.metadata().await, Self::Sftp(entry) => entry.metadata().await, } } @@ -26,6 +29,7 @@ impl FileHolder for DirEntry { fn name(&self) -> StrandCow<'_> { match self { Self::Local(entry) => entry.name(), + Self::S3(entry) => entry.name(), Self::Sftp(entry) => entry.name(), } } @@ -33,6 +37,7 @@ impl FileHolder for DirEntry { fn path(&self) -> PathBufDyn { match self { Self::Local(entry) => entry.path(), + Self::S3(entry) => entry.path(), Self::Sftp(entry) => entry.path(), } } @@ -40,6 +45,7 @@ impl FileHolder for DirEntry { fn url(&self) -> UrlBuf { match self { Self::Local(entry) => entry.url(), + Self::S3(entry) => entry.url(), Self::Sftp(entry) => entry.url(), } } diff --git a/yazi-vfs/src/provider/gate.rs b/yazi-vfs/src/provider/gate.rs index a62f9465..dad54b97 100644 --- a/yazi-vfs/src/provider/gate.rs +++ b/yazi-vfs/src/provider/gate.rs @@ -49,6 +49,9 @@ impl FileBuilder for Gate { SchemeKind::Archive => { Err(io::Error::new(io::ErrorKind::Unsupported, "Unsupported filesystem: archive"))? } + SchemeKind::S3 => { + Err(io::Error::new(io::ErrorKind::Unsupported, "S3 provider does not expose file handles"))? + } SchemeKind::Sftp => self.build::().open(url).await?.into(), }) } diff --git a/yazi-vfs/src/provider/mod.rs b/yazi-vfs/src/provider/mod.rs index 375515cc..8f61bca6 100644 --- a/yazi-vfs/src/provider/mod.rs +++ b/yazi-vfs/src/provider/mod.rs @@ -1,5 +1,8 @@ -yazi_macro::mod_pub!(sftp); +yazi_macro::mod_pub!(s3 sftp); yazi_macro::mod_flat!(calculator copier dir_entry gate provider providers read_dir rw_file); -pub(super) fn init() { sftp::init(); } +pub(super) fn init() { + s3::init(); + sftp::init(); +} diff --git a/yazi-vfs/src/provider/provider.rs b/yazi-vfs/src/provider/provider.rs index 52d9a011..195f3d75 100644 --- a/yazi-vfs/src/provider/provider.rs +++ b/yazi-vfs/src/provider/provider.rs @@ -52,6 +52,9 @@ where V: AsUrl, { let (from, to) = (from.as_url(), to.as_url()); + if matches!(from, Url::S3 { .. }) { + return super::s3::copy_impl(from, to, attrs).await; + } match (from.kind().is_local(), to.kind().is_local()) { (true, true) => Local::new(from).await?.copy(to.loc(), attrs).await, @@ -73,6 +76,9 @@ where A: Into, { let (from, to) = (from.as_url(), to.as_url()); + if matches!(from, Url::S3 { .. }) { + return Ok(super::s3::copy_with_progress_impl(from.to_owned(), to.to_owned(), attrs.into())); + } match (from.kind().is_local(), to.kind().is_local()) { (true, true) => Local::new(from).await?.copy_with_progress(to.loc(), attrs), @@ -262,6 +268,7 @@ where match url.as_url() { Url::Regular(_) | Url::Search { .. } => yazi_fs::provider::local::try_absolute(url), Url::Archive { .. } => None, // TODO + Url::S3 { .. } => crate::provider::s3::try_absolute(url), Url::Sftp { .. } => crate::provider::sftp::try_absolute(url), } } diff --git a/yazi-vfs/src/provider/providers.rs b/yazi-vfs/src/provider/providers.rs index 48aefbbc..8fcd710e 100644 --- a/yazi-vfs/src/provider/providers.rs +++ b/yazi-vfs/src/provider/providers.rs @@ -7,6 +7,7 @@ use yazi_shared::{path::{AsPath, PathBufDyn}, strand::AsStrand, url::{Url, UrlBu #[derive(Clone)] pub(super) enum Providers<'a> { Local(yazi_fs::provider::local::Local<'a>), + S3(super::s3::S3<'a>), Sftp(super::sftp::Sftp<'a>), } @@ -20,6 +21,7 @@ impl<'a> Provider for Providers<'a> { async fn absolute(&self) -> io::Result { match self { Self::Local(p) => p.absolute().await, + Self::S3(p) => p.absolute().await, Self::Sftp(p) => p.absolute().await, } } @@ -27,6 +29,7 @@ impl<'a> Provider for Providers<'a> { async fn canonicalize(&self) -> io::Result { match self { Self::Local(p) => p.canonicalize().await, + Self::S3(p) => p.canonicalize().await, Self::Sftp(p) => p.canonicalize().await, } } @@ -34,6 +37,7 @@ impl<'a> Provider for Providers<'a> { fn capabilities(&self) -> Capabilities { match self { Self::Local(p) => p.capabilities(), + Self::S3(p) => p.capabilities(), Self::Sftp(p) => p.capabilities(), } } @@ -41,6 +45,7 @@ impl<'a> Provider for Providers<'a> { async fn casefold(&self) -> io::Result { match self { Self::Local(p) => p.casefold().await, + Self::S3(p) => p.casefold().await, Self::Sftp(p) => p.casefold().await, } } @@ -51,6 +56,7 @@ impl<'a> Provider for Providers<'a> { { match self { Self::Local(p) => p.copy(to, attrs).await, + Self::S3(p) => p.copy(to, attrs).await, Self::Sftp(p) => p.copy(to, attrs).await, } } @@ -62,6 +68,7 @@ impl<'a> Provider for Providers<'a> { { match self { Self::Local(p) => p.copy_with_progress(to, attrs), + Self::S3(p) => p.copy_with_progress(to, attrs), Self::Sftp(p) => p.copy_with_progress(to, attrs), } } @@ -69,6 +76,9 @@ impl<'a> Provider for Providers<'a> { async fn create(&self) -> io::Result { Ok(match self { Self::Local(p) => p.create().await?.into(), + Self::S3(_) => { + Err(io::Error::new(io::ErrorKind::Unsupported, "S3 provider is read-only"))? + } Self::Sftp(p) => p.create().await?.into(), }) } @@ -76,6 +86,7 @@ impl<'a> Provider for Providers<'a> { async fn create_dir(&self) -> io::Result<()> { match self { Self::Local(p) => p.create_dir().await, + Self::S3(p) => p.create_dir().await, Self::Sftp(p) => p.create_dir().await, } } @@ -83,6 +94,7 @@ impl<'a> Provider for Providers<'a> { async fn create_dir_all(&self) -> io::Result<()> { match self { Self::Local(p) => p.create_dir_all().await, + Self::S3(p) => p.create_dir_all().await, Self::Sftp(p) => p.create_dir_all().await, } } @@ -90,6 +102,9 @@ impl<'a> Provider for Providers<'a> { async fn create_new(&self) -> io::Result { Ok(match self { Self::Local(p) => p.create_new().await?.into(), + Self::S3(_) => { + Err(io::Error::new(io::ErrorKind::Unsupported, "S3 provider is read-only"))? + } Self::Sftp(p) => p.create_new().await?.into(), }) } @@ -100,6 +115,7 @@ impl<'a> Provider for Providers<'a> { { match self { Self::Local(p) => p.hard_link(to).await, + Self::S3(p) => p.hard_link(to).await, Self::Sftp(p) => p.hard_link(to).await, } } @@ -107,6 +123,7 @@ impl<'a> Provider for Providers<'a> { async fn metadata(&self) -> io::Result { match self { Self::Local(p) => p.metadata().await, + Self::S3(p) => p.metadata().await, Self::Sftp(p) => p.metadata().await, } } @@ -119,6 +136,7 @@ impl<'a> Provider for Providers<'a> { K::Archive => { Err(io::Error::new(io::ErrorKind::Unsupported, "Unsupported filesystem: archive"))? } + K::S3 => Self::Me::S3(super::s3::S3::new(url).await?), K::Sftp => Self::Me::Sftp(super::sftp::Sftp::new(url).await?), }) } @@ -126,6 +144,10 @@ impl<'a> Provider for Providers<'a> { async fn open(&self) -> io::Result { Ok(match self { Self::Local(p) => p.open().await?.into(), + Self::S3(_) => Err(io::Error::new( + io::ErrorKind::Unsupported, + "S3 provider does not expose file handles", + ))?, Self::Sftp(p) => p.open().await?.into(), }) } @@ -133,6 +155,7 @@ impl<'a> Provider for Providers<'a> { async fn read_dir(self) -> io::Result { Ok(match self { Self::Local(p) => Self::ReadDir::Local(p.read_dir().await?), + Self::S3(p) => Self::ReadDir::S3(p.read_dir().await?), Self::Sftp(p) => Self::ReadDir::Sftp(p.read_dir().await?), }) } @@ -140,6 +163,7 @@ impl<'a> Provider for Providers<'a> { async fn read_link(&self) -> io::Result { match self { Self::Local(p) => p.read_link().await, + Self::S3(p) => p.read_link().await, Self::Sftp(p) => p.read_link().await, } } @@ -147,6 +171,7 @@ impl<'a> Provider for Providers<'a> { async fn remove_dir(&self) -> io::Result<()> { match self { Self::Local(p) => p.remove_dir().await, + Self::S3(p) => p.remove_dir().await, Self::Sftp(p) => p.remove_dir().await, } } @@ -154,6 +179,7 @@ impl<'a> Provider for Providers<'a> { async fn remove_dir_all(&self) -> io::Result<()> { match self { Self::Local(p) => p.remove_dir_all().await, + Self::S3(p) => p.remove_dir_all().await, Self::Sftp(p) => p.remove_dir_all().await, } } @@ -161,6 +187,7 @@ impl<'a> Provider for Providers<'a> { async fn remove_file(&self) -> io::Result<()> { match self { Self::Local(p) => p.remove_file().await, + Self::S3(p) => p.remove_file().await, Self::Sftp(p) => p.remove_file().await, } } @@ -171,6 +198,7 @@ impl<'a> Provider for Providers<'a> { { match self { Self::Local(p) => p.rename(to).await, + Self::S3(p) => p.rename(to).await, Self::Sftp(p) => p.rename(to).await, } } @@ -182,6 +210,7 @@ impl<'a> Provider for Providers<'a> { { match self { Self::Local(p) => p.symlink(original, is_dir).await, + Self::S3(p) => p.symlink(original, is_dir).await, Self::Sftp(p) => p.symlink(original, is_dir).await, } } @@ -192,6 +221,7 @@ impl<'a> Provider for Providers<'a> { { match self { Self::Local(p) => p.symlink_dir(original).await, + Self::S3(p) => p.symlink_dir(original).await, Self::Sftp(p) => p.symlink_dir(original).await, } } @@ -202,6 +232,7 @@ impl<'a> Provider for Providers<'a> { { match self { Self::Local(p) => p.symlink_file(original).await, + Self::S3(p) => p.symlink_file(original).await, Self::Sftp(p) => p.symlink_file(original).await, } } @@ -209,6 +240,7 @@ impl<'a> Provider for Providers<'a> { async fn symlink_metadata(&self) -> io::Result { match self { Self::Local(p) => p.symlink_metadata().await, + Self::S3(p) => p.symlink_metadata().await, Self::Sftp(p) => p.symlink_metadata().await, } } @@ -216,6 +248,7 @@ impl<'a> Provider for Providers<'a> { async fn trash(&self) -> io::Result<()> { match self { Self::Local(p) => p.trash().await, + Self::S3(p) => p.trash().await, Self::Sftp(p) => p.trash().await, } } @@ -223,6 +256,7 @@ impl<'a> Provider for Providers<'a> { fn url(&self) -> Url<'_> { match self { Self::Local(p) => p.url(), + Self::S3(p) => p.url(), Self::Sftp(p) => p.url(), } } @@ -233,6 +267,7 @@ impl<'a> Provider for Providers<'a> { { match self { Self::Local(p) => p.write(contents).await, + Self::S3(p) => p.write(contents).await, Self::Sftp(p) => p.write(contents).await, } } diff --git a/yazi-vfs/src/provider/read_dir.rs b/yazi-vfs/src/provider/read_dir.rs index 55e63159..22afa723 100644 --- a/yazi-vfs/src/provider/read_dir.rs +++ b/yazi-vfs/src/provider/read_dir.rs @@ -4,6 +4,7 @@ use yazi_fs::provider::DirReader; pub enum ReadDir { Local(yazi_fs::provider::local::ReadDir), + S3(super::s3::ReadDir), Sftp(super::sftp::ReadDir), } @@ -13,6 +14,7 @@ impl DirReader for ReadDir { async fn next(&mut self) -> io::Result> { Ok(match self { Self::Local(reader) => reader.next().await?.map(Self::Entry::Local), + Self::S3(reader) => reader.next().await?.map(Self::Entry::S3), Self::Sftp(reader) => reader.next().await?.map(Self::Entry::Sftp), }) } diff --git a/yazi-vfs/src/provider/s3/absolute.rs b/yazi-vfs/src/provider/s3/absolute.rs new file mode 100644 index 00000000..3992a061 --- /dev/null +++ b/yazi-vfs/src/provider/s3/absolute.rs @@ -0,0 +1,29 @@ +use yazi_fs::CWD; +use yazi_shared::{loc::LocBuf, pool::InternStr, url::{AsUrl, Url, UrlBuf, UrlCow, UrlLike}}; + +pub fn try_absolute<'a, U>(url: U) -> Option> +where + U: Into>, +{ + let url = url.into(); + if url.is_absolute() { + Some(url) + } else if let Url::S3 { domain, .. } = url.as_url() { + let raw = url.loc().to_string_lossy(); + let raw = raw.trim_start_matches('/'); + let absolute = if raw.is_empty() { "/".to_owned() } else { format!("/{raw}") }; + Some( + UrlBuf::S3 { + loc: LocBuf::::zeroed(typed_path::UnixPathBuf::from(absolute)), + domain: domain.intern(), + } + .into(), + ) + } else if let cwd = CWD.load() + && cwd.scheme().covariant(url.scheme()) + { + Some(cwd.try_join(url.loc()).ok()?.into()) + } else { + None + } +} diff --git a/yazi-vfs/src/provider/s3/metadata.rs b/yazi-vfs/src/provider/s3/metadata.rs new file mode 100644 index 00000000..3b7d455d --- /dev/null +++ b/yazi-vfs/src/provider/s3/metadata.rs @@ -0,0 +1,97 @@ +use std::time::SystemTime; + +use object_store::ObjectMeta; +use yazi_fs::cha::{Cha, ChaKind, ChaMode}; +use yazi_shared::strand::AsStrand; + +pub(super) fn dir(_name: impl AsStrand) -> Cha { + Cha { + kind: ChaKind::empty(), + mode: default_dir_mode(), + len: 0, + atime: None, + btime: None, + ctime: None, + mtime: None, + dev: 0, + uid: 0, + gid: 0, + nlink: 0, + } +} + +pub(super) fn object(_name: impl AsStrand, meta: &ObjectMeta) -> Cha { + Cha { + kind: ChaKind::empty(), + mode: default_file_mode(), + len: meta.size, + atime: None, + btime: None, + ctime: None, + mtime: Some(SystemTime::from(meta.last_modified)), + dev: 0, + uid: 0, + gid: 0, + nlink: 0, + } +} + +#[inline] +const fn default_dir_mode() -> ChaMode { + ChaMode::T_DIR + .union(ChaMode::U_READ) + .union(ChaMode::U_WRITE) + .union(ChaMode::U_EXEC) + .union(ChaMode::G_READ) + .union(ChaMode::G_EXEC) + .union(ChaMode::O_READ) + .union(ChaMode::O_EXEC) +} + +#[inline] +const fn default_file_mode() -> ChaMode { + ChaMode::T_FILE + .union(ChaMode::U_READ) + .union(ChaMode::U_WRITE) + .union(ChaMode::G_READ) + .union(ChaMode::O_READ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn object_uses_readable_file_permissions() { + let cha = object( + "producer.8-2.json", + &ObjectMeta { + location: "producer.8-2.json".into(), + last_modified: chrono::Utc::now().into(), + size: 61, + e_tag: None, + version: None, + }, + ); + + assert!(cha.mode.contains(ChaMode::T_FILE)); + assert!(cha.mode.contains(ChaMode::U_READ)); + assert!(cha.mode.contains(ChaMode::U_WRITE)); + assert!(cha.mode.contains(ChaMode::G_READ)); + assert!(cha.mode.contains(ChaMode::O_READ)); + } + + #[test] + fn dir_uses_traversable_directory_permissions() { + let cha = dir("e2e_test"); + + assert!(cha.mode.contains(ChaMode::T_DIR)); + assert!(cha.mode.contains(ChaMode::U_READ)); + assert!(cha.mode.contains(ChaMode::U_WRITE)); + assert!(cha.mode.contains(ChaMode::U_EXEC)); + assert!(cha.mode.contains(ChaMode::G_READ)); + assert!(cha.mode.contains(ChaMode::G_EXEC)); + assert!(cha.mode.contains(ChaMode::O_READ)); + assert!(cha.mode.contains(ChaMode::O_EXEC)); + } +} diff --git a/yazi-vfs/src/provider/s3/mod.rs b/yazi-vfs/src/provider/s3/mod.rs new file mode 100644 index 00000000..a9a6395f --- /dev/null +++ b/yazi-vfs/src/provider/s3/mod.rs @@ -0,0 +1,10 @@ +yazi_macro::mod_flat!(absolute metadata read_dir s3); + +pub use absolute::try_absolute; +pub use read_dir::{DirEntry, ReadDir}; +pub use s3::S3; +pub(crate) use s3::{copy_impl, copy_with_progress_impl}; + +type DynStore = std::sync::Arc; + +pub(super) fn init() {} diff --git a/yazi-vfs/src/provider/s3/read_dir.rs b/yazi-vfs/src/provider/s3/read_dir.rs new file mode 100644 index 00000000..620321e3 --- /dev/null +++ b/yazi-vfs/src/provider/s3/read_dir.rs @@ -0,0 +1,125 @@ +use std::{collections::VecDeque, io, sync::Arc}; + +use object_store::{ListResult, ObjectMeta, path::Path}; +use yazi_fs::provider::{DirReader, FileHolder}; +use yazi_shared::{path::PathBufDyn, strand::StrandCow, url::{UrlBuf, UrlLike}}; + +use super::DynStore; + +pub struct ReadDir { + pub(super) dir: Arc, + pub(super) store: DynStore, + pub(super) prefix: String, + pub(super) token: Option, + pub(super) finished: bool, + pub(super) page_size: usize, + pub(super) buffer: VecDeque, +} + +impl DirReader for ReadDir { + type Entry = DirEntry; + + async fn next(&mut self) -> io::Result> { + loop { + if let Some(entry) = self.buffer.pop_front() { + return Ok(Some(entry)); + } + if self.finished { + return Ok(None); + } + self.fetch_next_page().await?; + } + } +} + +impl ReadDir { + async fn fetch_next_page(&mut self) -> io::Result<()> { + use object_store::list::{PaginatedListOptions, PaginatedListStore}; + + let result = self + .store + .list_paginated( + Some(&self.prefix), + PaginatedListOptions { + delimiter: Some("/".into()), + max_keys: Some(self.page_size), + page_token: self.token.take(), + ..Default::default() + }, + ) + .await + .map_err(super::s3::to_io)?; + + self.extend(result.result); + self.token = result.page_token; + self.finished = self.token.is_none(); + Ok(()) + } + + fn extend(&mut self, result: ListResult) { + for prefix in result.common_prefixes { + self.buffer.push_back(DirEntry { + dir: self.dir.clone(), + name: basename(&prefix), + kind: Kind::Dir, + }); + } + for object in result.objects { + self.buffer.push_back(DirEntry { + dir: self.dir.clone(), + name: basename(&object.location), + kind: Kind::File(object), + }); + } + } +} + +pub enum Kind { + Dir, + File(ObjectMeta), +} + +pub struct DirEntry { + pub(super) dir: Arc, + pub(super) name: String, + pub(super) kind: Kind, +} + +impl FileHolder for DirEntry { + async fn file_type(&self) -> io::Result { + Ok(match self.kind { + Kind::Dir => yazi_fs::cha::ChaType::Dir, + Kind::File(_) => yazi_fs::cha::ChaType::File, + }) + } + + async fn metadata(&self) -> io::Result { + match &self.kind { + Kind::Dir => Ok(super::metadata::dir(self.name.as_str())), + Kind::File(meta) => Ok(super::metadata::object(self.name.as_str(), meta)), + } + } + + fn name(&self) -> StrandCow<'_> { self.name.as_str().into() } + + fn path(&self) -> PathBufDyn { self.url().into_loc() } + + fn url(&self) -> UrlBuf { + self.dir.try_join(self.name.as_str()).expect("entry name is valid S3 path component") + } +} + +pub(super) fn basename(path: &Path) -> String { + path.to_string().rsplit('/').next().unwrap_or_default().to_owned() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn basename_returns_last_segment() { + assert_eq!(basename(&Path::from("foo/bar/baz.txt")), "baz.txt"); + assert_eq!(basename(&Path::from("prefix/dir")), "dir"); + } +} diff --git a/yazi-vfs/src/provider/s3/s3.rs b/yazi-vfs/src/provider/s3/s3.rs new file mode 100644 index 00000000..84823d3b --- /dev/null +++ b/yazi-vfs/src/provider/s3/s3.rs @@ -0,0 +1,407 @@ +use std::{collections::VecDeque, io, pin::Pin, sync::Arc, task::{Context, Poll}}; + +use object_store::{ObjectStore, path::Path}; +use tokio::{io::{AsyncRead, AsyncSeek, AsyncWrite, AsyncWriteExt, BufWriter, ReadBuf}, sync::mpsc}; +use typed_path::Component; +use yazi_config::vfs::{ServiceS3, Vfs}; +use yazi_fs::provider::{Capabilities, FileBuilder, Provider}; +use yazi_shared::{path::{AsPath, PathBufDyn}, strand::AsStrand, url::{AsUrl, Url, UrlBuf, UrlCow}}; + +use super::{DynStore, read_dir::ReadDir}; + +const PAGE_SIZE: usize = 500; +const COPY_BUF_SIZE: usize = 512 * 1024; +const COPY_CHUNK: usize = 64 * 1024; + +#[derive(Clone)] +pub struct S3<'a> { + url: Url<'a>, + key: Path, + store: DynStore, +} + +pub struct File; + +#[derive(Clone, Copy, Default)] +pub struct Gate; + +impl<'a> Provider for S3<'a> { + type File = File; + type Gate = Gate; + type Me<'b> = S3<'b>; + type ReadDir = ReadDir; + type UrlCow = UrlCow<'a>; + + async fn absolute(&self) -> io::Result { + Ok(if let Some(u) = super::absolute::try_absolute(self.url) { u } else { self.url.to_owned().into() }) + } + + async fn canonicalize(&self) -> io::Result { Ok(self.url.to_owned()) } + + fn capabilities(&self) -> Capabilities { Capabilities { symlink: false } } + + async fn casefold(&self) -> io::Result { Ok(self.url.to_owned()) } + + async fn copy

(&self, _to: P, _attrs: yazi_fs::provider::Attrs) -> io::Result + where + P: AsPath, + { + Err(io::Error::new(io::ErrorKind::Unsupported, "S3 provider is read-only")) + } + + fn copy_with_progress(&self, _to: P, _attrs: A) -> io::Result>> + where + P: AsPath, + A: Into, + { + Err(io::Error::new(io::ErrorKind::Unsupported, "S3 provider is read-only")) + } + + async fn create(&self) -> io::Result { + Err(io::Error::new(io::ErrorKind::Unsupported, "S3 provider is read-only")) + } + + async fn create_dir(&self) -> io::Result<()> { Err(io::Error::new(io::ErrorKind::Unsupported, "S3 provider is read-only")) } + + async fn create_new(&self) -> io::Result { + Err(io::Error::new(io::ErrorKind::Unsupported, "S3 provider is read-only")) + } + + async fn hard_link

(&self, _to: P) -> io::Result<()> + where + P: AsPath, + { Err(io::Error::new(io::ErrorKind::Unsupported, "S3 provider is read-only")) } + + async fn metadata(&self) -> io::Result { + if self.key.to_string().is_empty() { + return Ok(super::metadata::dir(self.url.name().unwrap_or_default())); + } + + match self.store.head(&self.key).await { + Ok(meta) => Ok(super::metadata::object(self.url.name().unwrap_or_default(), &meta)), + Err(object_store::Error::NotFound { .. }) if self.dir_exists().await? => { + Ok(super::metadata::dir(self.url.name().unwrap_or_default())) + } + Err(error) => Err(to_io(error)), + } + } + + async fn new<'b>(url: Url<'b>) -> io::Result> { + match url { + Url::S3 { loc, domain } => { + let (_name, config) = Vfs::service::<&ServiceS3>(domain).await?; + let (bucket, key) = split_bucket_and_key(loc.as_inner())?; + let store = build_store(config, &bucket)?; + Ok(Self::Me { url, key, store }) + } + _ => Err(io::Error::new(io::ErrorKind::InvalidInput, format!("Not an S3 URL: {url:?}"))), + } + } + + async fn open(&self) -> io::Result { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "S3 provider does not expose file handles", + )) + } + + async fn read_dir(self) -> io::Result { + let prefix = self.list_prefix(); + let dir = Arc::new(self.url.to_owned()); + Ok(ReadDir { + dir, + store: self.store, + prefix, + token: None, + finished: false, + page_size: PAGE_SIZE, + buffer: VecDeque::new(), + }) + } + + async fn read_link(&self) -> io::Result { Err(io::Error::new(io::ErrorKind::Unsupported, "S3 has no symlinks")) } + async fn remove_dir(&self) -> io::Result<()> { Ok(()) } + async fn remove_file(&self) -> io::Result<()> { self.store.delete(&self.key).await.map_err(to_io) } + async fn rename

(&self, _to: P) -> io::Result<()> + where + P: AsPath, + { Err(io::Error::new(io::ErrorKind::Unsupported, "S3 provider is read-only")) } + async fn symlink(&self, _original: S, _is_dir: F) -> io::Result<()> + where + S: AsStrand, + F: AsyncFnOnce() -> io::Result, + { Err(io::Error::new(io::ErrorKind::Unsupported, "S3 has no symlinks")) } + async fn symlink_metadata(&self) -> io::Result { self.metadata().await } + async fn trash(&self) -> io::Result<()> { Err(io::Error::new(io::ErrorKind::Unsupported, "S3 provider is read-only")) } + fn url(&self) -> Url<'_> { self.url } + + async fn write(&self, _contents: C) -> io::Result<()> + where + C: AsRef<[u8]>, + { Err(io::Error::new(io::ErrorKind::Unsupported, "S3 provider is read-only")) } +} + +impl<'a> S3<'a> { + pub(super) async fn read_bytes(&self) -> io::Result> { + let bytes = self.store.get(&self.key).await.map_err(to_io)?.bytes().await.map_err(to_io)?; + Ok(bytes.to_vec()) + } + + async fn dir_exists(&self) -> io::Result { + use object_store::list::{PaginatedListOptions, PaginatedListStore}; + + let prefix = self.list_prefix(); + let result = self + .store + .list_paginated( + Some(&prefix), + PaginatedListOptions { + delimiter: Some("/".into()), + max_keys: Some(1), + ..Default::default() + }, + ) + .await + .map_err(to_io)?; + + Ok(!result.result.common_prefixes.is_empty() || !result.result.objects.is_empty()) + } + + fn list_prefix(&self) -> String { + let key = self.key.to_string(); + if key.is_empty() || key.ends_with('/') { + key + } else { + format!("{key}/") + } + } +} + +impl FileBuilder for Gate { + type File = File; + + fn append(&mut self, _append: bool) -> &mut Self { self } + + fn attrs(&mut self, _attrs: yazi_fs::provider::Attrs) -> &mut Self { self } + + fn create(&mut self, _create: bool) -> &mut Self { self } + + fn create_new(&mut self, _create_new: bool) -> &mut Self { self } + + async fn open(&self, _url: U) -> io::Result + where + U: yazi_shared::url::AsUrl, + { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "S3 provider does not expose file handles", + )) + } + + fn read(&mut self, _read: bool) -> &mut Self { self } + + fn truncate(&mut self, _truncate: bool) -> &mut Self { self } + + fn write(&mut self, _write: bool) -> &mut Self { self } +} + +impl AsyncRead for File { + fn poll_read(self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &mut ReadBuf<'_>) -> Poll> { + Poll::Ready(Err(io::Error::new( + io::ErrorKind::Unsupported, + "S3 provider does not expose file handles", + ))) + } +} + +impl AsyncSeek for File { + fn start_seek(self: Pin<&mut Self>, _position: io::SeekFrom) -> io::Result<()> { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "S3 provider does not expose file handles", + )) + } + + fn poll_complete(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Err(io::Error::new( + io::ErrorKind::Unsupported, + "S3 provider does not expose file handles", + ))) + } +} + +impl AsyncWrite for File { + fn poll_write(self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &[u8]) -> Poll> { + Poll::Ready(Err(io::Error::new( + io::ErrorKind::Unsupported, + "S3 provider does not expose file handles", + ))) + } + + fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Err(io::Error::new( + io::ErrorKind::Unsupported, + "S3 provider does not expose file handles", + ))) + } + + fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Err(io::Error::new( + io::ErrorKind::Unsupported, + "S3 provider does not expose file handles", + ))) + } +} + +pub(crate) async fn copy_impl(from: Url<'_>, to: Url<'_>, attrs: yazi_fs::provider::Attrs) -> io::Result { + let provider = S3::new(from).await?; + let bytes = provider.read_bytes().await?; + let dist = crate::provider::create(to).await?; + + let mut writer = BufWriter::with_capacity(COPY_BUF_SIZE, dist); + writer.write_all(&bytes).await?; + writer.flush().await?; + writer.get_ref().set_attrs(attrs).await.ok(); + writer.shutdown().await.ok(); + Ok(bytes.len() as u64) +} + +pub(crate) fn copy_with_progress_impl( + from: UrlBuf, + to: UrlBuf, + attrs: yazi_fs::provider::Attrs, +) -> mpsc::Receiver> { + let (tx, rx) = mpsc::channel(10); + + tokio::spawn(async move { + let result = async { + let provider = S3::new(from.as_url()).await?; + let bytes = provider.read_bytes().await?; + let dist = crate::provider::create(&to).await?; + + let mut writer = BufWriter::with_capacity(COPY_BUF_SIZE, dist); + for chunk in bytes.chunks(COPY_CHUNK) { + writer.write_all(chunk).await?; + tx.send(Ok(chunk.len() as u64)).await.ok(); + } + writer.flush().await?; + + let mut file = writer.into_inner(); + file.set_attrs(attrs).await.ok(); + file.shutdown().await.ok(); + Ok(()) + } + .await; + + match result { + Ok(()) => { + tx.send(Ok(0)).await.ok(); + } + Err(error) => { + tx.send(Err(error)).await.ok(); + } + } + }); + + rx +} + +fn split_bucket_and_key(path: &typed_path::UnixPath) -> io::Result<(String, Path)> { + let mut components = path.components().filter(|component| !component.is_root()); + let Some(bucket) = components.next() else { + return Err(io::Error::new( + io::ErrorKind::Unsupported, + "S3 service root is not supported; use s3:////", + )); + }; + + let bucket = String::from_utf8_lossy(bucket.as_bytes()).into_owned(); + if bucket.is_empty() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "S3 bucket name must not be empty", + )); + } + + let key = components + .map(|component| String::from_utf8_lossy(component.as_bytes()).into_owned()) + .filter(|segment| !segment.is_empty()) + .collect::>() + .join("/"); + + Ok((bucket, Path::from(key))) +} + +fn build_store(config: &ServiceS3, bucket: &str) -> io::Result { + let mut builder = object_store::aws::AmazonS3Builder::new().with_bucket_name(bucket); + if let Some(region) = &config.region { + builder = builder.with_region(region); + } + if let Some(endpoint) = &config.endpoint { + builder = builder.with_endpoint(endpoint); + } + if let Some(key) = &config.access_key_id { + builder = builder.with_access_key_id(key); + } + if let Some(secret) = &config.secret_access_key { + builder = builder.with_secret_access_key(secret); + } + if let Some(token) = &config.session_token { + builder = builder.with_token(token); + } + builder = builder.with_allow_http(config.allow_http); + builder = builder.with_virtual_hosted_style_request(!config.force_path_style); + Ok(Arc::new(builder.build().map_err(io::Error::other)?)) +} + +pub(super) fn to_io(error: object_store::Error) -> io::Error { + match error { + object_store::Error::NotFound { .. } => io::Error::from(io::ErrorKind::NotFound), + object_store::Error::PermissionDenied { .. } + | object_store::Error::Unauthenticated { .. } => io::Error::from(io::ErrorKind::PermissionDenied), + other => io::Error::other(other), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use object_store::aws::AmazonS3Builder; + use yazi_shared::loc::Loc; + + #[test] + fn split_bucket_and_key_uses_first_segment_as_bucket() { + let (bucket, key) = split_bucket_and_key(typed_path::UnixPath::new("/srgdata/foo/bar")).unwrap(); + assert_eq!(bucket, "srgdata"); + assert_eq!(key.to_string(), "foo/bar"); + } + + #[test] + fn split_bucket_and_key_returns_empty_key_for_bucket_root() { + let (bucket, key) = split_bucket_and_key(typed_path::UnixPath::new("/srgdata")).unwrap(); + assert_eq!(bucket, "srgdata"); + assert_eq!(key.to_string(), ""); + } + + #[test] + fn provider_uses_bucket_root_as_empty_key() { + let url = Url::S3 { loc: Loc::zeroed(typed_path::UnixPath::new("/srgdata")), domain: "yabos" }; + let s3 = S3 { + url, + key: Path::from(""), + store: Arc::new(AmazonS3Builder::new().with_bucket_name("srgdata").with_region("us-east-1").build().unwrap()), + }; + assert_eq!(s3.key.to_string(), ""); + } + + #[test] + fn list_prefix_adds_trailing_slash_for_directories() { + let url = Url::S3 { loc: Loc::zeroed(typed_path::UnixPath::new("/srgdata/data")), domain: "yabos" }; + let s3 = S3 { + url, + key: Path::from("data"), + store: Arc::new(AmazonS3Builder::new().with_bucket_name("srgdata").with_region("us-east-1").build().unwrap()), + }; + assert_eq!(s3.list_prefix(), "data/"); + } +} diff --git a/yazi-vfs/src/provider/sftp/sftp.rs b/yazi-vfs/src/provider/sftp/sftp.rs index 364e97ed..bab42d0e 100644 --- a/yazi-vfs/src/provider/sftp/sftp.rs +++ b/yazi-vfs/src/provider/sftp/sftp.rs @@ -146,7 +146,7 @@ impl<'a> Provider for Sftp<'a> { async fn new<'b>(url: Url<'b>) -> io::Result> { match url { - Url::Regular(_) | Url::Search { .. } | Url::Archive { .. } => { + Url::Regular(_) | Url::Search { .. } | Url::Archive { .. } | Url::S3 { .. } => { Err(io::Error::new(io::ErrorKind::InvalidInput, format!("Not a SFTP URL: {url:?}"))) } Url::Sftp { loc, domain } => { diff --git a/yazi-watcher/src/reporter.rs b/yazi-watcher/src/reporter.rs index 7d74b1a3..97586db1 100644 --- a/yazi-watcher/src/reporter.rs +++ b/yazi-watcher/src/reporter.rs @@ -22,6 +22,7 @@ impl Reporter { match url.as_url().kind() { SchemeKind::Regular | SchemeKind::Search => self.report_local(url), SchemeKind::Archive => {} + SchemeKind::S3 => self.report_remote(url), SchemeKind::Sftp => self.report_remote(url), } }