diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..a908bbf --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,114 @@ +# The checks of each push and of each pull request: +# +# - npm test: the tests, with the oldest Node.js of package.json and the +# newer versions. The job installs rclone and Go, thus the tests that +# compare rfetch with rclone and with Go run too. +# - deno fmt --check: the format of the files. +# - wasm: the tests of the crate of backend/crypt/secretbox.wasm, and its +# build from its source gives the same bytes as the file in the repository. +# +# This workflow deploys nothing: the Pages project deploys the branches. +name: CI + +on: + push: + pull_request: + +permissions: + contents: read + +# A new push to a branch stops the old run of that branch. +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + node: [22, 24, 26] + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node }} + cache: npm + + # Go runs the tests of backend/http/list.js against encoding/csv. + - uses: actions/setup-go@v5 + with: + go-version: stable + cache: false + + # rclone reads each variable RCLONE_NAME as the flag --name. Thus the + # version is not in a variable RCLONE_VERSION of the job: rclone reads + # it as the flag --version, and fails. + - name: Install rclone + env: + VERSION: 1.75.1 + SHA256: 982b5aa772841168f8e380f139e9e787b2a105403e32b94da8676a0e1c0a13ab + run: | + mkdir -p "$HOME/bin" "$RUNNER_TEMP/rclone" + cd "$RUNNER_TEMP/rclone" + zip="rclone-v$VERSION-linux-amd64.zip" + curl -fsSLO "https://downloads.rclone.org/v$VERSION/$zip" + echo "$SHA256 $zip" | sha256sum -c - + unzip -q "$zip" + install -m 755 "rclone-v$VERSION-linux-amd64/rclone" "$HOME/bin/rclone" + "$HOME/bin/rclone" version + echo "$HOME/bin" >> "$GITHUB_PATH" + echo "RCLONE=$HOME/bin/rclone" >> "$GITHUB_ENV" + + - run: npm ci + + # A test against rclone or Go skips when the tool is not installed. + # Here the tools are installed, thus such a skip is an error. + - name: npm test + shell: bash + run: | + npm test 2>&1 | tee "$RUNNER_TEMP/test.log" + if grep -E "(rclone|Go) is not installed" "$RUNNER_TEMP/test.log"; then + echo "::error::A test skipped because a tool is not installed." + exit 1 + fi + + format: + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + + - uses: denoland/setup-deno@v2 + with: + deno-version: 2.9.6 + + - run: deno fmt --check + + wasm: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + defaults: + run: + working-directory: backend/crypt/native + steps: + - uses: actions/checkout@v4 + + # The compiler of rust-toolchain.toml, with the target of WebAssembly. + - name: Install Rust + run: | + channel=$(sed -n 's/^channel = "\(.*\)"$/\1/p' rust-toolchain.toml) + rustup toolchain install "$channel" --profile minimal \ + --target wasm32-unknown-unknown + + # The tests compare each box with the crate crypto_secretbox, on the + # machine of the job. + - run: cargo test --locked + + - run: ./build.sh + + - name: Compare with the file in the repository + run: git diff --exit-code -- ../secretbox.wasm diff --git a/.npmrc b/.npmrc deleted file mode 100644 index 41583e3..0000000 --- a/.npmrc +++ /dev/null @@ -1 +0,0 @@ -@jsr:registry=https://npm.jsr.io diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index aa1c94e..0000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "deno.enable": true, - "deno.unstable": true -} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 37b131e..3fab2e6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -9,15 +9,18 @@ Choose a name. The docs here will use `remote` as an example. Note that in rfetch terminology a file system backend is called a remote or an fs. -- `deno init backend/remote` -- Edit "backend/remote/main.js" and "backend/remote/main_test.js" for the new - backend. +- Make the folder "backend/remote" with the files "package.json", "main.js" and + "main_test.js" for the new backend. The tests use the test runner of Node.js + (`node:test`). +- Add the backend to `backends` in `backend/main.js`, as + `remote: () => import("@rfetch/remote")`, and to the `dependencies` of the + root "package.json". rfetch loads a backend at its first use. - Implements a `fetch` default export function that handles "GET", "PUT" and "DELETE". - Uses `backend/local/main.js` as reference, or the boilerplate below: ```ts -import {} from "../../deps.js"; +import { join } from "node:path"; function router(request: Request) { const { method, url } = request; @@ -62,18 +65,325 @@ export default { }; ``` +A backend imports nothing of rfetch, for example `mod.js`. It is a Worker of +Cloudflare: it gets all from the request and from `env`, the second argument of +`fetch`. A backend on top of another remote, for example `crypt` or `chunker`, +sends its requests to that remote with the global `fetch`, for example +`fetch(":memory:/crypt/file.bin")`. The host sets the global `fetch` that reads +such a path: the import of `fetch.js` sets it. `mod.js`, the commands and +`_worker.js` import `fetch.js`. The tests of such a backend import `mod.js`: + +```js +// Sets the global `fetch` that reads the paths of remotes. +import "../../mod.js"; + +backendSuite("crypt", crypt, { + params: { remote: ":memory:/crypt", password }, +}); +``` + +To count or change the requests to the remote in a test, replace the global +`fetch` with `t.mock.method(globalThis, "fetch", …)`. The test runner puts the +original `fetch` back at the end of the test. + +## Options of a backend + +A backend reads its options with `optionsOf(request, "remote", env)` of +`@rfetch/lib/options`, in this order: + +1. a field of the query, for example `?password=…`; +2. the header `RFETCH_REMOTE_NAME` of the request, for example + `RFETCH_CRYPT_PASSWORD`; +3. the variable `RFETCH_REMOTE_NAME` of `env`. + +The prefix `RCLONE_` of rclone also works, for compatibility, for example +`RCLONE_CRYPT_PASSWORD`. In each place, `RFETCH_` takes the place of `RCLONE_`. +Thus a client can give a secret in a header, and a Worker can give it in `env`: +then the secret is not in a URL. A backend makes the links of a list from the +query of the request only, with no secret option. Thus an option of a header or +of `env` is never in a link. + +The module of a backend exports its options with their types, for example +`export const options = { boolean: ["no_head"], string: ["url"] }`. An option in +`default` and in no list is a `string`. The command line makes a flag of each +option, for example `--http-no-head` and `--http-url URL`. An option that is not +in `options` has no flag: its flag is an unknown flag. + +A backend on top of another remote sends the headers +`remoteHeaders(request.headers, "remote", remote)` to that remote. The other +options of the request go on, for example a remote `RFETCH_CONFIG_NAME_TYPE` +that only the headers tell. The options of the backend itself do not go. When +the remote is a URL, no option goes: a secret must not go out to a server. + +The `fetch` of `fetch.js` reads the headers `RFETCH_*` and `RCLONE_*` of a +request as the variables, in the place of the variables of the process. A remote +can come only from variables: `RFETCH_CONFIG_NAME_TYPE` and its options, as in +rclone. The backend gets the options of the path, the connection string and the +flags, in its query, and all the other options in `env`, as +`RFETCH_REMOTE_NAME`. The headers `RFETCH_*` and `RCLONE_*` never go out to a +URL. + +A local path, for example `file.txt`, `../file.txt` or `/tmp/file.txt`, is +relative to the current folder, as in rclone. The `fetch` of `fetch.js` gives +the absolute path to the local backend, with `RFETCH_LOCAL_ROOT: /` in `env`. +`RFETCH_LOCAL_ROOT` is not an option: a config file, a variable, a header or the +query cannot set it. Thus `:local:path` stays in the current folder. + +`rfetch serve http REMOTE:PATH` serves the folder `REMOTE:PATH`, as +`rclone serve http`: the path of a request is a path in that folder, and each +request goes to the `fetch` of `fetch.js`. Thus the remote gets all its options, +from the config file, the variables and the flags. A client only reads: the +server refuses each method but `GET` and `HEAD`, and `TRACE` most of all, +because the answer of `fetch.js` to `TRACE` tells the options of the remote. A +client sets no option: the server removes its query and its headers `RFETCH_*` +and `RCLONE_*`, and a name with a `/` after the decoding gives 400. + +## Workspaces + +Each backend is an npm workspace with the name `@rfetch/NAME`, and can be a +Worker of its own: "main.js" is the entry of the Worker. The "package.json" of a +new backend is: + +```json +{ + "name": "@rfetch/remote", + "version": "0.1.0", + "type": "module", + "exports": { ".": "./main.js" }, + "scripts": { "test": "node --test" }, + "dependencies": { "@rfetch/lib": "^0.1.0" } +} +``` + +Then run `npm install`: npm links the workspace into `node_modules`. + +A backend imports only: + +- the modules `node:` of Node.js, +- the packages of its "package.json", for example `@rfetch/lib/listing`, +- the files in its folder. + +The tests and the benches of a backend can also import its `devDependencies`, +and `../../mod.js` for the global `fetch`. The test `backend/main_test.js` makes +sure of this rule. + +The code that two or more backends use is in the workspace `lib`, the package +`@rfetch/lib`: for example `listing`, `path`, `range`, `mime` and `obscure`. The +shared tests of the backends are in it too, in `@rfetch/lib/testing/suite`. The +code that only one backend uses is in the folder of that backend, for example +`backend/s3/xml.js`. + ## Architecture - `GET /folder/`: displays HTML page with folder content. - `GET /file`: fetches file. +- `HEAD /folder/` and `HEAD /file`: the same headers as `GET`, with no body. - `PUT /folder/`: creates folder. -- `PUT /file`: uploads file. -- `DELETE /folder/`: deletes folder. +- `PUT /file`: uploads file. `PUT` creates the parent folders that do not exist. +- `DELETE /folder/`: deletes folder, and all the files and folders in it. - `DELETE /file`: deletes file. +The answers: + +- `PUT` gives 201 with the path in `Content-Location`. `DELETE` gives 204. +- A file or a folder that does not exist gives 404. +- `GET` of a file gives the headers `Content-Type`, `Content-Length`, + `Last-Modified` and `Accept-Ranges: bytes`. +- `GET` with `Range: bytes=START-END` gives 206 with the bytes of the range, + `Content-Range` and the `Content-Length` of the range, as RFC 9110. A range + with no byte of the file gives 416. A request with more ranges, or with a + range that is not correct, gets all the file. Use `parseRange` of + `@rfetch/lib/range`. + +The path of a request has percent-encoding. Use `decodePath` of +`@rfetch/lib/path` to get the path with no percent-encoding. It gives `null` for +a path that is not safe, for example `/..%2Fsecret.txt`, which becomes +`/../secret.txt` after the decoding. For `null`, give 400. + For displaying folder content, any HTML can be used, but the listing itself should be in a ``, whose each rows are for each file or folder inside. Each items should have a `` whose `href` attribute points to the file or folder, and optionally a `type` attribute to tell the item's mime-type. A `` element should be used for file size, and `lost.txt +sub/ +sub/deep.txt +top +`; + const LAST_MODIFIED = "Sat, 17 Oct 2026 10:00:00 GMT"; - const url = new URL("/main.js", "file:"); - url.searchParams.set("url", remote.href); + const requests = []; + let index; + let fs; - // Fetch the remote file directly - let response = await fetch(`${remote}/main.js`); - const expected = await response.text(); + test("starts the server", async () => { + index = await listen((request) => { + const pathname = decodeURIComponent(new URL(request.url).pathname); + requests.push(`${request.method} ${pathname}`); + if (pathname === "/files/") { + return new Response(INDEX, { + headers: { "Content-Type": "text/html; charset=utf-8" }, + }); + } + if (pathname in FILES) { + // Bytes, thus the server sends no Content-Type. + const body = bytes(FILES[pathname]); + return new Response(request.method === "HEAD" ? null : body, { + headers: { + "Content-Length": `${body.length}`, + "Last-Modified": LAST_MODIFIED, + }, + }); + } + return new Response(null, { status: 404 }); + }); + fs = remote(backend, { params: { url: `${index.url}files/` } }); + }); + after(() => index?.close()); - // Then compares with result from the backend - response = await backend.fetch(new Request(url)); - const actual = await response.text(); - assertEquals( - actual, - expected, - "should be the same as fetching the remote file directly", + test("lists each link to a child of the folder one time", async () => { + const entries = await fs.list(""); + // The time is to the second, as the precision of http in rclone. + const time = "2026-10-17T10:00:00Z"; + assert.deepEqual( + entries.map(({ Name, IsDir, Size, ModTime }) => ({ + Name, + IsDir, + Size, + ModTime, + })), + [ + { Name: "a.txt", IsDir: false, Size: 1, ModTime: time }, + { Name: "b c.json", IsDir: false, Size: 2, ModTime: time }, + { Name: "d.bin", IsDir: false, Size: 3, ModTime: time }, + { Name: "e&f.txt", IsDir: false, Size: 4, ModTime: time }, + // A file whose HEAD request fails is not in the list, as in rclone. + { Name: "sub", IsDir: true, Size: -1, ModTime: "" }, + ], ); + assert.equal(entries[1].MimeType, "application/json"); + }); + + test("sends no HEAD request with no_head", async () => { + const fs = remote(backend, { + params: { url: `${index.url}files/`, no_head: "true" }, + }); + requests.length = 0; + const entries = await fs.list(""); + assert.deepEqual(requests, ["GET /files/"]); + assert.deepEqual( + entries.map(({ Name, Size, MimeType }) => ({ Name, Size, MimeType })), + [ + { Name: "a.txt", Size: -1, MimeType: mimeType("a.txt") }, + { Name: "b c.json", Size: -1, MimeType: "application/json" }, + { Name: "d.bin", Size: -1, MimeType: "application/octet-stream" }, + { Name: "e&f.txt", Size: -1, MimeType: mimeType("e&f.txt") }, + { Name: "lost.txt", Size: -1, MimeType: mimeType("lost.txt") }, + { Name: "sub", Size: -1, MimeType: "" }, + ], + ); + }); + + test("gets a file with the headers of the server", async () => { + const response = await fs.get("b c.json"); + assert.equal(response.headers.get("Content-Length"), "2"); + assert.equal(response.headers.get("Last-Modified"), LAST_MODIFIED); + assert.equal(response.headers.get("Content-Type"), "application/json"); + assert.equal(await response.text(), "{}"); + }); +}); + +describe("http redirects with the headers option", () => { + test("do not send the headers to a different host", async () => { + /** The Cookie of each request, by the port of its server. */ + const cookies = []; + const other = await listen((request) => { + cookies.push(["other", request.headers.get("Cookie")]); + // Back to the first server. + return Response.redirect(`${first.url}end.txt`, 302); + }); + const first = await listen((request) => { + const { pathname } = new URL(request.url); + cookies.push([pathname, request.headers.get("Cookie")]); + if (pathname === "/same.txt") { + return Response.redirect(`${first.url}end.txt`, 301); + } + if (pathname === "/away.txt") { + return Response.redirect(`${other.url}x.txt`, 307); + } + return new Response("end"); + }); + try { + const fs = remote(backend, { + params: { url: `${first.url}`, headers: "Cookie,secret" }, + }); + assert.equal(await (await fs.get("same.txt")).text(), "end"); + assert.deepEqual(cookies, [ + ["/same.txt", "secret"], + ["/end.txt", "secret"], + ]); + + cookies.length = 0; + assert.equal(await (await fs.get("away.txt")).text(), "end"); + assert.deepEqual(cookies, [ + ["/away.txt", "secret"], + ["other", null], + // The other host chose this URL, thus it gets no headers too. + ["/end.txt", null], + ]); + } finally { + await first.close(); + await other.close(); + } + }); + + test("stop at a redirect from HTTPS to HTTP", async (t) => { + const urls = []; + t.mock.method(globalThis, "fetch", (url) => { + urls.push(`${url}`); + return Promise.resolve( + new Response(null, { + status: 302, + headers: { Location: "http://example.com/a.txt" }, + }), + ); + }); + const fs = remote(backend, { + params: { url: "https://example.com/", headers: "Cookie,secret" }, + }); + const response = await fs.get("a.txt"); + await response.body?.cancel(); + assert.equal(response.status, 502); + assert.deepEqual(urls, ["https://example.com/a.txt"]); }); - await t.test("a directory", async () => { - // TODO: implement + test("stop after 10 redirects", async () => { + let count = 0; + const loop = await listen((request) => { + count++; + return Response.redirect(request.url, 302); + }); + try { + const fs = remote(backend, { + params: { url: `${loop.url}`, headers: "Cookie,secret" }, + }); + const response = await fs.get("a.txt"); + await response.body?.cancel(); + assert.equal(response.status, 502); + assert.equal(count, 11); + } finally { + await loop.close(); + } }); }); diff --git a/backend/http/package.json b/backend/http/package.json new file mode 100644 index 0000000..c9643c8 --- /dev/null +++ b/backend/http/package.json @@ -0,0 +1,31 @@ +{ + "name": "@rfetch/http", + "version": "0.1.0", + "description": "The http backend of rfetch: it reads the folders of a web site, as rclone.", + "type": "module", + "exports": { + ".": "./main.js" + }, + "scripts": { + "test": "node --test" + }, + "engines": { + "node": ">=22.2.0" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/sntran/rfetch.git", + "directory": "backend/http" + }, + "author": "Trần Nguyễn Sơn", + "license": "ISC", + "dependencies": { + "@rfetch/lib": "^0.1.0" + }, + "devDependencies": { + "@rfetch/chunker": "^0.1.0", + "@rfetch/crypt": "^0.1.0", + "@rfetch/memory": "^0.1.0", + "fast-check": "^3.19.0" + } +} diff --git a/backend/http/rclone_test.js b/backend/http/rclone_test.js new file mode 100644 index 0000000..12d2377 --- /dev/null +++ b/backend/http/rclone_test.js @@ -0,0 +1,279 @@ +/** + * The tests of the http backend against the http backend of rclone: the two + * list the same server, and they must give the same files. The tests run + * when rclone is in the PATH, or when RCLONE is the path of rclone. + */ +import { after, before, describe, test } from "node:test"; +import assert from "node:assert/strict"; +import { spawn, spawnSync } from "node:child_process"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import process from "node:process"; + +import { remote } from "@rfetch/lib/testing/remote"; +import { listen } from "@rfetch/lib/testing/server"; +// Sets the global `fetch` that reads the paths of remotes. +import "../../mod.js"; +import chunker from "@rfetch/chunker"; +import crypt from "@rfetch/crypt"; +import memory from "@rfetch/memory"; +import http from "./main.js"; + +const RCLONE = process.env.RCLONE || "rclone"; +const hasRclone = spawnSync(RCLONE, ["version"]).status === 0; + +/** The files, with hard names. */ +const FILES = { + "empty.txt": 0, + "a/one.bin": 1, + "a/b/c.json": 100, + "file name #1.txt": 2048, + [`hello? sausage/êé/Hello, 世界/ " ' @ < > & ? + ≠/z.txt`]: 3000, +}; + +/** An index page as Apache makes it, at /index/. */ +const INDEX = ` +

Index of /index

+Name Size
+Parent Directory
+a.txt      17-Oct-2026 10:00    1
+b c.json 17-Oct-2026 10:00    2
+x.txt
+a.txt again
+e&f g.txt 17-Oct-2026 10:00    3
+café.txt 17-Oct-2026 10:00    4
+
+
+lost.txt
+sub/
+sub/deep.txt
+top
+
`; + +/** The files of the index page. */ +const INDEX_FILES = { + "/index/a.txt": "a", + "/index/b c.json": "{}", + "/index/e&f g.txt": "efg", + "/index/café.txt": "café", +}; + +/** @param {number} size */ +function content(size) { + return new Uint8Array(size).map((_, index) => (index * 11 + size) & 0xff); +} + +describe("http and rclone", { + skip: !hasRclone && "rclone is not installed. Set RCLONE to its path.", +}, () => { + const folder = mkdtempSync(join(tmpdir(), "rfetch-http-")); + let server; + + before(async () => { + server = await listen((request) => { + const pathname = decodeURIComponent(new URL(request.url).pathname); + if (pathname === "/index/" || pathname === "/index/sub/") { + const html = pathname === "/index/" ? INDEX : ""; + return new Response(html, { + headers: { "Content-Type": "text/html; charset=utf-8" }, + }); + } + if (pathname in INDEX_FILES) { + const body = new TextEncoder().encode(INDEX_FILES[pathname]); + return new Response(request.method === "HEAD" ? null : body, { + headers: { + "Content-Length": `${body.length}`, + "Last-Modified": "Sat, 17 Oct 2026 10:00:00 GMT", + }, + }); + } + return memory.fetch(request); + }); + const files = remote(memory, { root: "/rclone/" }); + for (const [path, size] of Object.entries(FILES)) { + const response = await files.put(path, content(size)); + assert.equal(response.status, 201, path); + } + }); + after(async () => { + await server?.close(); + rmSync(folder, { recursive: true, force: true }); + }); + + /** + * The output of `rclone lsjson -R` for the http backend of rclone. + * Node does not answer the requests of rclone when spawnSync waits. + * @param {string} url + */ + function lsjson(url) { + return new Promise((resolve, reject) => { + const child = spawn( + RCLONE, + ["lsjson", "-R", "--http-url", url, ":http:"], + { env: { PATH: process.env.PATH, HOME: folder } }, + ); + let stdout = "", stderr = ""; + child.stdout.on("data", (data) => stdout += data); + child.stderr.on("data", (data) => stderr += data); + child.on("error", reject); + child.on("close", (status) => { + if (status === 0) resolve(JSON.parse(stdout)); + else reject(new Error(`rclone lsjson ${url}: ${stderr}`)); + }); + }); + } + + /** + * The entries of all the folders, through the http backend of rfetch. + * @param {string} url + */ + async function walk(url) { + const fs = remote(http, { params: { url } }); + const entries = []; + const next = async (path) => { + for (const entry of await fs.list(path)) { + entries.push({ ...entry, Path: `${path}${entry.Name}` }); + if (entry.IsDir) await next(`${path}${entry.Name}/`); + } + }; + await next(""); + return entries; + } + + /** The fields to compare, in the order of the paths. */ + function normalize(entries) { + return entries + .map(({ Path, IsDir, Size, MimeType, ModTime }) => ({ + Path, + IsDir, + Size: IsDir ? -1 : Size, + MimeType: IsDir ? "" : MimeType, + // rclone has the time of Last-Modified, to the second. + ModTime: IsDir ? 0 : Math.floor(Date.parse(ModTime) / 1000), + })) + .sort((a, b) => a.Path < b.Path ? -1 : +(a.Path > b.Path)); + } + + test("list the lists of rfetch with the same files", async () => { + const url = `${server.url}rclone/`; + assert.deepEqual( + normalize(await walk(url)), + normalize(await lsjson(url)), + ); + }); + + test("list an index page that is not of rfetch with the same files", async () => { + const url = `${server.url}index/`; + const expected = normalize(await lsjson(url)); + assert.deepEqual(expected.map(({ Path }) => Path), [ + "a.txt", + "b c.json", + "café.txt", + "e&f g.txt", + "sub", + ]); + assert.deepEqual(normalize(await walk(url)), expected); + }); +}); + +describe("rclone reads ranges of rfetch", { + skip: !hasRclone && "rclone is not installed. Set RCLONE to its path.", +}, () => { + const folder = mkdtempSync(join(tmpdir(), "rfetch-range-")); + /** The backends under the server, with their options. */ + const BACKENDS = { + crypt: [crypt, { + remote: ":memory:/rclone-range-crypt", + // "test-password", obscured by rclone. + password: "4OSlj-ZYMGdEIGjcWg_gc-DL4O3TFAvr4ct_yV4", + }], + chunker: [chunker, { + remote: ":memory:/rclone-range-chunker", + chunk_size: "64k", + }], + }; + const data = content(3 * 65536 + 1000); + let server; + + before(async () => { + server = await listen((request) => { + const url = new URL(request.url); + const [, name] = url.pathname.split("/"); + const [backend, params] = BACKENDS[name]; + url.pathname = url.pathname.slice(name.length + 1); + for (const [key, value] of Object.entries(params)) { + url.searchParams.set(key, value); + } + return backend.fetch(new Request(url, request)); + }); + for (const [backend, params] of Object.values(BACKENDS)) { + const response = await remote(backend, { params }).put( + "data.bin", + data, + ); + assert.equal(response.status, 201); + } + }); + after(async () => { + await server?.close(); + rmSync(folder, { recursive: true, force: true }); + }); + + /** + * The output of `rclone cat` for a range of a file of the server. + * @param {string} url + * @param {number} offset + * @param {number} count + */ + function cat(url, offset, count) { + return new Promise((resolve, reject) => { + const child = spawn( + RCLONE, + [ + "cat", + "--http-url", + url, + ":http:data.bin", + "--offset", + `${offset}`, + "--count", + `${count}`, + ], + { env: { PATH: process.env.PATH, HOME: folder } }, + ); + const chunks = []; + let stderr = ""; + child.stdout.on("data", (chunk) => chunks.push(chunk)); + child.stderr.on("data", (chunk) => stderr += chunk); + child.on("error", reject); + child.on("close", (status) => { + if (status === 0) resolve(new Uint8Array(Buffer.concat(chunks))); + else reject(new Error(`rclone cat ${url}: ${stderr}`)); + }); + }); + } + + for (const name of Object.keys(BACKENDS)) { + test(`${name}: rclone cat --offset --count gives the bytes`, async () => { + const url = `${server.url}${name}/`; + for ( + const [offset, count] of [ + [0, 1], + [65535, 2], + [65536, 65536], + [100000, 70000], + [data.length - 1, 1], + [70000, -1], + ] + ) { + const end = count < 0 ? data.length : offset + count; + assert.deepEqual( + await cat(url, offset, count), + data.subarray(offset, end), + `${name} --offset ${offset} --count ${count}`, + ); + } + }); + } +}); diff --git a/backend/local/main.js b/backend/local/main.js index 06378d5..a3497a7 100644 --- a/backend/local/main.js +++ b/backend/local/main.js @@ -1,29 +1,53 @@ -#!/usr/bin/env -S deno serve --allow-all - import process from "node:process"; -import { contentType, extname, formatBytes, join } from "../../deps.js"; +import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; +import { formatBytes } from "@rfetch/lib/format"; +import { escapeHTML } from "@rfetch/lib/html"; +import { mimeType } from "@rfetch/lib/mime"; +import { decodePath } from "@rfetch/lib/path"; +import { notSatisfiable, parseRange, partialHeaders } from "@rfetch/lib/range"; /** - * Serves a local remote + * Serves a local remote: the files and the folders in its root. The root is + * the current folder, or `RFETCH_LOCAL_ROOT` of `env`. + * + * The fetch of rfetch sets the root `/` only for a local path of the command + * line or of the code, for example `/tmp/file.txt`: an option, a header or + * the query cannot set it. Thus a request from a client of `rfetch serve`, + * which has no `env`, stays in the current folder. * @param {Request} request + * @param {Record} [env] * @returns {Promise} */ -async function local(request) { +async function local(request, env = {}) { const { method, url } = request; const { pathname, searchParams } = new URL(url); - const absolutePath = join(process.cwd?.() || ".", pathname); + const cwd = process.cwd?.() || "."; + const root = typeof env.RFETCH_LOCAL_ROOT === "string" + ? env.RFETCH_LOCAL_ROOT + : cwd; + const path = decodePath(pathname); + const absolutePath = path === null ? null : join(root, path); + + // A path must not go out of the root, also on Windows, where `\` is a + // separator too. + if (absolutePath === null || isOutside(root, absolutePath)) { + return new Response(null, { status: 400 }); + } // Have to use dynamic import here because `fs/promises` is not available in // Worker environments. const { mkdir, open, readdir, rm, stat } = await import("node:fs/promises"); - let stats; + /** @type {import("node:fs").Stats | null} */ + const stats = await stat(absolutePath).catch(() => null); const headers = new Headers(); let status = 200, body = null; if (method === "HEAD" || method === "GET") { - stats = await stat(absolutePath); + if (!stats) { + return new Response(null, { status: 404 }); + } if (stats.isDirectory()) { /** @@ -31,10 +55,23 @@ async function local(request) { */ const files = await readdir(absolutePath, { withFileTypes: true }); files.sort((a, b) => a.name.localeCompare(b.name)); + // The stat of each file, at the same time: one after the other takes + // much more time. A file that goes away before its stat is not in + // the list. + // The times in nanoseconds, as rclone gives them. + const fileStats = await Promise.all( + files.map((file) => + stat(join(absolutePath, file.name), { bigint: true }).catch(() => + null + ) + ), + ); if (method === "GET") { + const query = escapeHTML(`${searchParams}`); + body = new ReadableStream({ - async start(controller) { + start(controller) { controller.enqueue(`
@@ -58,49 +95,45 @@ async function local(request) { - + `); - for await (const file of files) { + for (const [index, file] of files.entries()) { + if (!fileStats[index]) continue; const name = file.name; - let filePath = file.name; const isDirectory = file.isDirectory(); - if (isDirectory) { - filePath += "/"; - } + const href = `${encodeURIComponent(name)}${ + isDirectory ? "/" : "" + }`; - const { size, mtime } = await stat( - join(absolutePath, filePath), - ); + const { size, mtimeNs } = fileStats[index]; + const mtime = timeOf(Number(mtimeNs / 1000000n)); controller.enqueue(` - - - `); } @@ -112,59 +145,91 @@ async function local(request) { headers.set("Content-Type", "text/html;charset=utf-8"); } else { - headers.append("Content-Type", contentType(extname(pathname)) || ""); + headers.append("Content-Type", mimeType(path)); headers.append("Content-Length", stats.size.toString()); + headers.set("Accept-Ranges", "bytes"); if (method === "GET") { + let range; + try { + range = parseRange(request.headers.get("Range"), stats.size); + } catch { + return notSatisfiable(stats.size); + } + if (range) { + status = 206; + for ( + const [name, value] of Object.entries( + partialHeaders(range, stats.size), + ) + ) { + headers.set(name, value); + } + } + // The position of the next byte, and the number of bytes to read. + let position = range?.start ?? 0; + let remaining = range ? range.end - range.start + 1 : stats.size; const file = await open(absolutePath); body = new ReadableStream({ async pull(controller) { - const chunk = new Uint8Array(64 * 1024); - const { bytesRead } = await file.read(chunk, 0); - controller.enqueue(chunk.slice(0, bytesRead)); + // A read of 1 MiB: each read is a call to the system, and parts + // of 64 KiB make a read of a large file much slower. + const chunk = new Uint8Array(Math.min(1024 * 1024, remaining)); + const { bytesRead } = remaining > 0 + ? await file.read(chunk, 0, chunk.length, position) + : { bytesRead: 0 }; if (bytesRead === 0) { controller.close(); - file.close(); + await file.close(); + return; } + position += bytesRead; + remaining -= bytesRead; + controller.enqueue(chunk.subarray(0, bytesRead)); }, - cancel() { - file.close(); + async cancel() { + await file.close(); }, }); } } - const mtime = stats.mtime?.toUTCString(); - if (mtime) headers.append("Last-Modified", mtime); - } - - /** `PUT` upserts a file at `pathname`. */ - if (method === "PUT") { + if (stats.mtimeMs !== undefined) { + headers.append("Last-Modified", timeOf(stats.mtimeMs).toUTCString()); + } + } else if (method === "PUT") { // Folder ending with trailing slash. - if (pathname.endsWith("/")) { + if (path.endsWith("/")) { await mkdir(absolutePath, { recursive: true }); } else { + await mkdir(dirname(absolutePath), { recursive: true }); const file = await open(absolutePath, "w"); - await request.body.pipeTo( - new WritableStream({ - async write(chunk) { - await file.write(chunk); - }, - async close() { - await file.close(); - }, - }), - ); + try { + for await (const chunk of request.body ?? []) { + await file.write(chunk); + } + } finally { + await file.close(); + } } status = 201; headers.append("Content-Location", pathname); - } + } else if (method === "DELETE") { + if (!stats) { + return new Response(null, { status: 404 }); + } + // The root and the current folder must stay. + if (path === "/" || resolve(absolutePath) === resolve(cwd)) { + return new Response(null, { status: 403 }); + } - if (method === "DELETE") { await rm(absolutePath, { recursive: true }); status = 204; + } else { + status = 405; + headers.set("Allow", "GET, HEAD, PUT, DELETE"); } return new Response(body, { @@ -173,6 +238,43 @@ async function local(request) { }); } +/** + * Tells if a path is out of the root folder. + * @param {string} root + * @param {string} path + * @returns {boolean} + */ +function isOutside(root, path) { + const rest = relative(root, path); + return rest === ".." || rest.startsWith(`..${sep}`) || isAbsolute(rest); +} + +/** + * The time of a file in ISO 8601, with 9 digits after the second: the + * format of rclone for a remote with a precision of 1 ns. + * @param {bigint} mtimeNs The nanoseconds after 1970. + * @returns {string} + */ +function isoStringOf(mtimeNs) { + const second = 1000000000n; + // The floor, also for a time before 1970. + let nanoseconds = mtimeNs % second; + if (nanoseconds < 0n) nanoseconds += second; + const seconds = (mtimeNs - nanoseconds) / second; + const date = new Date(Number(seconds) * 1000).toISOString().slice(0, 19); + return `${date}.${String(nanoseconds).padStart(9, "0")}Z`; +} + +/** + * The time of a file to the millisecond. As in Go, the time is cut, not + * rounded: the `mtime` of Node.js is rounded to the nearest millisecond. + * @param {number} mtimeMs + * @returns {Date} + */ +function timeOf(mtimeMs) { + return new Date(Math.floor(mtimeMs)); +} + export default { fetch: local, }; diff --git a/backend/local/main_test.js b/backend/local/main_test.js index 966f892..dd5056f 100644 --- a/backend/local/main_test.js +++ b/backend/local/main_test.js @@ -1,203 +1,70 @@ -import { test } from "node:test"; +import { after, describe, it } from "node:test"; +import assert from "node:assert/strict"; import process from "node:process"; -import { readdir, readFile, stat } from "node:fs/promises"; -import { contentType, extname, join } from "../../deps.js"; import { - assert, - assertEquals, - assertHeader, - assertRejects, -} from "../../dev_deps.js"; - + mkdir, + mkdtemp, + readFile, + rm, + stat, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { backendSuite } from "@rfetch/lib/testing/suite"; import backend from "./main.js"; -const fetch = backend.fetch; +// The backend serves the current folder. The tests use a new folder as the +// current folder, with a secret file next to it, out of the root of the +// backend. +const folder = await mkdtemp(join(tmpdir(), "rfetch-local-")); +const root = join(folder, "root"); +await mkdir(root); +await writeFile(join(folder, "secret.txt"), "secret"); +process.chdir(root); -const encoder = new TextEncoder(); -const cwd = process.cwd(); +backendSuite("local", backend); -test("GET", async (t) => { - const requestInit = { - method: "GET", - }; +describe("local", () => { + after(() => rm(folder, { recursive: true, force: true })); /** - * @type {string[]} + * @param {string} method + * @param {string} path */ - const files = []; - - await t.test("/", async () => { - const url = new URL("/", "local://"); - const request = new Request(url, requestInit); - const response = await fetch(request); - const { headers, body } = response; - - assert(body, "should have body"); - assert(headers.get("Content-Type").includes("text/html")); - const html = await response.text(); - - const entries = await readdir(cwd, { withFileTypes: true }); - for await (const entry of entries) { - let name = entry.name; - if (entry.isDirectory()) { - name += "/"; + async function send(method, path) { + const response = await backend.fetch( + new Request(`http://rfetch.test${path}`, { + method, + body: method === "PUT" ? "evil" : null, + }), + ); + await response.body?.cancel(); + return response; + } + + it("does not go out of its root folder", async () => { + const paths = [ + "/..%2Fsecret.txt", + "/a/..%2F..%2Fsecret.txt", + "/%2E%2E%2Fsecret.txt", + "/..%2F", + ]; + for (const path of paths) { + for (const method of ["GET", "HEAD", "PUT", "DELETE"]) { + const { status } = await send(method, path); + assert.equal(status, 400, `${method} ${path}`); } - - assert( - html.includes(` href="${name}`), - "should have the link in the HTML", - ); - - files.push(name); } - }); - - await t.test("a file", async () => { - for await (const name of files) { - // Checking links for now - if (name.endsWith("/")) continue; - - const path = join(cwd, name); - const file = await stat(path); - const url = new URL(`/${name}`, "local://"); - const request = new Request(url, requestInit); - const response = await fetch(request); - const { headers, body } = response; - - assertHeader(headers, "Content-Type", contentType(extname(name)) || ""); - assertHeader(headers, "Content-Length", `${file.size}`); - assertHeader(headers, "Last-Modified", file.mtime?.toUTCString() || ""); - - assert( - body instanceof ReadableStream, - "should have body as a ReadableStream", - ); - assertEquals( - await response.text(), - await readFile(path, "utf8"), - "should have the same content", - ); - } + assert.equal(await readFile(join(folder, "secret.txt"), "utf8"), "secret"); + assert.ok((await stat(root)).isDirectory()); }); -}); - -const newFile = "dummy.file"; -const newFolder = "dummy.folder/"; // Must have trailing slash. - -test("PUT", async (t) => { - await t.test("a new file", async () => { - const targetFile = join(cwd, newFile); - const body = "Hello World"; - - const url = new URL(`/${newFile}`, "local://"); - // Sends the PUT request with the body. - const request = new Request(url, { - method: "PUT", - headers: { - "Content-Type": contentType(extname(newFile)) || - "application/octet-stream", - "Content-Length": `${encoder.encode(body).byteLength}`, - }, - body, - }); - const { status, headers } = await fetch(request); - - assertEquals(status, 201, "should respond with 201 Created"); - assertHeader(headers, "Content-Location", `/${newFile}`); - - assertEquals( - await readFile(targetFile, "utf8"), - body, - "should have the same content", - ); - - // Also checks the content with a GET request. - const response = await fetch(new Request(url)); - assertEquals(await response.text(), body, "should have the same content"); - }); - - await t.test("a new folder", async () => { - const targetFolder = join(cwd, newFolder); - - const url = new URL(`/${newFolder}`, "local://"); - // Sends the PUT request with no body. - const request = new Request(url, { - method: "PUT", - headers: { - "Content-Type": "text/directory", - "Content-Length": "0", - }, - }); - const { status, headers } = await fetch(request); - - assertEquals(status, 201, "should respond with 201 Created"); - assertHeader(headers, "Content-Location", `/${newFolder}`); - - const stats = await stat(targetFolder); - assert(stats.isDirectory, "should be a directory"); - }); - - await t.test("a new nested file", async () => { - const targetFile = join(cwd, newFolder, newFile); - const body = "Hello Nested World"; - - const url = new URL(`/${join(newFolder, newFile)}`, "local://"); - // Sends the PUT request with the body. - const request = new Request(url, { - method: "PUT", - headers: { - "Content-Type": contentType(extname(newFile)) || - "application/octet-stream", - "Content-Length": `${encoder.encode(body).byteLength}`, - }, - body, - }); - const { status, headers } = await fetch(request); - - assertEquals(status, 201, "should respond with 201 Created"); - assertHeader(headers, "Content-Location", `/${join(newFolder, newFile)}`); - - assertEquals( - await readFile(targetFile, "utf8"), - body, - "should have the same content", - ); - - // Also checks the content with a GET request. - const response = await fetch(new Request(url)); - assertEquals(await response.text(), body, "should have the same content"); - }); -}); - -test("DELETE", async (t) => { - await t.test("a file", async () => { - const targetFile = join(cwd, newFile); - const url = new URL(`/${newFile}`, "local://"); - // Sends the PUT request with the body. - const request = new Request(url, { - method: "DELETE", - }); - const { status } = await fetch(request); - assertEquals(status, 204, "should respond with 204 No Content"); - - assertRejects(() => { - return stat(targetFile); - }); - }); - - await t.test("a folder", async () => { - const targetFolder = join(cwd, newFolder); - const url = new URL(`/${newFolder}`, "local://"); - // Sends the PUT request with the body. - const request = new Request(url, { - method: "DELETE", - }); - const { status } = await fetch(request); - assertEquals(status, 204, "should respond with 204 No Content"); - assertRejects(() => { - return stat(targetFolder); - }); + it("does not delete its root folder", async () => { + const { status } = await send("DELETE", "/"); + assert.equal(status, 403); + assert.ok((await stat(root)).isDirectory()); }); }); diff --git a/backend/local/package.json b/backend/local/package.json new file mode 100644 index 0000000..01dedc5 --- /dev/null +++ b/backend/local/package.json @@ -0,0 +1,25 @@ +{ + "name": "@rfetch/local", + "version": "0.1.0", + "description": "The local backend of rfetch: the files of the local disk.", + "type": "module", + "exports": { + ".": "./main.js" + }, + "scripts": { + "test": "node --test" + }, + "engines": { + "node": ">=22.2.0" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/sntran/rfetch.git", + "directory": "backend/local" + }, + "author": "Trần Nguyễn Sơn", + "license": "ISC", + "dependencies": { + "@rfetch/lib": "^0.1.0" + } +} diff --git a/backend/main.js b/backend/main.js index b3d7954..b096851 100644 --- a/backend/main.js +++ b/backend/main.js @@ -2,16 +2,44 @@ * A backend is an interface to remote storage provider. * * Each backend implements the `fetch` method which takes a request and returns - * a response, or a promise of a response. + * a response, or a promise of a response, as a Worker of Cloudflare. A backend + * imports nothing of rfetch: it gets all from the request and from `env`. + * + * A backend on top of another remote, for example crypt, sends its requests + * to that remote with the global `fetch`, for example + * `fetch(":memory:/path")`. The host sets the global `fetch` that reads + * such a path: the import of `fetch.js` sets it, in Node.js and in a Worker. * * @typedef {Object} Backend - * @property {(request: Request) => Response | Promise} fetch + * @property {(request: Request, env?: Object) => Response | Promise} fetch + */ + +/** + * The backends of rfetch, by type. Each backend is the npm package + * `@rfetch/TYPE`. It loads at its first use, thus rfetch loads only the + * backends that it needs. The names of the packages are literal, thus a + * bundler, for example the bundler of wrangler, can find them. + * @type {Record Promise<{ default: Backend }>>} */ +export const backends = { + alias: () => import("@rfetch/alias"), + chunker: () => import("@rfetch/chunker"), + crypt: () => import("@rfetch/crypt"), + drive: () => import("@rfetch/drive"), + http: () => import("@rfetch/http"), + local: () => import("@rfetch/local"), + memory: () => import("@rfetch/memory"), + s3: () => import("@rfetch/s3"), +}; -export { default as alias } from "./alias/main.js"; -export { default as chunker } from "./chunker/main.js"; -export { default as crypt } from "./crypt/main.js"; -export { default as drive } from "./drive/main.js"; -export { default as http } from "./http/main.js"; -export { default as local } from "./local/main.js"; -export { default as memory } from "./memory/main.js"; +/** + * Loads a backend. + * @param {string} type For example `crypt`. + * @returns {Promise} `undefined` for a type that is not + * a backend of rfetch. + */ +export async function load(type) { + if (!Object.hasOwn(backends, type)) return undefined; + const { default: backend } = await backends[type](); + return backend; +} diff --git a/backend/main_test.js b/backend/main_test.js new file mode 100644 index 0000000..17256b1 --- /dev/null +++ b/backend/main_test.js @@ -0,0 +1,133 @@ +/** + * The tests of the registry of the backends, and of the rule of the + * workspaces: a backend or a package imports only `node:` modules, the + * packages of its `package.json`, and the files in its folder. Thus each one + * is standalone, and can be a Worker of its own. + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { existsSync, readdirSync, readFileSync } from "node:fs"; +import { dirname, join, relative, resolve, sep } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { backends, load } from "./main.js"; + +const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), ".."); + +/** The folders of the workspaces, for example `lib` and `backend/crypt`. */ +const WORKSPACES = JSON.parse(readFileSync(join(ROOT, "package.json"))) + .workspaces.flatMap((pattern) => { + if (!pattern.endsWith("/*")) return [pattern]; + const parent = pattern.slice(0, -2); + return readdirSync(join(ROOT, parent), { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => `${parent}/${entry.name}`) + .filter((folder) => existsSync(join(ROOT, folder, "package.json"))); + }); + +/** + * The files of a test or a bench. They can import the devDependencies, and + * `mod.js` of rfetch: it sets the global `fetch` that reads the paths of + * remotes, for the tests of a backend on top of other remotes. + * @param {string} file + */ +function isTest(file) { + return /(_test|_bench)\.js$|^testing\.js$/.test(file.split(sep).at(-1)); +} + +/** @param {string} folder @returns {string[]} The JavaScript files. */ +function filesOf(folder) { + return readdirSync(folder, { recursive: true }) + .filter((file) => file.endsWith(".js")) + .filter((file) => !file.split(sep).includes("node_modules")); +} + +/** @param {string} source @returns {string[]} The specifiers of imports. */ +function importsOf(source) { + const pattern = + /\bfrom\s+"([^"]+)"|\bimport\s+"([^"]+)"|\bimport\(\s*"([^"]+)"\s*\)/g; + return [...source.matchAll(pattern)].map((match) => + match[1] ?? match[2] ?? match[3] + ); +} + +/** @param {string} specifier For example `@rfetch/lib/listing`. */ +function packageOf(specifier) { + const parts = specifier.split("/"); + return specifier.startsWith("@") ? parts.slice(0, 2).join("/") : parts[0]; +} + +test("each backend is in the registry, with its package name", () => { + const folders = WORKSPACES.filter((folder) => folder.startsWith("backend/")) + .map((folder) => folder.slice("backend/".length)); + assert.deepEqual(Object.keys(backends).sort(), folders.sort()); + for (const name of folders) { + const { name: packageName } = JSON.parse( + readFileSync(join(ROOT, "backend", name, "package.json")), + ); + assert.equal(packageName, `@rfetch/${name}`); + } +}); + +test("load gives each backend, and nothing for other names", async () => { + for (const type of Object.keys(backends)) { + const backend = await load(type); + assert.equal(typeof backend?.fetch, "function", type); + } + for (const type of ["lib", "testing", "constructor", "__proto__", ""]) { + assert.equal(await load(type), undefined, type); + } +}); + +test("each workspace imports only node:, its packages, and its files", () => { + for (const folder of WORKSPACES) { + const path = join(ROOT, folder); + const { dependencies = {}, devDependencies = {}, imports = {} } = JSON + .parse(readFileSync(join(path, "package.json"))); + for (const file of filesOf(path)) { + const source = readFileSync(join(path, file), "utf8"); + for (const specifier of importsOf(source)) { + const where = `${folder}/${file}: ${specifier}`; + if (specifier.startsWith("node:")) continue; + if (specifier.startsWith(".")) { + const target = relative( + path, + resolve(path, dirname(file), specifier), + ); + const isHost = isTest(file) && + resolve(path, dirname(file), specifier) === + join(ROOT, "mod.js"); + assert.ok( + isHost || !target.startsWith(".."), + `${where} is out of the folder`, + ); + continue; + } + if (specifier.startsWith("#")) { + // A file of the folder, from the field `imports` of package.json. + assert.ok( + Object.hasOwn(imports, specifier), + `${where} is not in imports`, + ); + const targets = JSON.stringify(imports[specifier]).match( + /"\.[^"]*"/g, + ); + for (const target of targets ?? []) { + const inside = relative(path, resolve(path, JSON.parse(target))); + assert.ok( + !inside.startsWith(".."), + `${where} is out of the folder`, + ); + } + continue; + } + const name = packageOf(specifier); + assert.ok( + Object.hasOwn(dependencies, name) || + (isTest(file) && Object.hasOwn(devDependencies, name)), + `${where} is not in package.json`, + ); + } + } + } +}); diff --git a/backend/memory/main.js b/backend/memory/main.js index 518bdac..e477628 100644 --- a/backend/memory/main.js +++ b/backend/memory/main.js @@ -1,5 +1,3 @@ -#!/usr/bin/env -S deno serve --allow-all - /** * Memory Backend * @@ -9,14 +7,32 @@ * Because it has no parameters you can just use it with the `:memory:` remote * name. */ -import { contentType, extname, formatBytes } from "../../deps.js"; +import { renderListing } from "@rfetch/lib/listing"; +import { mimeType } from "@rfetch/lib/mime"; +import { decodePath } from "@rfetch/lib/path"; +import { notSatisfiable, parseRange, partialHeaders } from "@rfetch/lib/range"; + +/** + * A file: its bytes, its media type, and the time of its last change. + * @typedef {Object} File + * @property {Uint8Array} bytes + * @property {string} type + * @property {number} modified In milliseconds. + */ + +/** + * The files, by their path with no percent-encoding. + * @type {Map} + */ +const files = new Map(); /** - * @type {Map} + * The folders, by their path with no percent-encoding and with `/` at the + * end. The value is the names of the children: `name` for a file and + * `name/` for a folder. Thus a list reads only its folder. + * @type {Map>} */ -const cache = new Map(); -// Creates the root folder. -cache.set("/", null); +const folders = new Map([["/", new Set()]]); /** * Serves a memory remote @@ -26,160 +42,93 @@ cache.set("/", null); async function memory(request) { const { method, url } = request; const { pathname, searchParams } = new URL(url); + const path = decodePath(pathname); + + if (path === null) { + return new Response(null, { status: 400 }); + } const headers = new Headers(); let status = 200, body = null; - if (method === "GET") { - // Retrieves file content. - for (const [key, value] of cache) { - if (key === pathname) { - // File. - if (value !== null) { - body = value.stream(); - break; + if (method === "HEAD" || method === "GET") { + const file = files.get(path); + if (file) { + const { bytes } = file; + headers.set("Content-Type", file.type || mimeType(path)); + headers.set("Content-Length", `${bytes.length}`); + headers.set("Last-Modified", new Date(file.modified).toUTCString()); + headers.set("Accept-Ranges", "bytes"); + if (method === "GET") { + let range; + try { + range = parseRange(request.headers.get("Range"), bytes.length); + } catch { + return notSatisfiable(bytes.length); } - - const regex = new RegExp(pathname + "([^/]+/?)"); - /** - * @type {string[]} - */ - const children = []; - for (const key of cache.keys()) { - const [, child] = key.match(regex) || []; - if (child) { - if (!children.includes(child)) { - children.push(child); - } + // The body is a copy of the bytes, with a length. A Worker reads a + // Blob in chunks of 4 KiB, which is slow. + body = bytes; + if (range) { + status = 206; + for ( + const [name, value] of Object.entries( + partialHeaders(range, bytes.length), + ) + ) { + headers.set(name, value); } + body = bytes.subarray(range.start, range.end + 1); } - - children.sort((a, b) => a.localeCompare(b)); - - // Folder listing - body = new ReadableStream({ - start(controller) { - controller.enqueue(` -
📁Go upGo up — —
${isDirectory ? "📁" : "📄"} ${name} + href="${href}?${query}" + type="${isDirectory ? "" : escapeHTML(mimeType(name))}" + >${escapeHTML(name)} - ${ + isDirectory ? "—" : `${ - size ? formatBytes(Number(size), { locale: true }) : "—" - } + >${formatBytes(Number(size), { locale: true })}` + } ${ mtime ? `` : "-" }
- - - - - - - - - - - - - - - - `); - - for (const name of children) { - const value = cache.get(pathname + name); - const isDirectory = value === null; - - let filePath = name; - if (isDirectory) { - filePath += "/"; - } - - const type = isDirectory - ? "" - : contentType(extname(pathname)) || ""; - const size = Number(isDirectory ? "-1" : value.size); - const lastModified = value?.lastModified - ? new Date(value?.lastModified) - : ""; - - controller.enqueue(` - - - - - `); - } - - controller.enqueue("
NameSizeModified
📁Go up——
${isDirectory ? "📁" : "📄"} - ${name} - - ${ - size - ? `${formatBytes(size, { locale: true })}` - : "—" - } - ${ - lastModified - ? `` - : "-" - } -
"); - controller.close(); - }, - }).pipeThrough(new TextEncoderStream()); - - headers.set("Content-Type", "text/html;charset=utf-8"); - - break; } - } - - if (!body) { - status = 404; - } - } - - if (method === "PUT") { - const folders = pathname.split("/").slice(1, -1); - // Creates ancestor folders, recursively. - let href = "/"; - for (const segment of folders) { - href += `${segment}/`; - cache.set(`${href}`, null); - } - - // Folder ending with trailing slash. - if (pathname.endsWith("/")) { - // Nothing, as we already created it. + } else if (folders.has(path)) { + headers.set("Content-Type", "text/html;charset=utf-8"); + if (method === "GET") { + body = listing(path, searchParams); + } } else { - // Creates file. - const file = new File([await request.arrayBuffer()], pathname, { - type: request.headers.get("Content-Type") || "", - lastModified: Date.now(), + return new Response(null, { status: 404 }); + } + } else if (method === "PUT") { + const slash = path.lastIndexOf("/") + 1; + // Creates the parent folders, then the folder or the file. + makeFolder(path.slice(0, slash)); + if (slash < path.length) { + const name = path.slice(slash); + files.set(path, { + bytes: new Uint8Array(await request.arrayBuffer()), + type: request.headers.get("Content-Type") || mimeType(name), + modified: Date.now(), }); - cache.set(pathname, file); + folders.get(path.slice(0, slash))?.add(name); } status = 201; headers.append("Content-Location", pathname); - } - - if (method === "DELETE") { - // Folder ending with trailing slash. - if (pathname.endsWith("/")) { - // Deletes files in that folder. - for (const key of cache.keys()) { - if (key.startsWith(pathname)) { - cache.delete(key); - } + } else if (method === "DELETE") { + if (files.has(path)) { + files.delete(path); + const slash = path.lastIndexOf("/") + 1; + folders.get(path.slice(0, slash))?.delete(path.slice(slash)); + } else if (folders.has(path)) { + // Deletes the folder and all in it. The root folder stays. + removeFolder(path); + if (path === "/") { + folders.set("/", new Set()); + } else { + const parent = path.slice( + 0, + path.lastIndexOf("/", path.length - 2) + 1, + ); + folders.get(parent)?.delete(path.slice(parent.length)); } - // Deletes the folder itself - cache.delete(pathname); } else { - // Deletes file. - cache.delete(pathname); + return new Response(null, { status: 404 }); } status = 204; + } else { + status = 405; + headers.set("Allow", "GET, HEAD, PUT, DELETE"); } return new Response(body, { @@ -188,6 +137,63 @@ async function memory(request) { }); } +/** + * Creates a folder and its parent folders. + * @param {string} path The path of the folder, with `/` at the end. + */ +function makeFolder(path) { + if (folders.has(path)) return; + const parent = path.slice(0, path.lastIndexOf("/", path.length - 2) + 1); + makeFolder(parent); + folders.set(path, new Set()); + folders.get(parent)?.add(path.slice(parent.length)); +} + +/** + * Removes a folder, its files and its folders. + * @param {string} path The path of the folder, with `/` at the end. + */ +function removeFolder(path) { + for (const name of folders.get(path) ?? []) { + if (name.endsWith("/")) removeFolder(`${path}${name}`); + else files.delete(`${path}${name}`); + } + folders.delete(path); +} + +/** + * Gives the HTML list of the files and the folders in a folder. + * @param {string} path The path of the folder, with `/` at the end. + * @param {URLSearchParams} searchParams The query, for each link. + * @returns {ReadableStream} + */ +function listing(path, searchParams) { + /** @type {import("@rfetch/lib/listing").Entry[]} */ + const entries = []; + for (const child of folders.get(path) ?? []) { + if (child.endsWith("/")) { + entries.push({ + Name: child.slice(0, -1), + IsDir: true, + Size: -1, + MimeType: "", + ModTime: "", + }); + } else { + const file = files.get(`${path}${child}`); + if (!file) continue; + entries.push({ + Name: child, + IsDir: false, + Size: file.bytes.length, + MimeType: file.type || mimeType(child), + ModTime: new Date(file.modified).toISOString(), + }); + } + } + return renderListing(entries, { query: `${searchParams}` }); +} + export default { fetch: memory, }; diff --git a/backend/memory/main_test.js b/backend/memory/main_test.js index e357f07..db5019f 100644 --- a/backend/memory/main_test.js +++ b/backend/memory/main_test.js @@ -1,149 +1,4 @@ -import { test } from "node:test"; -import { join } from "../../deps.js"; -import { assert, assertEquals, assertHeader } from "../../dev_deps.js"; - +import { backendSuite } from "@rfetch/lib/testing/suite"; import backend from "./main.js"; -const fetch = backend.fetch; - -const encoder = new TextEncoder(); - -const tree = { - "/": ["A/", "B/", "C/", "a", "b", "c"], - "/A/": ["a", "b", "c"], - "/B/": ["a", "c"], - "/C/": [], -}; - -/** - * Returns an array of descendants from `root` to the leaf nodes. - * - * Example: - * - * ```ts - * descendants("/"); - * // ["/A/a", "/A/b", "/B/a", "/B/c", "/C/", "/a", "/b"] - * ``` - * - * @param {string} [root="/"] The node to retrieve the descendants from. - * @returns {string[]} flat array of descendants with full path from `root`. - */ -function descendants(root = "/") { - /** - * @type {string[]} - */ - const results = []; - /** - * @type {string[]} - */ - const children = tree[root] || []; - children.forEach((child) => { - const path = join(root, child); - const grandChildren = descendants(path); - if (!grandChildren.length) { - grandChildren.push(path); - } - results.push(...grandChildren); - }); - - return results; -} - -test("PUT", async (t) => { - await t.test("a new file or folder", async () => { - for (const fixture of descendants("/")) { - const url = new URL(fixture, "memory:/"); - // Sends the PUT request with the body. - const request = new Request(url, { - method: "PUT", - headers: { - "Content-Type": "text/plain", - "Content-Length": `${encoder.encode(fixture).byteLength}`, - }, - body: fixture, - }); - const { status, headers } = await fetch(request); - - assertEquals(status, 201, "should respond with 201 Created"); - assertHeader(headers, "Content-Location", fixture); - } - }); -}); - -test("GET", async (t) => { - await t.test("folder", async () => { - const pathname = "/"; // Root folder - let url = new URL(pathname, "memory:/"); - let response = await fetch(new Request(url)); - assert(response.headers.get("Content-Type")?.includes("text/html")); - const html = await response.text(); - - for (const child of tree[pathname]) { - assert( - html.includes(` href="${child}`), - `should have link to ${child}`, - ); - - if (child.endsWith("/")) { - const childPathname = pathname + child; - - url = new URL(childPathname, "memory:/"); - response = await fetch(new Request(url)); - assert(response.headers.get("Content-Type")?.includes("text/html")); - const childHtml = await response.text(); - - for (const grandchild of tree[childPathname]) { - assert( - childHtml.includes(` href="${grandchild}`), - `should have link to ${grandchild}`, - ); - } - } - } - }); - - await t.test("file", async () => { - const pathname = "/A/"; // Folder with only files - for (const fixture of descendants(pathname)) { - const url = new URL(fixture, "memory:/"); - const response = await fetch(new Request(url)); - const responseText = await response.text(); - assertEquals(responseText, fixture, "should have the same content"); - } - }); -}); - -test("DELETE", async (t) => { - await t.test("file", async () => { - const pathname = "/B/"; // Folder with only files - for await (const fixture of descendants(pathname)) { - const url = new URL(`${pathname}${fixture}`, "memory:/"); - let response = await fetch( - new Request(url, { - method: "DELETE", - }), - ); - assertEquals(response.status, 204, "should respond with 204 No Content"); - - response = await fetch(new Request(url)); - assertEquals(response.status, 404, "should respond with 404 Not Found"); - } - }); - - await t.test("folder", async () => { - const pathname = "/A/"; // Folder with many files - const url = new URL(pathname, "memory:/"); - const response = await fetch( - new Request(url, { - method: "DELETE", - }), - ); - assertEquals(response.status, 204, "should respond with 204 No Content"); - - for await (const fixture of descendants(pathname)) { - const url = new URL(fixture, "memory:/"); - const response = await fetch(new Request(url)); - assertEquals(response.status, 404, "should respond with 404 Not Found"); - } - }); -}); +backendSuite("memory", backend); diff --git a/backend/memory/package.json b/backend/memory/package.json new file mode 100644 index 0000000..71a9c93 --- /dev/null +++ b/backend/memory/package.json @@ -0,0 +1,25 @@ +{ + "name": "@rfetch/memory", + "version": "0.1.0", + "description": "The memory backend of rfetch: files in memory.", + "type": "module", + "exports": { + ".": "./main.js" + }, + "scripts": { + "test": "node --test" + }, + "engines": { + "node": ">=22.2.0" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/sntran/rfetch.git", + "directory": "backend/memory" + }, + "author": "Trần Nguyễn Sơn", + "license": "ISC", + "dependencies": { + "@rfetch/lib": "^0.1.0" + } +} diff --git a/backend/s3/main.js b/backend/s3/main.js new file mode 100644 index 0000000..8c02691 --- /dev/null +++ b/backend/s3/main.js @@ -0,0 +1,894 @@ +/** + * Amazon S3 Backend + * + * The s3 backend reads and writes the objects of Amazon S3, and of the + * services that are compatible with S3, for example Cloudflare R2, MinIO and + * `rclone serve s3`, as the s3 backend of rclone. + * + * A path is `/bucket/path/to/file`. The path `/` is the list of the + * buckets, and `/bucket/` is the root of a bucket. + * + * The options, with the names of rclone: + * + * - `access_key_id`, `secret_access_key` and `session_token`: the + * credentials. With no credentials, the requests have no signature. These + * options are not in the links of a list, because they are secret. + * - `region`: `us-east-1` by default. For Cloudflare R2, `auto`. + * - `endpoint`: the URL of the service, by default the URL of Amazon S3 for + * the region. + * - `force_path_style`: `true` by default, for the URL + * `ENDPOINT/bucket/key`. With `false`, the URL is `bucket.ENDPOINT/key`. + * - `upload_cutoff`: 200 MiB by default. A file of a known size up to this + * size goes in one PUT. A larger file, or a file of unknown size that is + * larger than `chunk_size`, goes in parts. + * - `chunk_size`: 5 MiB by default, the size of a part. + * - `list_chunk`: 1000 by default, the number of objects of each list. + * - `list_url_encode`: `true` by default. The service puts the + * percent-encoding in the names of a list, thus a name can have control + * characters. + * - `directory_markers`: `false` by default, thus a folder exists only when + * it has a file, as in rclone. With `true`, the PUT of a folder or of a + * file puts a marker for the folder and for each parent folder that has + * no marker. A marker is an empty object with the name of the folder and + * a `/`, thus a folder can be empty. + * + * Each option can also come from a header or from `env`, with the name of + * the variable of rfetch, for example `RFETCH_S3_SECRET_ACCESS_KEY`, or of + * rclone, `RCLONE_S3_SECRET_ACCESS_KEY`. Put the credentials in a header or + * in `env`, not in a URL. + */ +import { createHash } from "node:crypto"; +import { renderListing } from "@rfetch/lib/listing"; +import { mimeType } from "@rfetch/lib/mime"; +import { optionsOf } from "@rfetch/lib/options"; +import { decodePath } from "@rfetch/lib/path"; +import { + notSatisfiable, + parseRange, + partialHeaders, + sliceStream, +} from "@rfetch/lib/range"; +import { parseSize } from "@rfetch/lib/size"; +import { sized } from "@rfetch/lib/sized"; +import { childrenOf, escapeXML, isXMLText, parseXML, textOf } from "./xml.js"; +import { EMPTY_PAYLOAD, sign, UNSIGNED_PAYLOAD, uriEncode } from "./sigv4.js"; + +export const options = { + string: [ + "access_key_id", + "secret_access_key", + "session_token", + "region", + "endpoint", + "upload_cutoff", + "chunk_size", + "list_chunk", + ], + boolean: ["force_path_style", "list_url_encode", "directory_markers"], + default: { + region: "us-east-1", + upload_cutoff: "200Mi", + chunk_size: "5Mi", + list_chunk: "1000", + force_path_style: "true", + list_url_encode: "true", + directory_markers: "false", + }, +}; + +/** The options that are secret. They are not in the links of a list. */ +const SECRETS = ["access_key_id", "secret_access_key", "session_token"]; + +/** The largest number of parts of an upload in parts. */ +const MAX_PARTS = 10000; + +/** The smallest size of a part, but not of the last part. */ +const MIN_CHUNK_SIZE = 5 * 1024 * 1024; + +/** + * The options of a request. + * @typedef {Object} Config + * @property {URL} endpoint + * @property {boolean} pathStyle + * @property {{ accessKeyId: string, secretAccessKey: string, + * sessionToken?: string, region: string } | null} credentials + * @property {number} uploadCutoff + * @property {number} chunkSize + * @property {number} listChunk + * @property {boolean} urlEncode + * @property {boolean} markers + */ + +/** An error with the status of the answer. */ +class S3Error extends Error { + /** @param {string} message @param {number} status */ + constructor(message, status) { + super(message); + this.status = status; + } +} + +/** + * Serves an S3 remote. + * @param {Request} request + * @param {Record} [env] The options from the variables + * `RFETCH_S3_*` or `RCLONE_S3_*`. + * @returns {Promise} + */ +async function s3(request, env = {}) { + const { method } = request; + const { pathname, searchParams } = new URL(request.url); + + let config; + try { + config = configOf(request, env); + } catch (error) { + return new Response(error.message, { status: 400 }); + } + const path = decodePath(pathname); + if (path === null) { + return new Response(null, { status: 400 }); + } + + const slash = path.indexOf("/", 1); + const bucket = slash === -1 ? path.slice(1) : path.slice(1, slash); + const key = slash === -1 ? "" : path.slice(slash + 1); + const isFolder = path.endsWith("/"); + if (!isFolder && key === "") { + // A file is in a bucket. + return new Response(null, { status: method === "PUT" ? 400 : 404 }); + } + + try { + if (method === "GET" || method === "HEAD") { + if (!isFolder) return await getFile(config, bucket, key, request); + const query = new URLSearchParams(searchParams); + for (const name of SECRETS) query.delete(name); + return await getFolder(config, bucket, key, method, `${query}`); + } + if (method === "PUT") { + if (bucket === "") { + return new Response(null, { status: 405, headers: { Allow: "GET" } }); + } + await (isFolder + ? putFolder(config, bucket, key) + : putFile(config, bucket, key, request)); + return new Response(null, { + status: 201, + headers: { "Content-Location": pathname }, + }); + } + if (method === "DELETE") { + if (bucket === "") { + return new Response(null, { status: 405, headers: { Allow: "GET" } }); + } + await (isFolder + ? deleteFolder(config, bucket, key) + : deleteFile(config, bucket, key)); + return new Response(null, { status: 204 }); + } + } catch (error) { + if (error instanceof S3Error) { + return new Response(error.message, { status: error.status }); + } + throw error; + } + return new Response(null, { + status: 405, + headers: { Allow: "GET, HEAD, PUT, DELETE" }, + }); +} + +/** + * Reads the options of a request, as `NewFs` of the s3 backend of rclone. + * @param {Request} request + * @param {Record} env + * @returns {Config} + * @throws {Error} When an option is not correct. + */ +function configOf(request, env) { + const values = optionsOf(request, "s3", env); + /** @param {string} name */ + const get = (name) => values.get(name) ?? options.default[name] ?? ""; + const region = get("region") || "us-east-1"; + let endpoint = get("endpoint") || + `https://s3.${region}.amazonaws.com`; + if (!/^https?:\/\//i.test(endpoint)) endpoint = `https://${endpoint}`; + let url; + try { + url = new URL(endpoint); + } catch { + throw new Error("The endpoint is not a URL"); + } + + const accessKeyId = get("access_key_id"); + const secretAccessKey = get("secret_access_key"); + if (Boolean(accessKeyId) !== Boolean(secretAccessKey)) { + throw new Error("access_key_id and secret_access_key go together"); + } + const sessionToken = get("session_token"); + + const chunkSize = parseSize(get("chunk_size")); + if (chunkSize < MIN_CHUNK_SIZE) { + throw new Error("chunk_size must be at least 5 MiB"); + } + const listChunk = Number(get("list_chunk")); + if (!Number.isInteger(listChunk) || listChunk < 1) { + throw new Error("list_chunk must be a number of 1 or more"); + } + + return { + endpoint: url, + pathStyle: get("force_path_style") !== "false", + credentials: accessKeyId + ? { accessKeyId, secretAccessKey, sessionToken, region } + : null, + uploadCutoff: parseSize(get("upload_cutoff")), + chunkSize, + listChunk, + urlEncode: get("list_url_encode") !== "false", + markers: get("directory_markers") === "true", + }; +} + +/** + * Sends a request to S3, with the signature of the credentials. + * @param {Config} config + * @param {string} method + * @param {string} bucket "" for the service. + * @param {string} key "" for the bucket. + * @param {Object} [init] + * @param {Record} [init.query] + * @param {Record} [init.headers] + * @param {BodyInit | null} [init.body] + * @param {number} [init.length] The size of the body. + * @param {string} [init.payload] The SHA-256 of the body, else the body has + * no signature. + * @returns {Promise} + * @throws {S3Error} When the service does not answer. + */ +async function send( + config, + method, + bucket, + key, + { query = {}, headers = {}, body = null, length, payload } = {}, +) { + const url = new URL(config.endpoint); + let path = url.pathname.replace(/\/$/, ""); + if (bucket && !config.pathStyle) { + url.hostname = `${bucket}.${url.hostname}`; + } else if (bucket) { + path += `/${uriEncode(bucket)}`; + } + if (key) path += `/${uriEncode(key, true)}`; + url.pathname = path || "/"; + // The query has the encoding of AWS, as the signature. + url.search = Object.entries(query) + .map(([name, value]) => `${uriEncode(name)}=${uriEncode(value)}`) + .join("&"); + + let signed = new Headers(headers); + if (config.credentials) { + signed = sign({ + method, + url, + headers: signed, + payload: payload ?? (body === null ? EMPTY_PAYLOAD : UNSIGNED_PAYLOAD), + }, config.credentials); + } + if (length !== undefined) signed.set("Content-Length", `${length}`); + // The size of a file is the size of its body: a service such as R2 can + // compress a text file for a client that accepts gzip. + signed.set("Accept-Encoding", "identity"); + + /** @type {RequestInit} */ + const init = { method, headers: signed }; + if (body !== null) { + init.body = body; + if (body instanceof ReadableStream) init.duplex = "half"; + } + try { + return await fetch(url, init); + } catch { + throw new S3Error("The S3 service did not answer", 502); + } +} + +/** + * Makes sure that an answer of S3 is a success, else throws its error. + * @param {Response} response + * @returns {Promise} + * @throws {S3Error} + */ +async function check(response) { + if (response.ok) return response; + const text = await response.text(); + let code = ""; + try { + code = textOf(parseXML(text), "Code"); + } catch { + // An answer with no XML, for example of HEAD. + } + throw new S3Error(code || response.statusText, response.status); +} + +/** + * Reads a name of a list, as rclone: with no percent-encoding when the + * list has the encoding `url`. As `url.QueryUnescape` of Go, a `+` is a + * space. + * @param {string} name + * @param {boolean} encoded + * @returns {string | null} null for an encoding that is not correct. + */ +function nameOf(name, encoded) { + if (!encoded) return name; + try { + return decodeURIComponent(name.replaceAll("+", " ")); + } catch { + return null; + } +} + +/** + * Gets a file, or only its headers for HEAD, with the range of the request. + * + * S3 gets a range that is correct, and its answer 206 goes to the client. + * When the service does not give a 206 with `Content-Range`, the backend + * makes the answer of RFC 9110 with the size of a HEAD request: some + * services, for example `rclone serve s3`, give the bytes of a range with + * 200. + * @param {Config} config + * @param {string} bucket + * @param {string} key + * @param {Request} request + * @returns {Promise} + */ +async function getFile(config, bucket, key, request) { + const { method } = request; + const header = method === "GET" ? request.headers.get("Range") : null; + let range = null; + try { + // Only the form of the range: the size is not known yet. + range = parseRange(header, Number.MAX_SAFE_INTEGER); + } catch { + // For example bytes=-0: no byte of a file is in the range. + return await getRange(config, bucket, key, header); + } + if (range) { + const response = await send(config, "GET", bucket, key, { + headers: { Range: header }, + }); + if (response.status === 206 && response.headers.has("Content-Range")) { + return fileResponse(response, key); + } + await response.body?.cancel(); + if (response.ok || response.status === 416) { + return await getRange(config, bucket, key, header); + } + return new Response(null, { status: response.status }); + } + const response = await send(config, method, bucket, key); + if (!response.ok) { + await response.body?.cancel(); + return new Response(null, { status: response.status }); + } + return fileResponse(response, key, method === "HEAD"); +} + +/** + * Gets a range of a file with the size of a HEAD request, as RFC 9110. + * @param {Config} config + * @param {string} bucket + * @param {string} key + * @param {string} header The `Range` of the request. + * @returns {Promise} + */ +async function getRange(config, bucket, key, header) { + const head = await send(config, "HEAD", bucket, key); + await head.body?.cancel(); + if (!head.ok) return new Response(null, { status: head.status }); + const size = Number(head.headers.get("Content-Length")); + let range; + try { + range = parseRange(header, size); + } catch { + return notSatisfiable(size); + } + if (!range) { + const response = await send(config, "GET", bucket, key); + if (!response.ok) { + await response.body?.cancel(); + return new Response(null, { status: response.status }); + } + return fileResponse(response, key); + } + + const response = await send(config, "GET", bucket, key, { + headers: { Range: `bytes=${range.start}-${range.end}` }, + }); + if (!response.ok) { + await response.body?.cancel(); + return new Response(null, { status: response.status }); + } + const length = range.end - range.start + 1; + let body = response.body; + // A service that sends all the file sends the bytes before the range too. + if ( + Number(response.headers.get("Content-Length")) === size && size !== length + ) { + body = sliceStream(body, range.start, length); + } + const result = fileHeaders(response, key); + for (const [name, value] of Object.entries(partialHeaders(range, size))) { + result.set(name, value); + } + return new Response(sized(body, length), { status: 206, headers: result }); +} + +/** + * The answer for a file, with the headers of the answer of S3. + * @param {Response} response + * @param {string} key + * @param {boolean} [head] True for no body. + * @returns {Response} + */ +function fileResponse(response, key, head = false) { + const headers = fileHeaders(response, key); + if (head) { + response.body?.cancel(); + return new Response(null, { headers }); + } + return new Response(response.body, { status: response.status, headers }); +} + +/** + * The headers of a file, from the answer of S3. + * @param {Response} response + * @param {string} key + * @returns {Headers} + */ +function fileHeaders(response, key) { + const headers = new Headers({ + "Content-Type": response.headers.get("Content-Type") || mimeType(key), + "Accept-Ranges": "bytes", + }); + for (const name of ["Content-Length", "Content-Range", "Last-Modified"]) { + const value = response.headers.get(name); + if (value) headers.set(name, value); + } + return headers; +} + +/** + * Gets the list of a folder, or only its headers for HEAD. + * @param {Config} config + * @param {string} bucket "" for the list of the buckets. + * @param {string} prefix The path of the folder in the bucket, with a `/` + * at the end, or "" for the root of the bucket. + * @param {string} method + * @param {string} query The query of each link. + * @returns {Promise} + */ +async function getFolder(config, bucket, prefix, method, query) { + const entries = bucket === "" + ? await listBuckets(config) + : await listFolder(config, bucket, prefix); + if (entries === null) return new Response(null, { status: 404 }); + const headers = { "Content-Type": "text/html;charset=utf-8" }; + if (method === "HEAD") return new Response(null, { headers }); + return new Response(renderListing(entries, { query }), { headers }); +} + +/** + * The buckets, as folders. + * @param {Config} config + * @returns {Promise} + */ +async function listBuckets(config) { + const response = await check(await send(config, "GET", "", "")); + const root = parseXML(await response.text()); + return childrenOf(childrenOf(root, "Buckets")[0], "Bucket").map((bucket) => ({ + Name: textOf(bucket, "Name"), + IsDir: true, + Size: -1, + MimeType: "", + ModTime: textOf(bucket, "CreationDate"), + })); +} + +/** + * The files and the folders in a folder, as `list` of the s3 backend of + * rclone with a delimiter. + * @param {Config} config + * @param {string} bucket + * @param {string} prefix "" for the root of the bucket. + * @returns {Promise} null when + * the folder does not exist. + */ +async function listFolder(config, bucket, prefix) { + /** @type {import("@rfetch/lib/listing").Entry[]} */ + const entries = []; + let found = false; + let token = ""; + do { + const query = { + "list-type": "2", + delimiter: "/", + prefix, + "max-keys": `${config.listChunk}`, + }; + if (config.urlEncode) query["encoding-type"] = "url"; + if (token) query["continuation-token"] = token; + const response = await send(config, "GET", bucket, "", { query }); + if (response.status === 404) { + await response.body?.cancel(); + return null; + } + const root = parseXML(await (await check(response)).text()); + const encoded = textOf(root, "EncodingType") === "url"; + + for (const common of childrenOf(root, "CommonPrefixes")) { + const name = nameOf(textOf(common, "Prefix"), encoded); + if (name === null || !name.startsWith(prefix)) continue; + found = true; + const folder = name.slice(prefix.length).replace(/\/$/, ""); + if (folder === "") continue; + entries.push({ + Name: folder, + IsDir: true, + Size: -1, + MimeType: "", + ModTime: "", + }); + } + for (const object of childrenOf(root, "Contents")) { + const name = nameOf(textOf(object, "Key"), encoded); + if (name === null || !name.startsWith(prefix)) continue; + found = true; + const file = name.slice(prefix.length); + // The marker of the folder is not a file in it. + if (file === "" || file.endsWith("/")) continue; + entries.push({ + Name: file, + IsDir: false, + Size: Number(textOf(object, "Size")), + MimeType: mimeType(file), + ModTime: textOf(object, "LastModified"), + }); + } + token = textOf(root, "IsTruncated") === "true" + ? textOf(root, "NextContinuationToken") + : ""; + } while (token); + + // The root of a bucket exists with no file. An empty folder exists when it + // has a marker, as rclone checks it. + if (!found && prefix !== "") { + const response = await send(config, "HEAD", bucket, prefix); + await response.body?.cancel(); + if (!response.ok) return null; + } + return entries; +} + +/** + * Makes a bucket, or a marker of a folder with the option + * `directory_markers`. With no marker, a folder exists when it has a file. + * @param {Config} config + * @param {string} bucket + * @param {string} prefix + */ +async function putFolder(config, bucket, prefix) { + if (prefix === "") { + const head = await send(config, "HEAD", bucket, ""); + await head.body?.cancel(); + if (head.ok) return; + const response = await send(config, "PUT", bucket, "", { + body: new Uint8Array(0), + length: 0, + payload: EMPTY_PAYLOAD, + }); + await check(response); + await response.body?.cancel(); + return; + } + if (config.markers) await putMarkers(config, bucket, prefix); +} + +/** + * Puts the markers of a folder and of its parent folders, as rclone: from + * the folder to the root of the bucket, until a marker exists. + * @param {Config} config + * @param {string} bucket + * @param {string} prefix The folder, with a `/` at the end. + */ +async function putMarkers(config, bucket, prefix) { + while (prefix !== "") { + const head = await send(config, "HEAD", bucket, prefix); + await head.body?.cancel(); + if (head.ok) return; + const response = await send(config, "PUT", bucket, prefix, { + body: new Uint8Array(0), + length: 0, + payload: EMPTY_PAYLOAD, + }); + await check(response); + await response.body?.cancel(); + prefix = prefix.slice(0, prefix.lastIndexOf("/", prefix.length - 2) + 1); + } +} + +/** + * Puts a file: in one PUT up to `upload_cutoff`, else in parts, as rclone. + * With `directory_markers`, the parent folders get markers first. + * @param {Config} config + * @param {string} bucket + * @param {string} key + * @param {Request} request + */ +async function putFile(config, bucket, key, request) { + if (config.markers) { + await putMarkers(config, bucket, key.slice(0, key.lastIndexOf("/") + 1)); + } + const header = request.headers.get("Content-Length"); + const length = header !== null && /^\d+$/.test(header) + ? Number(header) + : null; + const headers = { + "Content-Type": request.headers.get("Content-Type") || mimeType(key), + }; + + if (length !== null && length <= config.uploadCutoff) { + const body = sized(request.body ?? new Blob([]).stream(), length); + const response = await send(config, "PUT", bucket, key, { + headers, + body, + length, + }); + await check(response); + await response.body?.cancel(); + return; + } + + // A part has the size of chunk_size, or more for a large file. + const partSize = length === null + ? config.chunkSize + : Math.max(config.chunkSize, Math.ceil(length / MAX_PARTS)); + const parts = new Parts(request.body ?? new Blob([]).stream(), partSize); + const first = await parts.next(); + if (length === null && parts.done) { + // A small file of unknown size. + const response = await send(config, "PUT", bucket, key, { + headers, + body: first, + length: first.length, + payload: sha256(first), + }); + await check(response); + await response.body?.cancel(); + return; + } + + const create = await check( + await send(config, "POST", bucket, key, { + query: { uploads: "" }, + headers, + }), + ); + const uploadId = textOf(parseXML(await create.text()), "UploadId"); + const upload = { uploadId }; + try { + /** @type {string[]} */ + const etags = []; + for (let part = first; part; part = await parts.next()) { + if (etags.length === MAX_PARTS) { + throw new S3Error("too many parts", 413); + } + const response = await check( + await send(config, "PUT", bucket, key, { + query: { partNumber: `${etags.length + 1}`, ...upload }, + body: part, + length: part.length, + payload: sha256(part), + }), + ); + await response.body?.cancel(); + etags.push(response.headers.get("ETag") ?? ""); + if (parts.done) break; + } + const xml = new TextEncoder().encode( + `${ + etags.map((etag, index) => + `${index + 1}${ + escapeXML(etag) + }` + ).join("") + }`, + ); + const complete = await check( + await send(config, "POST", bucket, key, { + query: upload, + body: xml, + length: xml.length, + payload: sha256(xml), + }), + ); + // S3 can give an error in an answer 200. + const result = parseXML(await complete.text()); + if (result.name === "Error") { + throw new S3Error(textOf(result, "Code"), 502); + } + } catch (error) { + const abort = await send(config, "DELETE", bucket, key, { query: upload }); + await abort.body?.cancel(); + throw error; + } +} + +/** + * Deletes a file. As the other backends, a file that does not exist gives + * 404: S3 gives no error for it. + * @param {Config} config + * @param {string} bucket + * @param {string} key + */ +async function deleteFile(config, bucket, key) { + const head = await send(config, "HEAD", bucket, key); + await head.body?.cancel(); + if (head.status === 404) throw new S3Error("Not Found", 404); + await check(head); + const response = await check(await send(config, "DELETE", bucket, key)); + await response.body?.cancel(); +} + +/** + * Deletes a folder and all that is in it, with lists with no delimiter and + * DeleteObjects of 1000 objects. The root of a bucket deletes the bucket + * too, as `rclone purge`. + * @param {Config} config + * @param {string} bucket + * @param {string} prefix + */ +async function deleteFolder(config, bucket, prefix) { + let found = false; + let token = ""; + do { + const query = { + "list-type": "2", + prefix, + "max-keys": `${Math.min(config.listChunk, 1000)}`, + }; + if (config.urlEncode) query["encoding-type"] = "url"; + if (token) query["continuation-token"] = token; + const response = await send(config, "GET", bucket, "", { query }); + if (response.status === 404) throw new S3Error("Not Found", 404); + const root = parseXML(await (await check(response)).text()); + const encoded = textOf(root, "EncodingType") === "url"; + const keys = childrenOf(root, "Contents") + .map((object) => nameOf(textOf(object, "Key"), encoded)) + .filter((key) => key !== null); + if (keys.length) { + found = true; + await deleteObjects(config, bucket, keys); + } + token = textOf(root, "IsTruncated") === "true" + ? textOf(root, "NextContinuationToken") + : ""; + } while (token); + + if (prefix === "") { + const response = await check(await send(config, "DELETE", bucket, "")); + await response.body?.cancel(); + return; + } + if (!found) { + // An empty folder with a marker. + const head = await send(config, "HEAD", bucket, prefix); + await head.body?.cancel(); + if (!head.ok) throw new S3Error("Not Found", 404); + const response = await check(await send(config, "DELETE", bucket, prefix)); + await response.body?.cancel(); + } +} + +/** + * Deletes objects with one request. + * @param {Config} config + * @param {string} bucket + * @param {string[]} keys At most 1000 keys. + */ +async function deleteObjects(config, bucket, keys) { + // A key that XML cannot have, for example with a vertical tab, gets a + // DeleteObject. + for (const key of keys.filter((key) => !isXMLText(key))) { + const response = await check(await send(config, "DELETE", bucket, key)); + await response.body?.cancel(); + } + keys = keys.filter(isXMLText); + if (keys.length === 0) return; + const xml = new TextEncoder().encode( + `true${ + keys.map((key) => `${escapeXML(key)}`) + .join("") + }`, + ); + const response = await check( + await send(config, "POST", bucket, "", { + query: { delete: "" }, + headers: { + "Content-MD5": createHash("md5").update(xml).digest("base64"), + }, + body: xml, + length: xml.length, + payload: sha256(xml), + }), + ); + const result = parseXML(await response.text()); + const error = childrenOf(result, "Error")[0]; + if (error) { + throw new S3Error(`DeleteObjects: ${textOf(error, "Code")}`, 502); + } +} + +/** @param {Uint8Array} bytes */ +function sha256(bytes) { + return createHash("sha256").update(bytes).digest("hex"); +} + +/** + * Cuts a stream into parts of a size: each part has the size, but the last + * part can be smaller. + */ +class Parts { + /** @param {ReadableStream} stream @param {number} size */ + constructor(stream, size) { + this.reader = stream.getReader(); + this.size = size; + /** @type {Uint8Array} */ + this.pending = new Uint8Array(0); + /** True when the stream has no more data. */ + this.done = false; + } + + /** + * The next part, or null at the end. + * @returns {Promise} + */ + async next() { + const chunks = this.pending.length ? [this.pending] : []; + let length = this.pending.length; + this.pending = new Uint8Array(0); + while (length < this.size && !this.done) { + const { done, value } = await this.reader.read(); + if (done) { + this.done = true; + break; + } + chunks.push(value); + length += value.length; + } + if (length === 0 && this.done) return null; + const bytes = new Uint8Array(length); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.length; + } + if (length > this.size) { + this.pending = bytes.slice(this.size); + return bytes.subarray(0, this.size); + } + // A part of the size can be the last part: the next read tells it. + if (!this.done && this.pending.length === 0) { + const { done, value } = await this.reader.read(); + if (done) this.done = true; + else this.pending = value; + } + return bytes; + } +} + +export default { + fetch: s3, +}; diff --git a/backend/s3/main_test.js b/backend/s3/main_test.js new file mode 100644 index 0000000..0938389 --- /dev/null +++ b/backend/s3/main_test.js @@ -0,0 +1,208 @@ +/** + * The tests of the s3 backend, on `rclone serve s3`: a server of the S3 API + * on a local folder. They run when rclone is in the PATH, or when RCLONE is + * the path of rclone. + */ +import { after, describe, test } from "node:test"; +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import process from "node:process"; + +import { backendSuite } from "@rfetch/lib/testing/suite"; +import { remote } from "@rfetch/lib/testing/remote"; +import { listen } from "@rfetch/lib/testing/server"; +import s3 from "./main.js"; +import { KEYS, serveS3 } from "./testing.js"; + +const RCLONE = process.env.RCLONE || "rclone"; +const hasRclone = spawnSync(RCLONE, ["version"]).status === 0; + +if (hasRclone) { + const server = await serveS3(); + after(() => server.stop()); + const params = { ...KEYS, endpoint: server.endpoint, region: "us-east-1" }; + + // S3 has no empty folders with no markers, as rclone. + backendSuite("s3", s3, { + type: "s3", + params, + parent: "/rfetch/", + emptyFolders: false, + }); + // Files in parts: upload_cutoff of 100 KiB, and parts of 5 MiB. + backendSuite("s3 in parts", s3, { + type: "s3", + params: { ...params, upload_cutoff: "100k" }, + parent: "/rfetch/", + emptyFolders: false, + }); + + describe("s3", () => { + const fs = remote(s3, { params, root: "/rfetch/s3-test/" }); + + test("puts no credentials in the links of a list", async () => { + await fs.put("a b.txt", new TextEncoder().encode("a")); + const html = await fs.get("").then((response) => response.text()); + assert.ok(html.includes(' href="a%20b.txt?endpoint=')); + assert.ok(!html.includes("rfetch-key")); + assert.ok(!html.includes("rfetch-secret")); + }); + + test("lists the buckets", async () => { + const root = remote(s3, { params }); + assert.deepEqual( + (await root.list("")).map(({ Name, IsDir }) => ({ Name, IsDir })), + [{ Name: "rfetch", IsDir: true }], + ); + }); + + test("puts a file of unknown size in parts", async () => { + const size = 5 * 1024 * 1024 + 1000; + const data = new Uint8Array(size).map((_, i) => (i * 31) & 0xff); + const response = await fs.send("parts.bin", { + method: "PUT", + body: new Blob([data]).stream(), + }); + assert.equal(response.status, 201); + const got = await fs.get("parts.bin"); + assert.equal(got.headers.get("Content-Length"), `${size}`); + assert.deepEqual(new Uint8Array(await got.arrayBuffer()), data); + }); + + test("takes the options from env", async () => { + const fromEnv = remote(s3, { + params: { endpoint: server.endpoint }, + env: { + RCLONE_S3_ACCESS_KEY_ID: KEYS.access_key_id, + RCLONE_S3_SECRET_ACCESS_KEY: KEYS.secret_access_key, + }, + root: "/rfetch/s3-test/", + }); + const response = await fromEnv.get("a b.txt"); + assert.equal(await response.text(), "a"); + }); + + test("gives 403 with wrong credentials", async () => { + const wrong = remote(s3, { + params: { ...params, secret_access_key: "wrong" }, + root: "/rfetch/s3-test/", + }); + const response = await wrong.get("a b.txt"); + await response.body?.cancel(); + assert.equal(response.status, 403); + }); + + test("gives 400 for options that are not correct", async () => { + for ( + const options of [ + { secret_access_key: "" }, + { chunk_size: "1M" }, + { list_chunk: "0" }, + { endpoint: "http://[" }, + ] + ) { + const response = await remote(s3, { params: { ...params, ...options } }) + .get(""); + await response.body?.cancel(); + assert.equal(response.status, 400, JSON.stringify(options)); + } + }); + + test("deletes the files of the tests", async () => { + const response = await fs.delete(""); + assert.equal(response.status, 204); + }); + }); +} else { + test("s3", { skip: "rclone is not installed. Set RCLONE to its path." }); +} + +test("puts the markers of the parent folders, as rclone", async () => { + // A service that keeps the names of the objects, and records the requests. + const keys = new Set(); + const log = []; + const server = await listen(async (request) => { + const { pathname } = new URL(request.url); + log.push(`${request.method} ${pathname}`); + await request.body?.cancel(); + if (request.method === "HEAD") { + return new Response(null, { status: keys.has(pathname) ? 200 : 404 }); + } + keys.add(pathname); + return new Response(null, { headers: { ETag: '"0"' } }); + }); + after(() => server.close()); + + const params = { endpoint: `${server.url}`, directory_markers: "true" }; + const fs = remote(s3, { params, root: "/bucket/" }); + /** @param {string} path @param {string[]} requests */ + const putGives = async (path, requests) => { + log.length = 0; + const response = await fs.put(path, new Uint8Array(0)); + assert.equal(response.status, 201, path); + assert.deepEqual(log, requests, path); + }; + + await putGives("a/b/c.txt", [ + "HEAD /bucket/a/b/", + "PUT /bucket/a/b/", + "HEAD /bucket/a/", + "PUT /bucket/a/", + "PUT /bucket/a/b/c.txt", + ]); + // The parent folder a/ has a marker. + await putGives("a/d/e.txt", [ + "HEAD /bucket/a/d/", + "PUT /bucket/a/d/", + "HEAD /bucket/a/", + "PUT /bucket/a/d/e.txt", + ]); + await putGives("a/f/", [ + "HEAD /bucket/a/f/", + "PUT /bucket/a/f/", + "HEAD /bucket/a/", + ]); + await putGives("g.txt", ["PUT /bucket/g.txt"]); + + // With no markers, a file is one PUT. + log.length = 0; + const plain = remote(s3, { + params: { endpoint: `${server.url}` }, + root: "/bucket/", + }); + assert.equal((await plain.put("h/i.txt", new Uint8Array(0))).status, 201); + assert.deepEqual(log, ["PUT /bucket/h/i.txt"]); +}); + +// A run on a real service, for example Cloudflare R2, when S3_TEST_ENDPOINT +// and S3_TEST_BUCKET are set. The credentials are in RFETCH_S3_ACCESS_KEY_ID +// and RFETCH_S3_SECRET_ACCESS_KEY, or in the variables of rclone with the +// prefix RCLONE_, thus they are not in a URL. +const { S3_TEST_ENDPOINT, S3_TEST_BUCKET, S3_TEST_REGION } = process.env; +if (S3_TEST_ENDPOINT && S3_TEST_BUCKET) { + /** @param {string} name */ + const variable = (name) => + process.env[`RFETCH_S3_${name}`] ?? process.env[`RCLONE_S3_${name}`]; + const env = { + RFETCH_S3_ACCESS_KEY_ID: variable("ACCESS_KEY_ID"), + RFETCH_S3_SECRET_ACCESS_KEY: variable("SECRET_ACCESS_KEY"), + }; + const params = { + endpoint: S3_TEST_ENDPOINT, + region: S3_TEST_REGION ?? "auto", + }; + const parent = `/${S3_TEST_BUCKET}/`; + backendSuite("s3 service", s3, { + type: "s3", + params, + env, + parent, + emptyFolders: false, + }); + backendSuite("s3 service with folder markers", s3, { + type: "s3", + params: { ...params, directory_markers: "true" }, + env, + parent, + }); +} diff --git a/backend/s3/package.json b/backend/s3/package.json new file mode 100644 index 0000000..6c3aba3 --- /dev/null +++ b/backend/s3/package.json @@ -0,0 +1,28 @@ +{ + "name": "@rfetch/s3", + "version": "0.1.0", + "description": "The s3 backend of rfetch: Amazon S3 and the services with the same API, for example Cloudflare R2.", + "type": "module", + "exports": { + ".": "./main.js" + }, + "scripts": { + "test": "node --test" + }, + "engines": { + "node": ">=22.2.0" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/sntran/rfetch.git", + "directory": "backend/s3" + }, + "author": "Trần Nguyễn Sơn", + "license": "ISC", + "dependencies": { + "@rfetch/lib": "^0.1.0" + }, + "devDependencies": { + "fast-check": "^3.19.0" + } +} diff --git a/backend/s3/rclone_test.js b/backend/s3/rclone_test.js new file mode 100644 index 0000000..70d75e7 --- /dev/null +++ b/backend/s3/rclone_test.js @@ -0,0 +1,131 @@ +/** + * The tests of the s3 backend against the s3 backend of rclone, on the same + * `rclone serve s3`: each one reads what the other writes. The tests run + * when rclone is in the PATH, or when RCLONE is the path of rclone. + */ +import { after, before, describe, test } from "node:test"; +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import process from "node:process"; + +import { remote } from "@rfetch/lib/testing/remote"; +import { KEYS, serveS3 } from "./testing.js"; +import s3 from "./main.js"; + +const RCLONE = process.env.RCLONE || "rclone"; +const hasRclone = spawnSync(RCLONE, ["version"]).status === 0; + +/** The files: hard names, and sizes in one PUT and in parts. */ +const FILES = { + "empty.txt": 0, + "a/one.bin": 1, + "a/b/c.json": 100, + "file name #1.txt": 2048, + [`hello? sausage/êé/Hello, 世界/ " ' @ < > & ? + ≠/z.txt`]: 3000, + "parts/large.bin": 6 * 1024 * 1024 + 7, +}; + +/** @param {number} size */ +function content(size) { + return new Uint8Array(size).map((_, index) => (index * 13 + size) & 0xff); +} + +describe("s3 and rclone", { + skip: !hasRclone && "rclone is not installed. Set RCLONE to its path.", +}, () => { + let server; + /** The connection string of the s3 backend of rclone for the server. */ + let target; + /** @param {string[]} args */ + const rclone = (...args) => { + const { status, stdout, stderr } = spawnSync(RCLONE, args, { + env: { PATH: process.env.PATH, HOME: server.folder }, + maxBuffer: 64 * 1024 * 1024, + }); + assert.equal(status, 0, `rclone ${args[0]}: ${stderr}`); + return stdout; + }; + + before(async () => { + server = await serveS3(); + target = `:s3,provider=Other,access_key_id=${KEYS.access_key_id},` + + `secret_access_key=${KEYS.secret_access_key},` + + `endpoint='${server.endpoint}':rfetch`; + for (const [path, size] of Object.entries(FILES)) { + mkdirSync(dirname(join(server.folder, "plain", path)), { + recursive: true, + }); + writeFileSync(join(server.folder, "plain", path), content(size)); + } + }); + after(() => server?.stop()); + + const rfetch = (root) => + remote(s3, { + params: { endpoint: server.endpoint, upload_cutoff: "1M" }, + env: { + RCLONE_S3_ACCESS_KEY_ID: KEYS.access_key_id, + RCLONE_S3_SECRET_ACCESS_KEY: KEYS.secret_access_key, + }, + root, + }); + + /** The paths and the sizes of the files in a folder, from the lists. */ + async function walk(fs) { + const files = []; + const next = async (path) => { + for (const entry of await fs.list(path)) { + if (entry.IsDir) await next(`${path}${entry.Name}/`); + else files.push({ Path: `${path}${entry.Name}`, Size: entry.Size }); + } + }; + await next(""); + return files.sort((a, b) => a.Path < b.Path ? -1 : +(a.Path > b.Path)); + } + + test("rfetch reads what rclone writes", async () => { + rclone("copy", join(server.folder, "plain"), `${target}/by-rclone`); + const fs = rfetch("/rfetch/by-rclone/"); + for (const [path, size] of Object.entries(FILES)) { + const response = await fs.get(path); + assert.equal(response.status, 200, path); + assert.equal(response.headers.get("Content-Length"), `${size}`, path); + assert.deepEqual( + new Uint8Array(await response.arrayBuffer()), + content(size), + path, + ); + } + const expected = JSON.parse( + rclone("lsjson", "-R", "--files-only", `${target}/by-rclone`), + ).map(({ Path, Size }) => ({ Path, Size })) + .sort((a, b) => a.Path < b.Path ? -1 : +(a.Path > b.Path)); + assert.deepEqual(await walk(fs), expected); + }); + + test("rclone reads what rfetch writes", async () => { + const fs = rfetch("/rfetch/by-rfetch/"); + for (const [path, size] of Object.entries(FILES)) { + const response = await fs.put(path, content(size)); + assert.equal(response.status, 201, path); + } + rclone("check", join(server.folder, "plain"), `${target}/by-rfetch`); + rclone("copy", `${target}/by-rfetch`, join(server.folder, "out")); + for (const [path, size] of Object.entries(FILES)) { + assert.deepEqual( + new Uint8Array(readFileSync(join(server.folder, "out", path))), + content(size), + path, + ); + } + }); + + test("rclone purge leaves nothing that rfetch lists", async () => { + const fs = rfetch("/rfetch/by-rfetch/"); + assert.equal((await fs.delete("")).status, 204); + rclone("purge", `${target}/by-rclone`); + assert.deepEqual(await rfetch("/rfetch/").list(""), []); + }); +}); diff --git a/backend/s3/sigv4.js b/backend/s3/sigv4.js new file mode 100644 index 0000000..d1ed5d4 --- /dev/null +++ b/backend/s3/sigv4.js @@ -0,0 +1,107 @@ +/** + * The Signature Version 4 of AWS for S3: the header `Authorization` of a + * request, from the method, the path, the query, the headers and the hash of + * the body. + */ +import { createHash, createHmac } from "node:crypto"; + +/** The hash of a body that the signature does not cover. */ +export const UNSIGNED_PAYLOAD = "UNSIGNED-PAYLOAD"; + +/** The hash of an empty body. */ +export const EMPTY_PAYLOAD = + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + +/** + * Puts the percent-encoding of AWS in a text: all the bytes except + * `A-Z a-z 0-9 - . _ ~`, and also `/` when `slash` is false. + * @param {string} text + * @param {boolean} [slash] True to keep `/`, for a path. + * @returns {string} + */ +export function uriEncode(text, slash = false) { + const encoded = encodeURIComponent(text).replace( + /[!'()*]/g, + (c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`, + ); + return slash ? encoded.replaceAll("%2F", "/") : encoded; +} + +/** + * Signs a request. + * @param {Object} request + * @param {string} request.method + * @param {URL} request.url The path of the URL has the percent-encoding of + * `uriEncode`. The query is any. + * @param {Headers} request.headers The headers to sign. They get + * `x-amz-date`, `x-amz-content-sha256`, `x-amz-security-token` for a + * session token, and `Authorization`. + * @param {string} [request.payload] The SHA-256 of the body in hexadecimal, + * or `UNSIGNED-PAYLOAD`. + * @param {Object} credentials + * @param {string} credentials.accessKeyId + * @param {string} credentials.secretAccessKey + * @param {string} [credentials.sessionToken] + * @param {string} credentials.region + * @param {string} [credentials.service] + * @param {Date} [now] + * @returns {Headers} The headers of the request. + */ +export function sign( + { method, url, headers, payload = EMPTY_PAYLOAD }, + { accessKeyId, secretAccessKey, sessionToken, region, service = "s3" }, + now = new Date(), +) { + const amzDate = now.toISOString().replace(/[-:]|\.\d{3}/g, ""); + const date = amzDate.slice(0, 8); + headers = new Headers(headers); + headers.set("host", url.host); + headers.set("x-amz-date", amzDate); + headers.set("x-amz-content-sha256", payload); + if (sessionToken) headers.set("x-amz-security-token", sessionToken); + + // The query, sorted by the names and the values, with the encoding of AWS. + const query = [...url.searchParams] + .map(([name, value]) => [uriEncode(name), uriEncode(value)]) + .sort(([a, x], [b, y]) => a < b ? -1 : a > b ? 1 : x < y ? -1 : +(x > y)) + .map(([name, value]) => `${name}=${value}`) + .join("&"); + const names = [...headers.keys()].sort(); + const canonical = [ + method, + url.pathname, + query, + ...names.map((name) => + `${name}:${headers.get(name).trim().replace(/\s+/g, " ")}` + ), + "", + names.join(";"), + payload, + ].join("\n"); + + const scope = `${date}/${region}/${service}/aws4_request`; + const toSign = [ + "AWS4-HMAC-SHA256", + amzDate, + scope, + createHash("sha256").update(canonical).digest("hex"), + ].join("\n"); + + let key = hmac(`AWS4${secretAccessKey}`, date); + for (const part of [region, service, "aws4_request"]) key = hmac(key, part); + const signature = createHmac("sha256", key).update(toSign).digest("hex"); + + headers.set( + "Authorization", + `AWS4-HMAC-SHA256 Credential=${accessKeyId}/${scope}, ` + + `SignedHeaders=${names.join(";")}, Signature=${signature}`, + ); + // The host of the URL goes in the request. + headers.delete("host"); + return headers; +} + +/** @param {string | Buffer} key @param {string} data */ +function hmac(key, data) { + return createHmac("sha256", key).update(data).digest(); +} diff --git a/backend/s3/sigv4_test.js b/backend/s3/sigv4_test.js new file mode 100644 index 0000000..f20c2ba --- /dev/null +++ b/backend/s3/sigv4_test.js @@ -0,0 +1,105 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { sign, uriEncode } from "./sigv4.js"; + +/** + * The examples of "Signature Calculations for the Authorization Header" in + * the documentation of Amazon S3: the key, the date, and the signatures. + */ +const CREDENTIALS = { + accessKeyId: "AKIAIOSFODNN7EXAMPLE", + secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + region: "us-east-1", +}; +const NOW = new Date("2013-05-24T00:00:00Z"); + +/** @param {Headers} headers */ +function signatureOf(headers) { + return headers.get("Authorization").match(/Signature=([0-9a-f]+)$/)[1]; +} + +test("signs the GET of an object with a range", () => { + const headers = sign( + { + method: "GET", + url: new URL("https://examplebucket.s3.amazonaws.com/test.txt"), + headers: new Headers({ Range: "bytes=0-9" }), + }, + CREDENTIALS, + NOW, + ); + assert.equal( + headers.get("Authorization"), + "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request, " + + "SignedHeaders=host;range;x-amz-content-sha256;x-amz-date, " + + "Signature=f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41", + ); +}); + +test("signs the PUT of an object", () => { + const headers = sign( + { + method: "PUT", + url: new URL( + `https://examplebucket.s3.amazonaws.com/${ + uriEncode("test$file.text", true) + }`, + ), + headers: new Headers({ + Date: "Fri, 24 May 2013 00:00:00 GMT", + "x-amz-storage-class": "REDUCED_REDUNDANCY", + }), + payload: + "44ce7dd67c959e0d3524ffac1771dfbba87d2b6b4b4e99e42034a8b803f8b072", + }, + CREDENTIALS, + NOW, + ); + assert.equal( + signatureOf(headers), + "98ad721746da40c64f1a55b78f14c238d841ea1380cd77a1b5971af0ece108bd", + ); +}); + +test("signs a list with a query", () => { + const headers = sign( + { + method: "GET", + url: new URL( + "https://examplebucket.s3.amazonaws.com/?max-keys=2&prefix=J", + ), + headers: new Headers(), + }, + CREDENTIALS, + NOW, + ); + assert.equal( + signatureOf(headers), + "34b48302e7b5fa45bde8084f4b7868a86f0a534bc59db6670ed5711ef69dc6f7", + ); +}); + +test("signs a query with a name and no value", () => { + const headers = sign( + { + method: "GET", + url: new URL("https://examplebucket.s3.amazonaws.com/?lifecycle"), + headers: new Headers(), + }, + CREDENTIALS, + NOW, + ); + assert.equal( + signatureOf(headers), + "fea454ca298b7da1c68078a5d1bdbfbbe0d65c699e0f91ac7a200a0136783543", + ); +}); + +test("uriEncode keeps only the bytes that AWS keeps", () => { + assert.equal(uriEncode("a-z_A.Z~0 9"), "a-z_A.Z~0%209"); + assert.equal(uriEncode("!'()*"), "%21%27%28%29%2A"); + assert.equal(uriEncode("a/b c", true), "a/b%20c"); + assert.equal(uriEncode("a/b"), "a%2Fb"); + assert.equal(uriEncode("é+&="), "%C3%A9%2B%26%3D"); +}); diff --git a/backend/s3/testing.js b/backend/s3/testing.js new file mode 100644 index 0000000..477198f --- /dev/null +++ b/backend/s3/testing.js @@ -0,0 +1,60 @@ +/** + * A test server of the S3 API: `rclone serve s3` on a local folder, with the + * bucket `rfetch`. + */ +import { spawn } from "node:child_process"; +import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import { createServer } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import process from "node:process"; + +const RCLONE = process.env.RCLONE || "rclone"; + +/** The credentials of the test server. */ +export const KEYS = { + access_key_id: "rfetch-key", + secret_access_key: "rfetch-secret", +}; + +/** + * Starts `rclone serve s3` on a free port, with the bucket `rfetch`. + * @returns {Promise<{ endpoint: string, folder: string, stop: () => void }>} + */ +export async function serveS3() { + const folder = mkdtempSync(join(tmpdir(), "rfetch-s3-")); + mkdirSync(join(folder, "data", "rfetch"), { recursive: true }); + const port = await new Promise((resolve) => { + const server = createServer().listen(0, "127.0.0.1", () => { + const { port } = server.address(); + server.close(() => resolve(port)); + }); + }); + const child = spawn(RCLONE, [ + "serve", + "s3", + "--auth-key", + `${KEYS.access_key_id},${KEYS.secret_access_key}`, + "--addr", + `127.0.0.1:${port}`, + join(folder, "data"), + ], { env: { PATH: process.env.PATH, HOME: folder }, stdio: "ignore" }); + const endpoint = `http://127.0.0.1:${port}`; + // Waits until the server answers. + for (let i = 0; i < 100; i++) { + try { + await fetch(endpoint); + break; + } catch { + await new Promise((resolve) => setTimeout(resolve, 100)); + } + } + return { + endpoint, + folder, + stop() { + child.kill(); + rmSync(folder, { recursive: true, force: true }); + }, + }; +} diff --git a/backend/s3/xml.js b/backend/s3/xml.js new file mode 100644 index 0000000..3886da7 --- /dev/null +++ b/backend/s3/xml.js @@ -0,0 +1,121 @@ +/** + * A small reader of XML, for the answers of APIs such as S3: elements, text, + * CDATA and the entities. It does not read a DTD, and it does not keep the + * attributes. A name has no namespace prefix: `s3:Key` is `Key`. + */ + +/** + * An element of an XML document. + * @typedef {Object} XMLElement + * @property {string} name The name with no namespace prefix. + * @property {XMLElement[]} children The elements in the element. + * @property {string} text The text in the element, with no text of the + * elements in it. + */ + +const TOKENS = + /|<\?[\s\S]*?\?>||]*>|<(\/?)([^\s/>]+)[^>]*?(\/?)>|([^<]+)/g; + +const ENTITIES = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'" }; + +/** + * Reads an XML document. + * @param {string} text + * @returns {XMLElement} The root element. + * @throws {SyntaxError} When the elements do not close in order. + */ +export function parseXML(text) { + /** @type {XMLElement} */ + const document = { name: "", children: [], text: "" }; + const stack = [document]; + for (const match of text.matchAll(TOKENS)) { + const [, cdata, close, name, empty, chars] = match; + const top = stack.at(-1); + if (cdata !== undefined) { + top.text += cdata; + } else if (chars !== undefined) { + top.text += decodeEntities(chars); + } else if (name !== undefined) { + const local = name.slice(name.indexOf(":") + 1); + if (close) { + if (stack.length === 1 || top.name !== local) { + throw new SyntaxError(`unexpected `); + } + stack.pop(); + } else { + const element = { name: local, children: [], text: "" }; + top.children.push(element); + if (!empty) stack.push(element); + } + } + } + if (stack.length !== 1 || document.children.length !== 1) { + throw new SyntaxError("not one root element"); + } + return document.children[0]; +} + +/** + * The elements with a name in an element. + * @param {XMLElement | undefined} element + * @param {string} name + * @returns {XMLElement[]} + */ +export function childrenOf(element, name) { + return element?.children.filter((child) => child.name === name) ?? []; +} + +/** + * The text of the first element with a name in an element, or "". + * @param {XMLElement | undefined} element + * @param {string} name + * @returns {string} + */ +export function textOf(element, name) { + return childrenOf(element, name)[0]?.text ?? ""; +} + +/** + * Tells if XML 1.0 can have a text: it cannot have the control characters + * other than tab and line breaks, also as a character reference. + * @param {string} text + * @returns {boolean} + */ +export function isXMLText(text) { + return !/[\0-\x08\x0b\x0c\x0e-\x1f\ufffe\uffff]|[\ud800-\udbff](?![\udc00-\udfff])|(?"'\r]/g, + (c) => + ({ + "&": "&", + "<": "<", + ">": ">", + '"': """, + "'": "'", + "\r": " ", + })[c], + ); +} + +/** @param {string} text */ +function decodeEntities(text) { + return text.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (entity, body) => { + if (body[0] === "#") { + const code = body[1] === "x" || body[1] === "X" + ? parseInt(body.slice(2), 16) + : parseInt(body.slice(1), 10); + return code <= 0x10ffff ? String.fromCodePoint(code) : entity; + } + return ENTITIES[body] ?? entity; + }); +} diff --git a/backend/s3/xml_test.js b/backend/s3/xml_test.js new file mode 100644 index 0000000..b385018 --- /dev/null +++ b/backend/s3/xml_test.js @@ -0,0 +1,58 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fc from "fast-check"; + +import { childrenOf, escapeXML, isXMLText, parseXML, textOf } from "./xml.js"; + +test("parseXML reads an answer of S3", () => { + const root = parseXML(` + + bucket + false + a & b <c> é世10 + ]]>0 + d/ + + +`); + assert.equal(root.name, "ListBucketResult"); + assert.equal(textOf(root, "Name"), "bucket"); + assert.deepEqual( + childrenOf(root, "Contents").map(( + c, + ) => [textOf(c, "Key"), textOf(c, "Size")]), + [["a & b é世", "10"], ["", "0"]], + ); + assert.equal(textOf(childrenOf(root, "CommonPrefixes")[0], "Prefix"), "d/"); + assert.equal(childrenOf(root, "Empty").length, 1); + assert.equal(textOf(root, "Missing"), ""); +}); + +test("parseXML reads the names with a namespace prefix", () => { + const root = parseXML("NoSuchKey"); + assert.equal(root.name, "Error"); + assert.equal(textOf(root, "Code"), "NoSuchKey"); +}); + +test("parseXML refuses elements that do not close in order", () => { + for (const text of ["", "", "", "", "text"]) { + assert.throws(() => parseXML(text), SyntaxError, text); + } +}); + +test("parseXML reads the text that escapeXML writes", () => { + fc.assert( + fc.property(fc.string(), (text) => { + // XML 1.0 has no control characters other than tab and line breaks. + fc.pre(isXMLText(text)); + assert.equal(parseXML(`${escapeXML(text)}`).text, text); + }), + ); +}); + +test("isXMLText refuses the characters that XML 1.0 cannot have", () => { + for (const text of ["a", "\t\n\r", "é世😀"]) assert.ok(isXMLText(text), text); + for (const text of ["\0", "\v", "\x1f", "\ud800", "a\udc00", "\uffff"]) { + assert.ok(!isXMLText(text), JSON.stringify(text)); + } +}); diff --git a/bench/cases.js b/bench/cases.js new file mode 100644 index 0000000..d5a840a --- /dev/null +++ b/bench/cases.js @@ -0,0 +1,169 @@ +/** + * The cases of the benchmark of rfetch. They use no API of Node.js only, + * thus they run in Node.js (`bench/node.js`) and in a Worker + * (`bench/worker.js`). + */ +// The import also sets the global `fetch`, which reads the paths of remotes. +import { fetch, lsjson } from "../mod.js"; +import { readListing, renderListing } from "@rfetch/lib/listing"; +import memory from "@rfetch/memory"; +import { childrenOf, parseXML, textOf } from "../backend/s3/xml.js"; +import { sign, UNSIGNED_PAYLOAD } from "../backend/s3/sigv4.js"; + +/** The size of a large file. */ +export const SIZE = 8 * 1024 * 1024; +/** The number of files in a list. */ +const FILES = 1000; + +/** "test-password", obscured by rclone. */ +const PASSWORD = "4OSlj-ZYMGdEIGjcWg_gc-DL4O3TFAvr4ct_yV4"; +const CRYPT = { + RFETCH_CRYPT_REMOTE: ":memory:/bench-crypt", + RFETCH_CRYPT_PASSWORD: PASSWORD, +}; +const CRYPT_LIST = { + ...CRYPT, + RFETCH_CRYPT_REMOTE: ":memory:/bench-crypt-list", +}; +const CHUNKER = { + RFETCH_CHUNKER_REMOTE: ":memory:/bench-chunker", + RFETCH_CHUNKER_CHUNK_SIZE: "1M", +}; + +/** Bytes that do not compress, with no cost of a random source. */ +function data(size) { + const bytes = new Uint8Array(size); + let x = 0x9e3779b9; + for (let i = 0; i < size; i++) { + x ^= x << 13; + x ^= x >>> 17; + x ^= x << 5; + bytes[i] = x; + } + return bytes; +} + +// The data of the cases. `setup` makes them: a Worker cannot do such work +// at the start of the module. +let big = new Uint8Array(0); +let small = new Uint8Array(0); +let html = ""; +let xml = ""; + +/** @param {Response} response */ +async function drain(response) { + if (!response.ok) { + throw new Error(`${response.status} ${await response.text()}`); + } + await response.arrayBuffer(); +} + +/** @param {string} location @param {Uint8Array} body @param {HeadersInit} [headers] */ +async function put(location, body, headers = {}) { + await drain(await fetch(location, { method: "PUT", body, headers })); +} + +/** @param {string} location @param {HeadersInit} [headers] */ +async function get(location, headers = {}) { + await drain(await fetch(location, { headers })); +} + +/** The entries of a list of `count` files. */ +function entries(count) { + return Array.from({ length: count }, (_, i) => ({ + Name: `file ${i} é.txt`, + IsDir: false, + Size: i * 100, + MimeType: "text/plain; charset=utf-8", + ModTime: new Date(1700000000000 + i * 1000).toISOString(), + })); +} + +/** The answer of ListObjectsV2 of S3 for `count` keys. */ +function listXML(count) { + const contents = Array.from( + { length: count }, + (_, i) => + `folder/file%20${i}.txt` + + `2026-10-18T12:00:00.000Z` + + `"0123456789abcdef"${i}` + + `STANDARD`, + ).join(""); + return `` + + `bucketfolder/${count}` + + `1000url` + + `false${contents}`; +} + +/** Makes the data, and puts the files that the cases read. */ +export async function setup() { + big = data(SIZE); + small = data(100); + html = await new Response(renderListing(entries(10 * FILES))).text(); + xml = listXML(FILES); + await put(":memory:/bench/big.bin", big); + await put(":crypt:big.bin", big, CRYPT); + await put(":chunker:big.bin", big, CHUNKER); + await put(":memory:/bench/small.txt", small); + for (let i = 0; i < FILES; i++) { + await put(`:memory:/bench-list/file${i}.txt`, small); + await put(`:crypt:file${i}.txt`, small, CRYPT_LIST); + } +} + +/** @type {Record Promise>} */ +export const cases = { + "memory: PUT 8 MiB": () => put(":memory:/bench/put.bin", big), + "memory: GET 8 MiB": () => get(":memory:/bench/big.bin"), + "crypt: PUT 8 MiB": () => put(":crypt:put.bin", big, CRYPT), + "crypt: GET 8 MiB": () => get(":crypt:big.bin", CRYPT), + "crypt: GET a range of 1 MiB": () => + get(":crypt:big.bin", { ...CRYPT, Range: "bytes=3000000-4048575" }), + "chunker: PUT 8 MiB in chunks of 1 MiB": () => + put(":chunker:put.bin", big, CHUNKER), + "chunker: GET 8 MiB in chunks of 1 MiB": () => + get(":chunker:big.bin", CHUNKER), + "memory: list 1000 files": () => get(":memory:/bench-list/"), + "crypt: list 1000 files": () => get(":crypt:", CRYPT_LIST), + "crypt: lsjson of 1000 files": async () => { + await drain(await lsjson(":crypt:", {}, { headers: CRYPT_LIST })); + }, + "lib: renderListing of 10000 entries": async () => { + await new Response(renderListing(entries(10 * FILES))).arrayBuffer(); + }, + "lib: readListing of 10000 entries": async () => { + await readListing(new Response(html), new URL("http://bench/folder/")); + }, + "s3: sign 1000 requests": async () => { + const credentials = { + accessKeyId: "AKIAIOSFODNN7EXAMPLE", + secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + region: "auto", + }; + for (let i = 0; i < FILES; i++) { + sign({ + method: "GET", + url: new URL(`https://bucket.example.com/folder/file%20${i}.txt`), + headers: new Headers({ Range: "bytes=0-99" }), + payload: UNSIGNED_PAYLOAD, + }, credentials); + } + }, + "s3: read a list of 1000 keys": async () => { + const root = parseXML(xml); + for (const object of childrenOf(root, "Contents")) { + decodeURIComponent(textOf(object, "Key").replaceAll("+", " ")); + Number(textOf(object, "Size")); + } + }, + "rfetch fetch: 1000 GET of a small file": async () => { + for (let i = 0; i < FILES; i++) await get(":memory:/bench/small.txt"); + }, + "memory, no rfetch fetch: 1000 GET of a small file": async () => { + for (let i = 0; i < FILES; i++) { + await drain( + await memory.fetch(new Request("http://bench/bench/small.txt")), + ); + } + }, +}; diff --git a/bench/node.js b/bench/node.js new file mode 100644 index 0000000..1b1b996 --- /dev/null +++ b/bench/node.js @@ -0,0 +1,28 @@ +/** + * Runs the cases of the benchmark in Node.js: `node bench/node.js`. + * + * Each case runs in a new process, as a command of rfetch does: the time of + * a case can change much with the work before it in the same process. + * `node bench/node.js NAME` runs only the case NAME. + */ +import { execFileSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { bench } from "@rfetch/lib/testing/bench"; +import { cases, setup } from "./cases.js"; + +const [name] = process.argv.slice(2); + +if (name) { + await setup(); + await bench("", { [name]: cases[name] }, { runs: 5 }); +} else { + console.log("rfetch in Node.js"); + for (const name of Object.keys(cases)) { + const output = execFileSync( + process.execPath, + [fileURLToPath(import.meta.url), name], + { encoding: "utf8" }, + ); + process.stdout.write(output.split("\n").slice(1).join("\n")); + } +} diff --git a/bench/rclone.js b/bench/rclone.js new file mode 100644 index 0000000..3b76065 --- /dev/null +++ b/bench/rclone.js @@ -0,0 +1,227 @@ +/** + * Compares the commands of rfetch with the commands of rclone: + * `node bench/rclone.js [RUNTIME...]`. rclone must be in the PATH, or in the + * variable RCLONE. + * + * Each RUNTIME runs rfetch: a program of Node.js, Deno or Bun, for example + * `node bench/rclone.js node deno bun /opt/node26/bin/node`. With no + * RUNTIME, the Node.js of this script runs rfetch. + * + * Each case runs the same command of each tool as a new process, with the + * same remotes in the variables RCLONE_CONFIG_*, on the same files in a + * temporary folder, which is the current folder of each process. The time is + * the time of the whole process, as a user sees it: the start of the runtime + * or of rclone is in it. The first case, a list of an empty folder, shows the + * time of the start. + */ +import { execFileSync, spawnSync } from "node:child_process"; +import { createHash, randomBytes } from "node:crypto"; +import { + closeSync, + mkdirSync, + mkdtempSync, + openSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const RCLONE = process.env.RCLONE || "rclone"; +const RFETCH = fileURLToPath(new URL("../main.js", import.meta.url)); +const RUNS = 5; +/** The size of a large file. */ +const SIZE = 64 * 1024 * 1024; + +const root = mkdtempSync(join(tmpdir(), "rfetch-bench-")); +const home = join(root, "home"); +mkdirSync(home); + +/** The environment of each tool: the remotes, and no config file. */ +const env = { + PATH: process.env.PATH, + HOME: home, + RCLONE_CONFIG_SECRET_TYPE: "crypt", + RCLONE_CONFIG_SECRET_REMOTE: "crypt", + RCLONE_CONFIG_SECRET_PASSWORD: execFileSync(RCLONE, ["obscure", "bench"], { + encoding: "utf8", + }).trim(), + RCLONE_CONFIG_CHUNKS_TYPE: "chunker", + RCLONE_CONFIG_CHUNKS_REMOTE: "chunks", + RCLONE_CONFIG_CHUNKS_CHUNK_SIZE: "8M", +}; + +/** + * The command of a runtime that runs rfetch, as a program and its arguments. + * @param {string} program + * @returns {(args: string[]) => [string, string[]]} + */ +function runtime(program) { + if (basename(program).startsWith("deno")) { + return (args) => [program, ["run", "-A", RFETCH, ...args]]; + } + return (args) => [program, [RFETCH, ...args]]; +} + +/** The name of a runtime, with its version, for example `node 22.22.0`. */ +function nameOf(program) { + const version = execFileSync(program, ["--version"], { encoding: "utf8" }) + .match(/\d+\.\d+\.\d+/)?.[0]; + const kind = basename(program).startsWith("deno") + ? "deno" + : basename(program).startsWith("bun") + ? "bun" + : "node"; + return `${kind} ${version}`; +} + +/** + * The command of each tool, as a program and its arguments: the runtimes of + * rfetch, then rclone. + * @type {Record [string, string[]]>} + */ +const tools = {}; +const runtimes = process.argv.slice(2); +for (const program of runtimes.length ? runtimes : [process.execPath]) { + tools[`rfetch on ${nameOf(program)}`] = runtime(program); +} +tools.rclone = (args) => [RCLONE, args]; + +/** + * Runs a command of a tool, and gives its time and the hash of its output. + * @param {string} tool + * @param {string[]} args + * @param {string} [input] A file for the standard input. + */ +function run(tool, args, input) { + const [program, argv] = tools[tool](args); + const output = join(root, "output"); + const stdin = input ? openSync(input, "r") : "ignore"; + const stdout = openSync(output, "w"); + const start = performance.now(); + const result = spawnSync(program, argv, { + cwd: root, + env, + stdio: [stdin, stdout, "pipe"], + }); + const time = performance.now() - start; + if (typeof stdin === "number") closeSync(stdin); + closeSync(stdout); + if (result.status !== 0) { + throw new Error(`${tool} ${args.join(" ")}: ${result.stderr}`); + } + const hash = createHash("sha256").update(readFileSync(output)).digest("hex"); + return { time, hash }; +} + +/** + * The lines of `lsjson`, in order, for a compare. The size of a folder is not + * in a line: rclone gives the size of a local folder, and rfetch gives -1. + * The time is to the millisecond: Deno gives the time of a file in + * milliseconds. cmd/lsjson/rclone_test.js compares the nanoseconds. + */ +function lines(tool, args) { + run(tool, args); + return JSON.parse(readFileSync(join(root, "output"), "utf8")) + .map(({ Path, Size, IsDir, ModTime }) => + `${Path} ${IsDir ? "folder" : Size} ${Date.parse(ModTime)}` + ) + .sort(); +} + +// The files of the cases. +const big = join(root, "big.bin"); +writeFileSync(big, randomBytes(SIZE)); +mkdirSync(join(root, "empty")); +for (let folder = 0; folder < 10; folder++) { + mkdirSync(join(root, "tree", `folder ${folder}`), { recursive: true }); + for (let file = 0; file < 100; file++) { + writeFileSync( + join(root, "tree", `folder ${folder}`, `file ${file}.txt`), + `${folder} ${file}`, + ); + } +} +// rclone writes the files that both tools read from crypt and chunker. +const setup = { cwd: root, env, stdio: "ignore" }; +execFileSync(RCLONE, ["copy", "tree", "secret:tree"], setup); +execFileSync(RCLONE, ["copyto", "big.bin", "secret:big.bin"], setup); +execFileSync(RCLONE, ["copyto", "big.bin", "chunks:big.bin"], setup); + +/** @type {Record string[] | [string[], string]>} */ +const cases = { + "lsjson of an empty folder (start)": () => ["lsjson", "empty"], + "local: cat 64 MiB": () => ["cat", "big.bin"], + "local: lsjson --recursive, 1000 files": () => [ + "lsjson", + "--recursive", + "tree", + ], + "crypt: cat 64 MiB": () => ["cat", "secret:big.bin"], + "crypt: rcat 64 MiB": (tool) => [ + ["rcat", `secret:${tool.replaceAll(" ", "-")}/put.bin`], + big, + ], + "crypt: lsjson --recursive, 1000 files": () => [ + "lsjson", + "--recursive", + "secret:tree", + ], + "chunker: cat 64 MiB in chunks of 8 MiB": () => ["cat", "chunks:big.bin"], + "chunker: rcat 64 MiB in chunks of 8 MiB": (tool) => [ + ["rcat", `chunks:${tool.replaceAll(" ", "-")}/put.bin`], + big, + ], +}; + +try { + // Each tool gives the same data and the same lists as rclone. + const hash = createHash("sha256").update(readFileSync(big)).digest("hex"); + for (const tool of Object.keys(tools)) { + for (const location of ["secret:big.bin", "chunks:big.bin"]) { + if (run(tool, ["cat", location]).hash !== hash) { + throw new Error(`${tool} cat ${location}: not the same data`); + } + } + const folder = tool.replaceAll(" ", "-"); + for (const location of ["secret", "chunks"]) { + run(tool, ["rcat", `${location}:${folder}/check.bin`], big); + if ( + run("rclone", ["cat", `${location}:${folder}/check.bin`]).hash !== hash + ) { + throw new Error(`${tool} rcat ${location}: rclone reads other data`); + } + } + for (const location of ["tree", "secret:tree"]) { + const args = ["lsjson", "--recursive", location]; + const [ours, theirs] = [lines(tool, args), lines("rclone", args)]; + if (`${ours}` !== `${theirs}`) { + const line = ours.find((line, index) => line !== theirs[index]); + throw new Error( + `${tool} lsjson ${location}: not the same list, at "${line}"`, + ); + } + } + } + + console.log(`The median of ${RUNS} runs, in ms:\n`); + const names = Object.keys(tools); + console.log(`| Case | ${names.join(" | ")} |`); + console.log(`|---|${names.map(() => "---:").join("|")}|`); + for (const [name, make] of Object.entries(cases)) { + const times = []; + for (const tool of names) { + const made = make(tool); + const [args, input] = Array.isArray(made[0]) ? made : [made]; + run(tool, args, input); + const list = []; + for (let i = 0; i < RUNS; i++) list.push(run(tool, args, input).time); + times.push(list.sort((a, b) => a - b)[Math.floor(RUNS / 2)]); + } + console.log(`| ${name} | ${times.map((t) => t.toFixed(0)).join(" | ")} |`); + } +} finally { + rmSync(root, { recursive: true, force: true }); +} diff --git a/bench/worker.js b/bench/worker.js new file mode 100644 index 0000000..230504f --- /dev/null +++ b/bench/worker.js @@ -0,0 +1,25 @@ +/** + * Runs the cases of the benchmark in a Worker: + * `npx wrangler dev -c bench/wrangler.json`, then `curl http://localhost:8787/`. + * The clock of `wrangler dev` moves during the work of a request, thus the + * times are times of the CPU. A Worker in production has no such clock. + */ +import { cases, setup } from "./cases.js"; + +const RUNS = 5; + +export default { + async fetch() { + await setup(); + let text = "rfetch in workerd\n"; + for (const [name, run] of Object.entries(cases)) { + await run(); + const start = performance.now(); + for (let i = 0; i < RUNS; i++) await run(); + text += ` ${name}: ${ + ((performance.now() - start) / RUNS).toFixed(1) + } ms\n`; + } + return new Response(text); + }, +}; diff --git a/bench/wrangler.json b/bench/wrangler.json new file mode 100644 index 0000000..741f4ac --- /dev/null +++ b/bench/wrangler.json @@ -0,0 +1,6 @@ +{ + "name": "rfetch-bench", + "main": "worker.js", + "compatibility_date": "2026-09-01", + "compatibility_flags": ["nodejs_compat"] +} diff --git a/cmd/backend/main.js b/cmd/backend/main.js index 255de00..cd07f29 100644 --- a/cmd/backend/main.js +++ b/cmd/backend/main.js @@ -1,3 +1,5 @@ +import { fetch } from "../../fetch.js"; + /** * Run a backend-specific command. * diff --git a/cmd/backend/main_test.js b/cmd/backend/main_test.js index 15ee5e6..f2aee2c 100644 --- a/cmd/backend/main_test.js +++ b/cmd/backend/main_test.js @@ -1,5 +1,5 @@ import { test } from "node:test"; -import { assert } from "../../dev_deps.js"; +import assert from "node:assert/strict"; import { backend } from "./main.js"; test("backend", () => { diff --git a/cmd/cat/main.js b/cmd/cat/main.js index a49f756..29df04b 100644 --- a/cmd/cat/main.js +++ b/cmd/cat/main.js @@ -1,4 +1,4 @@ -import { fetch } from "../../mod.js"; +import { fetch } from "../../fetch.js"; /** * @param {string} location diff --git a/cmd/cat/main_test.js b/cmd/cat/main_test.js index 1d15a05..02994f8 100644 --- a/cmd/cat/main_test.js +++ b/cmd/cat/main_test.js @@ -1,5 +1,5 @@ import { test } from "node:test"; -import { assert } from "../../dev_deps.js"; +import assert from "node:assert/strict"; import { cat } from "./main.js"; test("cat", () => { diff --git a/cmd/config/main.js b/cmd/config/main.js index 015c261..eff8a18 100644 --- a/cmd/config/main.js +++ b/cmd/config/main.js @@ -1,8 +1,15 @@ import { env } from "node:process"; -import { config_dir, INI, join } from "../../deps.js"; +import { join } from "node:path"; +import { CONFIG_NAME } from "@rfetch/lib/fspath"; +import * as INI from "@rfetch/lib/ini"; -const NAME_REGEX = /^[\w.][\w.\s-]*$/; +/** + * The names of remotes that rclone accepts (`fspath.CheckConfigName`): + * letters, numbers, `_`, `-`, `.`, `+`, `@` and space, but not `-` or space + * at the start, and not space at the end. + */ +export const NAME_REGEX = new RegExp(`^${CONFIG_NAME}$`, "u"); /** * @typedef {Object} Options @@ -40,7 +47,6 @@ const NAME_REGEX = /^[\w.][\w.\s-]*$/; export async function config(subcommand, name, options, init) { const { readFile, writeFile } = await import("node:fs/promises"); - let file = "", ini = ""; // Order as specified at https://rclone.org/docs/#config-config-file. const PATHS = [ "rfetch.conf", @@ -50,6 +56,8 @@ export async function config(subcommand, name, options, init) { join(config_dir(), "rclone", "rclone.conf"), join(env["HOME"], ".rclone.conf"), ]; + // With no file, the file is the default file of rclone. + let file = PATHS[4], ini = ""; for await (const path of PATHS) { try { ini = await readFile(path, { encoding: "utf8" }); @@ -60,35 +68,43 @@ export async function config(subcommand, name, options, init) { } } + const configFile = new INI.ConfigFile(ini); /** * @type {Object} */ - let config = INI.parse(ini); - const encodeOptions = { - section: "", - whitespace: true, - }; + let config = configFile.config; + + /** + * Changes the file, and answers with the remote. The other lines of the + * file do not change, thus the comments of the person stay. An option + * that the file cannot keep, for example a value with a line break, gives + * 400 and the file does not change. + * @param {(configFile: INI.ConfigFile) => void} change + * @param {string} name + */ + async function save(change, name) { + try { + change(configFile); + } catch (error) { + if (!(error instanceof TypeError)) throw error; + return new Response(error.message, { status: 400 }); + } + await writeFile(file, configFile.toString()); + const remote = configFile.config[name]; + return new Response(INI.stringify(remote, { section: name })); + } switch (subcommand) { case "create": { // Create a new remote with name, type and options. if (!NAME_REGEX.test(name)) { - return new Response(null, { - status: 400, - statusText: `Invalid remote name: ${name}`, - }); + return new Response(`Invalid remote name: ${name}`, { status: 400 }); } - config[name] = options; - await writeFile( - file, - INI.stringify(config, encodeOptions).trim(), - ); - encodeOptions.section = name; - return new Response(INI.stringify(config[name], encodeOptions).trim()); + return await save((configFile) => configFile.set(name, options), name); } case "delete": // Delete an existing remote. - delete config[name]; - await writeFile(file, INI.stringify(config, encodeOptions).trim()); + configFile.delete(name); + await writeFile(file, configFile.toString()); return new Response(); case "disconnect": // Disconnects user from remote @@ -108,35 +124,32 @@ export async function config(subcommand, name, options, init) { case "show": // Print (decrypted) config file, or the config for a single remote. if (typeof name === "string") { - config = config[name]; - - if (!config) { + // Only an own key: the name "toString" is not a remote. + if (!Object.hasOwn(config, name)) { return new Response("", { status: 404 }); } - - encodeOptions.section = name; + config = config[name]; } if (new Headers(init?.headers).get("Accept") === "application/json") { return Response.json(config); } - return new Response(INI.stringify(config, encodeOptions).trim()); + // The whole file, with its comments, as `rclone config show`. + if (typeof name !== "string") { + return new Response(configFile.toString()); + } + return new Response(INI.stringify(config, { section: name })); case "touch": // Ensure configuration file exists. case "update": { // Update options in an existing remote. /** * @type {Object} */ - const remote = config[name]; - if (!remote) { + if (!Object.hasOwn(config, name)) { return new Response("", { status: 404 }); } - - Object.assign(remote, options); - await writeFile(file, INI.stringify(config, encodeOptions).trim()); - encodeOptions.section = name; - return new Response(INI.stringify(config[name], encodeOptions).trim()); + return await save((configFile) => configFile.update(name, options), name); } case "userinfo": // Prints info about logged in user of remote. @@ -144,3 +157,22 @@ export async function config(subcommand, name, options, init) { return new Response(); } } + +/** + * Retrieves the configuration directory for current platform. + * @returns {string | null} + */ +export function config_dir() { + let dir = env["XDG_CONFIG_HOME"]; + if (!dir) { + dir = env["APPDATA"]; // Windows + } + if (!dir) { + dir = env["HOME"]; + if (dir) { + dir += "/.config"; + } + } + + return dir; +} diff --git a/cmd/config/main_test.js b/cmd/config/main_test.js index 93c3193..4528e6b 100644 --- a/cmd/config/main_test.js +++ b/cmd/config/main_test.js @@ -1,9 +1,21 @@ import { test } from "node:test"; -import { readFile, unlink, writeFile } from "node:fs/promises"; -import { config_dir, join } from "../../deps.js"; -import { assert, assertEquals, fc } from "../../dev_deps.js"; - -import { config as configure } from "./main.js"; +import { mkdtemp, readFile, rm, unlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import process from "node:process"; +import assert from "node:assert/strict"; +import fc from "fast-check"; + +import { config as configure, config_dir, NAME_REGEX } from "./main.js"; + +// Each test file runs in its own process. These tests use a new folder as the +// current folder and as the home folder. Thus they do not change the files of +// the person, or the rclone.conf of other tests that run at the same time. +const folder = await mkdtemp(join(tmpdir(), "rfetch-config-")); +process.chdir(folder); +process.env.HOME = folder; +delete process.env.XDG_CONFIG_HOME; +delete process.env.APPDATA; test("config", async (t) => { let command = "file"; @@ -11,7 +23,7 @@ test("config", async (t) => { const configDir = config_dir(); let path = await configure(command).then((res) => res.text()); - assertEquals( + assert.deepEqual( path, join(configDir, "rclone", "rclone.conf"), "should default to config directory", @@ -19,7 +31,7 @@ test("config", async (t) => { await writeFile("rclone.conf", new Uint8Array()); path = await configure(command).then((res) => res.text()); - assertEquals( + assert.deepEqual( path, "rclone.conf", "should use rclone.conf at current directory first", @@ -29,11 +41,11 @@ test("config", async (t) => { command = "create"; await t.test(command, async () => { let config = await readFile("rclone.conf", { encoding: "utf8" }); - assertEquals(config, "", "initial empty config"); + assert.deepEqual(config, "", "initial empty config"); await fc.assert( fc.asyncProperty(fc.string(), async (name) => { - fc.pre(!/^[\w.][\w.\s-]*$/.test(name)); + fc.pre(!NAME_REGEX.test(name)); const response = await configure(command, name, { type: "local", @@ -41,7 +53,7 @@ test("config", async (t) => { assert( !response.ok, - "name may only contain 0-9, A-Z, a-z, _, -, . and space, but not start with - or space", + "name may only contain letters, numbers, _, -, ., +, @ and space, but not start with - or space, and not end with space", ); }), ); @@ -51,14 +63,14 @@ test("config", async (t) => { }); config = await readFile("rclone.conf", { encoding: "utf8" }); - assertEquals( + assert.deepEqual( config, - "[source]\ntype = local", + "[source]\ntype = local\n", "should have the new remote added", ); - assertEquals( + assert.deepEqual( await response.text(), - config, + "[source]\ntype = local", "should return the new remote in response", ); @@ -68,49 +80,71 @@ test("config", async (t) => { }); config = await readFile("rclone.conf", { encoding: "utf8" }); - assertEquals( + assert.deepEqual( config, - "[source]\ntype = local\n\n[target]\ntype = local", + "[source]\ntype = local\n\n[target]\ntype = local\n", "should have both remotes", ); - assertEquals( + assert.deepEqual( await response.text(), `[target]\ntype = local`, "should have only the new remote in response", ); }); + await t.test("the same names as rclone", () => { + for (const name of ["a", "a b", "a-b", "a.b+c@d", "_", "世界", "é1"]) { + assert.ok(NAME_REGEX.test(name), name); + } + for (const name of ["", " a", "-a", "a ", "a\nb", "a:b", "a/b", "a=b"]) { + assert.ok(!NAME_REGEX.test(name), JSON.stringify(name)); + } + }); + + await t.test("400 for an option that the file cannot keep", async () => { + const before = await readFile("rclone.conf", { encoding: "utf8" }); + const response = await configure("create", "broken", { + type: "local", + key: "line 1\nline 2", + }); + assert.equal(response.status, 400); + assert.equal(await readFile("rclone.conf", { encoding: "utf8" }), before); + }); + command = "show"; await t.test(command, async () => { const config = await readFile("rclone.conf", { encoding: "utf8" }); let response = await configure(command); - assertEquals( + assert.deepEqual( await response.text(), config, "should return the full config when name is not specified", ); response = await configure(command, "source"); - assertEquals( + assert.deepEqual( await response.text(), "[source]\ntype = local", "should return only remote specified by name", ); response = await configure(command, "target"); - assertEquals( + assert.deepEqual( await response.text(), "[target]\ntype = local", "should return only remote specified by name", ); response = await configure(command, "404"); - assertEquals( + assert.deepEqual( response.status, 404, "should return 404 when remote not found", ); + response = await configure(command, "toString"); + assert.equal(response.status, 404, "a key of Object is not a remote"); + response = await configure(command, "source", undefined, { headers: { "Accept": "application/json", @@ -129,19 +163,19 @@ test("config", async (t) => { const remote = `[target]\ntype = alias\nremote = source:`; const config = await readFile("rclone.conf", { encoding: "utf8" }); - assertEquals( + assert.deepEqual( config, - `[source]\ntype = local\n\n${remote}`, + `[source]\ntype = local\n\n${remote}\n`, "should have the remote updated", ); - assertEquals( + assert.deepEqual( await response.text(), remote, "should return the updated remote in response", ); response = await configure(command, "404", {}); - assertEquals( + assert.deepEqual( response.status, 404, "should return 404 when remote not found", @@ -151,7 +185,7 @@ test("config", async (t) => { command = "dump"; await t.test(command, async () => { const response = await configure(command); - assertEquals(await response.json(), { + assert.deepEqual(await response.json(), { source: { type: "local", }, @@ -166,25 +200,47 @@ test("config", async (t) => { await t.test(command, async () => { let response = await configure(command, "target"); const config = await readFile("rclone.conf", { encoding: "utf8" }); - assertEquals( + assert.deepEqual( config, - "[source]\ntype = local", + "[source]\ntype = local\n", "should have the remote deleted", ); - assertEquals( + assert.deepEqual( await response.text(), "", "should not return the deleted remote in response", ); response = await configure(command, "404"); - assertEquals( + assert.deepEqual( response.status, 200, "should still return 200 when remote not found", ); }); + await t.test("keeps the comments of the person", async () => { + await writeFile( + "rclone.conf", + "# My remotes.\n\n# Old disk.\n[old]\ntype = local\n\n" + + "[work]\n# The type.\ntype = local\n", + ); + await configure("create", "new", { type: "memory" }); + await configure("update", "work", { type: "alias", remote: "new:" }); + await configure("delete", "old"); + assert.equal( + await readFile("rclone.conf", { encoding: "utf8" }), + "# My remotes.\n\n[work]\n# The type.\ntype = alias\n" + + "remote = new:\n\n[new]\ntype = memory\n", + ); + assert.equal( + await configure("show").then((response) => response.text()), + await readFile("rclone.conf", { encoding: "utf8" }), + "show gives the file with the comments", + ); + }); + // Cleans up await unlink("rclone.conf"); + await rm(folder, { recursive: true, force: true }); }); diff --git a/cmd/copy/main.js b/cmd/copy/main.js index 755d4b6..834e261 100644 --- a/cmd/copy/main.js +++ b/cmd/copy/main.js @@ -1,4 +1,4 @@ -import { fetch } from "../../mod.js"; +import { fetch } from "../../fetch.js"; /** * Copy files from source to dest, skipping identical files. * diff --git a/cmd/copy/main_test.js b/cmd/copy/main_test.js index 55c2303..cbfba36 100644 --- a/cmd/copy/main_test.js +++ b/cmd/copy/main_test.js @@ -1,5 +1,5 @@ import { test } from "node:test"; -import { assert } from "../../dev_deps.js"; +import assert from "node:assert/strict"; import { copy } from "./main.js"; test("copy", () => { diff --git a/cmd/copyurl/main.js b/cmd/copyurl/main.js index 5dc594b..a59b611 100644 --- a/cmd/copyurl/main.js +++ b/cmd/copyurl/main.js @@ -1,6 +1,6 @@ -import { basename, resolve } from "../../deps.js"; +import { basename, resolve } from "node:path"; -import { fetch } from "../../mod.js"; +import { fetch } from "../../fetch.js"; /** * @typedef {Object} Options diff --git a/cmd/copyurl/main_test.js b/cmd/copyurl/main_test.js index 92bc62b..298ee45 100644 --- a/cmd/copyurl/main_test.js +++ b/cmd/copyurl/main_test.js @@ -1,5 +1,5 @@ import { test } from "node:test"; -import { assert } from "../../dev_deps.js"; +import assert from "node:assert/strict"; import { copyurl } from "./main.js"; test("copyurl", () => { diff --git a/cmd/flags.js b/cmd/flags.js new file mode 100644 index 0000000..281925b --- /dev/null +++ b/cmd/flags.js @@ -0,0 +1,185 @@ +/** + * The flags of the command line, as in rclone. Each flag has a type: + * + * - A flag of type `boolean` takes no value, for example `--recursive`, or a + * value after `=`, for example `--dir-slash=false`. + * - A flag of type `string` takes the next argument as its value, for + * example `--addr :8080`, or a value after `=`, for example + * `--addr=:8080`. + * + * The `parseArgs` of Node.js takes the next argument as the value only for a + * flag that it knows as `string`: it reads an unknown flag as `boolean`, and + * its value as an argument of the command. Thus rfetch tells it the type of + * each flag, and an unknown flag is an error, as in rclone. + */ +import { env } from "node:process"; +import { parseArgs } from "node:util"; + +/** + * The value of a variable of rfetch, or of rclone for compatibility, for + * example `RFETCH_DRY_RUN` or `RCLONE_DRY_RUN`. + * @param {string} name For example `DRY_RUN`. + * @returns {string | undefined} + */ +function variable(name) { + return env[`RFETCH_${name}`] ?? env[`RCLONE_${name}`]; +} + +/** + * The flags of the commands, in the format of `parseArgs`. + * @type {Record} + */ +export const FLAGS = { + // `serve` + "addr": { type: "string" }, + // `copyurl` + "auto-filename": { type: "boolean", short: "a" }, + // `lsf` + "csv": { type: "boolean" }, + // `lsf` + "dir-slash": { type: "boolean", short: "d" }, + // `lsf` + "dirs-only": { type: "boolean" }, + "dry-run": { + type: "boolean", + short: "n", + default: variable("DRY_RUN") === "true", + }, + // `lsf` + "files-only": { type: "boolean" }, + // `lsf` + "format": { type: "string", short: "F" }, + "header": { type: "string", multiple: true }, + "header-download": { type: "string", multiple: true }, + // `copyurl` + "header-filename": { type: "boolean" }, + "header-upload": { type: "string", multiple: true }, + "help": { type: "boolean", short: "h" }, + "human-readable": { type: "boolean" }, + // The lists + "max-depth": { type: "string" }, + // `copyurl` + "no-clobber": { type: "boolean" }, + // `copyurl` + "print-filename": { type: "boolean", short: "p" }, + "progress": { + type: "boolean", + short: "P", + default: variable("PROGRESS") === "true", + }, + // The lists + "recursive": { type: "boolean", short: "R" }, + // `lsf` + "separator": { type: "string", short: "s" }, + // `copyurl` + "stdout": { type: "boolean" }, + "transfers": { type: "string", default: variable("TRANSFERS") || "4" }, +}; + +/** + * The options of a backend, for example `{ boolean: ["no_head"], string: + * ["url"] }`. An option of `default` that is in no list is a `string`. + * @typedef {Object} BackendOptions + * @property {string[]} [boolean] + * @property {string[]} [string] + * @property {Record} [default] + */ + +/** + * The values that `strconv.ParseBool` of Go reads, as rclone. + * @type {Record} + */ +const BOOLEANS = { + "1": true, + "t": true, + "T": true, + "true": true, + "TRUE": true, + "True": true, + "0": false, + "f": false, + "F": false, + "false": false, + "FALSE": false, + "False": false, +}; + +/** + * Reads the flags and the arguments of a command line. + * + * A flag of a backend has the type of the backend before its name, for + * example `--crypt-password` for the option `password` of `crypt`. Its type + * comes from the `options` of the module of the backend. Only the backends + * that the flags tell load. + * + * Each flag is in the result with its name, for example `dry-run`, and with + * `_` in the place of `-`, for example `dry_run`. + * @param {string[]} args The arguments, with no program, for example + * `["lsjson", "--max-depth", "2", "remote:"]`. + * @param {Record Promise<{ options?: BackendOptions }>>} backends + * The modules of the backends, by type. + * @returns {Promise<{ flags: Record, positionals: string[] }>} + * @throws {Error} For an unknown flag, a flag with no value, or a value that + * is not a boolean, with the message of rclone. + */ +export async function parseFlags(args, backends) { + /** @type {Record} */ + const options = { ...FLAGS }; + + for (const arg of args) { + if (arg === "--") break; + if (!arg.startsWith("--")) continue; + const name = arg.slice(2).split("=", 1)[0]; + if (Object.hasOwn(options, name)) continue; + const type = name.slice(0, name.indexOf("-")); + if (!type || !Object.hasOwn(backends, type)) continue; + const declared = (await backends[type]()).options ?? {}; + const booleans = declared.boolean ?? []; + const names = [ + ...booleans, + ...declared.string ?? [], + ...Object.keys(declared.default ?? {}), + ]; + for (const option of names) { + options[`${type}-${option.replaceAll("_", "-")}`] = { + type: booleans.includes(option) ? "boolean" : "string", + }; + } + } + + const { values, positionals, tokens } = parseArgs({ + args, + options, + strict: false, + allowPositionals: true, + tokens: true, + }); + + for (const token of tokens) { + if (token.kind !== "option") continue; + const option = options[token.name]; + if (!option) throw new Error(`unknown flag: ${token.rawName}`); + if (option.type === "string" && token.value === undefined) { + throw new Error(`flag needs an argument: ${token.rawName}`); + } + if (option.type === "boolean" && token.value !== undefined) { + const value = Object.hasOwn(BOOLEANS, token.value) + ? BOOLEANS[token.value] + : undefined; + if (value === undefined) { + throw new Error( + `invalid argument "${token.value}" for "${token.rawName}" flag`, + ); + } + values[token.name] = value; + } + } + + /** @type {Record} */ + const flags = {}; + for (const [name, value] of Object.entries(values)) { + flags[name] = value; + flags[name.replaceAll("-", "_")] = value; + } + return { flags, positionals }; +} diff --git a/cmd/flags_test.js b/cmd/flags_test.js new file mode 100644 index 0000000..f23d65a --- /dev/null +++ b/cmd/flags_test.js @@ -0,0 +1,157 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { backends } from "../backend/main.js"; +import { FLAGS, parseFlags } from "./flags.js"; + +/** + * Backends with options, as the modules of `backend/main.js`. `loaded` has + * the type of each backend that loads. + */ +function fakeBackends() { + /** @type {string[]} */ + const loaded = []; + const module = (type, options) => () => { + loaded.push(type); + return Promise.resolve({ options }); + }; + return { + loaded, + backends: { + crypt: module("crypt", { string: ["remote", "password"] }), + http: module("http", { boolean: ["no_head"], string: ["url"] }), + local: module("local", undefined), + s3: module("s3", { + boolean: ["force_path_style"], + default: { chunk_size: "5Mi" }, + }), + }, + }; +} + +test("parseFlags reads the value of a flag after a space or after =", async () => { + for ( + const args of [ + ["serve", "http", ".", "--addr", ":8080"], + ["serve", "--addr=:8080", "http", "."], + ["--addr", ":8080", "serve", "http", "."], + ] + ) { + const { flags, positionals } = await parseFlags(args, {}); + assert.equal(flags.addr, ":8080", args.join(" ")); + assert.deepEqual(positionals, ["serve", "http", "."], args.join(" ")); + } +}); + +test("parseFlags reads a flag of a backend with the type of its option", async () => { + const { backends, loaded } = fakeBackends(); + const { flags, positionals } = await parseFlags([ + "lsjson", + "--crypt-password", + "secret", + "--http-no-head", + "remote:", + "--s3-force-path-style=false", + "--s3-chunk-size", + "8Mi", + ], backends); + assert.deepEqual(positionals, ["lsjson", "remote:"]); + assert.equal(flags.crypt_password, "secret"); + assert.equal(flags["crypt-password"], "secret"); + assert.equal(flags.http_no_head, true); + assert.equal(flags.s3_force_path_style, false); + assert.equal(flags.s3_chunk_size, "8Mi"); + // Only the backends of the flags load. + assert.deepEqual(loaded.toSorted(), ["crypt", "http", "s3"]); +}); + +test("parseFlags reads the boolean flags as rclone", async () => { + const { flags, positionals } = await parseFlags([ + "lsf", + "-R", + "--dir-slash=false", + "--files-only=1", + "--no-clobber", + "remote:", + ], {}); + assert.deepEqual(positionals, ["lsf", "remote:"]); + assert.equal(flags.recursive, true); + assert.equal(flags.dir_slash, false); + assert.equal(flags.files_only, true); + // A flag with the name `no-...` is not the negation of a flag. + assert.equal(flags.no_clobber, true); + assert.equal(flags.clobber, undefined); +}); + +test("parseFlags gives the arguments after -- as they are", async () => { + const { flags, positionals } = await parseFlags( + ["cat", "--", "--addr"], + {}, + ); + assert.deepEqual(positionals, ["cat", "--addr"]); + assert.equal(flags.addr, undefined); +}); + +test("parseFlags gives the errors of rclone", async () => { + const { backends } = fakeBackends(); + for ( + const [args, message] of [ + [["lsjson", "--nosuch", "x"], "unknown flag: --nosuch"], + [["lsjson", "-x"], "unknown flag: -x"], + // The local backend has no option: the root is not a flag. + [["lsjson", "--local-root", "/"], "unknown flag: --local-root"], + [["lsjson", "--crypt-nosuch", "x"], "unknown flag: --crypt-nosuch"], + [["lsjson", "--max-depth"], "flag needs an argument: --max-depth"], + [ + ["lsjson", "--dry-run=maybe"], + 'invalid argument "maybe" for "--dry-run" flag', + ], + [ + ["lsjson", "--dry-run=constructor"], + 'invalid argument "constructor" for "--dry-run" flag', + ], + ] + ) { + await assert.rejects(parseFlags(args, backends), { message }, `${args}`); + } +}); + +test("parseFlags reads each option of each backend of rfetch as a flag", async () => { + for (const [type, load] of Object.entries(backends)) { + const { options } = await load(); + if (!options) continue; + const booleans = options.boolean ?? []; + const strings = [ + ...options.string ?? [], + ...Object.keys(options.default ?? {}), + ].filter((name) => !booleans.includes(name)); + for (const name of booleans) { + const flag = `--${type}-${name.replaceAll("_", "-")}`; + const { flags, positionals } = await parseFlags( + ["lsjson", flag, "remote:"], + backends, + ); + assert.equal(flags[`${type}_${name}`], true, flag); + assert.deepEqual(positionals, ["lsjson", "remote:"], flag); + } + for (const name of strings) { + const flag = `--${type}-${name.replaceAll("_", "-")}`; + const { flags, positionals } = await parseFlags( + ["lsjson", flag, "value", "remote:"], + backends, + ); + assert.equal(flags[`${type}_${name}`], "value", flag); + assert.deepEqual(positionals, ["lsjson", "remote:"], flag); + } + } +}); + +test("FLAGS has a type for each flag, and no short name two times", () => { + const shorts = Object.values(FLAGS).map(({ short }) => short).filter( + Boolean, + ); + assert.equal(new Set(shorts).size, shorts.length); + for (const [name, { type }] of Object.entries(FLAGS)) { + assert.ok(type === "boolean" || type === "string", name); + } +}); diff --git a/cmd/folders.js b/cmd/folders.js new file mode 100644 index 0000000..5f093a7 --- /dev/null +++ b/cmd/folders.js @@ -0,0 +1,21 @@ +/** + * The folder in `cmd` of each command. The command line loads only the + * module of its command: the load of all the commands takes much time. + */ +export const FOLDERS = { + backend: "backend", + cat: "cat", + config: "config", + copy: "copy", + copyurl: "copyurl", + help: "help", + ls: "ls", + lsd: "lsd", + lsf: "lsf", + lsjson: "lsjson", + lsl: "lsl", + obscure: "obscure", + rcat: "rcat", + reveal: "obscure", + serve: "serve", +}; diff --git a/cmd/folders_test.js b/cmd/folders_test.js new file mode 100644 index 0000000..ae30846 --- /dev/null +++ b/cmd/folders_test.js @@ -0,0 +1,16 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import * as commands from "./main.js"; +import { FOLDERS } from "./folders.js"; + +test("FOLDERS has each command, and the folder of its module", async () => { + assert.deepEqual( + Object.keys(FOLDERS).toSorted(), + Object.keys(commands).toSorted(), + ); + for (const [name, folder] of Object.entries(FOLDERS)) { + const module = await import(`./${folder}/main.js`); + assert.equal(module[name], commands[name], name); + } +}); diff --git a/cmd/help/main.js b/cmd/help/main.js index bcc23ce..54a937b 100644 --- a/cmd/help/main.js +++ b/cmd/help/main.js @@ -1,5 +1,6 @@ -import * as backends from "../../backend/main.js"; -import * as commands from "../main.js"; +import { backends } from "../../backend/main.js"; +import { FLAGS } from "../flags.js"; +import { FOLDERS } from "../folders.js"; /** * @type {Object string>} @@ -12,7 +13,7 @@ usage["commands"] = () => rfetch [command] Available Commands: - ${Object.keys(commands).join("\n ")} + ${Object.keys(FOLDERS).join("\n ")} Use "rfetch [command] --help" for more information about a command. Use "rfetch help flags" for to see the global flags. @@ -21,14 +22,21 @@ Use "rfetch help backends" for a list of supported services. usage["flags"] = () => `Usage: - rfetch help flags [] [flags] - -Flags: - -h, --help help for flags + rfetch help flags Global Flags: +${ + Object.entries(FLAGS).map(([name, { type, short }]) => + ` ${short ? `-${short}, ` : " "}--${name}${ + type === "string" ? " string" : "" + }` + ).join("\n") + } Backend Flags: + --TYPE-NAME string The option NAME of the backend TYPE, for example + --crypt-password or --s3-chunk-size. A boolean + option takes no value, for example --http-no-head. `; usage["backends"] = () => diff --git a/cmd/ls/main.js b/cmd/ls/main.js index 62ff879..bd85226 100644 --- a/cmd/ls/main.js +++ b/cmd/ls/main.js @@ -1,4 +1,6 @@ -import { lsf } from "../../mod.js"; +import { sizeField } from "@rfetch/lib/format"; +import { isTrue } from "@rfetch/lib/options"; +import { LineStream, lsjson } from "../lsjson/main.js"; /** * List the objects in the path with size and path. * @@ -22,14 +24,20 @@ import { lsf } from "../../mod.js"; * @param {number} [flags.max_depth=Infinity] - Maximum depth of recursion. * @returns {Promise} */ -export function ls(location, flags = {}) { - flags = { - recursive: "true", +export async function ls(location, flags = {}) { + const human = isTrue(flags.human_readable); + const response = await lsjson(location, { + recursive: true, max_depth: "Infinity", - files_only: "true", - format: "sp", - separator: "\t", ...flags, - }; - return lsf(location, flags); + }); + if (!response.ok) return response; + // As `rclone ls`: the size in 9 characters, then the path. + const lines = new LineStream((item) => { + if (item.IsDir) return; + return `${sizeField(item.Size, human, 9)} ${item.Path}`; + }); + return new Response(response.body.pipeThrough(lines), { + headers: { "Content-Type": "text/plain" }, + }); } diff --git a/cmd/ls/main_test.js b/cmd/ls/main_test.js index d751cc7..00f5263 100644 --- a/cmd/ls/main_test.js +++ b/cmd/ls/main_test.js @@ -1,7 +1,26 @@ import { test } from "node:test"; -import { assert } from "../../dev_deps.js"; +import assert from "node:assert/strict"; + +import { fetch } from "../../fetch.js"; import { ls } from "./main.js"; -test("ls", () => { - assert(ls); +await fetch(":memory:/ls/a.txt", { method: "PUT", body: "hello" }); +await fetch(":memory:/ls/sub/b.bin", { + method: "PUT", + body: new Uint8Array(123456), +}); + +test("ls gives the size in 9 characters and the path of each file, as rclone", async () => { + const text = await (await ls(":memory:/ls/")).text(); + assert.equal(text, " 5 a.txt\n 123456 sub/b.bin\n"); +}); + +test("ls --human-readable gives the size with a suffix of rclone", async () => { + const response = await ls(":memory:/ls/", { human_readable: true }); + assert.equal(await response.text(), " 5 a.txt\n120.562Ki sub/b.bin\n"); +}); + +test("ls --max-depth 1 lists only the files of the folder", async () => { + const response = await ls(":memory:/ls/", { max_depth: "1" }); + assert.equal(await response.text(), " 5 a.txt\n"); }); diff --git a/cmd/lsd/main.js b/cmd/lsd/main.js index 10ddde6..24af1cc 100644 --- a/cmd/lsd/main.js +++ b/cmd/lsd/main.js @@ -1,4 +1,7 @@ -import { lsf } from "../../mod.js"; +import { countField, sizeField } from "@rfetch/lib/format"; +import { isTrue } from "@rfetch/lib/options"; +import { toListTime } from "@rfetch/lib/time"; +import { LineStream, lsjson } from "../lsjson/main.js"; /** * List all directories/containers/buckets in the path. * @@ -10,13 +13,20 @@ import { lsf } from "../../mod.js"; * @param {boolean} [flags.recursive=false] - Only list directories. * @returns {Promise} */ -export function lsd(location, flags = {}) { - flags = { - dirs_only: "true", - format: "stp", - recursive: "false", - separator: "\t", - ...flags, - }; - return lsf(location, flags); +export async function lsd(location, flags = {}) { + const human = isTrue(flags.human_readable); + const response = await lsjson(location, { recursive: false, ...flags }); + if (!response.ok) return response; + // As `rclone lsd`: the size in 12 characters, the local time to the + // second, the count of items in 9 characters, then the path. A folder has + // no size and no count: -1 for both. + const lines = new LineStream((item) => { + if (!item.IsDir) return; + const time = toListTime(item.ModTime).padStart(13); + const count = countField(-1, human, 9); + return `${sizeField(item.Size, human, 12)} ${time} ${count} ${item.Path}`; + }); + return new Response(response.body.pipeThrough(lines), { + headers: { "Content-Type": "text/plain" }, + }); } diff --git a/cmd/lsd/main_test.js b/cmd/lsd/main_test.js index 90e0ff0..ad87040 100644 --- a/cmd/lsd/main_test.js +++ b/cmd/lsd/main_test.js @@ -1,7 +1,30 @@ import { test } from "node:test"; -import { assert } from "../../dev_deps.js"; +import assert from "node:assert/strict"; + +import { DEFAULT_TIME, toListTime } from "@rfetch/lib/time"; +import { fetch } from "../../fetch.js"; import { lsd } from "./main.js"; -test("lsd", () => { - assert(lsd); +await fetch(":memory:/lsd/a.txt", { method: "PUT", body: "a" }); +await fetch(":memory:/lsd/sub/deep/b.txt", { method: "PUT", body: "b" }); + +// A folder of memory has no time: it has the default time of rclone. +const time = toListTime(DEFAULT_TIME); + +test("lsd gives the folders of one level, as rclone", async () => { + const text = await (await lsd(":memory:/lsd/")).text(); + assert.equal(text, ` -1 ${time} -1 sub\n`); +}); + +test("lsd --recursive gives the folders of each level", async () => { + const response = await lsd(":memory:/lsd/", { recursive: true }); + assert.equal( + await response.text(), + ` -1 ${time} -1 sub\n -1 ${time} -1 sub/deep\n`, + ); +}); + +test("lsd --human-readable gives fields of 9 and 8 characters", async () => { + const response = await lsd(":memory:/lsd/", { human_readable: true }); + assert.equal(await response.text(), ` -1 ${time} -1 sub\n`); }); diff --git a/cmd/lsf/main.js b/cmd/lsf/main.js index 7458549..6ca9943 100644 --- a/cmd/lsf/main.js +++ b/cmd/lsf/main.js @@ -1,4 +1,6 @@ -import { lsjson } from "../../mod.js"; +import { isTrue } from "@rfetch/lib/options"; +import { toListTime } from "@rfetch/lib/time"; +import { LineStream, lsjson } from "../lsjson/main.js"; const FORMATS = { "p": "Path", @@ -115,63 +117,51 @@ const FORMATS = { * @returns {Promise} */ export async function lsf(location, flags = {}) { - const { - csv = false, - dir_slash: dirSlash = true, - dirs_only: dirsOnly = false, - files_only: filesOnly = false, - format = "p", - separator = ";", - } = flags; + const format = flags.format ?? "p"; + const csv = isTrue(flags.csv); + // As rclone, the separator of CSV is "," when the flag does not tell it. + const separator = flags.separator ?? (csv ? "," : ";"); + const dirSlash = flags.dir_slash === undefined || isTrue(flags.dir_slash); + const dirsOnly = isTrue(flags.dirs_only); + const filesOnly = isTrue(flags.files_only); + // CSV has a separator of one character. + const comma = [...separator][0] ?? ","; const response = await lsjson(location, flags); - let { ok, headers, body } = response; - - if (!ok) { - return response; - } - - headers = new Headers(response.headers); - headers.set("Content-Type", "text/plain"); - - body = body - .pipeThrough(new TextDecoderStream()) - .pipeThrough( - new TransformStream({ - transform(chunk, controller) { - // `lsjson returns each item on a new line, and except the - // last line, all lines ends with a comma. We strip that - // trailing comma and any new lines. - chunk = chunk.trim().replace(/,$/, ""); - - if (chunk.startsWith("{") && chunk.at(-1) === "}") { - /** - * @type {import("../../mod.js").File} - */ - const item = JSON.parse(chunk); - if (dirsOnly && !item.IsDir) return; - if (filesOnly && item.IsDir) return; - - if (item.IsDir && dirSlash) { - item.Path += "/"; - } - - chunk = [...format].map((f) => { - let value = item[FORMATS[f]]; - if (csv && typeof value === "string" && value.includes(",")) { - value = `"${value}"`; - } - return value; - }).join(separator); - - controller.enqueue(`${chunk}\n`); - } - }, - }), - ) - .pipeThrough(new TextEncoderStream()); - - return new Response(body, { - headers, + if (!response.ok) return response; + const lines = new LineStream((item) => { + if (dirsOnly && !item.IsDir) return; + if (filesOnly && item.IsDir) return; + const values = [...format].map((letter) => { + if (letter === "p") { + return item.IsDir && dirSlash ? `${item.Path}/` : item.Path; + } + if (letter === "t") return toListTime(item.ModTime); + const value = item[FORMATS[letter]] ?? ""; + return typeof value === "object" ? JSON.stringify(value) : `${value}`; + }); + if (!csv) return values.join(separator); + return values.map((value) => quoted(value, comma)).join(comma); + }); + return new Response(response.body.pipeThrough(lines), { + headers: { "Content-Type": "text/plain" }, }); } + +/** + * Puts a value of CSV in quotes when it must have them, as `csv.Writer` of + * Go: when it has the comma, a quote, a line break, or a space at the + * start. A quote in the value becomes two quotes. + * @param {string} value + * @param {string} comma + * @returns {string} + */ +function quoted(value, comma) { + const needs = value === "\\." || value.includes(comma) || + /["\r\n]/.test(value) || SPACE.test(value); + return needs ? `"${value.replaceAll('"', '""')}"` : value; +} + +/** A space at the start of a text, as `unicode.IsSpace` of Go. */ +const SPACE = + /^[\t\n\v\f\r \u0085\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000]/; diff --git a/cmd/lsf/main_test.js b/cmd/lsf/main_test.js index 28bf17d..d4b14c7 100644 --- a/cmd/lsf/main_test.js +++ b/cmd/lsf/main_test.js @@ -1,7 +1,56 @@ import { test } from "node:test"; -import { assert } from "../../dev_deps.js"; +import assert from "node:assert/strict"; + +import { DEFAULT_TIME, toListTime } from "@rfetch/lib/time"; +import { fetch } from "../../fetch.js"; import { lsf } from "./main.js"; -test("lsf", () => { - assert(lsf); +for (const name of ["a,b.txt", 'q"t.txt', " space.txt", "plain.txt"]) { + await fetch(`:memory:/lsf/${encodeURIComponent(name)}`, { + method: "PUT", + body: "x", + }); +} +await fetch(":memory:/lsf/sub/c.txt", { method: "PUT", body: "x" }); + +/** The lines of lsf, in order. */ +async function lines(flags) { + const text = await (await lsf(":memory:/lsf/", flags)).text(); + return text.split("\n").slice(0, -1).sort(); +} + +test("lsf gives the paths, with a / at the end of a folder", async () => { + assert.deepEqual(await lines(), [ + " space.txt", + "a,b.txt", + "plain.txt", + 'q"t.txt', + "sub/", + ]); + assert.deepEqual(await lines({ dir_slash: false, dirs_only: true }), [ + "sub", + ]); +}); + +test("lsf --csv uses , and the quotes of CSV, as rclone", async () => { + assert.deepEqual( + await lines({ csv: true, format: "ps", files_only: true }), + [ + '" space.txt",1', + '"a,b.txt",1', + "plain.txt,1", + '"q""t.txt",1', + ].sort(), + ); + // With a separator, CSV uses it, and puts in quotes a value that has it. + assert.deepEqual( + await lines({ csv: true, format: "sp", separator: ";", files_only: true }), + ['1;" space.txt"', "1;a,b.txt", "1;plain.txt", '1;"q""t.txt"'].sort(), + ); +}); + +test("lsf --format t gives the local time to the second, as rclone", async () => { + const [line] = await lines({ format: "tp", separator: "|", dirs_only: true }); + assert.equal(line, `${toListTime(DEFAULT_TIME)}|sub/`); + assert.match(line, /^\d{4}-\d\d-\d\d \d\d:\d\d:\d\d\|sub\/$/); }); diff --git a/cmd/lsf/rclone_test.js b/cmd/lsf/rclone_test.js new file mode 100644 index 0000000..7b74195 --- /dev/null +++ b/cmd/lsf/rclone_test.js @@ -0,0 +1,138 @@ +/** + * The tests of the lists of text against the lists of rclone, on the same + * local folder: ls, lsl, lsd and lsf. The tests run when rclone is in the + * PATH, or when RCLONE is the path of rclone. + */ +import { after, before, describe, test } from "node:test"; +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import process from "node:process"; + +import { ls } from "../ls/main.js"; +import { lsd } from "../lsd/main.js"; +import { lsl } from "../lsl/main.js"; +import { lsf } from "./main.js"; + +const RCLONE = process.env.RCLONE || "rclone"; +const hasRclone = spawnSync(RCLONE, ["version"]).status === 0; + +/** The files, with names that CSV puts in quotes, and their sizes. */ +const FILES = { + "a,b.txt": 1, + 'q"t.txt': 1024, + " space.txt": 1536, + "semi;colon/big.bin": 123456, + "n o/ê é/Hello, 世界.bin": 5, +}; + +describe("the lists of text and the lists of rclone", { + skip: !hasRclone && "rclone is not installed. Set RCLONE to its path.", +}, () => { + const folder = mkdtempSync(join(tmpdir(), "rfetch-lsf-")); + const cwd = process.cwd(); + + before(() => { + for (const [path, size] of Object.entries(FILES)) { + mkdirSync(dirname(join(folder, "tree", path)), { recursive: true }); + writeFileSync(join(folder, "tree", path), new Uint8Array(size)); + } + // The local backend serves the current folder. + process.chdir(folder); + }); + after(() => { + process.chdir(cwd); + rmSync(folder, { recursive: true, force: true }); + }); + + /** + * The lines of rclone, in order: rclone lists the folders at the same + * time, thus its order can change. + * @param {string[]} args + */ + function rclone(args) { + const { status, stdout, stderr } = spawnSync(RCLONE, args, { + cwd: folder, + env: { + PATH: process.env.PATH, + HOME: folder, + // The same time zone as the test. + ...process.env.TZ ? { TZ: process.env.TZ } : {}, + }, + }); + assert.equal(status, 0, `${stderr}`); + return `${stdout}`.split("\n").slice(0, -1).sort(); + } + + /** @param {Promise} response */ + async function lines(response) { + const text = await (await response).text(); + return text.split("\n").slice(0, -1).sort(); + } + + test("ls", async () => { + assert.deepEqual(await lines(ls("tree/")), rclone(["ls", "tree"])); + assert.deepEqual( + await lines(ls("tree/", { human_readable: true })), + rclone(["ls", "--human-readable", "tree"]), + ); + }); + + test("lsl", async () => { + assert.deepEqual(await lines(lsl("tree/")), rclone(["lsl", "tree"])); + }); + + test("lsd", async () => { + // rclone gives the size of a local folder on the disk, and rfetch -1, + // as the other backends: the lines are the same after the size. + const cut = (list, width) => list.map((line) => line.slice(width)).sort(); + assert.deepEqual( + cut(await lines(lsd("tree/", { recursive: true })), 12), + cut(rclone(["lsd", "-R", "tree"]), 12), + ); + assert.deepEqual( + cut(await lines(lsd("tree/", { human_readable: true })), 9), + cut(rclone(["lsd", "--human-readable", "tree"]), 9), + ); + }); + + test("lsf", async () => { + for ( + const [flags, args] of [ + [{ recursive: true }, ["-R"]], + [{ recursive: true, format: "pst", files_only: true }, [ + "-R", + "--format", + "pst", + "--files-only", + ]], + [{ recursive: true, csv: true, format: "ps", files_only: true }, [ + "-R", + "--csv", + "--format", + "ps", + "--files-only", + ]], + [{ csv: true, separator: ";", format: "sp" }, [ + "--csv", + "--separator", + ";", + "--format", + "sp", + ]], + [{ dirs_only: true, dir_slash: false }, [ + "--dirs-only", + "--dir-slash=false", + ]], + ] + ) { + assert.deepEqual( + await lines(lsf("tree/", flags)), + rclone(["lsf", ...args, "tree"]), + args.join(" "), + ); + } + }); +}); diff --git a/cmd/lsjson/main.js b/cmd/lsjson/main.js index cc336d7..05ac369 100644 --- a/cmd/lsjson/main.js +++ b/cmd/lsjson/main.js @@ -1,9 +1,14 @@ -import { basename, HTMLRewriter, toLocaleISOString } from "../../deps.js"; +import { parse } from "@rfetch/lib/fspath"; +import { readListing } from "@rfetch/lib/listing"; +import { isTrue } from "@rfetch/lib/options"; +import { DEFAULT_TIME, toLocaleISOString } from "@rfetch/lib/time"; -import { fetch } from "../../mod.js"; +import { fetch } from "../../fetch.js"; const encoder = new TextEncoder(); -const decoder = new TextDecoder(); + +/** The URL of a folder: the links of its list are relative to it. */ +const FOLDER = new URL("http://remote/folder/"); /** * List directories and objects in the path in JSON format. @@ -50,229 +55,286 @@ const decoder = new TextDecoder(); * @param {Object} [flags={}] * @param {number} [flags.max_depth=Infinity] - Maximum depth of recursion. * @param {boolean} [flags.recursive=false] - Recurse into the listing. + * @param {RequestInit} [init] The init of each request of the list, for + * example the headers with the options `RFETCH_*`. * @returns {Promise} */ -export async function lsjson(location, flags = {}) { - const isDirectory = location.endsWith("/"); - const init = { method: "GET" }; - const segments = location.split("/"); +export async function lsjson(location, flags = {}, init = {}) { + // A path that ends with `.` or `..` is a folder, as in rclone. + if (/(^|[:/])\.\.?$/.test(location)) location = `${location}/`; + // Only the path in the remote has folders: a value of the connection + // string can have a "/" too, for example `:crypt,remote='a/b':path`. + const parsed = parse(location); + const prefix = location.slice(0, location.length - parsed.path.length); + const path = parsed.path; + const isDirectory = path === "" || path.endsWith("/"); + init = { ...init, method: "GET" }; + let folder = path; let filename; - // If file, remove the last segment to query the parent directory. + // If file, list the parent directory and keep only the file. if (!isDirectory) { - filename = segments.pop(); + const slash = path.lastIndexOf("/") + 1; + folder = path.slice(0, slash); + filename = nameOf(path.slice(slash)); } const params = new URLSearchParams(flags); - const url = `${segments.join("/")}?${params}`; - let response = await fetch(url, init); + const url = `${prefix}${folder}?${params}`; + const response = await fetch(url, init); if (!response.ok) { return response; } - let maxDepth = Number(flags.max_depth || (flags.recursive ? Infinity : 1)); - if (!isDirectory || !flags.recursive) { + /** @type {import("@rfetch/lib/listing").Entry[] | undefined} */ + let entries; + if (!isDirectory) { + // A path with no "/" at the end is a folder when no file has the name, + // as in rclone: then the list is the list of the folder. + entries = await readListing(response, FOLDER); + const named = entries.filter(({ Name }) => Name === filename); + // As rclone, a path that names no file and no folder is not found. + if (named.length === 0) { + return new Response("directory not found", { status: 404 }); + } + if (named.every(({ IsDir }) => IsDir)) { + return lsjson(`${location}/`, flags, init); + } + } + + const recursive = isTrue(flags.recursive); + let maxDepth = Number(flags.max_depth || (recursive ? Infinity : 1)); + if (!isDirectory || !recursive) { maxDepth = 1; } + /** @type {Walk} */ + const options = { location, depth: maxDepth, flags, init, filename }; + const listed = entries + ? Promise.resolve(folderOf(entries, options)) + : readListing(response, FOLDER).then((list) => folderOf(list, options)); + // The error comes when the stream gets to the folder. + listed.catch(() => {}); + const items = walk(listed, ""); + /** - * A stream of JSON-encoded file objects, one per chunk/line. - * Each JSON stringified line is followed by a comma and a newline. - * Note: This stream does not wrap the output in an array. - * @type {ReadableStream} + * The JSON of the item before, with no comma: only the last item has no + * comma after it. + * @type {string | null} */ - let body = new ReadableStream({ - start(controller) { - /** - * @type {import("../../mod.js").File} - */ - let item; - let transformed = Promise.resolve(); - /** - * @type {Promise[]} - */ - const promises = []; - - const rewriter = new HTMLRewriter(); - rewriter.on(`tbody`, { - element(element) { - element.onEndTag(() => { - Promise.all(promises).then(() => { - controller.close(); - }); - }); - }, - }); - rewriter.on(`tbody tr`, { - element(element) { - // Start a new item. - item = { - Path: "", - Name: "", - Size: -1, - MimeType: "", - ModTime: "", - IsDir: true, - }; - - element.onEndTag(() => { - if (item) { - // We always enqueue the item with trailing comma. - // Another transform will remove the trailing comma. - const line = JSON.stringify(item) + ",\n"; - controller.enqueue(encoder.encode(line)); - } - }); - }, - }); - // TODO: match only links relative to the current location - rewriter.on(`tr a[href]`, { - element(element) { - const href = element.getAttribute("href"); - const { pathname } = new URL(href, "file:"); + let previous = null; - if (pathname === "/") { - item = null; - return; - } - - const name = basename(pathname); - - // If listing a file, skip other files. - if (filename && name !== filename) { - item = null; - return; - } - - item.Name = name; - - /** - * The Path field will only show folders below the remote - * path being listed. If "remote:path" contains the file - * "subfolder/file.txt", the Path for "file.txt" will be - * "subfolder/file.txt", not "remote:path/subfolder/file.txt". - * When used without --recursive the Path will always be the - * same as Name. - */ - item.Path = name; - - const isDir = item.IsDir = pathname.endsWith("/"); - - const type = element.getAttribute("type"); - if (type) { - item.MimeType = type; + /** + * A stream of JSON-encoded file objects, one per chunk and line, in an + * array. + * @type {ReadableStream} + */ + const body = new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode("[\n")); + }, + async pull(controller) { + for (;;) { + const { done, value } = await items.next(); + if (done) { + if (previous !== null) { + controller.enqueue(encoder.encode(`${previous}\n`)); } + controller.enqueue(encoder.encode("]\n")); + controller.close(); + return; + } + const line = JSON.stringify(value); + if (previous !== null) { + controller.enqueue(encoder.encode(`${previous},\n`)); + previous = line; + return; + } + previous = line; + } + }, + async cancel() { + await items.return(undefined); + }, + }); - if (isDir) { - item.MimeType = "inode/directory"; - item.Size = -1; - - // Recursively list subdirectories if --recursive - if (maxDepth > 1) { - // We don't want to block the current iteration, so we - // start a new promise for each subdirectory using IIFE. - // Inside, we await for the `done` promise to be resolved - // before enqueueing the next item. - const promise = (async () => { - const response = await lsjson(`${location}${name}/`, { - ...flags, - max_depth: String(maxDepth - 1), - }); + return new Response(body, { + headers: { + "Content-Type": "application/json", + }, + }); +} - await response.body?.pipeThrough( - new TransformStream({ - async transform(chunk) { - let line = decoder.decode(chunk).trim(); +/** + * An item of the output of `lsjson`. + * @typedef {Object} Item + * @property {string} Path The path below the listed folder. + * @property {string} Name + * @property {number} Size + * @property {string} MimeType + * @property {string | undefined} ModTime + * @property {boolean} IsDir + */ - // skips the `[` line. - if (line === "[" || line === "]") { - return; - } +/** + * The list of a folder: its items, and the lists of its folders. + * @typedef {Object} Folder + * @property {Item[]} items + * @property {{ name: string, folder: Promise }[]} folders + */ - const child = JSON.parse(line.replace(/,$/, "")); - child.Path = `${name}/${child.Path}`; - line = JSON.stringify(child); +/** + * @typedef {Object} Walk + * @property {string} location The location of the folder, with a `/` at the + * end. + * @property {number} depth The levels to list: 1 is only this folder. + * @property {Object} flags + * @property {RequestInit} init + * @property {string} [filename] When set, only the file with this name. + */ - await transformed; - controller.enqueue(encoder.encode(line + ",\n")); - }, - }), - ).pipeTo(new WritableStream({})); - })(); - promises.push(promise); - } - } - }, - }); - rewriter.on(`tr data[value]`, { - element(element) { - if (!item) return; - const value = element.getAttribute("value"); - item.Size = Number(value); - }, - }); - rewriter.on(`tr time[datetime]`, { - element(element) { - if (!item) return; - const value = element.getAttribute("datetime"); - if (value) { - item.ModTime = toLocaleISOString(value); - } - }, +/** + * Gives the folder of the entries of its list. When `depth` is more than 1, + * it starts to read the lists of the folders in it at once, thus the + * requests go at the same time. + * @param {import("@rfetch/lib/listing").Entry[]} entries + * @param {Walk} options + * @returns {Folder} + */ +function folderOf(entries, { location, depth, flags, init, filename }) { + /** @type {Folder} */ + const result = { items: [], folders: [] }; + for (const { Name, IsDir, Size, MimeType, ModTime } of entries) { + // If listing a file, skip other files. + if (filename && Name !== filename) continue; + result.items.push({ + Path: Name, + Name, + Size, + MimeType: IsDir ? "inode/directory" : MimeType, + // A folder with no time has the default time of rclone. + ModTime: ModTime ? toLocaleISOString(ModTime) : IsDir ? DEFAULT_TIME : "", + IsDir, + }); + if (IsDir && depth > 1) { + // The name of the folder in the path has percent-encoding, else a "?" + // or a "#" in the name starts a query. + const child = `${location}${encodeURIComponent(Name)}/`; + const folder = list(Name, { + location: child, + depth: depth - 1, + flags: { ...flags, max_depth: String(depth - 1) }, + init, }); + // The error comes when the walk gets to this folder. + folder.catch(() => {}); + result.folders.push({ name: Name, folder }); + } + } + return result; +} - response = rewriter.transform(response); - transformed = response.body.pipeTo(new WritableStream()); - }, - }); +/** + * Requests the list of a folder, and reads it. + * @param {string} name The name of the folder, for the error. + * @param {Walk} options + * @returns {Promise} + */ +async function list(name, options) { + const params = new URLSearchParams(options.flags); + const response = await fetch(`${options.location}?${params}`, options.init); + if (!response.ok) { + await response.body?.cancel(); + throw new Error(`Cannot list ${name}: ${response.status}`); + } + return folderOf(await readListing(response, FOLDER), options); +} - // Because the JSON objects always have trailing commas, we need to - // do a transform to remove the trailing comma from the last object. +/** + * Gives the items of a folder, then the items of each folder in it. The + * `Path` of an item has the names of the folders above it, below the listed + * folder. + * @param {Promise} folder + * @param {string} path The path of the folder, or "" for the listed folder. + * @returns {AsyncGenerator} + */ +async function* walk(folder, path) { + const { items, folders } = await folder; + for (const item of items) { + if (path) item.Path = `${path}/${item.Path}`; + yield item; + } + for (const { name, folder } of folders) { + yield* walk(folder, path ? `${path}/${name}` : name); + } +} + +/** + * A stream that reads the body of `lsjson`, in chunks of any size, and gives + * the body of a list of text: a line for each item. It keeps only the part of + * a line that has no "\n" yet. The lists of text `ls`, `lsl`, `lsd` and `lsf` + * use it. + * + * Example + * + * ```js + * import { LineStream, lsjson } from "./main.js"; + * const response = await lsjson("remote:path"); + * const paths = response.body.pipeThrough( + * new LineStream((item) => item.Path), + * ); + * ``` + * @extends {TransformStream} + */ +export class LineStream extends TransformStream { /** - * @type {Uint8Array} + * @param {(item: Item) => string | undefined} line Gives the line of an + * item, with no line break, or no line. */ - let previousChunk; - body = body.pipeThrough( - new TransformStream({ - start(controller) { - controller.enqueue(encoder.encode("[\n")); - }, - // Because we want to detect the last chunk, we need to delay the - // transform by one step, so we can handle it in `flush`. + constructor(line) { + const decoder = new TextDecoder(); + let rest = ""; + /** + * Writes the lines of the items of the lines of `lsjson`. + * @param {string[]} texts + * @param {TransformStreamDefaultController} controller + */ + const write = (texts, controller) => { + let lines = ""; + for (let text of texts) { + // `lsjson` gives each item on a line, with a comma at the end of + // each line but the last. + text = text.trim().replace(/,$/, ""); + if (!text.startsWith("{")) continue; + const value = line(JSON.parse(text)); + if (value !== undefined) lines += `${value}\n`; + } + if (lines) controller.enqueue(encoder.encode(lines)); + }; + super({ transform(chunk, controller) { - if (previousChunk) { - controller.enqueue(previousChunk); - } - previousChunk = chunk; + const texts = (rest + decoder.decode(chunk, { stream: true })) + .split("\n"); + rest = texts.pop() ?? ""; + write(texts, controller); }, flush(controller) { - if (previousChunk) { - const chunk = concat( - previousChunk.slice(0, -2), - encoder.encode("\n"), - ); - controller.enqueue(chunk); - } - controller.enqueue(encoder.encode("]\n")); + write([rest + decoder.decode()], controller); }, - }), - ); - - return new Response(body, { - headers: { - "Content-Type": "application/json", - }, - }); + }); + } } /** - * Concats two Uint8Arrays - * @param {Uint8Array} a - * @param {Uint8Array} b - * @returns {Uint8Array} + * The name of a file in the path of a location, as a backend reads it: with + * no percent-encoding. + * @param {string} segment For example `a%20b.txt` or `a b.txt`. + * @returns {string} */ -function concat(a, b) { - const result = new Uint8Array(a.byteLength + b.byteLength); - result.set(a, 0); - result.set(b, a.byteLength); - return result; +function nameOf(segment) { + const { pathname } = new URL(segment, "file:///"); + try { + return decodeURIComponent(pathname.slice(1)); + } catch { + return pathname.slice(1); + } } diff --git a/cmd/lsjson/main_bench.js b/cmd/lsjson/main_bench.js index 30f1a2c..e4b5502 100644 --- a/cmd/lsjson/main_bench.js +++ b/cmd/lsjson/main_bench.js @@ -1,61 +1,31 @@ /** - * Benchmark `lsjson` command vs `rclone lsjson` command. + * Benchmark of the `lsjson` command against `rclone lsjson`. * - * `rclone` must be installed and available in the PATH. + * `rclone` must be in the PATH. Run with `node cmd/lsjson/main_bench.js`. * - * This benchmark skips the time to spawn `rclone` process for more - * accurate comparison. + * This benchmark leaves out the time to start the `rclone` process, for a + * more accurate comparison. */ - +import { spawn } from "node:child_process"; +import { Readable } from "node:stream"; +import { bench } from "@rfetch/lib/testing/bench"; import { lsjson } from "./main.js"; const remotes = ["./", "./backend/", "./cmd/"]; for (const remote of remotes) { - Deno.bench( - `rfetch lsjson ${remote} --recursive`, - { - baseline: true, - group: `timing ${remote} --recursive`, - permissions: { - env: true, - read: true, - }, - }, - async () => { + await bench(`timing ${remote} --recursive`, { + [`rfetch lsjson ${remote} --recursive`]: async () => { const response = await lsjson(remote, { recursive: true }); await response.arrayBuffer(); }, - ); - - Deno.bench( - `rclone lsjson ${remote} --recursive`, - { - group: `timing ${remote} --recursive`, - permissions: { - env: true, - read: true, - run: true, - }, - }, - async (b) => { - const command = new Deno.Command("rclone", { - args: [ - "lsjson", - remote, - "--recursive", - ], - stdout: "piped", + [`rclone lsjson ${remote} --recursive`]: async (timer) => { + const rclone = spawn("rclone", ["lsjson", remote, "--recursive"], { + stdio: ["ignore", "pipe", "inherit"], }); - - const process = command.spawn(); - - b.start(); - - const response = new Response(process.stdout); - await response.arrayBuffer(); - - b.end(); + timer.start(); + await new Response(Readable.toWeb(rclone.stdout)).arrayBuffer(); + timer.end(); }, - ); + }); } diff --git a/cmd/lsjson/main_test.js b/cmd/lsjson/main_test.js index a125098..741ac2b 100644 --- a/cmd/lsjson/main_test.js +++ b/cmd/lsjson/main_test.js @@ -1,6 +1,9 @@ import { test } from "node:test"; -import { assert, assertEquals } from "../../dev_deps.js"; -import { lsjson } from "./main.js"; +import assert from "node:assert/strict"; +import { listen } from "@rfetch/lib/testing/server"; +import memory from "@rfetch/memory"; +import { fetch } from "../../mod.js"; +import { LineStream, lsjson } from "./main.js"; const encoder = new TextEncoder(); const decoder = new TextDecoder(); @@ -55,44 +58,44 @@ test(`lsjson("${pathname}")`, async (t) => { const text = await response.clone().text(); const lines = text.trim().split("\n"); - assertEquals(lines.shift(), "["); + assert.deepEqual(lines.shift(), "["); for (const child of tree[pathname]) { const line = lines.shift().replace(/,$/, ""); const file = JSON.parse(line); - assertEquals(file.Name, child.replace("/", "")); + assert.deepEqual(file.Name, child.replace("/", "")); } - assertEquals(lines[0], "]"); + assert.deepEqual(lines[0], "]"); }); await t.test("Path field is same as Name", async () => { const result = await response.clone().json(); for (const file of result) { - assertEquals(file.Path, file.Name); + assert.deepEqual(file.Path, file.Name); } }); await t.test("each object is a chunk in body stream", async () => { const reader = response.body.getReader(); let result = await reader.read(); - assertEquals(result.done, false); - assertEquals(decoder.decode(result.value).trim(), "["); + assert.deepEqual(result.done, false); + assert.deepEqual(decoder.decode(result.value).trim(), "["); for (const child of tree[pathname]) { result = await reader?.read(); - assertEquals(result.done, false); + assert.deepEqual(result.done, false); const line = decoder.decode(result.value).trim().replace(/,$/, ""); const file = JSON.parse(line); - assertEquals(file.Name, child.replace("/", "")); + assert.deepEqual(file.Name, child.replace("/", "")); } result = await reader?.read(); - assertEquals(result.done, false); - assertEquals(decoder.decode(result.value).trim(), "]"); + assert.deepEqual(result.done, false); + assert.deepEqual(decoder.decode(result.value).trim(), "]"); result = await reader?.read(); - assertEquals(result.done, true); + assert.deepEqual(result.done, true); }); }); @@ -132,3 +135,166 @@ test(`lsjson("${pathname}", { recursive: true })`, async (t) => { }, ); }); + +test("lsjson gives the names with no percent-encoding", async () => { + const name = "a b?#%.txt"; + await fetch(`:memory:/names/${encodeURIComponent(name)}`, { + method: "PUT", + body: "a", + }); + + const result = await lsjson(":memory:/names/").then((res) => res.json()); + assert.deepEqual(result.map(({ Name, Path }) => ({ Name, Path })), [ + { Name: name, Path: name }, + ]); +}); + +/** Puts the files, with percent-encoding in each name of each path. */ +async function putAll(location, paths) { + for (const path of paths) { + const encoded = path.split("/").map(encodeURIComponent).join("/"); + const response = await fetch(`${location}${encoded}`, { + method: "PUT", + headers: { "Content-Length": `${encoder.encode(path).byteLength}` }, + body: path, + }); + assert.equal(response.status, 201, path); + } +} + +/** The paths of the entries of `lsjson --recursive`, in order. */ +async function walk(location) { + const result = await lsjson(location, { recursive: "true" }) + .then((response) => response.json()); + return result.map(({ Path, IsDir, Size }) => ({ Path, IsDir, Size })) + .sort((a, b) => a.Path < b.Path ? -1 : +(a.Path > b.Path)); +} + +/** Folders and files with names that have "?", "#", "%" and spaces. */ +const HARD = ["a?b/c#d/e%f.txt", "a?b/g h.txt", "i%20j/k?.txt", "l#/m"]; + +/** The entries of the files of HARD and of their folders. */ +function entries(paths) { + const all = new Map(); + for (const path of paths) { + const names = path.split("/"); + for (let i = 1; i < names.length; i++) { + const folder = names.slice(0, i).join("/"); + all.set(folder, { Path: folder, IsDir: true, Size: -1 }); + } + all.set(path, { + Path: path, + IsDir: false, + Size: encoder.encode(path).length, + }); + } + return [...all.values()].sort((a, b) => + a.Path < b.Path ? -1 : +(a.Path > b.Path) + ); +} + +test("lsjson --recursive keeps the names with ? and # in folders", async () => { + await putAll(":memory:/recursive/", HARD); + assert.deepEqual(await walk(":memory:/recursive/"), entries(HARD)); +}); + +test("lsjson lists one file with a hard name", async () => { + const result = await lsjson( + `:memory:/recursive/a%3Fb/${encodeURIComponent("g h.txt")}`, + ) + .then((response) => response.json()); + assert.deepEqual(result.map(({ Name }) => Name), ["g h.txt"]); +}); + +test("lsjson reads a connection string with a / in a value", async () => { + // "test-password", obscured by rclone. + const password = "4OSlj-ZYMGdEIGjcWg_gc-DL4O3TFAvr4ct_yV4"; + const location = + `:crypt,remote=':memory:/lsjson/crypt',password=${password}:`; + await putAll(location, HARD); + assert.deepEqual(await walk(location), entries(HARD)); +}); + +test("lsjson --recursive gives an error for a folder it cannot list", async () => { + // A server of the memory backend, with no list of the folder "bad". + const server = await listen((request) => + new URL(request.url).pathname === "/lsjson-error/bad/" + ? new Response("no list", { status: 500 }) + : memory.fetch(request) + ); + try { + await putAll(":memory:/lsjson-error/", ["bad/a.txt", "good/b.txt"]); + const response = await lsjson(`:http,url='${server.url}lsjson-error/':`, { + recursive: "true", + }); + assert.equal(response.status, 200); + await assert.rejects(response.text(), /Cannot list bad: 50\d/); + } finally { + await server.close(); + } +}); + +test("lsjson lists a folder with no / at the end, as rclone", async () => { + await putAll(":memory:/no-slash/", ["tree/a.txt", "tree/b/c.txt", "file"]); + assert.deepEqual(await walk(":memory:/no-slash/tree"), [ + { Path: "a.txt", IsDir: false, Size: 10 }, + { Path: "b", IsDir: true, Size: -1 }, + { Path: "b/c.txt", IsDir: false, Size: 12 }, + ]); + // A file of the name is a file, as for rclone. + const result = await lsjson(":memory:/no-slash/file") + .then((response) => response.json()); + assert.deepEqual(result.map(({ Path, IsDir }) => ({ Path, IsDir })), [ + { Path: "file", IsDir: false }, + ]); +}); + +test("lsjson gives 404 for a path that is not found, as rclone", async () => { + await putAll(":memory:/not-found/", ["a.txt"]); + for (const path of ["nosuch", "nosuch.txt", "nosuch/"]) { + const response = await lsjson(`:memory:/not-found/${path}`); + assert.equal(response.status, 404, path); + await response.body?.cancel(); + } +}); + +test("lsjson gives a folder with no time the default time of rclone", async () => { + const items = await lsjson(":memory:/A/").then((response) => response.json()); + const folder = items.find(({ Name }) => Name === "D"); + assert.equal(folder.IsDir, true); + assert.equal(folder.ModTime, "2000-01-01T00:00:00Z"); +}); + +test("LineStream reads the body of lsjson in chunks of any size", async () => { + const text = [ + "[", + '{"Path":"a","Name":"a","IsDir":false},', + '{"Path":"日本語 b","Name":"日本語 b","IsDir":false},', + '{"Path":"C","Name":"C","IsDir":true}', + "]", + "", + ].join("\n"); + const bytes = encoder.encode(text); + // All in one chunk, a chunk for each byte (thus a character of three + // bytes is in three chunks), and chunks of 7 bytes. + for (const size of [bytes.length, 1, 7]) { + const body = new ReadableStream({ + start(controller) { + for (let i = 0; i < bytes.length; i += size) { + controller.enqueue(bytes.slice(i, i + size)); + } + controller.close(); + }, + }); + const lines = body.pipeThrough( + new LineStream(({ Path, IsDir }) => IsDir ? undefined : Path), + ); + assert.equal(await new Response(lines).text(), "a\n日本語 b\n", `${size}`); + } +}); + +test("LineStream gives an error for a line that is not an item", async () => { + const body = new Response('[\n{"Path":"a",\n]\n').body; + const lines = body.pipeThrough(new LineStream(({ Path }) => Path)); + await assert.rejects(new Response(lines).text(), SyntaxError); +}); diff --git a/cmd/lsjson/rclone_test.js b/cmd/lsjson/rclone_test.js new file mode 100644 index 0000000..094206d --- /dev/null +++ b/cmd/lsjson/rclone_test.js @@ -0,0 +1,74 @@ +/** + * The tests of lsjson against rclone lsjson, on the same local folder. The + * tests run when rclone is in the PATH, or when RCLONE is the path of rclone. + */ +import { after, before, describe, test } from "node:test"; +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import process from "node:process"; + +import { lsjson } from "./main.js"; + +const RCLONE = process.env.RCLONE || "rclone"; +const hasRclone = spawnSync(RCLONE, ["version"]).status === 0; + +/** The files, with names that have "?", "#", "%", spaces and Unicode. */ +const FILES = [ + "a?b/c#d/e%f.txt", + "a?b/g h.json", + "i%20j/k?.html", + "l#/m", + "n o/ê é/Hello, 世界.bin", + "p.txt", +]; + +describe("lsjson and rclone lsjson", { + skip: !hasRclone && "rclone is not installed. Set RCLONE to its path.", +}, () => { + const folder = mkdtempSync(join(tmpdir(), "rfetch-lsjson-")); + const cwd = process.cwd(); + + before(() => { + for (const [index, path] of FILES.entries()) { + mkdirSync(dirname(join(folder, "tree", path)), { recursive: true }); + writeFileSync(join(folder, "tree", path), "x".repeat(index * 10)); + } + // The local backend serves the current folder. + process.chdir(folder); + }); + after(() => { + process.chdir(cwd); + rmSync(folder, { recursive: true, force: true }); + }); + + /** The fields to compare, in the order of the paths. */ + function normalize(entries) { + return entries + .map(({ Path, Name, Size, IsDir, MimeType, ModTime }) => ({ + Path, + Name, + // rclone gives the size of a local folder on the disk. + Size: IsDir ? -1 : Size, + IsDir, + MimeType, + // The same digits as rclone: 9 for the nanoseconds of local files. + ModTime, + })) + .sort((a, b) => a.Path < b.Path ? -1 : +(a.Path > b.Path)); + } + + test("lsjson --recursive gives the entries of rclone lsjson -R", async () => { + const { status, stdout, stderr } = spawnSync( + RCLONE, + ["lsjson", "-R", join(folder, "tree")], + { env: { PATH: process.env.PATH, HOME: folder } }, + ); + assert.equal(status, 0, `${stderr}`); + const actual = await lsjson("tree/", { recursive: "true" }) + .then((response) => response.json()); + assert.deepEqual(normalize(actual), normalize(JSON.parse(stdout))); + }); +}); diff --git a/cmd/lsl/main.js b/cmd/lsl/main.js index d1d7368..107f321 100644 --- a/cmd/lsl/main.js +++ b/cmd/lsl/main.js @@ -1,4 +1,7 @@ -import { lsf } from "../../mod.js"; +import { sizeField } from "@rfetch/lib/format"; +import { isTrue } from "@rfetch/lib/options"; +import { toListTime } from "@rfetch/lib/time"; +import { LineStream, lsjson } from "../lsjson/main.js"; /** * List the objects in path with modification time, size and path. @@ -23,14 +26,22 @@ import { lsf } from "../../mod.js"; * @param {number} [flags.max_depth=Infinity] - Maximum depth of recursion. * @returns {Promise} */ -export function lsl(location, flags = {}) { - flags = { - recursive: "true", +export async function lsl(location, flags = {}) { + const human = isTrue(flags.human_readable); + const response = await lsjson(location, { + recursive: true, max_depth: "Infinity", - files_only: "true", - format: "stp", - separator: "\t", ...flags, - }; - return lsf(location, flags); + }); + if (!response.ok) return response; + // As `rclone lsl`: the size in 9 characters, the local time to the + // nanosecond, then the path. + const lines = new LineStream((item) => { + if (item.IsDir) return; + const time = toListTime(item.ModTime, 9); + return `${sizeField(item.Size, human, 9)} ${time} ${item.Path}`; + }); + return new Response(response.body.pipeThrough(lines), { + headers: { "Content-Type": "text/plain" }, + }); } diff --git a/cmd/lsl/main_test.js b/cmd/lsl/main_test.js index 16454dc..4f5ab09 100644 --- a/cmd/lsl/main_test.js +++ b/cmd/lsl/main_test.js @@ -1,7 +1,19 @@ import { test } from "node:test"; -import { assert } from "../../dev_deps.js"; +import assert from "node:assert/strict"; + +import { toListTime } from "@rfetch/lib/time"; +import { fetch } from "../../fetch.js"; +import { lsjson } from "../lsjson/main.js"; import { lsl } from "./main.js"; -test("lsl", () => { - assert(lsl); +await fetch(":memory:/lsl/a.txt", { method: "PUT", body: "hello" }); + +test("lsl gives the size, the local time to the nanosecond and the path, as rclone", async () => { + const [{ ModTime }] = await (await lsjson(":memory:/lsl/")).json(); + const text = await (await lsl(":memory:/lsl/")).text(); + assert.equal(text, ` 5 ${toListTime(ModTime, 9)} a.txt\n`); + assert.match( + text, + /^ {8}5 \d{4}-\d\d-\d\d \d\d:\d\d:\d\d\.\d{9} a\.txt\n$/, + ); }); diff --git a/cmd/obscure/main.js b/cmd/obscure/main.js index 78faeb4..25fafd5 100644 --- a/cmd/obscure/main.js +++ b/cmd/obscure/main.js @@ -1,118 +1,3 @@ -const ALGORITHM = "AES-CTR"; - -const KEY = await crypto.subtle.importKey( - "raw", - // https://github.com/rclone/rclone/blob/10c884552caab3d3d8484ed3401e94f983a23f93/fs/config/obscure/obscure.go#L17-L22 - Uint8Array.from([ - 0x9c, - 0x93, - 0x5b, - 0x48, - 0x73, - 0x0a, - 0x55, - 0x4d, - 0x6b, - 0xfd, - 0x7c, - 0x63, - 0xc8, - 0x86, - 0xa9, - 0x2b, - 0xd3, - 0x90, - 0x19, - 0x8e, - 0xb8, - 0x12, - 0x8a, - 0xfb, - 0xf4, - 0xde, - 0x16, - 0x2b, - 0x8b, - 0x95, - 0xf6, - 0x38, - ]), - ALGORITHM, - false, - ["encrypt", "decrypt"], -); - -const BlockSize = 16; - -/** - * Obscrures a password by encrypting them and writing them out in base64. - * @param {string} password - * @returns {Promise} A Response containing the obscured password. - */ -export async function obscure(password) { - const plaintext = new TextEncoder().encode(password); - // Constructs the initial empty ciphertext buffer containing the IV and the encrypted text. - const ciphertext = new Uint8Array(BlockSize + plaintext.length); - // Constructs the IV buffer. - const iv = ciphertext.subarray(0, BlockSize); - // Fills IV buffer with random bytes. - crypto.getRandomValues(iv); - // Encrypts the plaintext into the encrypted buffer. - const encrypted = await crypto.subtle.encrypt( - { - name: ALGORITHM, - counter: iv, - length: 64, - }, - KEY, - plaintext, - ); - // Copies the encrypted buffer into the ciphertext buffer. - ciphertext.set(new Uint8Array(encrypted), BlockSize); - // Encodes the ciphertext buffer into a base64 string. - const encoded = btoa(String.fromCharCode(...ciphertext)); - - return new Response(encoded); -} - -/** - * Reveals obscured text by decrypting them - * @param {string} cipherText - * @returns {Promise} - */ -export async function reveal(cipherText) { - // Decodes the base64 string into a ciphertext buffer. - const bytes = base64(cipherText); - // Constructs the IV buffer from the first block of the ciphertext buffer. - const iv = bytes.subarray(0, BlockSize); - // Constructs the encrypted buffer from the remaining blocks of the ciphertext buffer. - const data = bytes.subarray(BlockSize); - // Decrypts the encrypted buffer into the plaintext buffer. - const plaintext = await crypto.subtle.decrypt( - { - name: ALGORITHM, - counter: iv, - length: 64, - }, - KEY, - data, - ); - - return new Response(plaintext); -} - -/** - * Encodes a string into a base64 Uint8Array. - * @param {string} s - * @returns {Uint8Array} - */ -function base64(s) { - // Replace non-url compatible chars with base64 standard chars - s = s.replace(/-/g, "+").replace(/_/g, "/"); - // Pad out with standard base64 required padding characters - const pad = s.length % 4; - if (pad != 0) { - s += "====".substring(0, 4 - pad); - } - return new Uint8Array(atob(s).split("").map((c) => c.charCodeAt(0))); -} +// The code of the commands obscure and reveal is in @rfetch/lib, because the +// backends crypt and drive also use reveal. +export { obscure, reveal } from "@rfetch/lib/obscure"; diff --git a/cmd/rcat/main.js b/cmd/rcat/main.js index bbeef0e..720ceab 100644 --- a/cmd/rcat/main.js +++ b/cmd/rcat/main.js @@ -1,5 +1,5 @@ import process from "node:process"; -import { fetch } from "../../mod.js"; +import { fetch } from "../../fetch.js"; /** * Copies standard input to file on remote. diff --git a/cmd/rcat/main_test.js b/cmd/rcat/main_test.js index 2f92719..6244f39 100644 --- a/cmd/rcat/main_test.js +++ b/cmd/rcat/main_test.js @@ -1,5 +1,5 @@ import { test } from "node:test"; -import { assert } from "../../dev_deps.js"; +import assert from "node:assert/strict"; import { rcat } from "./main.js"; test("rcat", () => { diff --git a/cmd/serve/http/main.js b/cmd/serve/http/main.js index 0e521d5..c216445 100644 --- a/cmd/serve/http/main.js +++ b/cmd/serve/http/main.js @@ -1,11 +1,30 @@ -import { serve as serveHTTP } from "@sntran/serve"; +import { createServer } from "node:http"; +import { once } from "node:events"; +import { relink } from "@rfetch/lib/listing"; +import { isOption } from "@rfetch/lib/options"; +import { decodePath } from "@rfetch/lib/path"; + +import { fetch } from "../../../fetch.js"; + +/** The methods of a client, as `rclone serve http`: it only reads. */ +const METHODS = ["GET", "HEAD"]; /** - * Serve the remote over HTTP. - * @param {(Request) => Response | Promise} fetch - * @returns {Response} + * Serves a folder of a remote over HTTP, as `rclone serve http`. The path of + * a request is a path in the folder. Each request goes to the fetch of + * rfetch, thus the remote gets all its options: the config file, the + * variables of the environment, and the flags of the backend, for example + * `--crypt-password`. + * + * A client sets no option of the remote: its query and its headers + * `RFETCH_*` and `RCLONE_*` go away. Else a client could change the remote, + * for example `?remote=/etc/` for alias, or tell a new remote in the headers. + * @param {string} remote The remote and the path of the folder, for example + * `remote:path/to/folder`. + * @param {Object} [flags] `addr`, and the flags of the backend. + * @returns {Response} The lines of the server, for example the address. */ -export function serve(fetch, flags = {}) { +export function serve(remote, flags = {}) { const { addr = "127.0.0.1:8080", ...backendFlags @@ -16,32 +35,103 @@ export function serve(fetch, flags = {}) { hostname = "0.0.0.0"; } - const abortController = new AbortController(); + // The path of each request goes after the folder. + const folder = /[:/]$/.test(remote) ? remote : `${remote}/`; + const query = `${new URLSearchParams(backendFlags)}`; + + /** + * The answer to a request of a client. + * @param {import("node:http").IncomingMessage} incoming + * @param {AbortSignal} signal + * @returns {Promise} + */ + async function answer(incoming, signal) { + // The check is before a Request: the Request of the Fetch standard + // refuses TRACE. TRACE of the fetch of rfetch also tells the options of + // the remote, with its secrets. + if (!METHODS.includes(incoming.method ?? "")) { + return new Response("Method Not Allowed", { + status: 405, + headers: { Allow: METHODS.join(", ") }, + }); + } + const { pathname } = new URL(incoming.url ?? "/", "http://localhost"); + // The URL parser removes the names `.` and `..`, but a name such as + // `..%2F..%2Fetc` must not go out of the folder. + if (decodePath(pathname) === null) { + return new Response("Bad Request", { status: 400 }); + } + const headers = new Headers(); + const { rawHeaders } = incoming; + for (let i = 0; i < rawHeaders.length; i += 2) { + if (!isOption(rawHeaders[i])) { + headers.append(rawHeaders[i], rawHeaders[i + 1]); + } + } + const path = `${folder}${pathname.slice(1)}`; + const response = await fetch(query ? `${path}?${query}` : path, { + method: incoming.method, + headers, + signal, + }); + // The links of a list have the options of the backend in their query. + // The server reads no query, thus the links of its lists have none. + const type = response.headers.get("Content-Type") || ""; + if (response.ok && pathname.endsWith("/") && type.startsWith("text/html")) { + return relink(response, ""); + } + return response; + } + + const server = createServer(async (incoming, outgoing) => { + const controller = new AbortController(); + outgoing.once("close", () => controller.abort()); + /** @type {Response} */ + let response; + try { + response = await answer(incoming, controller.signal); + } catch { + // An error does not stop the server. Its message can tell the + // config, thus the client does not get it. + response = new Response("Internal Server Error", { status: 500 }); + } + try { + for (const [name, value] of response.headers) { + if (name !== "set-cookie") outgoing.setHeader(name, value); + } + const cookies = response.headers.getSetCookie(); + if (cookies.length) outgoing.setHeader("Set-Cookie", cookies); + outgoing.writeHead(response.status, response.statusText || undefined); + if (incoming.method === "HEAD") { + await response.body?.cancel(); + } else if (response.body) { + for await (const chunk of response.body) { + // Waits when the client reads slowly: else the file stays in memory. + if (!outgoing.write(chunk)) await once(outgoing, "drain"); + } + } + outgoing.end(); + } catch { + // The client went away. + await response.body?.cancel().catch(() => {}); + outgoing.destroy(); + } + }); const body = new ReadableStream({ start(controller) { - serveHTTP({ - fetch(request) { - const url = new URL(request.url); - Object.entries(backendFlags).forEach(([key, value]) => { - if (!url.searchParams.has(key)) { - url.searchParams.set(key, value); - } - }); - - request = new Request(url, request); - return fetch(request); - }, - hostname, - port: Number(port), - signal: abortController.signal, - onListen({ hostname, port }) { - controller.enqueue(`Listening on http://${hostname}:${port}`); - }, + server.on("error", (error) => controller.error(error)); + server.listen(Number(port), hostname, () => { + const info = server.address(); + const where = typeof info === "string" + ? info + : `${info?.address}:${info?.port}`; + controller.enqueue(`Listening on http://${where}\n`); }); }, cancel() { - abortController.abort(); + server.close(); + server.closeAllConnections?.(); }, }).pipeThrough(new TextEncoderStream()); diff --git a/cmd/serve/main.js b/cmd/serve/main.js index a2f798a..7332134 100644 --- a/cmd/serve/main.js +++ b/cmd/serve/main.js @@ -1,33 +1,26 @@ -import { fetch } from "../../mod.js"; -import * as backends from "../../backend/main.js"; +import { fetch } from "../../fetch.js"; +import { load } from "../../backend/main.js"; /** * Serves a remote backend * @param {'http'} protocol The protocol to serve. - * @param {string} remote + * @param {string} remote The remote and the path of the folder to serve, for + * example `remote:path/to/folder`. * @param {Object} flags - * @returns {Response|Promise} + * @returns {Promise} */ export async function serve(protocol, remote, flags = {}) { - // TRACE the remote to retrive its type and configuration. + // The remote must have a backend of rfetch. TRACE gives the type of the + // remote, and does not go to the backend. const response = await fetch(remote, { method: "TRACE" }); + await response.body?.cancel(); const [, type] = response.headers.get("Via")?.match(/^(.*?)(\/.*)?$/) || []; - const body = await response.text(); - const [, url] = body.match(/^TRACE (.*) HTTP\/1.1$/m) || - []; - const { searchParams } = new URL(url, "file:"); - // Fills the options with the remote's configuration. - // TODO: remove prefix from flags specific for the remote. - searchParams.forEach((value, key) => { - if (!flags[key]) { - flags[key] = value; - } - }); - - const backend = backends[type]; + if (!type || !await load(type)) { + throw new Error(`Backend ${type} not found.`); + } if (protocol === "http") { const { serve } = await import("./http/main.js"); - return serve(backend.fetch, flags); + return serve(remote, flags); } } diff --git a/cmd/serve/main_test.js b/cmd/serve/main_test.js new file mode 100644 index 0000000..d98d89f --- /dev/null +++ b/cmd/serve/main_test.js @@ -0,0 +1,190 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { request as httpRequest } from "node:http"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import process from "node:process"; + +import { obscure } from "@rfetch/lib/obscure"; +import { fetch as rfetch } from "../../fetch.js"; +import { serve } from "./main.js"; + +/** + * Serves a remote over HTTP while `check` runs. `check` gets a function that + * sends a request with a path and gives the text of the answer, or its + * status when it is not ok. + * @param {string} remote + * @param {(get: (path: string, init?: RequestInit) => Promise, url: string) => Promise} check + * @param {Object} [flags] + */ +async function serving(remote, check, flags = {}) { + const response = await serve("http", remote, { + addr: "127.0.0.1:0", + ...flags, + }); + const reader = response.body.getReader(); + try { + const { value } = await reader.read(); + const [url] = new TextDecoder().decode(value).match(/http:\/\/\S+/); + await check(async (path, init = {}) => { + const answer = await fetch(new URL(path, url), init); + return answer.ok ? await answer.text() : answer.status; + }, url); + } finally { + await reader.cancel(); + } +} + +/** + * Sends a request with a method that fetch does not send, and gives its + * status. + * @param {string} url + * @param {string} method + * @returns {Promise} + */ +function statusOf(url, method) { + return new Promise((resolve, reject) => { + httpRequest(url, { method }, (response) => { + response.resume(); + resolve(response.statusCode); + }).on("error", reject).end(); + }); +} + +/** A folder with `b.txt`, and `sub/a.txt` in it. */ +async function folder() { + const root = await mkdtemp(join(tmpdir(), "rfetch-serve-")); + await mkdir(join(root, "sub")); + await writeFile(join(root, "b.txt"), "b"); + await writeFile(join(root, "sub", "a.txt"), "a"); + return root; +} + +test("serve serves only the folder of the remote", async () => { + const root = await folder(); + try { + await serving(join(root, "sub"), async (get) => { + assert.equal(await get("/a.txt"), "a"); + assert.equal(await get("/b.txt"), 404); + // The URL parser removes `..`, and a `/` after the decoding is bad. + assert.equal(await get("/../b.txt"), 404); + assert.equal(await get("/..%2Fb.txt"), 400); + assert.equal(await get("/x%2F..%2F..%2Fb.txt"), 400); + }); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test("serve gives the remote its options of the environment, of the config file and of the flags", async (t) => { + const root = await folder(); + const config = await mkdtemp(join(tmpdir(), "rfetch-serve-config-")); + const { XDG_CONFIG_HOME } = process.env; + t.after(async () => { + if (XDG_CONFIG_HOME === undefined) delete process.env.XDG_CONFIG_HOME; + else process.env.XDG_CONFIG_HOME = XDG_CONFIG_HOME; + delete process.env.RFETCH_CONFIG_SERVED_TYPE; + delete process.env.RFETCH_CONFIG_SERVED_REMOTE; + await rm(root, { recursive: true, force: true }); + await rm(config, { recursive: true, force: true }); + }); + + await t.test("the environment", async () => { + process.env.RFETCH_CONFIG_SERVED_TYPE = "alias"; + process.env.RFETCH_CONFIG_SERVED_REMOTE = `${root}/`; + await serving("served:sub", async (get) => { + assert.equal(await get("/a.txt"), "a"); + // A request that fails does not stop the server. + delete process.env.RFETCH_CONFIG_SERVED_TYPE; + assert.equal(await get("/a.txt"), 500); + process.env.RFETCH_CONFIG_SERVED_TYPE = "alias"; + assert.equal(await get("/a.txt"), "a"); + }); + }); + + const password = await (await obscure("serve password")).text(); + + await t.test("the config file", async () => { + process.env.XDG_CONFIG_HOME = config; + await mkdir(join(config, "rfetch")); + await writeFile( + join(config, "rfetch", "rfetch.conf"), + `[secret]\ntype = crypt\nremote = ${root}/encrypted/\npassword = ${password}\n`, + ); + const put = await rfetch("secret:sub/c.txt", { + method: "PUT", + body: "secret c", + }); + assert.equal(put.status, 201); + await serving("secret:sub", async (get) => { + assert.equal(await get("/c.txt"), "secret c"); + }); + }); + + await t.test("the flags", async () => { + await serving(`:crypt,remote='${root}/encrypted/':sub`, async (get) => { + assert.equal(await get("/c.txt"), "secret c"); + }, { crypt_password: password }); + }); +}); + +test("serve gives lists with no query in the links", async () => { + const root = await folder(); + try { + await serving(`:alias,remote='${root}/':`, async (get) => { + const html = await get("/"); + assert.match(html, /href="sub\/"/); + assert.match(html, /href="b\.txt"/); + assert.doesNotMatch(html, /href="[^"]*\?/); + }); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test("serve only reads, as rclone serve http", async () => { + const root = await folder(); + try { + await serving(join(root, "sub"), async (get, url) => { + // TRACE of the fetch of rfetch tells the options of the remote. + assert.equal(await statusOf(url, "TRACE"), 405); + assert.equal(await get("/a.txt", { method: "PUT", body: "x" }), 405); + assert.equal(await get("/a.txt", { method: "DELETE" }), 405); + assert.equal(await get("/", { method: "DELETE" }), 405); + assert.equal(await readFile(join(root, "sub", "a.txt"), "utf8"), "a"); + }); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test("a client of serve cannot set an option of the remote", async () => { + const inside = await mkdtemp(join(tmpdir(), "rfetch-serve-inside-")); + const outside = await mkdtemp(join(tmpdir(), "rfetch-serve-outside-")); + await writeFile(join(inside, "a.txt"), "inside"); + await writeFile(join(outside, "a.txt"), "outside"); + try { + await serving(`:alias,remote='${inside}/':`, async (get) => { + assert.equal(await get("/a.txt"), "inside"); + // The query of the client is not an option. + assert.equal( + await get(`/a.txt?remote=${encodeURIComponent(`${outside}/`)}`), + "inside", + ); + // A header of the client is not an option, and tells no remote. + assert.equal( + await get("/a.txt", { + headers: { + RFETCH_ALIAS_REMOTE: `${outside}/`, + RCLONE_ALIAS_REMOTE: `${outside}/`, + }, + }), + "inside", + ); + }); + } finally { + await rm(inside, { recursive: true, force: true }); + await rm(outside, { recursive: true, force: true }); + } +}); diff --git a/deno.json b/deno.json deleted file mode 100644 index 724bbe0..0000000 --- a/deno.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "tasks": { - "start": "deno run -A main.js", - "test": "deno test -A --fail-fast" - } -} diff --git a/deps.js b/deps.js deleted file mode 100644 index 79c4a83..0000000 --- a/deps.js +++ /dev/null @@ -1,98 +0,0 @@ -import { env } from "node:process"; - -export * as INI from "@std/ini"; -export { basename, extname, join, resolve } from "@std/path"; -export { getCookies } from "@std/http/cookie"; -export { contentType } from "@std/media-types"; -export { format as formatBytes } from "@std/fmt/bytes"; -export { format as formatDuration } from "@std/fmt/duration"; - -export { encodeBase64Url, encodeHex } from "@std/encoding"; -import { crypto } from "@std/crypto"; - -export { default as decodeBase32 } from "base32-decode"; -export { default as encodeBase32 } from "base32-encode"; - -import aesjs from "aes-js"; -// `aes-js` is a CommonJS module, so we need to use `default` to import it. -const { AES } = aesjs; -export { AES }; - -export { xsalsa20poly1305 } from "@noble/ciphers/salsa"; -export { scryptAsync as scrypt } from "@noble/hashes/scrypt"; - -import { HTMLRewriter } from "@sntran/html-rewriter"; -globalThis.HTMLRewriter = HTMLRewriter; -export { HTMLRewriter }; - -/** - * Calculates the hash of a file. - * @param {string} path - * @param {import("@std/crypto").DigestAlgorithm} [algorithm=""] - * @returns {string} - */ -async function digest(path, algorithm = "MD5") { - const { readFile } = await import("node:fs/promises"); - const payload = await readFile(path); - const buffer = await crypto.subtle.digest(algorithm, payload); - const hashArray = Array.from(new Uint8Array(buffer)); - return hashArray.map((b) => b.toString(16).padStart(2, "0")).join(""); -} - -export { crypto, digest }; - -/** - * Retrieves the configuration directory for current platform. - * @returns {string | null} - */ -export function config_dir() { - let dir = env["XDG_CONFIG_HOME"]; - if (!dir) { - dir = env["APPDATA"]; // Windows - } - if (!dir) { - dir = env["HOME"]; - if (dir) { - dir += "/.config"; - } - } - - return dir; -} - -/** - * Converts a date string to a locale ISO string. - * @param {string | null} value - * @returns {string} - */ -export function toLocaleISOString(value) { - if (!value) return undefined; - - let date = new Date(value); - const off = date.getTimezoneOffset() * -1; - const del = date.getMilliseconds() ? "Z" : "."; // have milliseconds ? - date = new Date(date.getTime() + off * 60000); // add or subtract time zone - return date - .toISOString() - .split(del)[0] + - (off < 0 ? "-" : "+") + - ("0" + Math.abs(Math.floor(off / 60))).substring(-2) + - ":" + - ("0" + Math.abs(off % 60)).substring(-2); -} - -/** - * Makes a buffer filled with random values. - * @param {number} sizeInBytes - * @param {(Uint8Array) => Uint8Array} fill - * @returns {Uint8Array} - */ -export function mkBuffer(sizeInBytes, fill = crypto.getRandomValues) { - const buffer = new Uint8Array(sizeInBytes); - const chunkSize = 65536; // maximum chunk size allowed by crypto.getRandomValues() - for (let i = 0; i < buffer.length; i += chunkSize) { - const chunk = buffer.subarray(i, i + chunkSize); - fill(chunk); // fill the chunk with random values - } - return buffer; -} diff --git a/dev_deps.js b/dev_deps.js deleted file mode 100644 index 0c396cc..0000000 --- a/dev_deps.js +++ /dev/null @@ -1,41 +0,0 @@ -import { - deepEqual as assertEquals, - notDeepEqual as assertNotEquals, - ok as assert, - rejects as assertRejects, -} from "node:assert/strict"; -export { assert, assertEquals, assertNotEquals, assertRejects }; - -/** - * Asserts a header - * @param {Headers} headers - * @param {string} name - * @param {string} [value] - */ -export function assertHeader(headers, name, value) { - if (value !== undefined) { - assertEquals( - headers.get(name), - value, - `should have ${name} header of ${value} instead of ${headers.get(name)}`, - ); - } else { - assert(headers.get(name), `should have ${name} header`); - } -} - -/** - * Checks if two byte arrays are equal - * @param {Uint8Array} a - * @param {Uint8Array} b - * @returns {boolean} - */ -export function equalBytes(a, b) { - // Should not happen - if (a.length !== b.length) return false; - let isSame = true; - for (let i = 0; i < a.length; i++) isSame &&= a[i] === b[i]; // Lets hope JIT won't optimize away. - return isSame; -} - -export * as fc from "fast-check"; diff --git a/fetch.js b/fetch.js new file mode 100644 index 0000000..062cc4a --- /dev/null +++ b/fetch.js @@ -0,0 +1,335 @@ +/** + * The `fetch` of rfetch, which reads the paths of remotes. The import of this + * module sets it as the global `fetch`. A command imports this module and + * not mod.js: mod.js exports all the commands, which take much time to load. + */ +import process, { env } from "node:process"; +import { backends, load } from "./backend/main.js"; +import { parse } from "@rfetch/lib/fspath"; +import { isOption, PREFIXES } from "@rfetch/lib/options"; + +/** + * @typedef {Object & { type: string }} Remote + */ + +const globalFetch = globalThis.fetch; + +/** + * A `fetch` with support for remote URL. The import of this module sets it + * as the global `fetch`. Thus a backend on top of other remotes, for example + * crypt, calls `fetch(":memory:/path")` and does not import rfetch. + * + * The syntax of the paths passed to the rfetch command are as follows. + * + * `/path/to/dir`` + * + * This refers to the local file system. + * + * On Windows `\` may be used instead of `/` in local paths only, non local + * paths must use `/`. + * + * These paths needn't start with a leading `/` - if they don't then they + * will be relative to the current directory. + * + * `remote:path/to/dir` + * + * This refers to a directory `path/to/dir` on `remote:` as defined in the + * config. + * + * `remote:/path/to/dir` + * + * On most backends this refers to the same directory as `remote:path/to/dir` + * and that format should be preferred. On a very small number of remotes + * (FTP, SFTP, ..) this will refer to a different directory. On these, paths + * without a leading `/` will refer to your "home" directory and paths with a + * leading `/` will refer to the root. + * + * `:backend:path/to/dir` + * + * This is an advanced form for creating remotes on the fly. `backend` should + * be the name or prefix of a backend (the `type` in the config file) and all + * the configuration for the backend should be provided on the command line + * (or in environment variables). + * + * ## Precedence + * + * The various different methods of backend configuration are read in this + * order and the first one with a value is used. + * + * - Parameters in connection strings, e.g. `myRemote,skip_links:` + * - Flag values as supplied on the command line, e.g. `--skip-links` + * - Remote specific environment vars, e.g. `RFETCH_CONFIG_MYREMOTE_SKIP_LINKS` + * - Backend-specific environment vars, e.g. `RFETCH_LOCAL_SKIP_LINKS` + * - Backend generic environment vars, e.g. `RFETCH_SKIP_LINKS` + * - Config file, e.g. `skip_links = true`. + * - Default values, e.g. `false` - these can't be changed. + * + * So if both `--skip-links` is supplied on the command line and an environment + * variable `RFETCH_LOCAL_SKIP_LINKS` is set, the command line flag will take + * preference. + * + * Each variable can also have the prefix `RCLONE_` of rclone, for + * compatibility, for example `RCLONE_LOCAL_SKIP_LINKS`. In each place, + * `RFETCH_` takes the place of `RCLONE_`. + * + * A header of the request with the name of a variable, for example + * `RFETCH_CONFIG_MYREMOTE_SKIP_LINKS`, takes the place of the variable of the + * environment. A remote can come only from variables, as in rclone: + * `RFETCH_CONFIG_MYREMOTE_TYPE` and its options. Thus a request can give + * all its configuration, with no config file. + * + * The backend gets the options of the path, the connection string and the + * flags, in the query of its request. It gets the other options in `env`, + * the second argument of its `fetch`, with the names of the variables of + * rfetch, for example `RFETCH_LOCAL_SKIP_LINKS`. A backend makes the links of + * a list from the query only, thus an option of `env` is never in a link. + * + * The headers `RFETCH_*` and `RCLONE_*` are only for the backends of rfetch: + * they never go + * out to a URL. + * + * For non backend configuration the order is as follows: + * + * - Flag values as supplied on the command line, e.g. `--stats 5s`. + * - Environment vars, e.g. `RFETCH_STATS=5s`. + * - Default values, e.g. `1m` - these can't be changed. + * + * ## Other environment variables + * + * @param {string | Request | URL} input + * @param { RequestInit} [init={}] + * @returns {Promise} + */ +export async function fetch(input, init = {}) { + if ( + typeof input !== "string" || input.startsWith("https://") || + input.startsWith("http://") || input.startsWith("file://") + ) { + return globalFetch(...withoutOptions(input, init)); + } + + // The parts of the path, as rclone reads them. It throws a SyntaxError + // for a path that is not correct. + const parsed = parse(input); + + const headers = new Headers(init.headers); + // The variables, by their names with no prefix, for example + // `CONFIG_R2_TYPE`: the environment of the process, then the headers of + // the request, which take their place. In each place, `RFETCH_` takes the + // place of `RCLONE_`. + /** @type {Record} */ + const variables = {}; + const places = [ + variablesOf(env), + [...headers].map(([key, value]) => [key.toUpperCase(), value]), + ]; + for (const entries of places) { + for (const prefix of ["RCLONE_", "RFETCH_"]) { + for (const [key, value] of entries) { + if (key.startsWith(prefix) && typeof value === "string") { + variables[key.slice(prefix.length)] = value; + } + } + } + } + + let { name } = parsed; + + /** + * @type {Remote} + */ + let config; + + if (name === "") { // A local path. + config = { type: "local" }; + } else if (name.startsWith(":")) { // `:type:path/to/file` + config = { type: name.slice(1) }; + name = ""; + } else { + // A remote of the config file, or of the variables, as rclone. + const type = variables[`CONFIG_${name.toUpperCase()}_TYPE`]; + // The command config reads the config file. It loads only for a remote + // with a name. + const { config: command } = await import("./cmd/config/main.js"); + const response = await command("show", name, undefined, { + headers: { "Accept": "application/json" }, + }).catch(() => null); + if (response?.ok) { + config = await response.json(); + } else if (type) { + config = {}; + } else { + throw new Error(`Remote ${name} not found in config.`); + } + if (type) config.type = type; + } + + const type = config.type; + delete config.type; + + // The options that are not in the path, from the lowest precedence: the + // config file, then the variables. + const options = new Map(Object.entries(config)); + + // Backend generic variables, for example `RFETCH_SKIP_LINKS`. A variable + // of a remote or of a backend is not generic: it is not an option of each + // backend. + const specific = ["CONFIG", ...Object.keys(backends)] + .map((prefix) => `${prefix.toUpperCase()}_`); + for (const [key, value] of Object.entries(variables)) { + if (specific.some((prefix) => key.startsWith(prefix))) continue; + options.set(key.toLowerCase(), value); + } + + // Backend-specific variables, for example `RFETCH_LOCAL_SKIP_LINKS`. + let prefix = `${type.toUpperCase()}_`; + for (const [key, value] of Object.entries(variables)) { + if (key.startsWith(prefix)) { + options.set(key.slice(prefix.length).toLowerCase(), value); + } + } + + // Remote specific variables, for example `RFETCH_CONFIG_MYREMOTE_SKIP_LINKS`. + if (name) { + prefix = `CONFIG_${name.toUpperCase()}_`; + for (const [key, value] of Object.entries(variables)) { + if (key.startsWith(prefix) && key !== `${prefix}TYPE`) { + options.set(key.slice(prefix.length).toLowerCase(), value); + } + } + } + + // A local path, of the command line or of the code, is relative to the + // current folder, as in rclone: for example `file.txt`, `../file.txt` or + // `/tmp/file.txt`. Only such a path gives the root `/` to the local + // backend. The Worker gives no such path: a path with no remote is a file + // of its site. + let location = new URL(parsed.path, "file:"); + /** @type {string | undefined} */ + let root; + if (parsed.name === "") { + const { pathToFileURL } = await import("node:url"); + const folder = pathToFileURL(process.cwd()); + if (!folder.pathname.endsWith("/")) folder.pathname += "/"; + location = new URL(parsed.path, folder); + root = "/"; + } + // `searchParams` may contain other flags not for the current backend. + const { pathname, searchParams } = location; + + // The query of the backend: the flags prefixed with the backend type, then + // the parameters in the connection string, which take their place. + const params = new URLSearchParams(); + searchParams.forEach((value, key) => { + if (key.startsWith(`${type}_`)) { + params.set(key.replace(`${type}_`, ""), value); + } + }); + for (const [key, value] of Object.entries(parsed.config)) { + params.set(key, value); + } + + // The other options go in `env`. The backend reads its query first, as a + // flag takes the place of a variable. + /** @type {Record} */ + const optionsEnv = {}; + for (const [key, value] of options) { + optionsEnv[`RFETCH_${type.toUpperCase()}_${key.toUpperCase()}`] = value; + } + // The root of the local backend is not an option: a config file, a + // variable, a header or the query cannot set it. + delete optionsEnv.RFETCH_LOCAL_ROOT; + if (root !== undefined) optionsEnv.RFETCH_LOCAL_ROOT = root; + // The headers of this backend are in its options now. + const own = PREFIXES.map((start) => `${start}${type.toLowerCase()}_`); + for (const key of [...headers.keys()]) { + if (own.some((start) => key.startsWith(start))) headers.delete(key); + } + + if (init.method === "TRACE") { + let body = `TRACE ${pathname}?${params} HTTP/1.1\r`; + // Reflects the request as response body, then the options in `env`. + for (const [name, value] of headers) { + body += `${name}: ${value}\r`; + } + for (const [name, value] of Object.entries(optionsEnv)) { + body += `${name}: ${value}\r`; + } + return new Response(body, { + status: 200, + headers: { + "Content-Type": "message/http", + "Via": `${type}/1.1 ${name}`.trim(), + }, + }); + } + + if (init.body instanceof ReadableStream) { + // @ts-ignore: Must have `duplex` for streaming body + init.duplex = "half"; // Must set this for stream body. + } + + const backend = await load(type); + if (!backend) { + throw new Error(`Backend ${type} not found.`); + } + + const url = new URL(`${pathname}?${params}`, import.meta.url || `file:`); + // Creates a new request with the initial init. + const request = new Request(url, init); + // Clones that request and updates the headers. + return backend.fetch( + new Request(request, { + headers, + }), + optionsEnv, + ); +} + +/** + * Gives the variables `RFETCH_*` and `RCLONE_*` of the environment of the + * process. It reads only the names, then the values of these variables: + * Node.js reads each value of `process.env` from the system, which takes + * much time. + * @param {Record} variables + * @returns {[string, string][]} + */ +function variablesOf(variables) { + /** @type {[string, string][]} */ + const entries = []; + for (const key of Object.keys(variables)) { + if (key.startsWith("RFETCH_") || key.startsWith("RCLONE_")) { + const value = variables[key]; + if (typeof value === "string") entries.push([key, value]); + } + } + return entries; +} + +/** + * Removes the options of rclone from a request to a URL: they are only for + * the backends of rfetch. + * @param {string | Request | URL} input + * @param {RequestInit} init + * @returns {[string | Request | URL, RequestInit]} + */ +function withoutOptions(input, init) { + /** @param {HeadersInit} [from] */ + const clean = (from) => { + const headers = new Headers(from); + for (const key of [...headers.keys()]) { + if (isOption(key)) headers.delete(key); + } + return headers; + }; + if (input instanceof Request) { + input = new Request(input, { headers: clean(input.headers) }); + } + if (init.headers !== undefined) { + init = { ...init, headers: clean(init.headers) }; + } + return [input, init]; +} + +// The other URLs go to the original `fetch`. +globalThis.fetch = fetch; diff --git a/lib/format.js b/lib/format.js new file mode 100644 index 0000000..2ef7fb3 --- /dev/null +++ b/lib/format.js @@ -0,0 +1,186 @@ +/** + * Formats of numbers for people: a size in bytes and a duration, and the + * sizes and the counts of the lists of rclone. + * + * They give the same text as `format` of `@std/fmt/bytes` and of + * `@std/fmt/duration` for the options that rfetch uses, but with `KiB` in + * place of `kiB`: the symbol of IEC 80000-13, as rclone writes it. + */ + +/** The first letters of the units above the byte. */ +const PREFIXES = "kMGTPEZY"; + +/** + * Formats a size in bytes, for example `1.5 kB`, or `1.46 KiB` with `binary`. + * The number has 3 significant digits, and a size of less than 1 byte has no + * unit prefix. + * @param {number} bytes + * @param {{ binary?: boolean, locale?: boolean }} [options] `binary` uses + * units of 1024 (KiB, MiB, ...) in place of 1000 (kB, MB, ...). `locale` + * writes the number in the format of the locale of the runtime. + * @returns {string} + */ +export function formatBytes(bytes, { binary = false, locale = false } = {}) { + if (!Number.isFinite(bytes)) { + throw new TypeError( + `Expected a finite number, got ${typeof bytes}: ${bytes}`, + ); + } + const sign = bytes < 0 ? "-" : ""; + let value = Math.abs(bytes); + const text = (number) => + locale ? localeFormat().format(number) : number.toString(); + if (value < 1) { + return `${sign}${text(value)} B`; + } + + const exponent = Math.min( + Math.floor( + binary ? Math.log(value) / Math.log(1024) : Math.log10(value) / 3, + ), + PREFIXES.length, + ); + value /= Math.pow(binary ? 1024 : 1000, exponent); + const number = text(Number(value.toPrecision(3))); + return `${sign}${number} ${unit(exponent, binary)}`; +} + +/** @type {Intl.NumberFormat | undefined} */ +let numberFormat; + +/** + * Gives the format of numbers of the locale of the runtime. It gives the + * same text as `toLocaleString()`, which makes a new format for each call. + */ +function localeFormat() { + numberFormat ??= new Intl.NumberFormat(); + return numberFormat; +} + +/** + * Gives the unit of a power of 1000 or of 1024: B, kB, MB, ... or B, KiB, + * MiB, ... + * @param {number} exponent + * @param {boolean} binary + * @returns {string} + */ +function unit(exponent, binary) { + if (exponent === 0) { + return "B"; + } + const prefix = PREFIXES[exponent - 1]; + return binary ? `${prefix === "k" ? "K" : prefix}iB` : `${prefix}B`; +} + +/** + * Formats a duration in milliseconds with the units that are not zero, from + * days to nanoseconds, for example `1m 5s 300ms`. + * @param {number} milliseconds + * @returns {string} + */ +export function formatDuration(milliseconds) { + const ms = Math.abs(milliseconds); + // The digits of the microseconds and of the nanoseconds. + const fraction = ms.toFixed(7).slice(-7, -1); + const parts = [ + [Math.trunc(ms / 86400000), "d"], + [Math.trunc(ms / 3600000) % 24, "h"], + [Math.trunc(ms / 60000) % 60, "m"], + [Math.trunc(ms / 1000) % 60, "s"], + [Math.trunc(ms) % 1000, "ms"], + [Number(fraction.slice(0, 3)), "µs"], + [Number(fraction.slice(3, 6)), "ns"], + ]; + return parts.filter(([value]) => value).map(([value, unit]) => + `${value}${unit}` + ) + .join(" "); +} + +/** The suffixes of the sizes of rclone: powers of 1024. */ +const SIZE_SUFFIXES = ["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei"]; + +/** The suffixes of the counts of rclone: powers of 1000. */ +const COUNT_SUFFIXES = ["", "k", "M", "G", "T", "P", "E"]; + +/** + * Writes a number that is 0 or more with the suffix of a power of `base`, as + * `SizeSuffix` and `CountSuffix` of rclone: with no digit after the point + * when the number is whole, else with 3 digits. The rounding is the rounding + * of `%.3f` of Go: to the nearest, and to the even digit for a half. + * @param {number} value + * @param {number} base + * @param {string[]} suffixes + * @returns {string} + */ +function suffixed(value, base, suffixes) { + const number = BigInt(value); + const step = BigInt(base); + let divisor = 1n; + let exponent = 0; + while (exponent < suffixes.length - 1 && number >= divisor * step) { + divisor *= step; + exponent++; + } + const suffix = suffixes[exponent]; + if (number % divisor === 0n) return `${number / divisor}${suffix}`; + let thousandths = number * 1000n / divisor; + const rest = number * 1000n % divisor * 2n; + if (rest > divisor || (rest === divisor && thousandths % 2n === 1n)) { + thousandths++; + } + const fraction = `${thousandths % 1000n}`.padStart(3, "0"); + return `${thousandths / 1000n}.${fraction}${suffix}`; +} + +/** + * Writes a size in bytes as the lists of rclone: the number, or with + * `humanReadable` the number with a suffix of a power of 1024, for example + * `120.562Ki`, as the flag `--human-readable` of rclone. + * @param {number} size A whole number of bytes, or -1 for no size. + * @param {boolean} [humanReadable] + * @returns {string} + */ +export function sizeString(size, humanReadable = false) { + if (!humanReadable) return `${size}`; + if (size < 0) return `-${suffixed(-size, 1024, SIZE_SUFFIXES)}`; + return suffixed(size, 1024, SIZE_SUFFIXES); +} + +/** + * Writes a count of items as the lists of rclone: the number, or with + * `humanReadable` the number with a suffix of a power of 1000, for example + * `1.235M`. + * @param {number} count A whole number, or -1 for no count. + * @param {boolean} [humanReadable] + * @returns {string} + */ +export function countString(count, humanReadable = false) { + if (!humanReadable) return `${count}`; + if (count < 0) return `-${suffixed(-count, 1000, COUNT_SUFFIXES)}`; + return suffixed(count, 1000, COUNT_SUFFIXES); +} + +/** + * Writes a size in a field of fixed width, as `SizeStringField` of rclone: + * on the right of `width` characters, or of 9 with `humanReadable`. + * @param {number} size + * @param {boolean} humanReadable + * @param {number} width + * @returns {string} + */ +export function sizeField(size, humanReadable, width) { + return sizeString(size, humanReadable).padStart(humanReadable ? 9 : width); +} + +/** + * Writes a count in a field of fixed width, as `CountStringField` of rclone: + * on the right of `width` characters, or of 8 with `humanReadable`. + * @param {number} count + * @param {boolean} humanReadable + * @param {number} width + * @returns {string} + */ +export function countField(count, humanReadable, width) { + return countString(count, humanReadable).padStart(humanReadable ? 8 : width); +} diff --git a/lib/format_test.js b/lib/format_test.js new file mode 100644 index 0000000..c94eaf4 --- /dev/null +++ b/lib/format_test.js @@ -0,0 +1,87 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { + countString, + formatBytes, + formatDuration, + sizeString, +} from "./format.js"; + +// The texts of `format` of @std/fmt 0.225, which these functions replace. +test("formatBytes", () => { + const cases = [ + [0, {}, "0 B"], + [0.5, {}, "0.5 B"], + [999, {}, "999 B"], + [1000, {}, "1 kB"], + [1536, {}, "1.54 kB"], + [1536, { binary: true }, "1.5 KiB"], + [1048576, { binary: true }, "1 MiB"], + [123456789, {}, "123 MB"], + [2 ** 40, { binary: true }, "1 TiB"], + [1e30, {}, "1000000 YB"], + [-1536, { binary: true }, "-1.5 KiB"], + [1234567, { locale: true }, `${(1.23).toLocaleString()} MB`], + ]; + for (const [bytes, options, text] of cases) { + assert.equal(formatBytes(bytes, options), text, `${bytes}`); + } + assert.throws(() => formatBytes(Infinity), TypeError); +}); + +test("formatDuration", () => { + assert.equal(formatDuration(0), ""); + assert.equal(formatDuration(61000), "1m 1s"); + assert.equal(formatDuration(90061001.5), "1d 1h 1m 1s 1ms 500µs"); + assert.equal(formatDuration(0.000001), "1ns"); + assert.equal(formatDuration(-3600000), "1h"); +}); + +test("sizeString writes the sizes of rclone", () => { + // The texts of SizeSuffix of rclone, from Go. + for ( + const [size, text] of [ + [0, "0"], + [1, "1"], + [1023, "1023"], + [1024, "1Ki"], + [1025, "1.001Ki"], + [1536, "1.500Ki"], + // 120.5625: Go rounds a half to the even digit. + [123456, "120.562Ki"], + [1408, "1.375Ki"], + [1048575, "1023.999Ki"], + [1048576, "1Mi"], + [5242881, "5.000Mi"], + [67108864, "64Mi"], + [1099511627776, "1Ti"], + [1125899906842624, "1Pi"], + [9007199254740991, "8.000Pi"], + ] + ) { + assert.equal(sizeString(size, true), text, `${size}`); + assert.equal(sizeString(size), `${size}`); + } + assert.equal(sizeString(-1, true), "-1"); + assert.equal(sizeString(-1), "-1"); +}); + +test("countString writes the counts of rclone", () => { + for ( + const [count, text] of [ + [0, "0"], + [999, "999"], + [1000, "1k"], + [1001, "1.001k"], + [1500, "1.500k"], + [999999, "999.999k"], + [1234567, "1.235M"], + [1000000, "1M"], + ] + ) { + assert.equal(countString(count, true), text, `${count}`); + } + assert.equal(countString(-1, true), "-1"); + assert.equal(countString(-1), "-1"); +}); diff --git a/lib/fspath.js b/lib/fspath.js new file mode 100644 index 0000000..047f5a4 --- /dev/null +++ b/lib/fspath.js @@ -0,0 +1,213 @@ +/** + * The paths of rclone, as the package fs/fspath of rclone 1.75.1: + * + * - `remote:path` is a path of the remote `remote` of the config file. + * - `:backend:path` is a path of a remote of the type `backend`, with no + * config file. + * - `remote,name=value,flag:path` sets the options `name` and `flag` of the + * remote. A value in `'` or `"` quotes can have `,` and `:`, and two + * quotes for one quote. + * - A path with no `:`, or with a `/` before the first `:` or `,`, is a local + * path. + */ + +/** The regular expression of the name of a remote in the config file. */ +export const CONFIG_NAME = String + .raw`[\w\p{L}\p{N}.+@]+(?:[ -]+[\w\p{L}\p{N}.+@-]+)*`; + +const CONFIG_NAME_MATCHER = new RegExp(`^${CONFIG_NAME}$`, "u"); +const REMOTE_NAME_MATCHER = new RegExp(`^:?${CONFIG_NAME}(?::$|,)`, "u"); + +/** The errors, with the text of rclone. */ +export const Errors = { + INVALID_CHARACTERS: + "config name contains invalid characters - may only contain numbers, letters, `_`, `-`, `.`, `+`, `@` and space, while not start with `-` or space, and not end with space", + CANT_BE_EMPTY: "can't use empty string as a path", + BAD_CONFIG_PARAM: + "config parameters may only contain `0-9`, `A-Z`, `a-z`, `_` and `.`", + EMPTY_CONFIG_PARAM: "config parameters can't be empty", + CONFIG_NAME_EMPTY: "config name can't be empty", + CONFIG_NAME: "config name needs a trailing `:`", + PARAM: "config parameter must end with `,` or `:`", + VALUE: "unquoted config value must end with `,` or `:`", + QUOTED_VALUE: "unterminated quoted config value", + AFTER_QUOTE: "expecting `:` or `,` or another quote after a quote", +}; + +/** + * Tells if a text is a correct name of a remote in the config file. + * @param {string} name + */ +export function isConfigName(name) { + return CONFIG_NAME_MATCHER.test(name); +} + +/** + * The parts of a path. + * @typedef {Object} Parsed + * @property {string} name The name of the remote, for example `remote`, or + * `:backend` for a remote with no config file, or "" for a local path. + * @property {string} configString The text before the `:`, for example + * `:backend,name=value`. + * @property {string} path The path in the remote, or the local path. + * @property {Record} config The options of the connection + * string, with no prototype. + */ + +/** + * Reads a path of rclone, as `Parse` of rclone. + * @param {string} text For example `remote,name='a:b':path/to/file`. + * @returns {Parsed} + * @throws {SyntaxError} When the path is not correct. + */ +export function parse(text) { + /** @type {Parsed} */ + const parsed = { + name: "", + configString: "", + path: text, + config: Object.create(null), + }; + if (text === "") throw new SyntaxError(Errors.CANT_BE_EMPTY); + // A path with no ":" is a local path. + if (!text.includes(":")) return parsed; + + const State = { + CONFIG_NAME: 0, + PARAM: 1, + VALUE: 2, + QUOTED_VALUE: 3, + AFTER_QUOTE: 4, + DONE: 5, + }; + let state = State.CONFIG_NAME; + let index = 0; + // The start of the current part. + let start = 0; + let quote = ""; + let param = ""; + let doubled = false; + + loop: + for (; index < text.length; index++) { + const c = text[index]; + switch (state) { + case State.CONFIG_NAME: + if (index === 0 && c === ":") continue; + if (c === "/" || c === "\\") { + // A ":" or a "," after a "/" is a part of a local path. A path + // that starts with ":" is for a remote with no config file. + if (text[0] === ":") { + throw new SyntaxError(Errors.INVALID_CHARACTERS); + } + return parsed; + } + if (c === ":" || c === ",") { + parsed.name = text.slice(0, index); + checkRemoteName(`${parsed.name}:`); + start = index + 1; + if (c === ":") { + state = State.DONE; + break loop; + } + state = State.PARAM; + } + break; + + case State.PARAM: + if (c === ":" || c === "," || c === "=") { + param = text.slice(start, index); + if (param === "") throw new SyntaxError(Errors.EMPTY_CONFIG_PARAM); + start = index + 1; + if (c === "=") { + state = State.VALUE; + break; + } + parsed.config[param] = "true"; + if (c === ":") { + state = State.DONE; + break loop; + } + } else if (!/[\w.]/.test(c)) { + throw new SyntaxError(Errors.BAD_CONFIG_PARAM); + } + break; + + case State.VALUE: + if (c === "'" || c === '"') { + // A quote starts a value in quotes only at the start of the value. + if (index === start) { + quote = c; + state = State.QUOTED_VALUE; + start = index + 1; + doubled = false; + } + } else if (c === ":" || c === ",") { + parsed.config[param] = text.slice(start, index); + start = index + 1; + if (c === ":") { + state = State.DONE; + break loop; + } + state = State.PARAM; + } + break; + + case State.QUOTED_VALUE: + if (c === quote) state = State.AFTER_QUOTE; + break; + + case State.AFTER_QUOTE: + if (c === ":" || c === ",") { + let value = text.slice(start, index - 1); + if (doubled) value = value.replaceAll(quote + quote, quote); + parsed.config[param] = value; + start = index + 1; + if (c === ":") { + state = State.DONE; + break loop; + } + state = State.PARAM; + } else if (c === quote) { + // Two quotes are one quote in the value. + state = State.QUOTED_VALUE; + doubled = true; + } else { + throw new SyntaxError(Errors.AFTER_QUOTE); + } + break; + } + } + + switch (state) { + case State.CONFIG_NAME: + throw new SyntaxError(Errors.CONFIG_NAME); + case State.PARAM: + throw new SyntaxError(Errors.PARAM); + case State.VALUE: + throw new SyntaxError(Errors.VALUE); + case State.QUOTED_VALUE: + throw new SyntaxError(Errors.QUOTED_VALUE); + case State.AFTER_QUOTE: + throw new SyntaxError(Errors.AFTER_QUOTE); + } + + parsed.configString = text.slice(0, index); + parsed.path = text.slice(index + 1); + return parsed; +} + +/** + * Makes sure that the start of a path is a correct name of a remote, as + * `checkRemoteName` of rclone. + * @param {string} name For example `remote:` or `:backend,`. + * @throws {SyntaxError} + */ +function checkRemoteName(name) { + if (name === ":" || name === "::") { + throw new SyntaxError(Errors.CONFIG_NAME_EMPTY); + } + if (!REMOTE_NAME_MATCHER.test(name)) { + throw new SyntaxError(Errors.INVALID_CHARACTERS); + } +} diff --git a/lib/fspath_test.js b/lib/fspath_test.js new file mode 100644 index 0000000..776594a --- /dev/null +++ b/lib/fspath_test.js @@ -0,0 +1,220 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fc from "fast-check"; + +import { Errors, isConfigName, parse } from "./fspath.js"; + +/** + * The cases of TestParse of rclone 1.75.1 (fs/fspath/path_test.go), with + * no case for Windows only: a path, and its parts or its error. + */ +const CASES = [ + ["", Errors.CANT_BE_EMPTY], + [":", Errors.CONFIG_NAME], + ["::", Errors.CONFIG_NAME_EMPTY], + [":/:", Errors.INVALID_CHARACTERS], + ["/:", { name: "", configString: "", path: "/:" }], + ["\\backslash:", { name: "", configString: "", path: "\\backslash:" }], + ["/slash:", { name: "", configString: "", path: "/slash:" }], + ["with\\backslash:", { + name: "", + configString: "", + path: "with\\backslash:", + }], + ["with/slash:", { name: "", configString: "", path: "with/slash:" }], + ["/path/to/file", { name: "", configString: "", path: "/path/to/file" }], + ["/path:/to/file", { name: "", configString: "", path: "/path:/to/file" }], + ["./path:/to/file", { name: "", configString: "", path: "./path:/to/file" }], + ["./:colon.txt", { name: "", configString: "", path: "./:colon.txt" }], + ["path/to/file", { name: "", configString: "", path: "path/to/file" }], + [".:", { name: ".", configString: ".", path: "" }], + ["..:", { name: "..", configString: "..", path: "" }], + [".:colon.txt", { name: ".", configString: ".", path: "colon.txt" }], + ["remote:path/to/file", { + name: "remote", + configString: "remote", + path: "path/to/file", + }], + ["rem*ote:path/to/file", Errors.INVALID_CHARACTERS], + ["remote:/path/to/file", { + name: "remote", + configString: "remote", + path: "/path/to/file", + }], + ["rem.ote:/path/to/file", { + name: "rem.ote", + configString: "rem.ote", + path: "/path/to/file", + }], + ["rem ote:/path/to/file", { + name: "rem ote", + configString: "rem ote", + path: "/path/to/file", + }], + ["remote :/path/to/file", Errors.INVALID_CHARACTERS], + [" remote:/path/to/file", Errors.INVALID_CHARACTERS], + [" remote :/path/to/file", Errors.INVALID_CHARACTERS], + ["rem#ote:/path/to/file", Errors.INVALID_CHARACTERS], + [":backend:/path/to/file", { + name: ":backend", + configString: ":backend", + path: "/path/to/file", + }], + [":back.end:/path/to/file", { + name: ":back.end", + configString: ":back.end", + path: "/path/to/file", + }], + [":bac*kend:/path/to/file", Errors.INVALID_CHARACTERS], + ["C:\\path\\to\\file", { + name: "C", + configString: "C", + path: "\\path\\to\\file", + }], + ["\\path\\to\\file", { + name: "", + configString: "", + path: "\\path\\to\\file", + }], + [".", { name: "", configString: "", path: "." }], + ["..", { name: "", configString: "", path: ".." }], + ["remote:\\path\\to\\file", { + name: "remote", + configString: "remote", + path: "\\path\\to\\file", + }], + ["D:/path/to/file", { name: "D", configString: "D", path: "/path/to/file" }], + [":backend,param1:/path/to/file", { + name: ":backend", + configString: ":backend,param1", + path: "/path/to/file", + config: { param1: "true" }, + }], + [":backend,param1=value:/path/to/file", { + name: ":backend", + configString: ":backend,param1=value", + path: "/path/to/file", + config: { param1: "value" }, + }], + [":backend,param1=value1,param2,param3=value3:/path/to/file", { + name: ":backend", + configString: ":backend,param1=value1,param2,param3=value3", + path: "/path/to/file", + config: { param1: "value1", param2: "true", param3: "value3" }, + }], + [":backend,param1=value1,param2=\"value2\",param3='value3':/path/to/file", { + name: ":backend", + configString: ":backend,param1=value1,param2=\"value2\",param3='value3'", + path: "/path/to/file", + config: { param1: "value1", param2: "value2", param3: "value3" }, + }], + [":backend,param-1=value:/path/to/file", Errors.BAD_CONFIG_PARAM], + [':backend,param1="value"x:/path/to/file', Errors.AFTER_QUOTE], + [":backend,", Errors.PARAM], + [":backend,param=value", Errors.VALUE], + [":backend,param=\"value'", Errors.QUOTED_VALUE], + [':backend,param1="value"', Errors.AFTER_QUOTE], + [":backend,=value:", Errors.EMPTY_CONFIG_PARAM], + [":backend,:", Errors.EMPTY_CONFIG_PARAM], + [":backend,,:", Errors.EMPTY_CONFIG_PARAM], + [":backend,param=:path", { + name: ":backend", + configString: ":backend,param=", + path: "path", + config: { param: "" }, + }], + [':backend,param="with""quote":path', { + name: ":backend", + configString: ':backend,param="with""quote"', + path: "path", + config: { param: 'with"quote' }, + }], + [":backend,param='''''':", { + name: ":backend", + configString: ":backend,param=''''''", + path: "", + config: { param: "''" }, + }], + [":backend,param=''bad'':", Errors.AFTER_QUOTE], +]; + +test("parse reads the paths of rclone (TestParse)", () => { + for (const [text, expected] of CASES) { + const what = JSON.stringify(text); + if (typeof expected === "string") { + assert.throws(() => parse(text), { + name: "SyntaxError", + message: expected, + }, what); + continue; + } + const { config = {}, ...parts } = expected; + const parsed = parse(text); + assert.deepEqual( + { ...parsed, config: { ...parsed.config } }, + { ...parts, config }, + what, + ); + } +}); + +test("isConfigName accepts the names of rclone (TestCheckConfigName)", () => { + for ( + const name of [ + "remote", + "REMOTE", + "r-emote-", + "_rem_ote_", + ".", + "..", + ".r.e.m.o.t.e.", + "rem ote", + "user@example.com", + "user+junkmail@example.com", + "blåbær", + "chữ Quốc ngữ", + ] + ) { + assert.ok(isConfigName(name), name); + } + for ( + const name of [ + "", + ":remote:", + "remote:", + "rem:ote", + "rem/ote", + "rem\\ote", + "[remote", + "*", + "-remote", + "---rem:::ote???", + "remote ", + " remote", + " remote ", + ] + ) { + assert.ok(!isConfigName(name), name); + } +}); + +test("parse gives back the options of a connection string", () => { + const param = fc.stringMatching(/^[\w.]{1,8}$/); + const value = fc.string({ maxLength: 12 }); + fc.assert( + fc.property( + fc.dictionary(param, value, { maxKeys: 4 }), + fc.constantFrom("'", '"'), + fc.string({ maxLength: 12 }), + (config, quote, path) => { + const options = Object.entries(config).map(([name, value]) => + `,${name}=${quote}${value.replaceAll(quote, quote + quote)}${quote}` + ); + const parsed = parse(`:backend${options.join("")}:${path}`); + assert.equal(parsed.name, ":backend"); + assert.equal(parsed.path, path); + assert.deepEqual({ ...parsed.config }, { ...config }); + }, + ), + ); +}); diff --git a/lib/html.js b/lib/html.js new file mode 100644 index 0000000..0eb6bd4 --- /dev/null +++ b/lib/html.js @@ -0,0 +1,19 @@ +/** The characters that have a meaning in HTML, and their references. */ +const REFERENCES = { + "&": "&", + "<": "<", + ">": ">", + '"': """, + "'": "'", +}; + +/** + * Puts a character reference in place of each character of the text that has + * a meaning in HTML. The result is safe as the text of an element, and as the + * value of an attribute in quotes. + * @param {string} text + * @returns {string} + */ +export function escapeHTML(text) { + return String(text).replace(/[&<>"']/g, (character) => REFERENCES[character]); +} diff --git a/lib/html_test.js b/lib/html_test.js new file mode 100644 index 0000000..cc1cb19 --- /dev/null +++ b/lib/html_test.js @@ -0,0 +1,17 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { escapeHTML } from "./html.js"; + +test("escapeHTML", async (t) => { + await t.test("puts a reference for each character of HTML", () => { + assert.equal( + escapeHTML(`Tom & Jerry's`), + "<a href="x">Tom & Jerry's</a>", + ); + }); + + await t.test("does not change other text", () => { + assert.equal(escapeHTML("Hello, 世界 ≠ ?+#"), "Hello, 世界 ≠ ?+#"); + }); +}); diff --git a/lib/ini.js b/lib/ini.js new file mode 100644 index 0000000..b6aef14 --- /dev/null +++ b/lib/ini.js @@ -0,0 +1,763 @@ +/** + * The configuration file of rclone, in the INI format of the library goconfig + * v1.0.0 (github.com/unknwon/goconfig), which rclone uses to read and write + * the file: + * + * ```ini + * # The comments of the person stay in the file. + * [remote] + * type = s3 + * provider = Cloudflare + * + * [secret] + * type = crypt + * remote = remote:bucket + * ``` + * + * The rules of goconfig: + * + * - A line ends with "\n". The white space at the two ends of a line goes + * away, with the white space of Go: also U+0085 and U+00A0. + * - An empty line, and a line that starts with `#` or `;` (a comment), give + * no key. + * - `[name]` starts a section. The keys before the first section go in the + * section `DEFAULT`. Two sections with the same name become one. + * - `key = value` and `key: value` set a key. The first `=` or `:` is the + * end of the key. + * - A key in quotes (`` `key` ``, `"key"` or `"""key"""`) keeps its white + * space, and can have `=` and `:`. + * - A value in quotes (`` `value` `` or `"""value"""`) keeps its white space. + * The value ends at the last quote of the line. + * - The key `-` gets a number in its section: `#1`, then `#2`, and so on. + * + * `parse` and `read` give the configuration. `ConfigFile` changes the + * configuration in the text of a file, and keeps the comments. + */ + +/** The section of the keys before the first section. */ +export const DEFAULT = "DEFAULT"; + +/** The white space that `strings.TrimSpace` of Go removes. */ +const SPACE = + "[\\t\\n\\v\\f\\r \\u0085\\u00A0\\u1680\\u2000-\\u200A\\u2028\\u2029\\u202F\\u205F\\u3000]"; +const TRIM = new RegExp(`^${SPACE}+|${SPACE}+$`, "g"); +const TRIM_START = new RegExp(`^${SPACE}+`); + +/** The byte order mark, which goconfig removes at the start of the file. */ +const BOM = "\uFEFF"; + +const encoder = new TextEncoder(); + +/** + * An item of a line of a configuration file. An empty line has no entry. + * + * - A `section` entry starts a section. The section `[]` has the name "": + * goconfig makes it the section DEFAULT, but it cannot have keys. + * - A `key` entry has a key and its value. `auto` is true for a key `-`. + * goconfig ignores the key "", and the key " " only makes the section. + * - A `comment` entry has the text of the comment, with its `#` or `;`. + * + * `section` is the current section of the line, and `line` is the number of + * the line, from 1. + * @typedef {{ section: string, line: number } & ( + * { type: "section" } | + * { type: "key", key: string, value: string, auto?: true } | + * { type: "comment", text: string } + * )} Entry + */ + +/** + * A configuration: the keys of each section, in the order of the file. + * @typedef {Record>} Config + */ + +/** + * Reads the text of a configuration file. + * @param {string} text + * @returns {Config} + * @throws {SyntaxError} For a line that goconfig does not accept. + */ +export function parse(text) { + const lines = new LineReader(); + const config = new ConfigBuilder(); + for (const line of text.split("\n")) { + config.add(lines.read(line)); + } + return config.value; +} + +/** + * Reads a configuration file from a stream of text, for example the body of + * a response through a `TextDecoderStream`. + * @param {ReadableStream} stream + * @returns {Promise} + */ +export async function read(stream) { + const config = new ConfigBuilder(); + for await (const entry of stream.pipeThrough(new ParseStream())) { + config.add(entry); + } + return config.value; +} + +/** + * A stream that reads the text of a configuration file, in chunks of any + * size, and gives an entry for each line with a section, a key or a comment. + * It keeps only the part of a line that has no "\n" yet. A line that goconfig + * does not accept makes an error of the stream. + * @extends {TransformStream} + */ +export class ParseStream extends TransformStream { + constructor() { + const lines = new LineReader(); + let rest = ""; + super({ + transform(chunk, controller) { + const parts = (rest + chunk).split("\n"); + rest = parts.pop(); + for (const part of parts) { + const entry = lines.read(part); + if (entry) controller.enqueue(entry); + } + }, + flush(controller) { + const entry = lines.read(rest); + if (entry) controller.enqueue(entry); + }, + }); + } +} + +/** + * The text of a configuration file, which changes only on the lines of a + * change. All other lines stay as they are: the comments, the empty lines and + * the format of each line. + * + * A comment directly above a section or a key, with no empty line between, + * is a comment of that line. It goes away only when that line goes away. All + * other comments always stay. + */ +export class ConfigFile { + #bom = false; + /** @type {Array<{ text: string, entry?: Entry, delimiter: number }>} */ + #lines = []; + /** "\r" when the lines of the file end with "\r\n". */ + #eol = ""; + + /** + * @param {string} [text] The text of the file. + * @throws {SyntaxError} For a line that goconfig does not accept. + */ + constructor(text = "") { + this.#load(text); + } + + /** @returns {Config} */ + get config() { + const config = new ConfigBuilder(); + for (const { entry } of this.#lines) { + config.add(entry); + } + return config.value; + } + + /** @returns {string} The text of the file. */ + toString() { + const text = this.#lines.map(({ text }) => text).join("\n"); + // A line that starts with U+FEFF needs a byte order mark before it when + // it is the first line. + return this.#bom || text.startsWith(BOM) ? BOM + text : text; + } + + /** + * Sets the keys of a section, and makes the section when it does not + * exist. A key with no value (undefined or null) goes away. The other keys + * stay. + * @param {string} name + * @param {Record} keys + * @throws {TypeError} For a name, a key or a value that rclone cannot read + * back the same, as `stringify`. Then the file does not change. + */ + update(name, keys) { + this.#change(name, keys, false); + } + + /** + * Sets the keys of a section as `update`, and removes the other keys of + * the section. + * @param {string} name + * @param {Record} keys + * @throws {TypeError} As `update`. + */ + set(name, keys) { + this.#change(name, keys, true); + } + + /** + * Removes a section: all its lines, and their comments. + * @param {string} name + */ + delete(name) { + const remove = new Set(); + for (const [index, { entry }] of this.#lines.entries()) { + if (entry && entry.type !== "comment" && sectionOf(entry) === name) { + remove.add(index); + } + } + if (remove.size) { + this.#write({ remove }); + } + } + + /** + * @param {string} name + * @param {Record} keys + * @param {boolean} replace True to remove the keys that are not in `keys`. + */ + #change(name, keys, replace) { + const lines = this.#lines; + /** Tells if a line has a key of the section. */ + const isKey = (index) => { + const entry = lines[index].entry; + return entry?.type === "key" && sectionOf(entry) === name && + entry.key !== "" && entry.key !== " "; + }; + + /** @type {Set} */ + const remove = new Set(); + /** @type {Map} The new values of lines. */ + const values = new Map(); + /** @type {string[]} The lines of the new keys. */ + const added = []; + + for (const [key, value] of Object.entries(keys)) { + const indexes = [...lines.keys()].filter((index) => + isKey(index) && lines[index].entry.key === key + ); + if (value === undefined || value === null) { + for (const index of indexes) remove.add(index); + } else if (indexes.length) { + // The last line of a key gives its value. + const index = indexes.at(-1); + withValue(lines[index], `${value}`); + if (lines[index].entry.value !== `${value}`) { + values.set(index, `${value}`); + } + } else { + added.push(writeLine(key, `${value}`) + this.#eol); + } + } + if (replace) { + for (const index of lines.keys()) { + if (!isKey(index)) continue; + const { key } = lines[index].entry; + // Only own keys: the key "toString" is not in `{}`. + if ((Object.hasOwn(keys, key) ? keys[key] : null) == null) { + remove.add(index); + } + } + } + + if (!Object.hasOwn(this.config, name)) { + const header = `[${writeSection(name)}]${this.#eol}`; + this.#write({ remove, values, section: [header, ...added] }); + return; + } + + /** @type {Map} */ + const insert = new Map(); + /** @type {Map} */ + const texts = new Map(); + + // The new keys go after the last line of the section that stays. + const after = lines.findLastIndex(({ entry }, index) => + entry && entry.type !== "comment" && sectionOf(entry) === name && + !(entry.type === "key" && entry.key === "") && !remove.has(index) + ); + + // DEFAULT has no `[DEFAULT]` line when its keys are at the start of the + // file. When all of them go away, a `[DEFAULT]` line in the place of the + // first one keeps the place of the section. + const top = lines.findIndex(({ entry }) => entry?.type === "section"); + const atTop = [...lines.keys()].filter((index) => + (top === -1 || index < top) && lines[index].entry?.type === "key" && + lines[index].entry.key !== "" + ); + if ( + name === DEFAULT && atTop.length && + atTop.every((index) => remove.has(index)) + ) { + const header = `[${DEFAULT}]${this.#eol}`; + insert.set(atTop[0] - 1, after === -1 ? [header, ...added] : [header]); + } + + if (after !== -1) { + insert.set(after, added); + // The section `[]` cannot have keys, but `[DEFAULT]` can. + const { entry, text } = lines[after]; + if (added.length && entry.type === "section" && entry.section === "") { + texts.set(after, `[${DEFAULT}]${eolOf(text)}`); + } + } + + this.#write({ remove, values, texts, insert }); + } + + /** + * Writes the new text of the file, and reads it again. + * @param {object} change + * @param {Set} change.remove The lines to remove, with their + * comments. + * @param {Map} [change.values] The new values of lines. + * @param {Map} [change.texts] The new texts of lines. + * @param {Map} [change.insert] The lines to put after a + * line, or at the start for -1. + * @param {string[]} [change.section] A new section for the end. + */ + #write({ + remove, + values = new Map(), + texts = new Map(), + insert = new Map(), + section = [], + }) { + const lines = this.#lines; + + // A comment directly above a line that goes away goes away too. + for (const index of [...remove]) { + for ( + let above = index - 1; + lines[above]?.entry?.type === "comment"; + above-- + ) { + remove.add(above); + } + } + + // A key `-` gets its number from the keys `-` before it in its part of + // the section. When one goes away, the keys `-` after it get their + // number as a key in quotes. + const explicit = new Set(); + let shift = false; + for (const [index, { entry }] of lines.entries()) { + if (entry?.type === "section") shift = false; + if (entry?.type !== "key" || !entry.auto) continue; + if (remove.has(index)) shift = true; + else if (shift) explicit.add(index); + } + + const result = [...insert.get(-1) ?? []]; + let removed = false; + for (const [index, line] of lines.entries()) { + if (!remove.has(index)) { + // Two empty lines do not come together where lines went away. + const blank = !line.entry && trim(line.text) === ""; + if ( + !(blank && removed && (!result.length || trim(result.at(-1)) === "")) + ) { + result.push(this.#text(index, values, texts, explicit.has(index))); + } + removed = false; + } else { + removed = true; + } + if (insert.get(index)?.length) { + result.push(...insert.get(index)); + removed = false; + } + } + + if (section.length) { + // The new section goes at the end, before the empty end of the file, + // after an empty line. + let end = result.length; + if (end && result[end - 1] === "") end--; + if (end && trim(result[end - 1]) !== "") section.unshift(this.#eol); + result.splice(end, 0, ...section); + } + + const text = result.join("\n"); + this.#load(this.#bom || text.startsWith(BOM) ? BOM + text : text); + } + + /** + * The new text of a line. + * @param {number} index + * @param {Map} values + * @param {Map} texts + * @param {boolean} explicit True to write the number of a key `-`. + */ + #text(index, values, texts, explicit) { + const line = this.#lines[index]; + const { entry } = line; + if (explicit) { + const value = values.get(index) ?? entry.value; + return writeLine(entry.key, value) + eolOf(line.text); + } + if (values.has(index)) { + return withValue(line, values.get(index)); + } + return texts.get(index) ?? line.text; + } + + /** @param {string} text */ + #load(text) { + this.#bom = text.startsWith(BOM); + const reader = new LineReader({ bom: false }); + this.#lines = (this.#bom ? text.slice(1) : text).split("\n").map(( + text, + ) => ({ text, entry: reader.read(text), delimiter: reader.delimiter })); + this.#eol = this.#lines.some(({ text }) => text.endsWith("\r")) ? "\r" : ""; + } +} + +/** + * Writes a configuration file that rclone reads as the same configuration. + * The keys of the section `DEFAULT` come first with no `[DEFAULT]` line, when + * `DEFAULT` is the first section. Then each section comes, with an empty line + * between the sections, as rclone writes them. A key with no value + * (undefined or null) is not written. To keep the comments of a file, use + * `ConfigFile`. + * + * A key or a value goes in quotes when rclone does not read it the same with + * no quotes, for example a value with a space at the start. In some of these + * cases, rclone writes no quotes, or quotes that it cannot read back. + * @param {Record>} config + * @param {{ section?: string }} [options] With a `section`, `config` is the + * keys of that one section. + * @returns {string} + * @throws {TypeError} For a name, a key or a value that rclone cannot read + * back the same: a line break in it, a section name that is empty or has + * white space at the start or at the end, and the keys "", " " and "-". + */ +export function stringify(config, { section } = {}) { + if (section) { + return stringify({ [section]: config }); + } + + const blocks = []; + for (const [index, [name, keys]] of Object.entries(config).entries()) { + if (typeof keys !== "object" || keys === null) { + throw new TypeError( + `INI: the section ${JSON.stringify(name)} is not an object.`, + ); + } + const lines = Object.entries(keys) + .filter(([, value]) => value !== undefined && value !== null) + .map(([key, value]) => writeLine(key, `${value}`)); + + if (name === DEFAULT && index === 0 && lines.length) { + blocks.push(lines); + } else { + blocks.push([`[${writeSection(name)}]`, ...lines]); + } + } + const text = blocks.map((lines) => lines.join("\n")).join("\n\n"); + // A key that starts with U+FEFF needs a byte order mark before it when it + // is the first line. + return text.startsWith(BOM) ? BOM + text : text; +} + +/** + * Reads the lines of one file, one at a time, as `ConfigFile.read` of + * goconfig. It keeps the current section and the number of the line. + */ +class LineReader { + #section = DEFAULT; + /** The number of the next key `-` in the section. */ + #count = 1; + #number = 0; + #bom; + + /** + * The position of the `=` or `:` after the key in the last line, or -1 + * when the last line has no key. + */ + delimiter = -1; + + /** + * @param {{ bom?: boolean }} [options] `bom: false` keeps a byte order mark + * at the start of the first line. + */ + constructor({ bom = true } = {}) { + this.#bom = bom; + } + + /** + * Reads one line, with no "\n". + * @param {string} raw + * @returns {Entry | undefined} + */ + read(raw) { + this.#number++; + this.delimiter = -1; + let text = raw; + if (this.#bom && this.#number === 1 && text.startsWith(BOM)) { + text = text.slice(1); + } + + const line = trim(text); + if (line === "") { + return; + } + if (line[0] === "#" || line[0] === ";") { + return { + type: "comment", + section: this.#section, + line: this.#number, + text: line, + }; + } + + if (line[0] === "[" && line.at(-1) === "]") { + this.#section = trim(line.slice(1, -1)); + this.#count = 1; + return { type: "section", section: this.#section, line: this.#number }; + } + + if (this.#section === "") { + throw this.#error("has a key in a section with no name"); + } + + let key, end; + const quote = startsWithQuotes(line) + ? '"""' + : line[0] === '"' || line[0] === "`" + ? line[0] + : ""; + if (quote) { + const close = line.indexOf(quote, quote.length); + if (close === -1) { + throw this.#error("has a key with no quote at the end"); + } + key = line.slice(quote.length, close); + end = delimiter(line, close); + } else { + end = delimiter(line, 0); + key = trim(line.slice(0, Math.max(end, 0))); + } + if (end <= 0) { + throw this.#error('has no "=" or ":" after a key'); + } + + /** @type {Entry} */ + const entry = { + type: "key", + section: this.#section, + line: this.#number, + key, + value: "", + }; + if (key === "-") { + entry.key = `#${this.#count++}`; + entry.auto = true; + } + + let value = trim(line.slice(end + 1)); + const valueQuote = value[0] === "`" && value.length >= 2 + ? "`" + : startsWithQuotes(value) + ? '"""' + : ""; + if (valueQuote) { + const close = value.lastIndexOf(valueQuote); + if (close < valueQuote.length) { + throw this.#error("has a value with no quote at the end"); + } + value = value.slice(valueQuote.length, close); + } + entry.value = value; + + // The line starts after the byte order mark and the white space. + this.delimiter = raw.length - text.replace(TRIM_START, "").length + end; + return entry; + } + + /** + * An error with the number of the line, but not the text of the line, + * which can have a password or a token. + * @param {string} problem + */ + #error(problem) { + return new SyntaxError(`INI: line ${this.#number} ${problem}.`); + } +} + +/** + * Makes the configuration from the entries, in the order of the file, as + * `ConfigFile.SetValue` of goconfig. + */ +class ConfigBuilder { + /** @type {Map>} */ + #sections = new Map(); + + /** @param {Entry | undefined} entry */ + add(entry) { + if (!entry || entry.type === "comment") return; + // goconfig ignores an empty key. + if (entry.type === "key" && entry.key === "") return; + + const section = sectionOf(entry); + if (!this.#sections.has(section)) { + this.#sections.set(section, new Map()); + } + // The key " " only makes the section. + if (entry.type === "key" && entry.key !== " ") { + this.#sections.get(section).set(entry.key, entry.value); + } + } + + /** @returns {Config} */ + get value() { + // Object.fromEntries makes own properties, so a name such as "__proto__" + // does not change the prototype. + return Object.fromEntries( + [...this.#sections].map(([name, keys]) => [ + name, + Object.fromEntries(keys), + ]), + ); + } +} + +/** + * The section of an entry. The section `[]` is DEFAULT. + * @param {Entry} entry + */ +function sectionOf(entry) { + return entry.section || DEFAULT; +} + +/** + * The line with a new value, with no change to the key and the `=` or `:`. + * @param {{ text: string, delimiter: number }} line + * @param {string} value + */ +function withValue({ text, delimiter }, value) { + const head = text.slice(0, delimiter + 1); + const after = text.slice(delimiter + 1); + const space = after === "" || TRIM_START.test(after) ? " " : ""; + return head + space + writeLineValue(head, value) + eolOf(text); +} + +/** + * Writes the line of a key and its value. + * @param {string} key + * @param {string} value + */ +function writeLine(key, value) { + const head = `${writeKey(key)} =`; + return `${head} ${writeLineValue(head, value)}`; +} + +/** + * Writes the value of a line. A line that starts with "[" and ends with "]" + * is a section for goconfig, thus the value goes in quotes then. + * @param {string} head The line before the value. + * @param {string} value + */ +function writeLineValue(head, value) { + const written = writeValue(value); + if (trim(head).startsWith("[") && written.endsWith("]")) { + return `\`${value}\``; + } + return written; +} + +/** "\r" when the line ends with "\r". @param {string} text */ +function eolOf(text) { + return text.endsWith("\r") ? "\r" : ""; +} + +/** + * Writes a section name, which goconfig reads with no white space at its two + * ends. + * @param {string} name + */ +function writeSection(name) { + if (name === "" || name.includes("\n") || trim(name) !== name) { + throw new TypeError( + `INI: the section name ${JSON.stringify(name)} cannot be written.`, + ); + } + return name; +} + +/** + * Writes a key, in quotes when goconfig does not read it the same with no + * quotes. The quotes are the quotes of goconfig for a key with "=" or ":". + * goconfig reads a key in quotes to the first quote at the end. + * @param {string} key + */ +function writeKey(key) { + if (key === "" || key === " " || key === "-" || key.includes("\n")) { + throw new TypeError( + `INI: the key ${JSON.stringify(key)} cannot be written.`, + ); + } + // A key that starts with "[" needs care for its value: see + // writeLineValue. + if (trim(key) === key && !/^[#;"`]|[=:]/.test(key)) { + return key; + } + if (!key.includes("`")) { + return `\`${key}\``; + } + if (!key.includes('"')) { + return `"${key}"`; + } + if (!key.includes('"""') && !key.endsWith('"')) { + return `"""${key}"""`; + } + throw new TypeError(`INI: the key ${JSON.stringify(key)} cannot be written.`); +} + +/** + * Writes a value, in quotes when goconfig does not read it the same with no + * quotes. goconfig reads a value in quotes to the last quote of the line, + * thus each value with no line break can go in quotes. + * @param {string} value + */ +function writeValue(value) { + if (value.includes("\n")) { + throw new TypeError("INI: a value cannot have a line break."); + } + // goconfig writes a value with "`" in """ quotes, but it writes the value + // in " quotes when the value has no '"'. Then it cannot read the value + // back, because a value in " quotes keeps its quotes. + if (value.includes("`")) { + return `"""${value}"""`; + } + if (trim(value) !== value || value.startsWith('"""')) { + return `\`${value}\``; + } + return value; +} + +/** + * Removes the white space at the two ends, as `strings.TrimSpace` of Go. + * @param {string} text + */ +function trim(text) { + return text.replace(TRIM, ""); +} + +/** + * Tells if a text starts with `"""` and has 6 bytes or more in UTF-8, as + * goconfig checks it for `"""` quotes. Thus `"""é` is not in quotes, but + * `"""€` is in quotes with no end, which is an error. + * @param {string} text + */ +function startsWithQuotes(text) { + return text.startsWith('"""') && encoder.encode(text).length >= 6; +} + +/** + * Gives the position of the first "=" or ":" from a position, or -1. + * @param {string} line + * @param {number} from + */ +function delimiter(line, from) { + const found = line.slice(from).search(/[=:]/); + return found === -1 ? -1 : from + found; +} diff --git a/lib/ini_test.js b/lib/ini_test.js new file mode 100644 index 0000000..a5c4861 --- /dev/null +++ b/lib/ini_test.js @@ -0,0 +1,821 @@ +import { after, describe, test } from "node:test"; +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import process from "node:process"; +import fc from "fast-check"; + +import { ConfigFile, parse, ParseStream, read, stringify } from "./ini.js"; + +describe("parse", () => { + test("reads sections and keys", () => { + assert.deepEqual(parse(""), {}); + assert.deepEqual( + parse( + "[a]\ntype = s3\n spaced key = a value \nurl=https://x/?a=b\n" + + "; a comment\n# a comment\n\n[b c]\nempty =\n", + ), + { + a: { type: "s3", "spaced key": "a value", url: "https://x/?a=b" }, + "b c": { empty: "" }, + }, + ); + }); + + test("puts the keys before the first section in DEFAULT", () => { + assert.deepEqual(parse("top = 1\n[a]\nx = 2"), { + DEFAULT: { top: "1" }, + a: { x: "2" }, + }); + assert.deepEqual(parse("[DEFAULT]\ntop = 1\n[]"), { + DEFAULT: { top: "1" }, + }); + }); + + test("makes one section of two sections with the same name", () => { + assert.deepEqual(parse("[a]\nx = 1\n[b]\n[a]\ny = 2\r\nx = 3"), { + a: { x: "3", y: "2" }, + b: {}, + }); + }); + + test('ends a key at the first "=" or ":"', () => { + assert.deepEqual(parse("[a]\nk1: v=1\nk2 = v:2\nk3:"), { + a: { k1: "v=1", k2: "v:2", k3: "" }, + }); + }); + + test("reads keys and values in quotes", () => { + assert.deepEqual( + parse( + '[a]\n`k=1` = ` v `\n"k`2" = """a"`b"""\n' + + '"""k"`3""" = `a`b` rest\nk4 = "q"\nk5 = """é\n', + ), + { + a: { + "k=1": " v ", + "k`2": 'a"`b', + 'k"`3': "a`b", + k4: '"q"', + k5: '"""é', + }, + }, + ); + }); + + test("gives numbers to the keys -", () => { + assert.deepEqual(parse("[a]\n- = x\n- = y\n[b]\n- = z"), { + a: { "#1": "x", "#2": "y" }, + b: { "#1": "z" }, + }); + }); + + test("ignores the empty key and the key ' '", () => { + assert.deepEqual(parse('[a]\n"" = x\n" " = y\nk = v'), { a: { k: "v" } }); + }); + + test("removes the white space of Go and the byte order mark", () => { + assert.deepEqual(parse("\uFEFF[a]\u0085\n\u00A0k = v\u3000\r\n"), { + a: { k: "v" }, + }); + }); + + test("gives an error with the line number and no text of the line", () => { + for ( + const [text, message] of [ + ["[a]\nno equal sign", 'INI: line 2 has no "=" or ":" after a key.'], + ["[a]\n= value", 'INI: line 2 has no "=" or ":" after a key.'], + [ + '[a]\n"key = value', + "INI: line 2 has a key with no quote at the end.", + ], + [ + "[a]\nk = `secret", + "INI: line 2 has a value with no quote at the end.", + ], + ['[a]\nk = """€', "INI: line 2 has a value with no quote at the end."], + ["[]\nk = v", "INI: line 2 has a key in a section with no name."], + ] + ) { + assert.throws(() => parse(text), { name: "SyntaxError", message }); + } + }); + + test("does not change the prototype", () => { + const config = parse("[__proto__]\npolluted = yes\n"); + assert.deepEqual(Object.keys(config), ["__proto__"]); + assert.equal({}.polluted, undefined); + }); +}); + +describe("stringify", () => { + test("writes the format of rclone", () => { + const config = { a: { type: "s3", key: "v=1" }, b: { type: "local" } }; + assert.equal( + stringify(config), + "[a]\ntype = s3\nkey = v=1\n\n[b]\ntype = local", + ); + assert.equal(stringify(config, { section: "" }), stringify(config)); + assert.equal(stringify(config.b, { section: "b" }), "[b]\ntype = local"); + assert.equal( + stringify({ DEFAULT: { top: "1" }, a: { x: "2" } }), + "top = 1\n\n[a]\nx = 2", + ); + assert.equal( + stringify({ a: {}, DEFAULT: { top: "1" } }), + "[a]\n\n[DEFAULT]\ntop = 1", + ); + assert.equal( + stringify({ a: { x: undefined, y: null, z: 0 } }), + "[a]\nz = 0", + ); + assert.equal(stringify({}), ""); + }); + + test("puts in quotes what rclone does not read the same with no quotes", () => { + assert.equal( + stringify({ + a: { + "k=1": " v ", + "#k": 'a"`b', + "k`:": "a`b", + 'k`"=': '"""x', + "k5": '"q"', + "[k6": "v]", + }, + }), + '[a]\n`k=1` = ` v `\n`#k` = """a"`b"""\n"k`:" = """a`b"""\n' + + '"""k`"=""" = `"""x`\nk5 = "q"\n[k6 = `v]`', + ); + }); + + test("throws for what rclone cannot read the same", () => { + for ( + const config of [ + { "": { k: "v" } }, + { " a": {} }, + { "a\nb": {} }, + { a: { "": "v" } }, + { a: { " ": "v" } }, + { a: { "-": "v" } }, + { a: { "k\n": "v" } }, + { a: { 'k`="': "v" } }, + { a: { k: "line 1\nline 2" } }, + { a: "not a section" }, + ] + ) { + assert.throws(() => stringify(config), TypeError); + } + }); +}); + +test("ParseStream gives an entry for each line with a section, a key or a comment", async () => { + const entries = await Array.fromAsync( + ReadableStream.from(["top = 1\n[a", "]\n# note\nk = v\n\n- = x", "\n[]"]) + .pipeThrough(new ParseStream()), + ); + assert.deepEqual(entries, [ + { type: "key", section: "DEFAULT", line: 1, key: "top", value: "1" }, + { type: "section", section: "a", line: 2 }, + { type: "comment", section: "a", line: 3, text: "# note" }, + { type: "key", section: "a", line: 4, key: "k", value: "v" }, + { type: "key", section: "a", line: 6, key: "#1", value: "x", auto: true }, + { type: "section", section: "", line: 7 }, + ]); +}); + +describe("ConfigFile", () => { + const text = [ + "# The remotes of the person.", + "", + "# The disk at work.", + "[work]", + "type: drive", + "# A note on the token.", + "token = {}", + "", + "[home]", + "type = local", + "# The last comment.", + "", + ].join("\n"); + + test("keeps the text when nothing changes", () => { + const file = new ConfigFile(text); + assert.equal(file.toString(), text); + assert.deepEqual(file.config, parse(text)); + }); + + test("changes only the value of a key, and keeps the comments", () => { + const file = new ConfigFile(text); + file.update("work", { type: "s3", token: "{}" }); + assert.equal(file.toString(), text.replace("type: drive", "type: s3")); + }); + + test("puts a new key after the last key of the section", () => { + const file = new ConfigFile(text); + file.update("home", { "a b": " x " }); + assert.equal( + file.toString(), + text.replace("type = local", "type = local\na b = ` x `"), + ); + }); + + test("puts a new section at the end, after an empty line", () => { + const file = new ConfigFile(text); + file.update("new", { type: "memory" }); + assert.equal(file.toString(), `${text}\n[new]\ntype = memory\n`); + assert.equal( + new ConfigFile("[a]").toString(), + "[a]", + ); + const empty = new ConfigFile(""); + empty.set("a", { type: "local" }); + assert.equal(empty.toString(), "[a]\ntype = local\n"); + }); + + test("removes a key with its comment, and keeps the other comments", () => { + const file = new ConfigFile(text); + file.update("work", { token: null }); + assert.equal( + file.toString(), + text.replace("# A note on the token.\ntoken = {}\n", ""), + ); + + const again = new ConfigFile(text); + again.set("work", { type: "drive" }); + assert.equal(again.toString(), file.toString()); + }); + + test("removes a section with its comments, but not the other comments", () => { + const file = new ConfigFile(text); + file.delete("work"); + assert.equal( + file.toString(), + "# The remotes of the person.\n\n[home]\ntype = local\n# The last comment.\n", + ); + // The last comment has an empty line below it, thus it stays. + file.delete("home"); + assert.equal( + file.toString(), + "# The remotes of the person.\n\n# The last comment.\n", + ); + }); + + test("keeps the numbers of the keys -", () => { + const file = new ConfigFile("[a]\n- = x\n- = y\n- = z"); + file.update("a", { "#1": null, "#3": "Z" }); + assert.equal(file.toString(), "[a]\n`#2` = y\n`#3` = Z"); + assert.deepEqual(file.config, { a: { "#2": "y", "#3": "Z" } }); + }); + + test("keeps the place of DEFAULT", () => { + const top = new ConfigFile("x = 1\n[a]"); + top.set("DEFAULT", { y: "2" }); + assert.equal(top.toString(), "[DEFAULT]\ny = 2\n[a]"); + + const empty = new ConfigFile("[]\n[a]"); + empty.update("DEFAULT", { y: "2" }); + assert.equal(empty.toString(), "[DEFAULT]\ny = 2\n[a]"); + }); + + test("keeps the ends of lines and the byte order mark", () => { + const file = new ConfigFile("\uFEFF[a]\r\nk = v\r\n"); + file.update("a", { k: "w", n: "1" }); + assert.equal(file.toString(), "\uFEFF[a]\r\nk = w\r\nn = 1\r\n"); + }); + + test("does not change for what rclone cannot read the same", () => { + const file = new ConfigFile(text); + for ( + const change of [ + () => file.update("work", { type: "line 1\nline 2" }), + () => file.update("work", { "-": "x" }), + () => file.set(" bad name", {}), + ] + ) { + assert.throws(change, TypeError); + assert.equal(file.toString(), text); + } + }); +}); + +// The generators for the properties. + +/** Pieces of text with the characters that have a meaning for goconfig. */ +const PIECES = [ + "=", + ":", + "#", + ";", + "[", + "]", + '"', + '"""', + "`", + "-", + " ", + "\t", + "\r", + "\u0085", + "\u00A0", + "\u3000", + "\uFEFF", + "é", + "€", + "😀", + "DEFAULT", + "a", + "b", + "type", + "s3:bucket", +]; + +/** A text with no line break. */ +const text = fc.array( + fc.oneof( + { weight: 4, arbitrary: fc.constantFrom(...PIECES) }, + { + weight: 1, + arbitrary: fc.fullUnicodeString({ maxLength: 4 }) + .map((text) => text.replaceAll("\n", "")), + }, + ), + { maxLength: 6 }, +).map((pieces) => pieces.join("")); + +const quote = fc.constantFrom("`", '"', '"""'); + +/** A key: often a name of an option, but sometimes any text. */ +const key = fc.oneof( + { weight: 3, arbitrary: fc.stringMatching(/^[a-z_][a-z0-9_]{0,8}$/) }, + { weight: 2, arbitrary: text }, +); + +/** A line of a configuration file, often a line that goconfig accepts. */ +const line = fc.oneof( + { weight: 2, arbitrary: text.map((name) => `[${name}]`) }, + { + weight: 4, + arbitrary: fc.tuple(key, fc.constantFrom(" = ", "=", ": ", " : "), text) + .map(([key, delimiter, value]) => `${key}${delimiter}${value}`), + }, + { + weight: 2, + arbitrary: fc.tuple(quote, key, quote, text) + .map(([q1, key, q2, value]) => `${q1}${key}${q1} = ${q2}${value}${q2}`), + }, + { + weight: 1, + arbitrary: fc.tuple(key, quote, text) + .map(([key, q, value]) => `${key} = ${q}${value}`), + }, + { weight: 1, arbitrary: text.map((value) => `- = ${value}`) }, + { weight: 1, arbitrary: text.map((comment) => `# ${comment}`) }, + { weight: 1, arbitrary: text }, +); + +/** The text of a configuration file. */ +const file = fc.tuple( + fc.constantFrom("", "\uFEFF"), + fc.array(fc.tuple(line, fc.constantFrom("\n", "\r\n")), { maxLength: 10 }), +).map(([start, lines]) => + start + lines.map(([line, end]) => line + end).join("") +); + +/** + * A configuration from the given names, keys and values. + * @param {fc.Arbitrary} name + * @param {fc.Arbitrary} key + * @param {fc.Arbitrary} value + */ +function configOf(name, key, value) { + const keys = fc.uniqueArray(fc.tuple(key, value), { + selector: ([key]) => key, + maxLength: 5, + }); + return fc.uniqueArray(fc.tuple(name, keys), { + selector: ([name]) => name, + maxLength: 4, + }).map((sections) => + Object.fromEntries( + sections.map(([name, keys]) => [name, Object.fromEntries(keys)]), + ) + ); +} + +/** A configuration from texts with the characters that have a meaning. */ +const anyConfig = configOf(text, text, text); + +/** + * A configuration as rclone makes it: the names of remotes as + * `fspath.CheckConfigName` accepts them, the names of options, and values of + * all characters but a line break. + */ +const rcloneConfig = configOf( + fc.stringMatching(/^[\w.+@é世]+(?:[ -]+[\w.+@é世-]+)*$/), + fc.stringMatching(/^[a-z][a-z0-9_]*$/), + fc.oneof( + text, + fc.fullUnicodeString().map((text) => text.replaceAll("\n", "")), + ), +); + +/** + * The sections and the keys in their order, for a comparison that checks + * the order too. + * @param {Record>} config + */ +function ordered(config) { + return Object.entries(config).map(([name, keys]) => [ + name, + Object.entries(keys), + ]); +} + +/** + * Gives the result of a function, or the message of its SyntaxError. + * @param {() => any} read + */ +async function outcome(read) { + try { + return { config: ordered(await read()) }; + } catch (error) { + if (!(error instanceof SyntaxError)) throw error; + return { error: error.message }; + } +} + +/** + * A file that parse reads, and changes for it: the names and the keys are + * often names and keys of the file. + */ +const edited = file.chain((source) => { + let config; + try { + config = parse(source); + } catch { + return fc.constant(null); + } + const names = Object.keys(config); + const keys = [...new Set(Object.values(config).flatMap(Object.keys))]; + const name = fc.oneof( + ...(names.length ? [fc.constantFrom(...names)] : []), + fc.constant("DEFAULT"), + text, + ); + const changes = fc.uniqueArray( + fc.tuple( + keys.length ? fc.oneof(fc.constantFrom(...keys), key) : key, + fc.option(text, { nil: null, freq: 3 }), + ), + { selector: ([key]) => key, maxLength: 4 }, + ).map((entries) => Object.fromEntries(entries)); + const change = fc.oneof( + fc.record({ type: fc.constantFrom("update", "set"), name, keys: changes }), + fc.record({ type: fc.constant("delete"), name }), + ); + return fc.record({ + text: fc.constant(source), + changes: fc.array(change, { maxLength: 4 }), + }); +}).filter((value) => value !== null); + +/** + * Makes a change in a ConfigFile, and in a model of the configuration with + * maps. A change that throws a TypeError must not change the file, and does + * not change the model. + * @param {ConfigFile} file + * @param {Map>} model + * @param {{ type: string, name: string, keys?: Record }} change + */ +function change(file, model, { type, name, keys }) { + const before = file.toString(); + try { + if (type === "delete") file.delete(name); + else file[type](name, keys); + } catch (error) { + assert.ok(error instanceof TypeError, error); + assert.equal( + file.toString(), + before, + "a change that throws changes nothing", + ); + return false; + } + + if (type === "delete") { + model.delete(name); + return true; + } + const old = model.get(name) ?? new Map(); + const next = new Map(old); + if (type === "set") { + for (const key of old.keys()) { + if ((Object.hasOwn(keys, key) ? keys[key] : null) === null) { + next.delete(key); + } + } + } + for (const [key, value] of Object.entries(keys)) { + if (value === null) next.delete(key); + else next.set(key, value); + } + model.set(name, next); + return true; +} + +/** @param {Record>} config */ +function toModel(config) { + return new Map( + Object.entries(config).map(( + [name, keys], + ) => [name, new Map(Object.entries(keys))]), + ); +} + +/** @param {Map>} model */ +function fromModel(model) { + return Object.fromEntries( + [...model].map(([name, keys]) => [name, Object.fromEntries(keys)]), + ); +} + +/** + * The entries of each line of a text, by the number of the line. + * @param {string} text + */ +async function entriesOf(text) { + const entries = await Array.fromAsync( + ReadableStream.from([text]).pipeThrough(new ParseStream()), + ); + return new Map(entries.map((entry) => [entry.line, entry])); +} + +describe("properties", () => { + test("parse reads what stringify writes, or stringify throws", () => { + fc.assert( + fc.property(anyConfig, (config) => { + let text; + try { + text = stringify(config); + } catch (error) { + assert.ok(error instanceof TypeError, error); + return; + } + assert.deepEqual(ordered(parse(text)), ordered(config)); + }), + { numRuns: 1000 }, + ); + }); + + test("stringify writes each configuration that rclone makes", () => { + fc.assert( + fc.property(rcloneConfig, (config) => { + assert.deepEqual(ordered(parse(stringify(config))), ordered(config)); + }), + { numRuns: 500 }, + ); + }); + + test("stringify writes each configuration that parse reads", () => { + fc.assert( + fc.property(file, (text) => { + let config; + try { + config = parse(text); + } catch (error) { + assert.ok(error instanceof SyntaxError, error); + return; + } + assert.deepEqual(ordered(parse(stringify(config))), ordered(config)); + }), + { numRuns: 1000 }, + ); + }); + + test("ConfigFile keeps the text of each file that parse reads", () => { + fc.assert( + fc.property(file, (text) => { + let config; + try { + config = parse(text); + } catch { + return; + } + const configFile = new ConfigFile(text); + assert.equal(configFile.toString(), text); + assert.deepEqual(ordered(configFile.config), ordered(config)); + }), + { numRuns: 1000 }, + ); + }); + + test("ConfigFile changes the configuration as a model of maps", () => { + fc.assert( + fc.property(edited, ({ text, changes }) => { + const configFile = new ConfigFile(text); + const model = toModel(configFile.config); + for (const item of changes) { + change(configFile, model, item); + const expected = ordered(fromModel(model)); + assert.deepEqual(ordered(configFile.config), expected); + assert.deepEqual(ordered(parse(configFile.toString())), expected); + } + }), + { numRuns: 1000 }, + ); + }); + + test("ConfigFile does not throw for the changes that rclone makes", () => { + fc.assert( + fc.property( + rcloneConfig, + fc.array(fc.tuple(fc.boolean(), fc.boolean()), { maxLength: 4 }), + (config, steps) => { + const configFile = new ConfigFile(); + const model = new Map(); + const names = Object.keys(config); + for (const [index, [set, remove]] of steps.entries()) { + const name = names[index % names.length]; + if (name === undefined) break; + const item = remove + ? { type: "delete", name } + : { type: set ? "set" : "update", name, keys: config[name] }; + assert.ok(change(configFile, model, item), "no TypeError"); + } + assert.deepEqual( + ordered(configFile.config), + ordered(fromModel(model)), + ); + }, + ), + { numRuns: 500 }, + ); + }); + + test("ConfigFile removes only the comments directly above a line that goes away", async () => { + await fc.assert( + fc.asyncProperty( + edited.filter(({ changes }) => changes.length > 0), + async ({ text, changes: [item] }) => { + const before = await entriesOf(text); + const configFile = new ConfigFile(text); + const changed = change(configFile, toModel(configFile.config), item); + + /** Tells if the line of an entry goes away with the change. */ + const goes = (entry) => { + if (!changed || entry.type === "comment") return false; + const section = entry.section || "DEFAULT"; + if (item.type === "delete") return section === item.name; + if (entry.type !== "key" || section !== item.name) return false; + if (entry.key === "" || entry.key === " ") return false; + const value = Object.hasOwn(item.keys, entry.key) + ? item.keys[entry.key] + : undefined; + return value === null || + (item.type === "set" && value === undefined); + }; + + const lines = text.split("\n").length; + const expected = []; + for (const entry of before.values()) { + if (entry.type !== "comment") continue; + // The comment is of the first line below it that is not a + // comment, when that line is not empty. + let below = entry.line + 1; + while (before.get(below)?.type === "comment") below++; + const target = below <= lines ? before.get(below) : undefined; + if (!target || !goes(target)) expected.push(entry.text); + } + + const after = await entriesOf(configFile.toString()); + const comments = [...after.values()] + .filter(({ type }) => type === "comment") + .map(({ text }) => text); + assert.deepEqual(comments, expected); + }, + ), + { numRuns: 1000 }, + ); + }); + + test("read gives the result of parse, for each split of the text", async () => { + await fc.assert( + fc.asyncProperty( + file, + fc.array(fc.nat(), { maxLength: 6 }), + async (text, cuts) => { + const points = [ + ...new Set(cuts.map((cut) => cut % (text.length + 1))), + ] + .sort((a, b) => a - b); + const chunks = []; + let start = 0; + for (const point of points) { + chunks.push(text.slice(start, point)); + start = point; + } + chunks.push(text.slice(start)); + + assert.deepEqual( + await outcome(() => read(ReadableStream.from(chunks))), + await outcome(() => parse(text)), + ); + }, + ), + { numRuns: 500 }, + ); + }); +}); + +// The tests against rclone, which uses goconfig. RCLONE is the path of rclone, +// or "rclone" in the PATH. +const RCLONE = process.env.RCLONE || "rclone"; +const hasRclone = spawnSync(RCLONE, ["version"]).status === 0; + +describe("the same as rclone", { + skip: !hasRclone && "rclone is not installed. Set RCLONE to its path.", +}, () => { + const folder = mkdtempSync(join(tmpdir(), "rfetch-ini-")); + const path = join(folder, "rclone.conf"); + after(() => rmSync(folder, { recursive: true, force: true })); + + /** + * Gives the configuration that rclone reads from the text, or the error of + * rclone. + * @param {string} text + */ + function rclone(text) { + writeFileSync(path, text); + const { status, stdout, stderr } = spawnSync( + RCLONE, + ["--config", path, "config", "dump"], + // No RCLONE_* variable of this environment changes the result. + { encoding: "utf8", env: { PATH: process.env.PATH, HOME: folder } }, + ); + return status === 0 ? { config: JSON.parse(stdout) } : { error: stderr }; + } + + test("parse reads each file as rclone reads it", () => { + fc.assert( + fc.property(file, (text) => { + const expected = rclone(text); + if (expected.error) { + assert.match(expected.error, /could not parse line|empty section/); + assert.throws(() => parse(text), SyntaxError); + } else { + // The JSON of rclone has the keys in the order of the alphabet. + assert.deepEqual(parse(text), expected.config); + } + }), + { numRuns: 100 }, + ); + }); + + test("parse reads each line as rclone reads it", () => { + fc.assert( + fc.property(line, (line) => { + const text = `[a]\n${line}\n`; + const expected = rclone(text); + if (expected.error) { + assert.throws(() => parse(text), SyntaxError); + } else { + assert.deepEqual(parse(text), expected.config); + } + }), + { numRuns: 100 }, + ); + }); + + test("rclone reads what stringify writes", () => { + fc.assert( + fc.property(fc.oneof(anyConfig, rcloneConfig), (config) => { + let text; + try { + text = stringify(config); + } catch { + return; + } + assert.deepEqual(rclone(text).config, config); + }), + { numRuns: 100 }, + ); + }); + test("rclone reads the file that ConfigFile changes", () => { + fc.assert( + fc.property(edited, ({ text, changes }) => { + const configFile = new ConfigFile(text); + const model = toModel(configFile.config); + for (const item of changes) change(configFile, model, item); + assert.deepEqual( + rclone(configFile.toString()).config, + fromModel(model), + ); + }), + { numRuns: 50 }, + ); + }); +}); diff --git a/lib/listing.js b/lib/listing.js new file mode 100644 index 0000000..db1cbe4 --- /dev/null +++ b/lib/listing.js @@ -0,0 +1,200 @@ +/** + * The list of a folder in HTML, as CONTRIBUTING.md tells: the rows of a + * ``. In each row, `` is the link to the file or the folder, + * with a `/` at the end for a folder, `type` is the media type, `` is the size in bytes, and ` + + + + + `; + }, + ); + + const html = ` +
${IsDir ? "📁" : "📄"}${escapeHTML(IsDir ? `${Name}/` : Name)}${ + IsDir || Size < 0 + ? "—" + : `${ + formatBytes(Size, { locale: true }) + }` + }${ + time && datetime + ? `` + : "-" + }
+ + + + + + + + + + + + + + + ${rows.join("")} + +
NameSizeModified
📁Go up——
+ `; + return new Blob([html]).stream(); +} + +/** + * Writes the list of another backend again, with a query in each link, for + * a backend on top of another backend. The links of the other backend have + * its own options, which are not the options of a request to this backend. + * @param {Response} response The list of the other backend. + * @param {string} query The query, with no secret option. + * @returns {Promise} + */ +export async function relink(response, query) { + const headers = new Headers(response.headers); + headers.delete("Content-Length"); + headers.set("Content-Type", "text/html;charset=utf-8"); + const init = { + status: response.status, + statusText: response.statusText, + headers, + }; + // The answer to HEAD has no body. + if (response.body === null) return new Response(null, init); + const entries = await readListing(response, new URL("http://remote/folder/")); + return new Response(renderListing(entries, { query }), init); +} + +/** + * The name of a file (`name`) or a folder (`name/`) directly in a folder, + * from a link, or null when the link is not to such a child. The query and + * the fragment of the link are not a part of the name. + * @param {string} href + * @param {URL} base The URL of the folder. + * @returns {string | null} + */ +export function childOf(href, base) { + const { origin, pathname } = new URL(href, base); + if (origin !== base.origin || !pathname.startsWith(base.pathname)) { + return null; + } + const rest = pathname.slice(base.pathname.length); + const isDir = rest.endsWith("/"); + const name = isDir ? rest.slice(0, -1) : rest; + if (name === "" || name.includes("/")) return null; + try { + return decodeURIComponent(name) + (isDir ? "/" : ""); + } catch { + return null; + } +} diff --git a/lib/listing_test.js b/lib/listing_test.js new file mode 100644 index 0000000..c33da1a --- /dev/null +++ b/lib/listing_test.js @@ -0,0 +1,178 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fc from "fast-check"; + +import { childOf, readListing, renderListing } from "./listing.js"; + +const base = new URL("http://remote/folder/"); + +/** @param {string} html */ +const read = (html) => readListing(new Response(html), base); + +test("readListing reads the list of renderListing", async () => { + const entries = [ + { + Name: "a b?#%.txt", + IsDir: false, + Size: 3, + MimeType: "text/plain", + ModTime: "2026-10-18T12:00:00.000Z", + }, + { Name: "folder é", IsDir: true, Size: -1, MimeType: "", ModTime: "" }, + { + Name: "&", + IsDir: false, + Size: 0, + MimeType: "application/octet-stream", + ModTime: "2026-10-18T12:00:01.000Z", + }, + ]; + const html = await new Response(renderListing(entries, { query: "x=1" })) + .text(); + const sorted = entries.toSorted((a, b) => a.Name.localeCompare(b.Name)); + assert.deepEqual(await read(html), sorted); +}); + +test("readListing reads only the links to the children of the folder", async () => { + const html = ` + + + + + +
Go up
a
b
c
d
`; + assert.deepEqual((await read(html)).map(({ Name }) => Name), ["d.txt"]); +}); + +test("readListing gives one entry for each row, with the last link", async () => { + const html = ` + + + + + +
7a
bc5
up9
e/ +
`; + assert.deepEqual(await read(html), [ + { Name: "a.txt", IsDir: false, Size: -1, MimeType: "", ModTime: "" }, + { + Name: "c.txt", + IsDir: false, + Size: 5, + MimeType: "text/plain", + ModTime: "2026-10-18T12:00:00Z", + }, + { Name: "e", IsDir: true, Size: -1, MimeType: "", ModTime: "" }, + ]); +}); + +test("childOf gives the name of a child, or null", () => { + assert.equal(childOf("a%20b.txt", base), "a b.txt"); + assert.equal(childOf("a/", base), "a/"); + assert.equal(childOf("a%2Fb", base), "a/b"); + assert.equal(childOf("../", base), null); + assert.equal(childOf("a/b", base), null); + assert.equal(childOf("%E0", base), null); + assert.equal(childOf("//example.com/folder/a", base), null); +}); + +/** + * A response with the bytes of a text in chunks, cut at the places. + * @param {string} html + * @param {number[]} cuts + */ +function inChunks(html, cuts) { + const bytes = new TextEncoder().encode(html); + const places = [...new Set(cuts.map((cut) => cut % (bytes.length + 1)))] + .sort((a, b) => a - b); + const chunks = []; + let start = 0; + for (const place of [...places, bytes.length]) { + chunks.push(bytes.subarray(start, place)); + start = place; + } + return new Response(new Blob(chunks).stream()); +} + +test("readListing gives back the entries of renderListing", async () => { + const entry = fc.record({ + // `.`, `..` and a name with `/` are not names of files: the link of `.` + // is the folder, the link of `..` is its parent, and `/` is between two + // names. + Name: fc.fullUnicodeString({ minLength: 1, maxLength: 12 }) + .filter((name) => name !== "." && name !== ".." && !name.includes("/")), + IsDir: fc.boolean(), + Size: fc.nat(), + MimeType: fc.constantFrom("", "text/plain; charset=utf-8", "image/png"), + ModTime: fc.constantFrom( + "", + "2026-10-18T12:00:00.000Z", + "2026-10-18T15:02:41.540476887Z", + ), + }); + await fc.assert( + fc.asyncProperty( + fc.array(entry, { maxLength: 20 }), + fc.constantFrom("", "a=1&b=%3C"), + // The places where the HTML is cut into chunks. + fc.array(fc.nat(), { maxLength: 8 }), + async (entries, query, cuts) => { + const html = await new Response(renderListing(entries, { query })) + .text(); + const expected = entries + .toSorted((a, b) => a.Name.localeCompare(b.Name)) + .map((entry) => + entry.IsDir ? { ...entry, Size: -1, MimeType: "" } : entry + ); + assert.deepEqual( + await readListing(inChunks(html, cuts), base), + expected, + ); + }, + ), + { numRuns: 50 }, + ); +}); + +test("readListing reads HTML as the HTML standard tells", async () => { + const html = ` + + + +
q + r12
t +
no link + dno value
xu`; + assert.deepEqual(await read(html), [ + { Name: "r&s.txt", IsDir: false, Size: 12, MimeType: "", ModTime: "" }, + { + Name: "té.txt", + IsDir: false, + Size: -1, + MimeType: "", + ModTime: "2026-10-18T12:00:00Z", + }, + { Name: "d", IsDir: true, Size: -1, MimeType: "", ModTime: "" }, + { Name: "u.txt", IsDir: false, Size: -1, MimeType: "", ModTime: "" }, + ]); +}); + +test("renderListing keeps the digits of a time, as the precision of the remote", async () => { + const entries = [ + "2026-10-18T15:02:41.540476887Z", + "2026-10-18T15:02:41.540Z", + "2026-10-18T22:02:41.540476+07:00", + ].map((ModTime, index) => ({ + Name: `${index}.txt`, + IsDir: false, + Size: 1, + MimeType: "text/plain", + ModTime, + })); + const html = await new Response(renderListing(entries)).text(); + assert.deepEqual((await read(html)).map(({ ModTime }) => ModTime), [ + "2026-10-18T15:02:41.540476887Z", + "2026-10-18T15:02:41.540Z", + "2026-10-18T15:02:41.540476Z", + ]); +}); diff --git a/lib/mime.js b/lib/mime.js new file mode 100644 index 0000000..64f387e --- /dev/null +++ b/lib/mime.js @@ -0,0 +1,75 @@ +/** + * The media type of a file from its name, as `MimeTypeFromName` of rclone: + * the built-in table of the `mime` package of Go, the types that rclone adds + * (fs/mimetype.go of rclone 1.75.1), and `text/plain` for `.txt`, as the + * `mime.types` file of most systems gives it. + */ + +/** The types for each extension, in lowercase. */ +const TYPES = { + // The built-in table of Go. + ".avif": "image/avif", + ".css": "text/css; charset=utf-8", + ".gif": "image/gif", + ".htm": "text/html; charset=utf-8", + ".html": "text/html; charset=utf-8", + ".jpeg": "image/jpeg", + ".jpg": "image/jpeg", + ".js": "text/javascript; charset=utf-8", + ".json": "application/json", + ".mjs": "text/javascript; charset=utf-8", + ".pdf": "application/pdf", + ".png": "image/png", + ".svg": "image/svg+xml", + ".wasm": "application/wasm", + ".webp": "image/webp", + ".xml": "text/xml; charset=utf-8", + // The types that rclone adds. + ".flac": "audio/flac", + ".mpga": "audio/mpeg", + ".mpega": "audio/mpeg", + ".mp2": "audio/mpeg", + ".mp3": "audio/mpeg", + ".m4a": "audio/mpeg", + ".oga": "audio/ogg", + ".ogg": "audio/ogg", + ".opus": "audio/ogg", + ".spx": "audio/ogg", + ".wav": "audio/x-wav", + ".tiff": "image/tiff", + ".tif": "image/tiff", + ".dif": "video/dv", + ".dv": "video/dv", + ".fli": "video/fli", + ".mpeg": "video/mpeg", + ".mpg": "video/mpeg", + ".mpe": "video/mpeg", + ".ts": "video/MP2T", + ".mp4": "video/mp4", + ".qt": "video/quicktime", + ".mov": "video/quicktime", + ".ogv": "video/ogg", + ".webm": "video/webm", + ".avi": "video/x-msvideo", + ".mpv": "video/x-matroska", + ".mkv": "video/x-matroska", + ".srt": "application/x-subrip", + // The type of most systems. + ".txt": "text/plain; charset=utf-8", +}; + +/** + * Gives the media type of a file from the extension of its name, or + * `application/octet-stream` for an unknown extension. The extension can be + * in each case: `A.PNG` is an image. + * @param {string} name A name or a path of a file. + * @returns {string} + */ +export function mimeType(name) { + const base = name.slice(name.lastIndexOf("/") + 1); + const dot = base.lastIndexOf("."); + const extension = dot === -1 ? "" : base.slice(dot).toLowerCase(); + return Object.hasOwn(TYPES, extension) + ? TYPES[extension] + : "application/octet-stream"; +} diff --git a/lib/mime_test.js b/lib/mime_test.js new file mode 100644 index 0000000..0f38593 --- /dev/null +++ b/lib/mime_test.js @@ -0,0 +1,19 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { mimeType } from "./mime.js"; + +test("mimeType", () => { + // The table of Go, the additions of rclone, and .txt. + assert.equal(mimeType("index.html"), "text/html; charset=utf-8"); + assert.equal(mimeType("photo.JPG"), "image/jpeg"); + assert.equal(mimeType("a/b.c/song.mp3"), "audio/mpeg"); + assert.equal(mimeType("movie.mkv"), "video/x-matroska"); + assert.equal(mimeType("hello.txt"), "text/plain; charset=utf-8"); + // An unknown extension, no extension, and a dot in a folder only. + for ( + const name of ["1MB.bin", "README", "a.b/c", "toString", ".constructor"] + ) { + assert.equal(mimeType(name), "application/octet-stream", name); + } +}); diff --git a/lib/obscure.js b/lib/obscure.js new file mode 100644 index 0000000..9d20f48 --- /dev/null +++ b/lib/obscure.js @@ -0,0 +1,130 @@ +const ALGORITHM = "AES-CTR"; + +/** @type {Promise | undefined} */ +let key; + +/** + * Gives the key of rclone for `obscure`. The import happens at the first use, + * not at the load of the module: a top-level await breaks the bundle of a + * Worker, and a Worker cannot use some APIs at the load. + * @returns {Promise} + */ +function getKey() { + key ??= crypto.subtle.importKey( + "raw", + // https://github.com/rclone/rclone/blob/10c884552caab3d3d8484ed3401e94f983a23f93/fs/config/obscure/obscure.go#L17-L22 + Uint8Array.from([ + 0x9c, + 0x93, + 0x5b, + 0x48, + 0x73, + 0x0a, + 0x55, + 0x4d, + 0x6b, + 0xfd, + 0x7c, + 0x63, + 0xc8, + 0x86, + 0xa9, + 0x2b, + 0xd3, + 0x90, + 0x19, + 0x8e, + 0xb8, + 0x12, + 0x8a, + 0xfb, + 0xf4, + 0xde, + 0x16, + 0x2b, + 0x8b, + 0x95, + 0xf6, + 0x38, + ]), + ALGORITHM, + false, + ["encrypt", "decrypt"], + ); + return key; +} + +const BlockSize = 16; + +/** + * Obscrures a password by encrypting them and writing them out in base64. + * @param {string} password + * @returns {Promise} A Response containing the obscured password. + */ +export async function obscure(password) { + const plaintext = new TextEncoder().encode(password); + // Constructs the initial empty ciphertext buffer containing the IV and the encrypted text. + const ciphertext = new Uint8Array(BlockSize + plaintext.length); + // Constructs the IV buffer. + const iv = ciphertext.subarray(0, BlockSize); + // Fills IV buffer with random bytes. + crypto.getRandomValues(iv); + // Encrypts the plaintext into the encrypted buffer. + const encrypted = await crypto.subtle.encrypt( + { + name: ALGORITHM, + counter: iv, + length: 64, + }, + await getKey(), + plaintext, + ); + // Copies the encrypted buffer into the ciphertext buffer. + ciphertext.set(new Uint8Array(encrypted), BlockSize); + // Encodes the ciphertext buffer into a base64 string. + const encoded = btoa(String.fromCharCode(...ciphertext)); + + return new Response(encoded); +} + +/** + * Reveals obscured text by decrypting them + * @param {string} cipherText + * @returns {Promise} + */ +export async function reveal(cipherText) { + // Decodes the base64 string into a ciphertext buffer. + const bytes = base64(cipherText); + // Constructs the IV buffer from the first block of the ciphertext buffer. + const iv = bytes.subarray(0, BlockSize); + // Constructs the encrypted buffer from the remaining blocks of the ciphertext buffer. + const data = bytes.subarray(BlockSize); + // Decrypts the encrypted buffer into the plaintext buffer. + const plaintext = await crypto.subtle.decrypt( + { + name: ALGORITHM, + counter: iv, + length: 64, + }, + await getKey(), + data, + ); + + return new Response(plaintext); +} + +/** + * Encodes a string into a base64 Uint8Array. + * @param {string} s + * @returns {Uint8Array} + */ +function base64(s) { + // Replace non-url compatible chars with base64 standard chars + s = s.replace(/-/g, "+").replace(/_/g, "/"); + // Pad out with standard base64 required padding characters + const pad = s.length % 4; + if (pad != 0) { + s += "====".substring(0, 4 - pad); + } + return new Uint8Array(atob(s).split("").map((c) => c.charCodeAt(0))); +} diff --git a/cmd/obscure/main_test.js b/lib/obscure_test.js similarity index 81% rename from cmd/obscure/main_test.js rename to lib/obscure_test.js index 180e355..1b21e32 100644 --- a/cmd/obscure/main_test.js +++ b/lib/obscure_test.js @@ -1,6 +1,7 @@ import { test } from "node:test"; -import { assertEquals, fc } from "../../dev_deps.js"; -import { obscure, reveal } from "./main.js"; +import assert from "node:assert/strict"; +import fc from "fast-check"; +import { obscure, reveal } from "./obscure.js"; /** * Asserts a text Response contains the expected string. @@ -9,7 +10,7 @@ import { obscure, reveal } from "./main.js"; */ async function assertResponse(response, expected) { response = await response; - assertEquals(await response.text(), expected); + assert.deepEqual(await response.text(), expected); } test("reveal", () => { diff --git a/lib/options.js b/lib/options.js new file mode 100644 index 0000000..17a31f1 --- /dev/null +++ b/lib/options.js @@ -0,0 +1,96 @@ +/** + * The options of a backend, from the three places of rclone: a flag, a + * variable of the request, and a variable of the environment. + * + * A variable has the name of the option with the prefix `RFETCH_` and the + * type of the backend, for example `RFETCH_S3_ACCESS_KEY_ID` for the option + * `access_key_id` of s3. The prefix `RCLONE_` of rclone also works, for + * compatibility: `RFETCH_` takes its place when both are set. In a request, + * a variable is a header. The names of headers are not case sensitive. + */ + +/** The prefixes of the options, the primary one first. */ +export const PREFIXES = ["rfetch_", "rclone_"]; + +/** + * Tells if a flag or an option is on: `true`, or the text `true` of a query + * or of a variable. + * @param {unknown} value + * @returns {boolean} + */ +export function isTrue(value) { + return value === true || value === "true"; +} + +/** + * Gives the options of a backend for a request: a field of the query + * first, then the header `RFETCH_TYPE_NAME` or `RCLONE_TYPE_NAME`, then the + * variable `RFETCH_TYPE_NAME` or `RCLONE_TYPE_NAME` of `env`, as a flag of + * rclone takes the place of a variable. + * + * Use the options to read an option, and the query of the request to make + * the links of a list: thus an option of a header or of `env` is never in a + * link. + * @param {Request} request + * @param {string} type The type of the backend, for example `s3`. + * @param {Record} [env] + * @returns {URLSearchParams} + */ +export function optionsOf(request, type, env = {}) { + const options = new URL(request.url).searchParams; + /** @param {string} key @param {string} value */ + const add = (key, value) => { + if (!options.has(key)) options.set(key, value); + }; + for ( + const prefix of PREFIXES.map((start) => `${start}${type.toLowerCase()}_`) + ) { + for (const [name, value] of request.headers) { + if (name.startsWith(prefix)) add(name.slice(prefix.length), value); + } + } + for ( + const prefix of PREFIXES.map((start) => `${start}${type}_`.toUpperCase()) + ) { + for (const [name, value] of Object.entries(env)) { + if (name.startsWith(prefix) && typeof value === "string") { + add(name.slice(prefix.length).toLowerCase(), value); + } + } + } + return options; +} + +/** + * Tells if a header is an option of rfetch or of rclone. + * @param {string} name + */ +export function isOption(name) { + const lower = name.toLowerCase(); + return PREFIXES.some((prefix) => lower.startsWith(prefix)); +} + +/** + * Gives the headers of a request of a backend to the remote under it. + * + * The other options of the request go to the remote, for example the + * remotes of `RFETCH_CONFIG_NAME_KEY`. The options of the backend itself do + * not go: they are not for the remote. When the remote is a URL, no option + * goes: a secret must not go out to a server. + * @param {HeadersInit} headers The headers of the request of the backend. + * @param {string} type The type of the backend, for example `crypt`. + * @param {string} remote The remote under the backend. + * @returns {Headers} + */ +export function remoteHeaders(headers, type, remote) { + const result = new Headers(headers); + const own = PREFIXES.map((prefix) => `${prefix}${type.toLowerCase()}_`); + const url = /^[a-z][a-z0-9+.-]*:\/\//i.test(remote); + for (const name of [...result.keys()]) { + const remove = url + ? isOption(name) + : own.some((prefix) => name.startsWith(prefix)); + if (remove) result.delete(name); + } + return result; +} diff --git a/lib/options_test.js b/lib/options_test.js new file mode 100644 index 0000000..1fa022c --- /dev/null +++ b/lib/options_test.js @@ -0,0 +1,128 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fc from "fast-check"; + +import { isOption, optionsOf, remoteHeaders } from "./options.js"; + +test("optionsOf takes the query, then the headers, then env", () => { + const request = new Request("https://rfetch.test/a?remote=query", { + headers: { + RCLONE_CRYPT_REMOTE: "header", + RCLONE_CRYPT_PASSWORD: "from-header", + RCLONE_S3_ENDPOINT: "not-for-crypt", + }, + }); + const options = optionsOf(request, "crypt", { + RCLONE_CRYPT_REMOTE: "env", + RCLONE_CRYPT_PASSWORD: "env", + RCLONE_CRYPT_PASSWORD2: "from-env", + RCLONE_CRYPT_NUMBER: 1, + }); + assert.equal(options.get("remote"), "query"); + assert.equal(options.get("password"), "from-header"); + assert.equal(options.get("password2"), "from-env"); + assert.equal(options.get("endpoint"), null); + // A value of env that is not a text, for example a binding, is not an + // option. + assert.equal(options.get("number"), null); +}); + +test("optionsOf takes RFETCH_ in the place of RCLONE_", () => { + const request = new Request("https://rfetch.test/", { + headers: { + RCLONE_S3_REGION: "rclone-header", + RFETCH_S3_REGION: "rfetch-header", + RCLONE_S3_ENDPOINT: "rclone-header", + }, + }); + const options = optionsOf(request, "s3", { + RCLONE_S3_ENDPOINT: "rclone-env", + RCLONE_S3_PROVIDER: "rclone-env", + RFETCH_S3_PROVIDER: "rfetch-env", + }); + assert.equal(options.get("region"), "rfetch-header"); + // A header takes the place of a variable of env, of each prefix. + assert.equal(options.get("endpoint"), "rclone-header"); + assert.equal(options.get("provider"), "rfetch-env"); +}); + +test("optionsOf reads the names of headers in each case", () => { + for ( + const name of ["RCLONE_S3_REGION", "rclone_s3_region", "Rclone_S3_Region"] + ) { + const request = new Request("https://rfetch.test/", { + headers: { [name]: "auto" }, + }); + assert.equal(optionsOf(request, "s3").get("region"), "auto", name); + } +}); + +test("optionsOf keeps an option of env with characters of each language", () => { + fc.assert( + fc.property(fc.string({ unit: "grapheme" }), (value) => { + const request = new Request("https://rfetch.test/"); + const options = optionsOf(request, "alias", { + RCLONE_ALIAS_REMOTE: value, + }); + assert.equal(options.get("remote"), value); + }), + ); +}); + +test("optionsOf does not change the query of the request", () => { + const request = new Request("https://rfetch.test/?a=1", { + headers: { RCLONE_ALIAS_REMOTE: "header" }, + }); + optionsOf(request, "alias"); + assert.equal(new URL(request.url).search, "?a=1"); +}); + +test("isOption tells the headers of rfetch and of rclone", () => { + assert.ok(isOption("RFETCH_CONFIG_R2_TYPE")); + assert.ok(isOption("RCLONE_CONFIG_R2_TYPE")); + assert.ok(isOption("rclone_s3_access_key_id")); + assert.ok(!isOption("Range")); + assert.ok(!isOption("X-Rclone")); +}); + +test("remoteHeaders keeps the other options for a remote of rfetch", () => { + const headers = remoteHeaders( + { + Range: "bytes=0-1", + RCLONE_CRYPT_PASSWORD: "secret", + RFETCH_CRYPT_PASSWORD2: "secret", + RCLONE_CONFIG_R2_TYPE: "s3", + RFETCH_S3_REGION: "auto", + }, + "crypt", + "r2:bucket", + ); + assert.deepEqual([...headers].sort(), [ + ["range", "bytes=0-1"], + ["rclone_config_r2_type", "s3"], + ["rfetch_s3_region", "auto"], + ]); +}); + +test("remoteHeaders gives no option to a URL", () => { + for ( + const remote of [ + "https://example.com/a", + "http://127.0.0.1:1", + "file:///tmp", + ] + ) { + const headers = remoteHeaders( + { + Range: "bytes=0-1", + RCLONE_CRYPT_PASSWORD: "secret", + RFETCH_CRYPT_PASSWORD: "secret", + RCLONE_CONFIG_R2_SECRET_ACCESS_KEY: "secret", + RFETCH_CONFIG_R2_SECRET_ACCESS_KEY: "secret", + }, + "crypt", + remote, + ); + assert.deepEqual([...headers], [["range", "bytes=0-1"]], remote); + } +}); diff --git a/lib/package.json b/lib/package.json new file mode 100644 index 0000000..ae09c86 --- /dev/null +++ b/lib/package.json @@ -0,0 +1,47 @@ +{ + "name": "@rfetch/lib", + "version": "0.1.0", + "description": "The code that rfetch and its backends share: the list of a folder, paths, ranges, types of files, sizes, the obscure of rclone, and the shared tests of the backends.", + "type": "module", + "exports": { + "./format": "./format.js", + "./fspath": "./fspath.js", + "./html": "./html.js", + "./ini": "./ini.js", + "./listing": "./listing.js", + "./mime": "./mime.js", + "./obscure": "./obscure.js", + "./options": "./options.js", + "./path": "./path.js", + "./progress": "./progress.js", + "./range": "./range.js", + "./rewriter": "./rewriter.js", + "./size": "./size.js", + "./sized": "./sized.js", + "./time": "./time.js", + "./testing/bench": "./testing/bench.js", + "./testing/remote": "./testing/remote.js", + "./testing/server": "./testing/server.js", + "./testing/suite": "./testing/suite.js" + }, + "scripts": { + "test": "node --test" + }, + "engines": { + "node": ">=22.2.0" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/sntran/rfetch.git", + "directory": "lib" + }, + "author": "Trần Nguyễn Sơn", + "license": "ISC", + "dependencies": { + "@sntran/lol-html": "^2.8.1", + "entities": "^8.1.0" + }, + "devDependencies": { + "fast-check": "^3.19.0" + } +} diff --git a/lib/path.js b/lib/path.js new file mode 100644 index 0000000..317bdc9 --- /dev/null +++ b/lib/path.js @@ -0,0 +1,46 @@ +/** + * Gives the path of a URL with no percent-encoding, for a backend to use as + * the path of a file or a folder. The path keeps its `/` at the start, and at + * the end for a folder. + * + * It gives null when the path is not safe: when a name has bad + * percent-encoding, or when a name is empty, is `.` or `..`, or has a `/` or a + * NUL character after the decoding. The URL parser removes the segments `.` + * and `..`, but it keeps `..%2F`, which becomes `../` after the decoding. Thus + * a backend that uses only the result of this function cannot go out of its + * root folder. + * @param {string} pathname The path of a URL, for example `/a%20b/c.txt`. + * @returns {string | null} The path, for example `/a b/c.txt`, or null. + */ +export function decodePath(pathname) { + if (!pathname.startsWith("/")) return null; + + const names = pathname.slice(1).split("/"); + // The path of a folder ends with "/", thus its last name is empty. + const last = names.length - 1; + for (let i = 0; i < names.length; i++) { + let name; + try { + name = decodeURIComponent(names[i]); + } catch { + return null; + } + if (name === "" && i === last) continue; + if (name === "" || name === "." || name === ".." || /[/\0]/.test(name)) { + return null; + } + names[i] = name; + } + + return `/${names.join("/")}`; +} + +/** + * Puts the percent-encoding in each name of a path, the reverse of + * `decodePath`. + * @param {string} path For example `/a b/c.txt`. + * @returns {string} For example `/a%20b/c.txt`. + */ +export function encodePath(path) { + return path.split("/").map(encodeURIComponent).join("/"); +} diff --git a/lib/path_test.js b/lib/path_test.js new file mode 100644 index 0000000..5889dd7 --- /dev/null +++ b/lib/path_test.js @@ -0,0 +1,64 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fc from "fast-check"; + +import { decodePath } from "./path.js"; + +test("decodePath", async (t) => { + await t.test("removes the percent-encoding of each name", () => { + assert.equal(decodePath("/"), "/"); + assert.equal(decodePath("/a%20b/c.txt"), "/a b/c.txt"); + assert.equal(decodePath("/a%20b/"), "/a b/"); + assert.equal(decodePath("/%E4%B8%96%E7%95%8C"), "/世界"); + assert.equal(decodePath("/a%3Fb%23c"), "/a?b#c"); + assert.equal(decodePath("/test%2546.txt"), "/test%46.txt"); + }); + + await t.test("gives the same name for each encoding of it", () => { + fc.assert( + fc.property(fc.string({ minLength: 1 }), (name) => { + fc.pre(!/[/\0]/.test(name) && name !== "." && name !== ".."); + let encoded; + try { + encoded = encodeURIComponent(name); + } catch { + fc.pre(false); // A string with a lone surrogate has no encoding. + } + assert.equal(decodePath(`/${encoded}`), `/${name}`); + assert.equal(decodePath(`/${encoded}/`), `/${name}/`); + }), + ); + }); + + await t.test("refuses a path that can go out of the root", () => { + for ( + const pathname of [ + "/..%2Fsecret.txt", + "/a/..%2F..%2Fsecret.txt", + "/%2E%2E%2Fsecret.txt", + "/..%2F", + "/%2E%2E", + "/.", + "/a/%2E/b", + ] + ) { + assert.equal(decodePath(pathname), null, pathname); + } + }); + + await t.test("refuses a path with a name that is not valid", () => { + for ( + const pathname of [ + "", + "a", + "//", + "/a//b", + "/a%2Fb", + "/a%00b", + "/%E0%A4%A", + ] + ) { + assert.equal(decodePath(pathname), null, pathname); + } + }); +}); diff --git a/lib/streams/progress.js b/lib/progress.js similarity index 86% rename from lib/streams/progress.js rename to lib/progress.js index 39095ee..d3bceac 100644 --- a/lib/streams/progress.js +++ b/lib/progress.js @@ -1,4 +1,4 @@ -import { formatBytes, formatDuration } from "../../deps.js"; +import { formatBytes, formatDuration } from "./format.js"; export class Progress extends TransformStream { completed = 0; @@ -48,7 +48,7 @@ export class Progress extends TransformStream { } get etah() { - return formatDuration(this.eta * 1000, { ignoreZero: true }); + return formatDuration(this.eta * 1000); } get rateh() { @@ -56,7 +56,7 @@ export class Progress extends TransformStream { } get timeh() { - return formatDuration(this.time * 1000, { ignoreZero: true }); + return formatDuration(this.time * 1000); } get totalh() { diff --git a/lib/range.js b/lib/range.js new file mode 100644 index 0000000..66ba956 --- /dev/null +++ b/lib/range.js @@ -0,0 +1,108 @@ +/** + * The ranges of bytes of a file, as RFC 9110 (section 14): the header + * `Range: bytes=START-END` of a GET request, and the answer 206 with + * `Content-Range`, or 416 when no byte of the file is in the range. + * + * Only one range is supported. A server can send all the file for a + * request with more ranges, or with a range that it cannot read. + */ + +/** + * Reads the range of a request, for a file of a size. + * @param {string | null} header The value of `Range`, for example + * `bytes=0-99`, `bytes=100-` or `bytes=-100`. + * @param {number} size The size of the file. + * @returns {{ start: number, end: number } | null} The first byte and the + * last byte, or null to send all the file: for no range, for more ranges, + * and for a range that is not correct. + * @throws {RangeError} When no byte of the file is in the range. Then the + * answer is 416. + */ +export function parseRange(header, size) { + const match = header?.trim().match(/^bytes=(\d*)-(\d*)$/i); + if (!match) return null; + const [, first, last] = match; + + if (first === "") { + // The last bytes of the file. + if (last === "") return null; + const length = Number(last); + if (length === 0 || size === 0) { + throw new RangeError("Range Not Satisfiable"); + } + return { start: Math.max(0, size - length), end: size - 1 }; + } + + const start = Number(first); + const end = last === "" ? size - 1 : Math.min(Number(last), size - 1); + if (last !== "" && Number(last) < start) return null; + if (start >= size) throw new RangeError("Range Not Satisfiable"); + return { start, end }; +} + +/** + * The headers of an answer 206, for a range of a file. + * @param {{ start: number, end: number }} range + * @param {number} size The size of the file. + * @returns {Record} + */ +export function partialHeaders({ start, end }, size) { + return { + "Content-Range": `bytes ${start}-${end}/${size}`, + "Content-Length": `${end - start + 1}`, + }; +} + +/** + * The answer 416, when no byte of the file is in the range. + * @param {number} size The size of the file. + * @returns {Response} + */ +export function notSatisfiable(size) { + return new Response(null, { + status: 416, + headers: { "Content-Range": `bytes */${size}` }, + }); +} + +/** + * Gives a part of a stream: the bytes after the first `skip` bytes, at most + * `length` bytes. It cancels the stream after the part. + * @param {ReadableStream} stream + * @param {number} skip + * @param {number} length + * @returns {ReadableStream} + */ +export function sliceStream(stream, skip, length) { + const reader = stream.getReader(); + return new ReadableStream({ + async pull(controller) { + while (true) { + if (length <= 0) { + controller.close(); + await reader.cancel(); + return; + } + const { done, value } = await reader.read(); + if (done) { + controller.close(); + return; + } + let chunk = value; + if (skip > 0) { + const dropped = Math.min(skip, chunk.length); + skip -= dropped; + chunk = chunk.subarray(dropped); + } + if (chunk.length === 0) continue; + if (chunk.length > length) chunk = chunk.subarray(0, length); + length -= chunk.length; + controller.enqueue(chunk); + return; + } + }, + cancel(reason) { + return reader.cancel(reason); + }, + }); +} diff --git a/lib/range_test.js b/lib/range_test.js new file mode 100644 index 0000000..97ad105 --- /dev/null +++ b/lib/range_test.js @@ -0,0 +1,81 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fc from "fast-check"; + +import { parseRange, sliceStream } from "./range.js"; + +test("parseRange reads one range of bytes", () => { + const size = 1000; + for ( + const [header, range] of [ + ["bytes=0-0", { start: 0, end: 0 }], + ["bytes=0-99", { start: 0, end: 99 }], + ["bytes=100-", { start: 100, end: 999 }], + ["bytes=999-", { start: 999, end: 999 }], + ["bytes=500-5000", { start: 500, end: 999 }], + ["bytes=-1", { start: 999, end: 999 }], + ["bytes=-100", { start: 900, end: 999 }], + ["bytes=-5000", { start: 0, end: 999 }], + ["Bytes=1-2", { start: 1, end: 2 }], + [" bytes=1-2 ", { start: 1, end: 2 }], + ] + ) { + assert.deepEqual(parseRange(header, size), range, header); + } +}); + +test("parseRange gives null to send all the file", () => { + for ( + const header of [ + null, + "", + "bytes=", + "bytes=-", + "bytes=5-1", + "bytes=0-1,5-6", + "items=0-1", + "bytes=a-b", + "bytes=1.5-2", + ] + ) { + assert.equal(parseRange(header, 1000), null, JSON.stringify(header)); + } +}); + +test("parseRange throws when no byte is in the range", () => { + for ( + const [header, size] of [ + ["bytes=1000-", 1000], + ["bytes=1000-2000", 1000], + ["bytes=-0", 1000], + ["bytes=0-", 0], + ["bytes=-1", 0], + ] + ) { + assert.throws(() => parseRange(header, size), RangeError, header); + } +}); + +test("sliceStream gives the bytes of the part, for any chunks", async () => { + await fc.assert( + fc.asyncProperty( + fc.uint8Array({ maxLength: 300 }), + fc.array(fc.nat(50), { maxLength: 20 }), + fc.nat(320), + fc.nat(320), + async (data, cuts, skip, length) => { + // The data in chunks of the sizes of `cuts`, then the rest. + const chunks = []; + let offset = 0; + for (const cut of cuts) { + chunks.push(data.subarray(offset, offset + cut)); + offset += cut; + } + chunks.push(data.subarray(offset)); + const part = sliceStream(ReadableStream.from(chunks), skip, length); + const bytes = new Uint8Array(await new Response(part).arrayBuffer()); + assert.deepEqual(bytes, data.slice(skip, skip + length)); + }, + ), + ); +}); diff --git a/lib/rewriter.js b/lib/rewriter.js new file mode 100644 index 0000000..fed5211 --- /dev/null +++ b/lib/rewriter.js @@ -0,0 +1,86 @@ +/** + * HTMLRewriter, to read the elements of an HTML document, for example the + * list of a folder: + * + * ```js + * await readElements(response, [ + * ["tr a[href]", { element(element) { attributeOf(element, "href"); } }], + * ]); + * ``` + * + * HTMLRewriter is lol-html, the HTML parser of Cloudflare, as the HTML + * standard tells: thus a list from a server reads the same in each runtime. + * In workerd and in Bun, HTMLRewriter is a part of the runtime. In Node.js and + * in Deno, it is the lol-html of `@sntran/lol-html` in WebAssembly, which + * loads at its first use: a command that reads no list does not need it. + */ +import { decodeHTMLAttribute } from "entities/decode"; + +/** + * The handlers of the elements of a selector. A handler only reads: it is + * not async, and it does not change the document. + * @typedef {Object} ElementHandlers + * @property {(element: Element) => void} [element] + */ + +/** + * An element of the document, in the handler of its start tag. + * @typedef {Object} Element + * @property {string} tagName + * @property {(name: string) => string | null | undefined} getAttribute + * @property {(handler: () => void) => void} onEndTag + */ + +/** + * The lol-html of WebAssembly, at its first use. + * @type {Promise<{ HTMLRewriter: any }> | undefined} + */ +let loaded; + +/** + * Reads an HTML document, and calls the handlers of each selector for its + * elements. The document is UTF-8, as for rclone. + * @param {Response} response + * @param {Array<[string, ElementHandlers]>} handlers + * @returns {Promise} + */ +export async function readElements(response, handlers) { + if (!response.body) return; + // @ts-ignore: HTMLRewriter is a global of workerd and of Bun. + const Native = globalThis.HTMLRewriter; + if (typeof Native === "function") { + const rewriter = new Native(); + for (const [selector, handler] of handlers) rewriter.on(selector, handler); + // Only the handlers read the document: its output goes away. The + // runtime reads the output with no JavaScript between the chunks. + await rewriter.transform(response).arrayBuffer(); + return; + } + + loaded ??= import("@sntran/lol-html"); + const { HTMLRewriter } = await loaded; + // The output of each chunk goes away. With no stream between the chunks, + // no queue of output chunks grows. + const rewriter = new HTMLRewriter("utf-8", () => {}); + try { + for (const [selector, handler] of handlers) rewriter.on(selector, handler); + for await (const chunk of response.body) rewriter.write(chunk); + rewriter.end(); + } finally { + rewriter.free(); + } +} + +/** + * The value of an attribute of an element, or null when the element does not + * have the attribute. HTMLRewriter gives the value as it is in the document: + * this function decodes its character references, as the HTML standard tells, + * for example `a&b` gives `a&b`. + * @param {Element} element + * @param {string} name + * @returns {string | null} + */ +export function attributeOf(element, name) { + const value = element.getAttribute(name); + return value == null ? null : decodeHTMLAttribute(value); +} diff --git a/lib/rewriter_test.js b/lib/rewriter_test.js new file mode 100644 index 0000000..ec0a574 --- /dev/null +++ b/lib/rewriter_test.js @@ -0,0 +1,116 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fc from "fast-check"; + +import { attributeOf, readElements } from "./rewriter.js"; +import { escapeHTML } from "./html.js"; + +/** + * The elements of a document, as `name href` lines, for the selector. + * @param {Response} response + * @param {string} selector + */ +async function linesOf(response, selector) { + const lines = []; + await readElements(response, [ + [selector, { + element(element) { + lines.push(`${element.tagName} ${attributeOf(element, "href")}`); + }, + }], + ]); + return lines; +} + +test("readElements reads HTML as the HTML standard tells", async () => { + const html = `<a href="title"> + 12 + + + + 3no link`; + assert.deepEqual(await linesOf(new Response(html), "a[href]"), [ + "a one", + "a two", + "a three", + ]); +}); + +test("readElements gives the same elements for each cut in chunks", async () => { + const html = ` +
é
c
`; + const bytes = new TextEncoder().encode(html); + const expected = await linesOf(new Response(html), "tr a[href]"); + assert.deepEqual(expected, ["a a%20b.txt", "a c&d/"]); + await fc.assert( + fc.asyncProperty(fc.array(fc.nat(bytes.length)), async (cuts) => { + const places = [...new Set([0, ...cuts, bytes.length])] + .sort((a, b) => a - b); + const chunks = places.slice(1).map((end, index) => + bytes.subarray(places[index], end) + ); + const response = new Response(new Blob(chunks).stream()); + assert.deepEqual(await linesOf(response, "tr a[href]"), expected); + }), + { numRuns: 50 }, + ); +}); + +test("readElements reads no element of a response with no body", async () => { + const response = new Response(null); + assert.deepEqual(await linesOf(response, "a[href]"), []); +}); + +test("readElements gives the error of a handler", async () => { + await assert.rejects( + readElements(new Response("x"), [ + ["a", { + element() { + throw new Error("no"); + }, + }], + ]), + ); +}); + +test("attributeOf", async (t) => { + /** The attribute `href` of the first link of the HTML. */ + const hrefOf = async (html) => { + let href; + await readElements(new Response(html), [ + ["a", { + element(element) { + href ??= attributeOf(element, "href"); + }, + }], + ]); + return href; + }; + + await t.test("decodes the character references", async () => { + assert.equal( + await hrefOf(`x`), + "a&b { + assert.equal( + await hrefOf(`x`), + "?a=1©=2&", + ); + }); + + await t.test("gives null for no attribute", async () => { + assert.equal(await hrefOf(`x`), null); + }); + + await t.test("is the inverse of escapeHTML", async () => { + await fc.assert( + fc.asyncProperty(fc.fullUnicodeString(), async (text) => { + assert.equal(await hrefOf(``), text); + }), + { numRuns: 50 }, + ); + }); +}); diff --git a/lib/size.js b/lib/size.js new file mode 100644 index 0000000..cb13d13 --- /dev/null +++ b/lib/size.js @@ -0,0 +1,60 @@ +/** The multipliers of the suffixes of a size, as rclone reads them. */ +const SYMBOLS = { + k: 2 ** 10, + m: 2 ** 20, + g: 2 ** 30, + t: 2 ** 40, + p: 2 ** 50, + e: 2 ** 60, +}; + +/** + * Reads a size as `SizeSuffix.Set` of rclone: a number with an optional + * suffix of 1024 (k), 1024² (M), and so on to E, with an optional "i" and + * "B" after it. A number with no suffix is in KiB, and "b" or "B" is a size + * in bytes. "off" is -1. + * @param {string} text For example `10M`, `2Gi`, `100KiB` or `4096b`. + * @returns {number} The size in bytes. + * @throws {RangeError} For a text that rclone does not accept. + */ +export function parseSize(text) { + if (text === "") throw new RangeError("Size: empty string."); + if (text.toLowerCase() === "off") return -1; + + let suffix = text.at(-1); + let suffixLength = 1; + let multiplier; + if (/[0-9.]/.test(suffix)) { + suffixLength = 0; + multiplier = SYMBOLS.k; + } else if (suffix === "b" || suffix === "B") { + if (text.length > 2 && text.at(-2) === "i") { + suffix = text.at(-3); + suffixLength = 3; + multiplier = SYMBOLS[suffix.toLowerCase()]; + } else { + multiplier = 1; + } + } else if (suffix === "i" || suffix === "I") { + if (text.length > 1) { + suffix = text.at(-2); + suffixLength = 2; + multiplier = SYMBOLS[suffix.toLowerCase()]; + } + } else { + multiplier = SYMBOLS[suffix.toLowerCase()]; + } + if (!multiplier) { + throw new RangeError(`Size: bad suffix ${JSON.stringify(suffix)}.`); + } + + const number = text.slice(0, text.length - suffixLength); + if (!/^[+-]?(\d+\.?\d*|\.\d+)([eE][+-]?\d+)?$/.test(number)) { + throw new RangeError(`Size: ${JSON.stringify(text)} is not a number.`); + } + const value = Number(number); + if (value < 0) { + throw new RangeError(`Size: ${JSON.stringify(text)} is negative.`); + } + return Math.trunc(value * multiplier); +} diff --git a/lib/size_test.js b/lib/size_test.js new file mode 100644 index 0000000..c257f13 --- /dev/null +++ b/lib/size_test.js @@ -0,0 +1,40 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { parseSize } from "./size.js"; + +test("parseSize reads the sizes of rclone (TestSizeSuffixSet)", () => { + for ( + const [text, size] of [ + ["0", 0], + ["1b", 1], + ["102B", 102], + ["0.1k", 102], + ["0.1", 102], + ["1K", 1024], + ["1k", 1024], + ["1KI", 1024], + ["1Ki", 1024], + ["1kI", 1024], + ["1ki", 1024], + ["1KiB", 1024], + ["1kib", 1024], + ["1", 1024], + ["2.5", 1024 * 2.5], + ["1M", 1024 * 1024], + ["1Mi", 1024 * 1024], + ["1MiB", 1024 * 1024], + ["1.g", 1024 * 1024 * 1024], + ["10G", 10 * 1024 ** 3], + ["10T", 10 * 1024 ** 4], + ["10P", 10 * 1024 ** 5], + ["off", -1], + ["OFF", -1], + ] + ) { + assert.equal(parseSize(text), size, text); + } + for (const text of ["", "1q", "1.q", "-1K", "1i", "1iB", "1MB"]) { + assert.throws(() => parseSize(text), RangeError, text); + } +}); diff --git a/lib/sized.js b/lib/sized.js new file mode 100644 index 0000000..f5e4e8e --- /dev/null +++ b/lib/sized.js @@ -0,0 +1,16 @@ +/** + * Gives a stream with a known size for the body of a response. In a Worker, + * the response of a stream has no Content-Length, unless the stream is a + * `FixedLengthStream`, which also makes an error when the stream has a + * different size. Other runtimes keep the Content-Length header, thus the + * stream does not change there. + * @param {ReadableStream} stream + * @param {number} size + * @returns {ReadableStream} + */ +export function sized(stream, size) { + const { FixedLengthStream } = globalThis; + return FixedLengthStream + ? stream.pipeThrough(new FixedLengthStream(size)) + : stream; +} diff --git a/lib/testing/bench.js b/lib/testing/bench.js new file mode 100644 index 0000000..500a080 --- /dev/null +++ b/lib/testing/bench.js @@ -0,0 +1,51 @@ +/** + * A small benchmark for the scripts `*_bench.js`, in place of `Deno.bench`. + * Run a script with `node FILE`. + * + * Each case is a function that gets a timer. By default the time of a run is + * the time of the whole function. A case can call `timer.start()` and + * `timer.end()` to time only a part, for example to leave out the start of a + * process. + */ + +/** + * @typedef {Object} Timer + * @property {() => void} start + * @property {() => void} end + */ + +/** + * Runs each case of the group one time to warm up, then `runs` times, and + * prints the mean time of a run. + * @param {string} group + * @param {Record Promise>} cases + * @param {{ runs?: number }} [options] + */ +export async function bench(group, cases, { runs = 10 } = {}) { + console.log(group); + for (const [name, run] of Object.entries(cases)) { + await time(run); + let total = 0; + for (let i = 0; i < runs; i++) { + total += await time(run); + } + console.log(` ${name}: ${(total / runs).toFixed(1)} ms`); + } +} + +/** + * Gives the time of one run, in milliseconds. + * @param {(timer: Timer) => Promise} run + * @returns {Promise} + */ +async function time(run) { + let start, end; + const timer = { + start: () => (start = performance.now()), + end: () => (end = performance.now()), + }; + const before = performance.now(); + await run(timer); + const after = performance.now(); + return (end ?? after) - (start ?? before); +} diff --git a/lib/testing/remote.js b/lib/testing/remote.js new file mode 100644 index 0000000..99cea94 --- /dev/null +++ b/lib/testing/remote.js @@ -0,0 +1,82 @@ +import { readListing } from "../listing.js"; + +/** The origin of the URLs of the requests. A backend reads only the path. */ +const ORIGIN = "http://rfetch.test"; + +/** + * A client of a backend in the same process, for the tests. A path is plain + * text, with `/` between the names and at the end of a folder. The client + * puts the percent-encoding in each name, and the parameters of the backend + * in the query. + * @param {{ fetch: (request: Request, env?: object) => Response | Promise }} backend + * @param {{ params?: Record, root?: string, env?: object, headers?: HeadersInit }} [options] + * `root` is a folder, for example `/rfetch-test-abc/`, under which all + * paths go. `env` is the env of the backend, for example the credentials of s3. + * `headers` go in each request, for example the options `RCLONE_S3_*`. + */ +export function remote( + backend, + { params = {}, root = "/", env = {}, headers = {} } = {}, +) { + /** + * Gives the URL of the path under the root. + * @param {string} path + * @returns {URL} + */ + function url(path) { + const encoded = `${root}${path}`.split("/").map(encodeURIComponent) + .join("/"); + const url = new URL(encoded, ORIGIN); + for (const [name, value] of Object.entries(params)) { + url.searchParams.set(name, value); + } + return url; + } + + /** + * Sends a request to the backend. + * @param {string} path + * @param {RequestInit} [init] + * @returns {Promise} + */ + async function send(path, init = {}) { + const all = new Headers(headers); + for (const [name, value] of new Headers(init.headers)) all.set(name, value); + return await backend.fetch( + new Request(url(path), { duplex: "half", ...init, headers: all }), + env, + ); + } + + return { + url, + send, + /** @param {string} path @param {BodyInit} [body] @param {HeadersInit} [headers] */ + put(path, body = new Uint8Array(0), headers = {}) { + headers = new Headers(headers); + // An HTTP client sends the size of a body that it knows. + if (body instanceof Uint8Array && !headers.has("Content-Length")) { + headers.set("Content-Length", `${body.length}`); + } + return send(path, { method: "PUT", body, headers }); + }, + /** @param {string} path @param {HeadersInit} [headers] */ + get: (path, headers = {}) => send(path, { headers }), + /** @param {string} path */ + head: (path) => send(path, { method: "HEAD" }), + /** @param {string} path */ + delete: (path) => send(path, { method: "DELETE" }), + /** + * Gives the entries of a folder, or throws with the status. + * @param {string} path A folder, with `/` at the end, or "" for the root. + */ + async list(path) { + const response = await send(path); + if (!response.ok) { + await response.body?.cancel(); + throw new Error(`GET ${JSON.stringify(path)} gave ${response.status}`); + } + return await readListing(response, url(path)); + }, + }; +} diff --git a/lib/testing/server.js b/lib/testing/server.js new file mode 100644 index 0000000..c3933cb --- /dev/null +++ b/lib/testing/server.js @@ -0,0 +1,55 @@ +import { createServer } from "node:http"; +import { Readable } from "node:stream"; +import { pipeline } from "node:stream/promises"; + +/** + * Serves a backend over HTTP on 127.0.0.1, for the tests of a backend that + * sends requests to a server. + * @param {(request: Request) => Response | Promise} handler + * @returns {Promise<{ url: URL, close: () => Promise }>} `url` is the + * root of the server, for example `http://127.0.0.1:40000/`. + */ +export async function listen(handler) { + const server = createServer(async (incoming, outgoing) => { + try { + const { method, headers } = incoming; + const request = new Request( + new URL(incoming.url, `http://${headers.host}`), + { + method, + headers: Object.entries(headers).flatMap(([name, value]) => + [value].flat().map((value) => [name, value]) + ), + body: method === "GET" || method === "HEAD" + ? null + : Readable.toWeb(incoming), + duplex: "half", + }, + ); + const response = await handler(request); + outgoing.writeHead( + response.status, + response.statusText, + [...response.headers].flat(), + ); + if (response.body) { + await pipeline(Readable.fromWeb(response.body), outgoing); + } else { + outgoing.end(); + } + } catch (error) { + if (!outgoing.headersSent) outgoing.writeHead(500); + outgoing.end(`${error}`); + } + }); + + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const { port } = server.address(); + return { + url: new URL(`http://127.0.0.1:${port}/`), + close() { + server.closeAllConnections(); + return new Promise((resolve) => server.close(() => resolve())); + }, + }; +} diff --git a/lib/testing/suite.js b/lib/testing/suite.js new file mode 100644 index 0000000..374dc1f --- /dev/null +++ b/lib/testing/suite.js @@ -0,0 +1,501 @@ +/** + * The tests that each backend must pass, as `fstests` of rclone + * (fstest/fstests/fstests.go). A test file of a backend calls + * `backendSuite(name, backend)` once. The tests run in order, and each test + * uses the files of the tests before it. + * + * The tests use a new folder `/rfetch-test-RANDOM/` and remove it at the end. + * They do not touch other files of the backend. + */ +import { after, describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { randomBytes } from "node:crypto"; + +import { mimeType } from "../mime.js"; +import { remote } from "./remote.js"; + +/** + * The names of `FsEncoding` of rclone that can be one name in the path of a + * URL. The names `.` and `..`, the names with a `/`, the name with all the + * control characters and the name with bad UTF-8 are not here: a URL or a + * string of JavaScript cannot have them as one name. + */ +const NAMES = [ + " leading space", + "~leading tilde", + "\rleading CR", + "\nleading LF", + "\tleading HT", + "\vleading VT", + ".leading dot", + "trailing space ", + "trailing CR\r", + "trailing LF\n", + "trailing HT\t", + "trailing VT\v", + "trailing dot.", + "test%46.txt", + "!\"#$%&'()*+,-.:;<=>?@[\\]^_`{|}~", + "êé", + "Hello, 世界", +]; + +/** The two files of `fstests`: one at the root, and one in folders with hard names. */ +const FILE1 = "file name.txt"; +const FILE2 = `hello? sausage/êé/Hello, 世界/ " ' @ < > & ? + ≠/z.txt`; + +/** + * Runs the tests of a backend. + * @param {string} name The name of the backend, for the report. + * @param {{ fetch: (request: Request) => Response | Promise }} backend + * @param {{ + * params?: Record, + * env?: object, + * skipName?: RegExp, + * writer?: { fetch: (request: Request) => Response | Promise }, + * parent?: string, + * emptyFolders?: boolean, + * type?: string, + * }} [options] `params` go in the query of each request, for example the + * options of the backend. `env` is the env of the backend, for example + * the credentials of s3. The tests skip the names that match `skipName`, + * for a limit that the backend has in rclone too. For a backend that is + * read only, the PUT and DELETE requests go to `writer`, with no `params`: + * a backend that writes the files that the backend under test reads. + * `parent` is the folder of the folder of the tests, for example the + * bucket `/bucket/` of s3. `emptyFolders` is false for a backend that + * cannot have an empty folder, as `CanHaveEmptyDirectories` of rclone: + * for example s3 with no folder markers. Then a folder with no file + * gives 404, as a folder that does not exist. `type` is the type of the + * backend in rclone, for example `s3`: with it, the tests also give the + * `params` in the headers `RFETCH_TYPE_NAME` and in the variables + * `RCLONE_TYPE_NAME` of `env`. + */ +export function backendSuite( + name, + backend, + { + params = {}, + env = {}, + skipName, + writer, + parent = "/", + emptyFolders = true, + type, + } = {}, +) { + const root = `${parent}rfetch-test-${randomBytes(6).toString("hex")}/`; + const fs = remote(backend, { params, root, env }); + if (writer) { + const { put, delete: remove } = remote(writer, { root }); + Object.assign(fs, { put, delete: remove }); + } + if (!emptyFolders) { + // A folder with no file is a folder that does not exist. + const { list } = fs; + fs.list = (path) => + list(path).catch((error) => { + if (/ gave 404$/.test(error.message)) return []; + throw error; + }); + } + + describe(`backend ${name}`, () => { + after(async () => { + // Removes what a test that failed did not remove. + const response = await fs.delete(""); + await response.body?.cancel(); + }); + + it("gives 404 for a folder that does not exist", async () => { + await expect(fs.get(""), 404); + }); + + it("makes a folder with PUT", async () => { + await put("", new Uint8Array(0)); + if (!emptyFolders) return; + + const response = await fs.get(""); + assert.equal(response.status, 200); + assert.match(response.headers.get("Content-Type"), /^text\/html\b/); + await response.body?.cancel(); + assert.deepEqual(await fs.list(""), [], "a new folder is empty"); + }); + + it("gives 404 for a file or a folder that does not exist", async () => { + await expect(fs.get("missing.txt"), 404); + await expect(fs.head("missing.txt"), 404); + await expect(fs.delete("missing.txt"), 404); + await expect(fs.get("missing/"), 404); + await expect(fs.delete("missing/"), 404); + await expect(fs.get("missing/missing.txt"), 404); + }); + + it("puts and gets a file of 0 bytes", async () => { + const start = Date.now(); + await put("zero.txt", new Uint8Array(0)); + const end = Date.now(); + + await assertFile("zero.txt", new Uint8Array(0), start, end); + await expect(fs.delete("zero.txt"), 204); + assert.deepEqual(await fs.list(""), []); + }); + + describe("names", () => { + for (const name of NAMES) { + const skip = skipName?.test(name) && "a limit of the backend in rclone"; + it(`keeps the name ${JSON.stringify(name)}`, { skip }, async () => { + // The name is a folder and a file in it. + const path = `${name}/${name}`; + const content = random(64); + try { + const start = Date.now(); + await put(path, content); + const end = Date.now(); + + assertEntries(await fs.list(""), [folder(name)]); + assertEntries(await fs.list(`${name}/`), [ + file(name, content.length), + ]); + await assertFile(path, content, start, end); + + await expect(fs.delete(`${name}/`), 204); + assertEntries(await fs.list(""), []); + } catch (error) { + // The next tests do not see the files of this test. + await fs.delete(`${name}/`).then((response) => + response.body?.cancel() + ); + throw error; + } + }); + } + }); + + describe("files", () => { + const files = [ + { path: FILE1, content: random(100), start: 0, end: 0 }, + // A file of more than one read of 64 KiB. + { path: FILE2, content: random(200 * 1024 + 1), start: 0, end: 0 }, + ]; + const [file1, file2] = files; + + it("puts the files", async () => { + for (const file of files) { + file.start = Date.now(); + await put(file.path, file.content); + file.end = Date.now(); + } + }); + + it("lists each folder", async () => { + assertEntries(await fs.list(""), [ + file(FILE1, file1.content.length), + folder("hello? sausage"), + ]); + + // Each folder above z.txt has one folder, then z.txt. + const names = FILE2.split("/"); + for (let i = 1; i < names.length; i++) { + const path = `${names.slice(0, i).join("/")}/`; + const name = names[i]; + const entry = i < names.length - 1 + ? folder(name) + : file(name, file2.content.length); + assertEntries(await fs.list(path), [entry]); + } + }); + + it("gets each file, with its headers", async () => { + for (const { path, content, start, end } of files) { + await assertFile(path, content, start, end); + } + }); + + it("gives a range of a file", async () => { + const { path, content } = file2; + const size = content.length; + for ( + const [header, start, end] of [ + ["bytes=0-0", 0, 0], + ["bytes=0-99", 0, 99], + // The ends of the blocks and of the chunks of 64 KiB. + ["bytes=65535-65536", 65535, 65536], + ["bytes=65536-131071", 65536, 131071], + ["bytes=100-", 100, size - 1], + [`bytes=${size - 1}-`, size - 1, size - 1], + ["bytes=-1", size - 1, size - 1], + ["bytes=-70000", size - 70000, size - 1], + ["bytes=-999999", 0, size - 1], + ["bytes=150000-999999", 150000, size - 1], + ] + ) { + const response = await fs.get(path, { Range: header }); + assert.equal(response.status, 206, header); + assert.equal( + response.headers.get("Content-Range"), + `bytes ${start}-${end}/${size}`, + header, + ); + assert.equal( + response.headers.get("Content-Length"), + `${end - start + 1}`, + header, + ); + const bytes = new Uint8Array(await response.arrayBuffer()); + assert.ok( + Buffer.from(bytes).equals(content.subarray(start, end + 1)), + `the bytes of the range ${header}`, + ); + } + }); + + it("gives 416 for a range with no byte of the file", async () => { + const size = file2.content.length; + const response = await fs.get(file2.path, { Range: `bytes=${size}-` }); + await response.body?.cancel(); + assert.equal(response.status, 416); + assert.equal(response.headers.get("Content-Range"), `bytes */${size}`); + }); + + it("gives all the file for a range that it does not read", async () => { + for (const header of ["bytes=5-1", "bytes=0-1,5-6", "items=0-1"]) { + const response = await fs.get(file1.path, { Range: header }); + assert.equal(response.status, 200, header); + const bytes = new Uint8Array(await response.arrayBuffer()); + assert.ok(Buffer.from(bytes).equals(file1.content), header); + } + }); + + it("gives the headers and no body for HEAD", async () => { + for (const { path, content } of files) { + const response = await fs.head(path); + assert.equal(response.status, 200, `HEAD ${JSON.stringify(path)}`); + assert.equal(response.body, null, "HEAD gives no body"); + assert.equal(response.headers.get("Content-Type"), mimeType(path)); + assert.equal( + response.headers.get("Content-Length"), + `${content.length}`, + ); + assert.ok(response.headers.get("Last-Modified")); + } + + const response = await fs.head(""); + assert.equal(response.status, 200, "HEAD of a folder"); + assert.equal(response.body, null, "HEAD gives no body"); + assert.match(response.headers.get("Content-Type"), /^text\/html\b/); + }); + + it("puts a file over a file", async () => { + file1.content = random(200); + file1.start = Date.now(); + const response = await fs.put(file1.path, file1.content); + file1.end = Date.now(); + assert.ok( + response.ok, + `PUT over a file gave ${response.status}, not 2xx`, + ); + await response.body?.cancel(); + + await assertFile(file1.path, file1.content, file1.start, file1.end); + assertEntries(await fs.list(""), [ + file(FILE1, file1.content.length), + folder("hello? sausage"), + ]); + }); + + it("deletes a file", async () => { + await expect(fs.delete(file2.path), 204); + await expect(fs.get(file2.path), 404); + const parent = file2.path.slice(0, file2.path.lastIndexOf("/") + 1); + assertEntries(await fs.list(parent), []); + }); + + it("deletes a folder and all that is in it", async () => { + await put("hello? sausage/êé/a.txt", random(10)); + await expect(fs.delete("hello? sausage/"), 204); + await expect(fs.get("hello? sausage/"), 404); + await expect(fs.get("hello? sausage/êé/"), 404); + await expect(fs.get("hello? sausage/êé/a.txt"), 404); + assertEntries(await fs.list(""), [file(FILE1, file1.content.length)]); + }); + }); + + it("reads the options from the headers and from env", { + skip: type && Object.keys(params).length ? false : "no options", + }, async () => { + /** @type {Record} */ + const variables = {}; + for (const [key, value] of Object.entries(params)) { + variables[`${type.toUpperCase()}_${key.toUpperCase()}`] = value; + } + /** @param {string} prefix */ + const named = (prefix) => + Object.fromEntries( + Object.entries(variables).map(( + [key, value], + ) => [prefix + key, value]), + ); + // The prefix of rfetch in the headers, and the prefix of rclone in + // env: both work. + const byHeaders = remote(backend, { + root, + env, + headers: named("RFETCH_"), + }); + const byEnv = remote(backend, { + root, + env: { ...env, ...named("RCLONE_") }, + }); + const content = random(100); + if (writer) { + await expect(fs.put("options.txt", content), 201); + } else { + await expect(byHeaders.put("options.txt", content), 201); + } + for (const client of [byHeaders, byEnv]) { + const response = await client.get("options.txt"); + assert.equal(response.status, 200); + assert.deepEqual(new Uint8Array(await response.arrayBuffer()), content); + // The options are not in the query, thus no link has a query. + const html = await (await client.get("")).text(); + assert.doesNotMatch(html, /href="[^"]*\?/, "a link has options"); + } + await expect(fs.delete("options.txt"), 204); + }); + + it("deletes the root folder of the tests", async () => { + await expect(fs.delete(""), 204); + await expect(fs.get(""), 404); + await expect(fs.get(FILE1), 404); + }); + }); + + /** + * Puts a file, or a folder for a path with a `/` at the end, and makes + * sure that the answer is 201 with the location of the path. + * @param {string} path + * @param {Uint8Array} content + */ + async function put(path, content) { + const response = await fs.put(path, content); + await response.body?.cancel(); + assert.equal(response.status, 201, `PUT ${JSON.stringify(path)}`); + + const location = response.headers.get("Content-Location"); + assert.ok(location, `PUT ${JSON.stringify(path)} gave no Content-Location`); + const url = fs.url(path); + assert.equal( + decodeURIComponent(new URL(location, url).pathname), + decodeURIComponent(url.pathname), + "Content-Location is the path of the file", + ); + } + + /** + * Gets a file and makes sure that its content and its headers are correct, + * and that the list of its folder has the same size, type and time. + * @param {string} path + * @param {Uint8Array} content + * @param {number} start The time before the PUT, in milliseconds. + * @param {number} end The time after the PUT, in milliseconds. + */ + async function assertFile(path, content, start, end) { + const what = `GET ${JSON.stringify(path)}`; + const response = await fs.get(path); + assert.equal(response.status, 200, what); + + const { headers } = response; + assert.equal(headers.get("Content-Type"), mimeType(path), what); + assert.equal(headers.get("Content-Length"), `${content.length}`, what); + assert.equal(headers.get("Accept-Ranges"), "bytes", what); + const lastModified = Date.parse(headers.get("Last-Modified")); + assertTime(lastModified, start, end, `Last-Modified of ${what}`); + + const bytes = new Uint8Array(await response.arrayBuffer()); + assert.equal(bytes.length, content.length, `the size of ${what}`); + assert.ok( + Buffer.from(bytes).equals(content), + `the bytes of ${what} are not the bytes of the PUT`, + ); + + const slash = path.lastIndexOf("/") + 1; + const name = path.slice(slash); + const entry = (await fs.list(path.slice(0, slash))) + .find((entry) => entry.Name === name); + assert.ok(entry, `the list of the folder has no ${JSON.stringify(name)}`); + assert.equal(entry.Size, content.length, `the size of ${name} in the list`); + assert.equal(entry.MimeType, mimeType(name), `the type of ${name}`); + const modTime = Date.parse(entry.ModTime); + assertTime(modTime, start, end, `the time of ${name} in the list`); + assert.ok( + Math.abs(modTime - lastModified) < 1000, + "the time in the list is the time of Last-Modified", + ); + } +} + +/** + * Sends a request, and makes sure that the answer has the status. + * @param {Promise} request + * @param {number} status + */ +async function expect(request, status) { + const response = await request; + await response.body?.cancel(); + assert.equal(response.status, status); +} + +/** + * Makes sure that the entries of a list are the expected entries, in any + * order. A folder has only its name. A file has its name and its size. + * @param {import("../listing.js").Entry[]} entries + * @param {Array<{ Name: string, IsDir: boolean, Size?: number }>} expected + */ +function assertEntries(entries, expected) { + const actual = entries.map(({ Name, IsDir, Size }) => + IsDir ? { Name, IsDir } : { Name, IsDir, Size } + ); + assert.deepEqual(sort(actual), sort(expected)); +} + +/** + * The time of a file must be between the times before and after its PUT. + * Some backends keep the time to the second, thus the time can be one + * second before. + * @param {number} time + * @param {number} start + * @param {number} end + * @param {string} what + */ +function assertTime(time, start, end, what) { + // `Last-Modified` has whole seconds, and the clock of a file system can be + // some milliseconds behind `Date.now()`: thus the second of the time can + // start before the second of `start`. + const first = Math.floor((start - 100) / 1000) * 1000; + assert.ok( + time >= first - 1000 && time <= end + 1000, + `${what} is ${new Date(time).toISOString()}, not between ` + + `${new Date(start).toISOString()} and ${new Date(end).toISOString()}`, + ); +} + +/** @param {string} Name */ +function folder(Name) { + return { Name, IsDir: true }; +} + +/** @param {string} Name @param {number} Size */ +function file(Name, Size) { + return { Name, IsDir: false, Size }; +} + +/** @template {{ Name: string }} T @param {T[]} entries */ +function sort(entries) { + return entries.toSorted((a, b) => a.Name < b.Name ? -1 : +(a.Name > b.Name)); +} + +/** @param {number} size */ +function random(size) { + return new Uint8Array(randomBytes(size)); +} diff --git a/lib/time.js b/lib/time.js new file mode 100644 index 0000000..4a1d5de --- /dev/null +++ b/lib/time.js @@ -0,0 +1,67 @@ +/** + * Writes a time as the ModTime of `rclone lsjson`, in the local time zone: + * the format `2006-01-02T15:04:05.000000000Z07:00` of Go, with the digits + * after the second of the remote. The time zone is `Z` for UTC, else + * `+hh:mm` or `-hh:mm`. + * + * As rclone, the number of digits after the second tells the precision of + * the remote: 9 for 1 ns, for example a local file, 3 for 1 ms, and none for + * 1 s, for example the http backend. A time in a string keeps its digits, + * from 0 to 9. A Date or a number has 3. + * @param {string | number | Date | null | undefined} value + * @param {number} [offset] The minutes after UTC of the time zone. By + * default, the offset of the local time zone at that time. + * @returns {string | undefined} No string for no time, or for a time that is + * not correct. + */ +export function toLocaleISOString(value, offset) { + if (value === null || value === undefined || value === "") return undefined; + /** The digits after the second of a string, or undefined for 3. */ + let digits; + if (typeof value === "string") { + const match = /^(.*T\d\d:\d\d:\d\d)(?:\.(\d+))?(Z|[+-]\d\d:\d\d)$/i + .exec(value); + if (match) { + digits = (match[2] ?? "").slice(0, 9); + value = `${match[1]}.${digits.slice(0, 3).padEnd(3, "0")}${match[3]}`; + } + } + const date = new Date(value); + if (isNaN(date.getTime())) return undefined; + + offset ??= -date.getTimezoneOffset(); + let local = new Date(date.getTime() + offset * 60000).toISOString() + .slice(0, -1); + // The time zone does not change the digits after the second. + if (digits !== undefined) { + local = local.slice(0, -4) + (digits ? `.${digits}` : ""); + } + if (offset === 0) return `${local}Z`; + const minutes = Math.abs(offset); + const pad = (number) => String(number).padStart(2, "0"); + return `${local}${offset < 0 ? "-" : "+"}${pad(Math.floor(minutes / 60))}:${ + pad(minutes % 60) + }`; +} + +/** + * The time of a folder that has no time, as the default of the flag + * `--default-time` of rclone. + */ +export const DEFAULT_TIME = "2000-01-01T00:00:00Z"; + +/** + * Writes a time as the lists of rclone, in the local time zone: the format + * `2006-01-02 15:04:05` of Go, for example for `rclone lsd` and `rclone lsf`, + * with `digits` digits after the second, for example 9 for `rclone lsl`. + * @param {string | number | Date | null | undefined} value + * @param {number} [digits] + * @returns {string} For no time, the zero time of Go, as rclone. + */ +export function toListTime(value, digits = 0) { + const iso = toLocaleISOString(value) ?? "0001-01-01T00:00:00Z"; + const [, day, time, fraction = ""] = + /^(.+)T(\d\d:\d\d:\d\d)(?:\.(\d+))?/.exec(iso) ?? []; + if (!digits) return `${day} ${time}`; + return `${day} ${time}.${fraction.padEnd(digits, "0").slice(0, digits)}`; +} diff --git a/lib/time_test.js b/lib/time_test.js new file mode 100644 index 0000000..db50af7 --- /dev/null +++ b/lib/time_test.js @@ -0,0 +1,79 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { toListTime, toLocaleISOString } from "./time.js"; + +test("toLocaleISOString writes the times of rclone lsjson", () => { + const time = "2026-10-17T03:56:05.819Z"; + for ( + const [offset, expected] of [ + [0, "2026-10-17T03:56:05.819Z"], + [60, "2026-10-17T04:56:05.819+01:00"], + [-300, "2026-10-16T22:56:05.819-05:00"], + [330, "2026-10-17T09:26:05.819+05:30"], + [-570, "2026-10-16T18:26:05.819-09:30"], + [840, "2026-10-17T17:56:05.819+14:00"], + ] + ) { + assert.equal(toLocaleISOString(time, offset), expected, `${offset}`); + } + // A time to the second stays a time to the second, as the precision of the + // remote, for example http. + assert.equal( + toLocaleISOString("2026-10-17T03:56:05Z", 0), + "2026-10-17T03:56:05Z", + ); + assert.equal( + toLocaleISOString("2026-10-17T03:56:05Z", 90), + "2026-10-17T05:26:05+01:30", + ); + // The milliseconds of a Date stay, also when they are zero. + assert.equal( + toLocaleISOString(new Date("2026-10-17T03:56:05Z"), 0), + "2026-10-17T03:56:05.000Z", + ); + for (const value of [null, undefined, "", "not a time"]) { + assert.equal(toLocaleISOString(value), undefined); + } +}); + +test("toLocaleISOString keeps the digits of the time, from 0 to 9", () => { + for ( + const [time, offset, expected] of [ + ["2026-10-18T15:02:41.540476887Z", 0, "2026-10-18T15:02:41.540476887Z"], + [ + "2026-10-18T15:02:41.540476887Z", + 420, + "2026-10-18T22:02:41.540476887+07:00", + ], + [ + "2026-10-18T00:02:41.000000001+00:00", + -300, + "2026-10-17T19:02:41.000000001-05:00", + ], + ["2026-10-18T15:02:41.540476Z", 0, "2026-10-18T15:02:41.540476Z"], + ["2026-10-18T15:02:41.1234567891Z", 0, "2026-10-18T15:02:41.123456789Z"], + ["2026-10-18T15:02:41.5Z", 0, "2026-10-18T15:02:41.5Z"], + ["2026-10-18T15:02:41.540Z", 0, "2026-10-18T15:02:41.540Z"], + ] + ) { + assert.equal(toLocaleISOString(time, offset), expected, time); + } +}); + +test("toListTime writes the times of the lists of rclone", () => { + const offset = new Date("2026-10-18T15:02:41Z").getTimezoneOffset(); + // The local time zone of the tests is not known: the test makes the + // expected time in it. + const local = new Date(Date.parse("2026-10-18T15:02:41Z") - offset * 60000) + .toISOString().slice(0, 19).replace("T", " "); + assert.equal(toListTime("2026-10-18T15:02:41.540476887Z"), local); + assert.equal( + toListTime("2026-10-18T15:02:41.540476887Z", 9), + `${local}.540476887`, + ); + assert.equal(toListTime("2026-10-18T15:02:41.5Z", 9), `${local}.500000000`); + assert.equal(toListTime("2026-10-18T15:02:41Z", 9), `${local}.000000000`); + assert.equal(toListTime(""), "0001-01-01 00:00:00"); + assert.equal(toListTime(undefined, 9), "0001-01-01 00:00:00.000000000"); +}); diff --git a/main.js b/main.js index d26f471..8e1049b 100755 --- a/main.js +++ b/main.js @@ -1,4 +1,4 @@ -#!/usr/bin/env -S deno run --allow-all +#!/usr/bin/env node /** * rfetch - fetching between remotes @@ -10,69 +10,59 @@ * ./main.js copy source:/path/to/file destination:/path/to/file * ``` */ -import { argv, env, stdout } from "node:process"; -import { parseArgs } from "node:util"; +import process, { argv, stderr, stdout } from "node:process"; +import { once } from "node:events"; -import * as commands from "./mod.js"; +// The import sets the global `fetch`, which reads the paths of remotes. +import "./fetch.js"; -/** Optional params for the subcommand. */ -const options = { - "dry-run": { - type: "boolean", - default: env["RCLONE_DRY_RUN"] === "true", - }, - "header": { - type: "string", - multiple: true, - }, - "header-download": { - type: "string", - multiple: true, - }, - "header-upload": { - type: "string", - multiple: true, - }, - "human-readable": { - type: "boolean", - }, - "progress": { - type: "boolean", - short: "P", - default: env["RCLONE_PROGRESS"] === "true", - }, - "transfers": { - type: "string", - default: env["RCLONE_TRANSFERS"] || "4", - } -}; - -const { - values: flags, - positionals: [ - subcommand = "help", - ...args - ], -} = parseArgs({ - args: argv.slice(2), - options, - strict: false, - allowPositionals: true, - allowNegative: true, - tokens: true, -}); - -for (const name of Object.keys(flags)) { - // Adds snake_case version of the flag. - flags[name.replace(/-/g, "_")] = flags[name]; -} +import { backends } from "./backend/main.js"; +import { parseFlags } from "./cmd/flags.js"; +import { FOLDERS } from "./cmd/folders.js"; (async () => { + /** @type {Awaited>} */ + let parsed; + try { + parsed = await parseFlags(argv.slice(2), backends); + } catch (error) { + // As rclone: for example `Error: unknown flag: --nosuch`. + stderr.write(`Error: ${error.message}\n`); + process.exitCode = 1; + return; + } + const { flags, positionals: [subcommand = "help", ...positionals] } = parsed; + + let name = subcommand; + let args = positionals; + if (!Object.hasOwn(FOLDERS, name)) { + stderr.write(`Error: unknown command "${name}" for "rfetch"\n`); + process.exitCode = 1; + [name, args] = ["help", []]; + } + // `rfetch COMMAND --help` shows the help. + if (flags.help) [name, args] = ["help", []]; + const module = await import(`./cmd/${FOLDERS[name]}/main.js`); /** TODO merge global flags into config */ - const response = await commands[subcommand](...args, flags); + const response = await module[name](...args, flags); + if (!response.ok) { + // As rclone: the exit code 3 is "directory not found", and 1 is an + // other error. + const text = (await response.text()).trim() || response.statusText || + (response.status === 404 ? "not found" : `status ${response.status}`); + stderr.write(`Error: ${text}\n`); + process.exitCode = response.status === 404 ? 3 : 1; + return; + } if (response.body) { + // A reader that closes the pipe, for example `head`, stops the command. + stdout.on("error", (error) => { + if (error.code === "EPIPE") process.exit(); + throw error; + }); for await (const chunk of response.body) { - stdout.write(chunk); + // Waits when the pipe is full: else the output stays in memory. + if (!stdout.write(chunk)) await once(stdout, "drain"); } } })(); diff --git a/main_test.js b/main_test.js index a14c261..51186ef 100644 --- a/main_test.js +++ b/main_test.js @@ -1,34 +1,40 @@ import { test } from "node:test"; import process from "node:process"; -import { unlink, writeFile } from "node:fs/promises"; -import { serve } from "@sntran/serve"; -import { assert, assertEquals } from "./dev_deps.js"; - -import { config } from "./mod.js"; +import { spawn } from "node:child_process"; +import { request } from "node:http"; +import { once } from "node:events"; +import { mkdir, mkdtemp, rm, stat, unlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { basename, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import assert from "node:assert/strict"; + +import { listen } from "@rfetch/lib/testing/server"; +import { config, fetch } from "./mod.js"; + +/** + * Gives an option in the `env` of the backend, from the body of an answer to + * `TRACE`: the options of the config file and of the variables are not in + * the query, thus they are never in a link. + * @param {string} body + * @param {string} name For example `RFETCH_ALIAS_REMOTE`. + */ +function variableOf(body, name) { + const line = body.split("\r").find((line) => line.startsWith(`${name}: `)); + return line?.slice(name.length + 2); +} test("fetch", async (t) => { - await t.test("global", () => { - const controller = new AbortController(); - serve({ - fetch: (_request) => new Response(), - port: 0, - signal: controller.signal, - async onListen({ port, hostname }) { - const url = `http://${hostname}:${port}/`; - let response = await fetch(url, { - method: "HEAD", - }); - assert(response.ok); - - const request = new Request(url, { - method: "HEAD", - }); - response = await fetch(request); - assert(response.ok); - - controller.abort(); - }, - }); + await t.test("global", async () => { + const server = await listen(() => new Response()); + try { + let response = await fetch(`${server.url}`, { method: "HEAD" }); + assert(response.ok); + response = await fetch(new Request(server.url, { method: "HEAD" })); + assert(response.ok); + } finally { + await server.close(); + } }); await t.test("TRACE", async (t) => { @@ -39,16 +45,16 @@ test("fetch", async (t) => { }); const { status, headers } = response; - assertEquals(status, 200); - assertEquals(headers.get("Content-Type"), "message/http"); + assert.deepEqual(status, 200); + assert.deepEqual(headers.get("Content-Type"), "message/http"); - assertEquals(headers.get("Via"), "local/1.1"); + assert.deepEqual(headers.get("Via"), "local/1.1"); const body = await response.text(); const [, _method, url] = body.match(/^(TRACE) (.*) HTTP\/1.1$/m) || []; const { pathname } = new URL(url, "file:"); - assertEquals(pathname, cwd); + assert.deepEqual(pathname, cwd); }); // Sets up @@ -65,20 +71,20 @@ test("fetch", async (t) => { const { status, headers } = await fetch(":local:", { method: "TRACE", }); - assertEquals(status, 200); - assertEquals(headers.get("Content-Type"), "message/http"); + assert.deepEqual(status, 200); + assert.deepEqual(headers.get("Content-Type"), "message/http"); - assertEquals(headers.get("Via"), "local/1.1"); + assert.deepEqual(headers.get("Via"), "local/1.1"); }); await t.test("name:", async () => { const { status, headers } = await fetch("source:", { method: "TRACE", }); - assertEquals(status, 200); - assertEquals(headers.get("Content-Type"), "message/http"); + assert.deepEqual(status, 200); + assert.deepEqual(headers.get("Content-Type"), "message/http"); - assertEquals(headers.get("Via"), "local/1.1 source"); + assert.deepEqual(headers.get("Via"), "local/1.1 source"); }); /** Connection string */ @@ -91,17 +97,18 @@ test("fetch", async (t) => { }); const { status, headers } = response; - assertEquals(status, 200); - assertEquals(headers.get("Content-Type"), "message/http"); + assert.deepEqual(status, 200); + assert.deepEqual(headers.get("Content-Type"), "message/http"); - assertEquals(headers.get("Via"), "alias/1.1 target"); + assert.deepEqual(headers.get("Via"), "alias/1.1 target"); const body = await response.text(); const [, _method, url] = body.match(/^(TRACE) (.*) HTTP\/1.1$/m) || []; const { pathname, searchParams } = new URL(url, "file:"); - assertEquals(pathname, "/"); - assertEquals(searchParams.get("remote"), `source:`); + assert.deepEqual(pathname, "/"); + assert(!searchParams.has("remote"), "the config is not in the query"); + assert.deepEqual(variableOf(body, "RFETCH_ALIAS_REMOTE"), `source:`); }); await t.test( @@ -116,8 +123,9 @@ test("fetch", async (t) => { []; const { searchParams } = new URL(url, "file:"); - assertEquals(searchParams.get("remote"), `source:`); - assertEquals(searchParams.get("skip_links"), `true`); + assert(!searchParams.has("skip_links"), "not in the query"); + assert.deepEqual(variableOf(body, "RFETCH_ALIAS_REMOTE"), `source:`); + assert.deepEqual(variableOf(body, "RFETCH_ALIAS_SKIP_LINKS"), `true`); }, ); @@ -133,7 +141,8 @@ test("fetch", async (t) => { []; const { searchParams } = new URL(url, "file:"); - assertEquals(searchParams.get("remote"), `/tmp/1`); + assert(!searchParams.has("remote"), "not in the query"); + assert.deepEqual(variableOf(body, "RFETCH_ALIAS_REMOTE"), `/tmp/1`); assert( !searchParams.has("alias_remote"), "should not have param prefixed with backend type", @@ -153,7 +162,8 @@ test("fetch", async (t) => { []; const { searchParams } = new URL(url, "file:"); - assertEquals(searchParams.get("remote"), `/tmp/2`); + assert(!searchParams.has("remote"), "not in the query"); + assert.deepEqual(variableOf(body, "RFETCH_ALIAS_REMOTE"), `/tmp/2`); assert( !searchParams.has("config_target_remote"), "should not have param prefixed with remote name", @@ -169,18 +179,18 @@ test("fetch", async (t) => { }); const { status, headers } = response; - assertEquals(status, 200); - assertEquals(headers.get("Content-Type"), "message/http"); + assert.deepEqual(status, 200); + assert.deepEqual(headers.get("Content-Type"), "message/http"); - assertEquals(headers.get("Via"), "alias/1.1 target"); + assert.deepEqual(headers.get("Via"), "alias/1.1 target"); const body = await response.text(); const [, _method, url] = body.match(/^(TRACE) (.*) HTTP\/1.1$/m) || []; const { pathname, searchParams } = new URL(url, "file:"); - assertEquals(pathname, "/"); - assertEquals(searchParams.get("remote"), `'/tmp/3'`); + assert.deepEqual(pathname, "/"); + assert.deepEqual(searchParams.get("remote"), `'/tmp/3'`); }, ); @@ -195,8 +205,9 @@ test("fetch", async (t) => { const [, _method, url] = body.match(/^(TRACE) (.*) HTTP\/1.1$/m) || []; const { pathname, searchParams } = new URL(url, "file:"); - assertEquals(pathname, "/"); - assertEquals(searchParams.get("remote"), `'${cwd}'`); + assert.deepEqual(pathname, "/"); + // As in rclone, the quotes are not a part of the value. + assert.deepEqual(searchParams.get("remote"), cwd); }); await t.test( @@ -213,8 +224,12 @@ test("fetch", async (t) => { []; const { pathname, searchParams } = new URL(url, "file:"); - assertEquals(pathname, "/"); - assertEquals(searchParams.get("remote"), `':local:${cwd}'`); + assert.deepEqual(pathname, "/"); + // The value in quotes can have "," and ":", as in rclone. + assert.deepEqual( + searchParams.get("remote"), + `:local,case_sensitive=true:${cwd}`, + ); }, ); }); @@ -226,3 +241,335 @@ test("fetch", async (t) => { delete process.env.RCLONE_CONFIG_TARGET_REMOTE; }); }); + +test("options in the headers of a request", async (t) => { + await t.test("a remote only from the headers, with no config file", async () => { + const headers = { RCLONE_CONFIG_MEM_TYPE: "memory" }; + const put = await fetch("mem:headers/a.txt", { + method: "PUT", + body: "from the headers", + headers, + }); + assert.equal(put.status, 201); + const response = await fetch("mem:headers/a.txt", { headers }); + assert.equal(await response.text(), "from the headers"); + // The same remote, as a remote on the fly. + const plain = await fetch(":memory:headers/a.txt"); + assert.equal(await plain.text(), "from the headers"); + await assert.rejects(fetch("mem:headers/a.txt"), /Remote mem not found/); + }); + + await t.test("a header takes the place of a variable", async () => { + process.env.RCLONE_CONFIG_PLACE_TYPE = "alias"; + process.env.RCLONE_CONFIG_PLACE_REMOTE = "/tmp/env"; + try { + const response = await fetch("place:", { + method: "TRACE", + headers: { RCLONE_CONFIG_PLACE_REMOTE: "/tmp/header" }, + }); + const body = await response.text(); + assert.equal(variableOf(body, "RFETCH_ALIAS_REMOTE"), "/tmp/header"); + } finally { + delete process.env.RCLONE_CONFIG_PLACE_TYPE; + delete process.env.RCLONE_CONFIG_PLACE_REMOTE; + } + }); + + await t.test("RFETCH_ takes the place of RCLONE_", async () => { + process.env.RCLONE_CONFIG_BOTH_TYPE = "alias"; + process.env.RCLONE_CONFIG_BOTH_REMOTE = "/tmp/rclone-env"; + process.env.RFETCH_CONFIG_BOTH_REMOTE = "/tmp/rfetch-env"; + try { + /** @param {HeadersInit} headers */ + const remoteOf = async (headers) => { + const response = await fetch("both:", { method: "TRACE", headers }); + return variableOf(await response.text(), "RFETCH_ALIAS_REMOTE"); + }; + assert.equal(await remoteOf({}), "/tmp/rfetch-env"); + // A header takes the place of a variable of the process. + assert.equal( + await remoteOf({ RCLONE_CONFIG_BOTH_REMOTE: "/tmp/rclone-header" }), + "/tmp/rclone-header", + ); + assert.equal( + await remoteOf({ + RCLONE_CONFIG_BOTH_REMOTE: "/tmp/rclone-header", + RFETCH_CONFIG_BOTH_REMOTE: "/tmp/rfetch-header", + }), + "/tmp/rfetch-header", + ); + } finally { + delete process.env.RCLONE_CONFIG_BOTH_TYPE; + delete process.env.RCLONE_CONFIG_BOTH_REMOTE; + delete process.env.RFETCH_CONFIG_BOTH_REMOTE; + } + }); + + await t.test("the variables of other remotes are not options", async () => { + process.env.RCLONE_CONFIG_OTHER_SECRET_ACCESS_KEY = "other-secret"; + try { + const response = await fetch(":memory:leak/", { + method: "TRACE", + headers: { RCLONE_CONFIG_THIRD_PASSWORD: "third-secret" }, + }); + const body = await response.text(); + assert(!body.includes("RCLONE_MEMORY_CONFIG_"), body); + const [, url] = body.match(/^TRACE (.*) HTTP\/1.1$/m); + assert.equal( + new URL(url, "file:").search, + "", + "an option is in the query", + ); + + await fetch(":memory:leak/a.txt", { method: "PUT", body: "a" }); + const html = await (await fetch(":memory:leak/", { + headers: { RCLONE_CONFIG_THIRD_PASSWORD: "third-secret" }, + })).text(); + assert(!html.includes("other-secret"), "a variable is in a link"); + assert(!html.includes("third-secret"), "a header is in a link"); + } finally { + delete process.env.RCLONE_CONFIG_OTHER_SECRET_ACCESS_KEY; + } + }); + + await t.test("no option goes out to a URL", async () => { + /** @type {Headers[]} */ + const seen = []; + const server = await listen((request) => { + seen.push(request.headers); + return new Response("ok"); + }); + try { + const headers = { + RCLONE_S3_SECRET_ACCESS_KEY: "secret", + RCLONE_CONFIG_R2_TYPE: "s3", + RFETCH_S3_SECRET_ACCESS_KEY: "secret", + RFETCH_CONFIG_R2_TYPE: "s3", + "X-Other": "other", + }; + await (await fetch(`${server.url}`, { headers })).text(); + await (await fetch(new Request(server.url, { headers }))).text(); + await (await fetch(server.url, { headers })).text(); + assert.equal(seen.length, 3); + for (const headers of seen) { + assert.equal(headers.get("x-other"), "other"); + assert.deepEqual( + [...headers.keys()].filter((name) => /^r(clone|fetch)_/.test(name)), + [], + ); + } + } finally { + await server.close(); + } + }); + + await t.test("two requests at the same time, each with its own remote", async () => { + await fetch(":memory:one/a.txt", { method: "PUT", body: "one" }); + await fetch(":memory:two/a.txt", { method: "PUT", body: "two" }); + /** @param {string} folder */ + const get = (folder) => + fetch("tenant:a.txt", { + headers: { + RCLONE_CONFIG_TENANT_TYPE: "alias", + RCLONE_CONFIG_TENANT_REMOTE: `:memory:${folder}`, + }, + }).then((response) => response.text()); + const folders = Array.from({ length: 20 }, (_, i) => i % 2 ? "one" : "two"); + assert.deepEqual(await Promise.all(folders.map(get)), folders); + }); +}); + +test("local paths", async (t) => { + const outside = await mkdtemp(join(tmpdir(), "rfetch-outside-")); + await writeFile(join(outside, "secret.txt"), "outside"); + const inside = await mkdtemp(join(tmpdir(), "rfetch-inside-")); + await writeFile(join(inside, "a.txt"), "inside"); + const cwd = process.cwd(); + process.chdir(inside); + try { + await t.test("a path is relative to the current folder", async () => { + assert.equal(await (await fetch("a.txt")).text(), "inside"); + assert.equal(await (await fetch("./a.txt")).text(), "inside"); + }); + + await t.test("an absolute path and ../ go out of the current folder", async () => { + const file = join(outside, "secret.txt"); + assert.equal(await (await fetch(file)).text(), "outside"); + const up = `../${basename(outside)}/secret.txt`; + assert.equal(await (await fetch(up)).text(), "outside"); + }); + + await t.test("a header, a variable or a remote cannot set the root", async () => { + const path = `${outside}/secret.txt`; + let response = await fetch(`:local:${path}`, { + headers: { RFETCH_LOCAL_ROOT: "/" }, + }); + assert.equal(response.status, 404); + process.env.RFETCH_LOCAL_ROOT = "/"; + try { + response = await fetch(`:local:${path}`); + assert.equal(response.status, 404); + } finally { + delete process.env.RFETCH_LOCAL_ROOT; + } + response = await fetch(`disk:${path}`, { + headers: { + RFETCH_CONFIG_DISK_TYPE: "local", + RFETCH_CONFIG_DISK_ROOT: "/", + }, + }); + assert.equal(response.status, 404); + }); + + await t.test("the current folder stays", async () => { + for (const path of ["./", ".", `${inside}/`, `../${basename(inside)}/`]) { + const response = await fetch(path, { method: "DELETE" }); + assert.equal(response.status, 403, JSON.stringify(path)); + } + assert.ok((await stat(join(inside, "a.txt"))).isFile()); + }); + } finally { + process.chdir(cwd); + await rm(outside, { recursive: true, force: true }); + await rm(inside, { recursive: true, force: true }); + } +}); + +const MAIN = fileURLToPath(new URL("./main.js", import.meta.url)); + +/** + * Runs the command line in a folder. + * @param {string} cwd + * @param {string[]} args + */ +async function run(cwd, args) { + const child = spawn(process.execPath, [MAIN, ...args], { cwd }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => stdout += chunk); + child.stderr.on("data", (chunk) => stderr += chunk); + const [code] = await once(child, "close"); + return { code, stdout, stderr }; +} + +test("the command line", async (t) => { + const folder = await mkdtemp(join(tmpdir(), "rfetch-cli-")); + await writeFile(join(folder, "a.txt"), "a"); + await mkdir(join(folder, "empty")); + try { + await t.test("gives the exit code 3 of rclone for a path that is not found", async () => { + for ( + const args of [ + ["lsjson", "nosuch"], + ["lsjson", "nosuch/"], + ["lsjson", "empty/nosuch.txt"], + ["cat", "nosuch.txt"], + ] + ) { + const { code, stdout, stderr } = await run(folder, args); + assert.equal(code, 3, args.join(" ")); + assert.equal(stdout, ""); + assert.match(stderr, /^Error: .*not found/); + } + }); + + await t.test("gives the exit code 0 for a path that is found", async () => { + const { code, stdout } = await run(folder, ["lsjson", "a.txt"]); + assert.equal(code, 0); + assert.equal(JSON.parse(stdout)[0].Name, "a.txt"); + }); + + await t.test("gives the exit code 1 and the help for an unknown command", async () => { + const { code, stdout, stderr } = await run(folder, ["nosuch"]); + assert.equal(code, 1); + assert.match(stderr, /unknown command "nosuch"/); + assert.match(stdout, /Available Commands:/); + }); + + await t.test("reads the value of a flag after a space", async () => { + await mkdir(join(folder, "deep", "deeper"), { recursive: true }); + await writeFile(join(folder, "deep", "deeper", "b.txt"), "b"); + const { code, stdout } = await run(folder, [ + "lsf", + "--max-depth", + "2", + "-R", + "--separator", + "|", + "--format", + "ps", + "deep", + ]); + assert.equal(code, 0); + assert.equal(stdout, "deeper/|-1\ndeeper/b.txt|1\n"); + }); + + await t.test("gives the exit code 1 for an unknown flag, as rclone", async () => { + const { code, stdout, stderr } = await run(folder, [ + "lsjson", + "--nosuch", + ".", + ]); + assert.equal(code, 1); + assert.equal(stdout, ""); + assert.equal(stderr, "Error: unknown flag: --nosuch\n"); + }); + + await t.test("lists one level with lsd, and . as a folder, as rclone", async () => { + await mkdir(join(folder, "deep", "deeper"), { recursive: true }); + const lsd = await run(join(folder, "deep"), ["lsd", "."]); + assert.equal(lsd.code, 0); + assert.match(lsd.stdout, /^ {10}-1 \S+ \S+ {8}-1 deeper\n$/); + assert.equal(lsd.stdout.split("\n").length, 2); + const lsjson = await run(join(folder, "deep"), ["lsjson", "."]); + assert.equal(lsjson.code, 0); + assert.deepEqual(JSON.parse(lsjson.stdout).map(({ Path }) => Path), [ + "deeper", + ]); + }); + + await t.test("stops with no error when the reader closes the pipe", async () => { + await writeFile( + join(folder, "big.bin"), + new Uint8Array(16 * 1024 * 1024), + ); + const child = spawn(process.execPath, [MAIN, "cat", "big.bin"], { + cwd: folder, + }); + let stderr = ""; + child.stderr.on("data", (chunk) => stderr += chunk); + await once(child.stdout, "data"); + child.stdout.destroy(); + const [code] = await once(child, "close"); + assert.equal(code, 0); + assert.equal(stderr, ""); + }); + + await t.test("serve reads --addr, and a TRACE does not stop it", async () => { + const child = spawn( + process.execPath, + [MAIN, "serve", "http", ".", "--addr", "127.0.0.1:0"], + { cwd: folder }, + ); + try { + const [line] = await once(child.stdout, "data"); + const [url] = `${line}`.match(/http:\/\/127\.0\.0\.1:\d+/) ?? []; + assert.ok(url, `${line}`); + const status = await new Promise((resolve, reject) => { + request(`${url}/`, { method: "TRACE" }, (response) => { + response.resume(); + resolve(response.statusCode); + }).on("error", reject).end(); + }); + assert.equal(status, 405); + const response = await fetch(new URL("a.txt", `${url}/`)); + assert.equal(await response.text(), "a"); + assert.equal(child.exitCode, null); + } finally { + child.kill(); + } + }); + } finally { + await rm(folder, { recursive: true, force: true }); + } +}); diff --git a/mod.js b/mod.js index 22b88df..bbdaf64 100644 --- a/mod.js +++ b/mod.js @@ -1,10 +1,8 @@ -import { env } from "node:process"; -import * as backends from "./backend/main.js"; -import * as commands from "./cmd/main.js"; - +// The import of fetch.js also sets the global `fetch`. +export { fetch } from "./fetch.js"; export * from "./cmd/main.js"; -export { Progress } from "./lib/streams/progress.js"; +export { Progress } from "@rfetch/lib/progress"; /** * @typedef {Object} Options @@ -34,210 +32,3 @@ export { Progress } from "./lib/streams/progress.js"; * @property {string} Tier * @property {Object} Metadata */ - -const REMOTE_REGEX = - /^(:)?(?:([\w.][\w.\s-]*(?:,[\w=,"'*#.:@%-_\/]+)?):)?([^:]*)$/; - -const globalFetch = globalThis.fetch; - -/** - * Extended global `fetch` with support for remote URL. - * - * The syntax of the paths passed to the rfetch command are as follows. - * - * `/path/to/dir`` - * - * This refers to the local file system. - * - * On Windows `\` may be used instead of `/` in local paths only, non local - * paths must use `/`. - * - * These paths needn't start with a leading `/` - if they don't then they - * will be relative to the current directory. - * - * `remote:path/to/dir` - * - * This refers to a directory `path/to/dir` on `remote:` as defined in the - * config. - * - * `remote:/path/to/dir` - * - * On most backends this refers to the same directory as `remote:path/to/dir` - * and that format should be preferred. On a very small number of remotes - * (FTP, SFTP, ..) this will refer to a different directory. On these, paths - * without a leading `/` will refer to your "home" directory and paths with a - * leading `/` will refer to the root. - * - * `:backend:path/to/dir` - * - * This is an advanced form for creating remotes on the fly. `backend` should - * be the name or prefix of a backend (the `type` in the config file) and all - * the configuration for the backend should be provided on the command line - * (or in environment variables). - * - * ## Precedence - * - * The various different methods of backend configuration are read in this - * order and the first one with a value is used. - * - * - Parameters in connection strings, e.g. `myRemote,skip_links:` - * - Flag values as supplied on the command line, e.g. `--skip-links` - * - Remote specific environment vars, e.g. `RCLONE_CONFIG_MYREMOTE_SKIP_LINKS` - * - Backend-specific environment vars, e.g. `RCLONE_LOCAL_SKIP_LINKS` - * - Backend generic environment vars, e.g. `RCLONE_SKIP_LINKS` - * - Config file, e.g. `skip_links = true`. - * - Default values, e.g. `false` - these can't be changed. - * - * So if both `--skip-links` is supplied on the command line and an environment - * variable `RCLONE_LOCAL_SKIP_LINKS` is set, the command line flag will take - * preference. - * - * For non backend configuration the order is as follows: - * - * - Flag values as supplied on the command line, e.g. `--stats 5s`. - * - Environment vars, e.g. `RCLONE_STATS=5s`. - * - Default values, e.g. `1m` - these can't be changed. - * - * ## Other environment variables - * - * @param {string | Request | URL} input - * @param { RequestInit} [init={}] - * @returns {Promise} - */ -export async function fetch(input, init = {}) { - if ( - typeof input !== "string" || input.startsWith("https://") || - input.startsWith("http://") || input.startsWith("file://") - ) { - return globalFetch(input, init); - } - - const [, colon, remote = "local", path] = input.match(REMOTE_REGEX) || []; - if (!remote && !path) { - return globalFetch(input, init); - } - - // The final params. - const params = new URLSearchParams(); - - /** - * Sets backend generic environment vars first. - */ - for (const [key, value] of Object.entries(env)) { - if (key.startsWith("RCLONE_")) { - const shortKey = key.slice(7).toLowerCase(); - params.set(shortKey, value); - } - } - - // Remote can be connection string with arguments separated by commas. - let [name, ...args] = remote.split(","); - - /** - * @type {Remote} - */ - let config; - - if (colon || remote === "local") { // the location has format `:type:path/to/file` - config = { type: name }; - name = ""; - } else { - const response = await commands.config("show", name, undefined, { - headers: { "Accept": "application/json" }, - }); - if (!response.ok) { - throw new Error(`Remote ${name} not found in config.`); - } - config = await response.json(); - } - - const type = config.type; - delete config.type; - - // Stores config into params if not already set, with lowest precedence. - Object.entries(config).forEach(([key, value]) => { - if (!params.has(key)) { - params.set(key, value); - } - }); - - // Overrides with backend-specific environment vars. - let envPrefix = `RCLONE_${type.toUpperCase()}_`; - for (const [key, value] of Object.entries(env)) { - if (key.startsWith(envPrefix)) { - const shortKey = key.slice(envPrefix.length).toLowerCase(); - params.set(shortKey, value); - - // Deletes any params that are already set by environment vars. - params.delete(`${type}_${shortKey}`); - } - } - - // Overrides with remote specific environment vars - envPrefix = `RCLONE_CONFIG_${name.toUpperCase()}_`; - for (const [key, value] of Object.entries(env)) { - if (key.startsWith(envPrefix)) { - const shortKey = key.slice(envPrefix.length).toLowerCase(); - params.set(shortKey, value); - - // Deletes any params that are already set by environment vars. - params.delete(`config_${name}_${shortKey}`); - } - } - - // `searchParams` may contain other flags not for the current backend. - const { pathname, searchParams } = new URL(path, "file:"); - - // Overrides with flags in request's search params. - searchParams.forEach((value, key) => { - if (key.startsWith(`${type}_`)) { - params.set(key.replace(`${type}_`, ""), value); - } - }); - - // Overrides with parameters in connection string. - args.forEach((arg) => { - const [key, value = "true"] = arg.split("="); - // `value` can be encoded through URLSearchParams. - params.set(key, decodeURIComponent(value)); - }); - - const headers = new Headers(init.headers); - - if (init.method === "TRACE") { - let body = `TRACE ${pathname}?${params} HTTP/1.1\r`; - // Reflects the request as response body. - for (const [name, value] of headers) { - body += `${name}: ${value}\r`; - } - return new Response(body, { - status: 200, - headers: { - "Content-Type": "message/http", - "Via": `${type}/1.1 ${name}`.trim(), - }, - }); - } - - if (init.body instanceof ReadableStream) { - // @ts-ignore: Must have `duplex` for streaming body - init.duplex = "half"; // Must set this for stream body. - } - - /** - * @type {import("./backend/main.js").Backend} - */ - const backend = backends[type]; - - const url = new URL(`${pathname}?${params}`, import.meta.url || `file:`); - // Creates a new request with the initial init. - const request = new Request(url, init); - // Clones that request and updates the headers. - return backend.fetch( - new Request(request, { - headers, - }), - ); -} - -globalThis.fetch = fetch; diff --git a/package-lock.json b/package-lock.json index 63eb940..5995018 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,21 +8,20 @@ "name": "rfetch", "version": "0.1.0", "license": "ISC", + "workspaces": [ + "lib", + "backend/*" + ], "dependencies": { - "@noble/ciphers": "^0.5.3", - "@noble/hashes": "^1.4.0", - "@sntran/html-rewriter": "^0.2.0", - "@sntran/serve": "^0.0.1", - "@std/crypto": "npm:@jsr/std__crypto@^1.0.0-rc.1", - "@std/encoding": "npm:@jsr/std__encoding@^1.0.0-rc.2", - "@std/fmt": "npm:@jsr/std__fmt@^0.225.4", - "@std/http": "npm:@jsr/std__http@^0.224.5", - "@std/ini": "npm:@jsr/std__ini@^0.225.1", - "@std/media-types": "npm:@jsr/std__media-types@^1.0.0-rc.1", - "@std/path": "npm:@jsr/std__path@^1.0.0-rc.2", - "aes-js": "^3.1.2", - "base32-decode": "^1.0.0", - "base32-encode": "^2.0.0" + "@rfetch/alias": "^0.1.0", + "@rfetch/chunker": "^0.1.0", + "@rfetch/crypt": "^0.1.0", + "@rfetch/drive": "^0.1.0", + "@rfetch/http": "^0.1.0", + "@rfetch/lib": "^0.1.0", + "@rfetch/local": "^0.1.0", + "@rfetch/memory": "^0.1.0", + "@rfetch/s3": "^0.1.0" }, "bin": { "rfetch": "main.js" @@ -31,155 +30,169 @@ "fast-check": "^3.19.0" }, "engines": { - "deno": ">=1.44.4", + "bun": ">=1.4.0", "node": ">=22.2.0" } }, - "node_modules/@jsr/std__async": { - "version": "1.0.0-rc.2", - "resolved": "https://npm.jsr.io/~/11/@jsr/std__async/1.0.0-rc.2.tgz", - "integrity": "sha512-mW3WDco4WkcsVhuwWRRRMSOW2odyXb1O80TMnfFjJZbxmlVXZKZmR9RkJvSQv8BcxopdZ3JCigkHNEmtRFsTXQ==" - }, - "node_modules/@jsr/std__bytes": { - "version": "1.0.0-rc.3", - "resolved": "https://npm.jsr.io/~/11/@jsr/std__bytes/1.0.0-rc.3.tgz", - "integrity": "sha512-4cHQrAO+8e6YcJVNHiPvkX9LVPhy4ugiatYUZnZ5kO1JG2dZia6wu9dYEsL6pInUQuBx9tFcPIowc0YmeNP+CQ==" - }, - "node_modules/@jsr/std__cli": { - "version": "0.224.7", - "resolved": "https://npm.jsr.io/~/11/@jsr/std__cli/0.224.7.tgz", - "integrity": "sha512-SSE7AMT/nqTUKeQXB+PZcmD6L5IjsOFDQdm88Ik2o6P4gS+gXewLMPZJu/X/gaOH5ux2C4xZ26Dgp3vCAIuB4g==" - }, - "node_modules/@jsr/std__encoding": { - "version": "1.0.0-rc.2", - "resolved": "https://npm.jsr.io/~/11/@jsr/std__encoding/1.0.0-rc.2.tgz", - "integrity": "sha512-ISvxD+exWVg/aESN3G1Tr0oszHRgUEEK1JjiRcuyCtd1uqSGXXMAoswYlYvPwA6mxZ7FQbFSZEbP3tN2J+fIUQ==" - }, - "node_modules/@jsr/std__fmt": { - "version": "0.225.4", - "resolved": "https://npm.jsr.io/~/11/@jsr/std__fmt/0.225.4.tgz", - "integrity": "sha512-lb8QcV3YnAu3GsrG7wd51Fb56RxOBp4szMRS/TCLnQptmF0Rd0v8ksQeTHmGkuD6MY/V8gEy/Sj0qulRvR9lqQ==" - }, - "node_modules/@jsr/std__io": { - "version": "0.224.2", - "resolved": "https://npm.jsr.io/~/11/@jsr/std__io/0.224.2.tgz", - "integrity": "sha512-7IbtJWWM13TaXkZR3jL0LLkAbEaublLR6WSMZuzg58IACDGnzcmxKlI57KjEBTuawX4u6x7g8f6zRleuRpSXVA==", + "backend/alias": { + "name": "@rfetch/alias", + "version": "0.1.0", + "license": "ISC", "dependencies": { - "@jsr/std__bytes": "^1.0.0-rc.3" + "@rfetch/lib": "^0.1.0" + }, + "engines": { + "node": ">=22.2.0" } }, - "node_modules/@jsr/std__media-types": { - "version": "1.0.0-rc.1", - "resolved": "https://npm.jsr.io/~/11/@jsr/std__media-types/1.0.0-rc.1.tgz", - "integrity": "sha512-0UdhqMLempJi8rHXJ4xSsRs79gMIm+X8/jAcBxUa9sdbsRALM148t/c2JIRWN4S58LQVBRHNK3WYd/YgR4fLNA==" - }, - "node_modules/@jsr/std__net": { - "version": "0.224.3", - "resolved": "https://npm.jsr.io/~/11/@jsr/std__net/0.224.3.tgz", - "integrity": "sha512-67B6bpkMjNSWtEPLy/k4OpuCb65w+L0mf9Amp1E2uUs4nt31vZZazW8IUykE9+1sv0tDVxa8nFZ2p9VwdiLXaA==" - }, - "node_modules/@jsr/std__path": { - "version": "1.0.0-rc.2", - "resolved": "https://npm.jsr.io/~/11/@jsr/std__path/1.0.0-rc.2.tgz", - "integrity": "sha512-qUHfVMqAtMGPFmdkOfmkCONqgpe4UFVAaD+wL6ZdlQxtvQiNxHQWsje+djri+0095s/kkoBOPH4CpYxBq+zh3A==" - }, - "node_modules/@jsr/std__streams": { - "version": "0.224.5", - "resolved": "https://npm.jsr.io/~/11/@jsr/std__streams/0.224.5.tgz", - "integrity": "sha512-LWs0LfBcyj4VzE3ff1cwoEyJ3AkoIT3CQubxeA0YQr3KEC/c6xB9Rq9nVit8gZ8RgtbMDlyfPtN1MkwTgPFj+g==", + "backend/chunker": { + "name": "@rfetch/chunker", + "version": "0.1.0", + "license": "ISC", "dependencies": { - "@jsr/std__bytes": "^1.0.0-rc.3", - "@jsr/std__io": "^0.224.1" + "@rfetch/lib": "^0.1.0" + }, + "engines": { + "node": ">=22.2.0" } }, - "node_modules/@noble/ciphers": { - "version": "0.5.3", - "resolved": "https://registry.npmjs.org/@noble/ciphers/-/ciphers-0.5.3.tgz", - "integrity": "sha512-B0+6IIHiqEs3BPMT0hcRmHvEj2QHOLu+uwt+tqDDeVd0oyVzh7BPrDcPjRnV1PV/5LaknXJJQvOuRGR0zQJz+w==", - "funding": { - "url": "https://paulmillr.com/funding/" + "backend/crypt": { + "name": "@rfetch/crypt", + "version": "0.1.0", + "license": "ISC", + "dependencies": { + "@rfetch/lib": "^0.1.0", + "base32-decode": "^1.0.0", + "base32-encode": "^2.0.0" + }, + "devDependencies": { + "@rfetch/memory": "^0.1.0", + "fast-check": "^3.19.0" + }, + "engines": { + "node": ">=22.2.0" } }, - "node_modules/@noble/hashes": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.4.0.tgz", - "integrity": "sha512-V1JJ1WTRUqHHrOSh597hURcMqVKVGL/ea3kv0gSnEdsEZ0/+VyPghM1lMNGc00z7CIQorSvbKpuJkxvuHbvdbg==", + "backend/drive": { + "name": "@rfetch/drive", + "version": "0.1.0", + "license": "ISC", + "dependencies": { + "@rfetch/lib": "^0.1.0" + }, "engines": { - "node": ">= 16" + "node": ">=22.2.0" + } + }, + "backend/http": { + "name": "@rfetch/http", + "version": "0.1.0", + "license": "ISC", + "dependencies": { + "@rfetch/lib": "^0.1.0" }, - "funding": { - "url": "https://paulmillr.com/funding/" + "devDependencies": { + "@rfetch/chunker": "^0.1.0", + "@rfetch/crypt": "^0.1.0", + "@rfetch/memory": "^0.1.0", + "fast-check": "^3.19.0" + }, + "engines": { + "node": ">=22.2.0" + } + }, + "backend/local": { + "name": "@rfetch/local", + "version": "0.1.0", + "license": "ISC", + "dependencies": { + "@rfetch/lib": "^0.1.0" + }, + "engines": { + "node": ">=22.2.0" + } + }, + "backend/memory": { + "name": "@rfetch/memory", + "version": "0.1.0", + "license": "ISC", + "dependencies": { + "@rfetch/lib": "^0.1.0" + }, + "engines": { + "node": ">=22.2.0" } }, - "node_modules/@sntran/html-rewriter": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/@sntran/html-rewriter/-/html-rewriter-0.2.0.tgz", - "integrity": "sha512-jK3XgNC9snm630DhVcaiyzziUV+RWJEaB0gxl+eqLw740NRnTTp0/ywmCbYeFMGc3BDUdy8Tn8Txj1gmIi6fBg==", + "backend/s3": { + "name": "@rfetch/s3", + "version": "0.1.0", + "license": "ISC", "dependencies": { - "html-rewriter-wasm": "^0.4.1" + "@rfetch/lib": "^0.1.0" + }, + "devDependencies": { + "fast-check": "^3.19.0" + }, + "engines": { + "node": ">=22.2.0" } }, - "node_modules/@sntran/serve": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/@sntran/serve/-/serve-0.0.1.tgz", - "integrity": "sha512-yejSB/mTWeRg7fK++lLZcz5Av407HZ7VN4l84FxxpCoERpGRL1fP43wNeRC4xeup7xPtnHXsjinxz6xEByglNA==" - }, - "node_modules/@std/crypto": { - "name": "@jsr/std__crypto", - "version": "1.0.0-rc.1", - "resolved": "https://npm.jsr.io/~/11/@jsr/std__crypto/1.0.0-rc.1.tgz", - "integrity": "sha512-HdqxgBx468H2ePsUUEYh4dyABELe1qxt4r2qK4UAR4gBrLxPRMBmAkZD0E7n/kUYA7+LTtLZ7OOhjZoQ9PPgrQ==" - }, - "node_modules/@std/encoding": { - "name": "@jsr/std__encoding", - "version": "1.0.0-rc.2", - "resolved": "https://npm.jsr.io/~/11/@jsr/std__encoding/1.0.0-rc.2.tgz", - "integrity": "sha512-ISvxD+exWVg/aESN3G1Tr0oszHRgUEEK1JjiRcuyCtd1uqSGXXMAoswYlYvPwA6mxZ7FQbFSZEbP3tN2J+fIUQ==" - }, - "node_modules/@std/fmt": { - "name": "@jsr/std__fmt", - "version": "0.225.4", - "resolved": "https://npm.jsr.io/~/11/@jsr/std__fmt/0.225.4.tgz", - "integrity": "sha512-lb8QcV3YnAu3GsrG7wd51Fb56RxOBp4szMRS/TCLnQptmF0Rd0v8ksQeTHmGkuD6MY/V8gEy/Sj0qulRvR9lqQ==" - }, - "node_modules/@std/http": { - "name": "@jsr/std__http", - "version": "0.224.5", - "resolved": "https://npm.jsr.io/~/11/@jsr/std__http/0.224.5.tgz", - "integrity": "sha512-PomDgsxwBoCkqTCG7e1Sz4QCAgNFhEGf71GWNercppQlgKtzp6zxsDgcJ4+vrqHIq5W8L1eqJWiceNkUUkMZjQ==", + "lib": { + "name": "@rfetch/lib", + "version": "0.1.0", + "license": "ISC", "dependencies": { - "@jsr/std__async": "^1.0.0-rc.1", - "@jsr/std__cli": "^0.224.7", - "@jsr/std__encoding": "1.0.0-rc.2", - "@jsr/std__fmt": "^0.225.4", - "@jsr/std__media-types": "^1.0.0-rc.1", - "@jsr/std__net": "^0.224.3", - "@jsr/std__path": "1.0.0-rc.2", - "@jsr/std__streams": "^0.224.5" + "@sntran/lol-html": "^2.8.1", + "entities": "^8.1.0" + }, + "devDependencies": { + "fast-check": "^3.19.0" + }, + "engines": { + "node": ">=22.2.0" } }, - "node_modules/@std/ini": { - "name": "@jsr/std__ini", - "version": "0.225.1", - "resolved": "https://npm.jsr.io/~/11/@jsr/std__ini/0.225.1.tgz", - "integrity": "sha512-K23bS0WRbZ2riDg9jQMnaJW4uW/KaQpFJZRmv8cGDL+7JhmMkI+yC3wm8zGxctggPR1+pxIXH0WmK44CbGkORQ==" - }, - "node_modules/@std/media-types": { - "name": "@jsr/std__media-types", - "version": "1.0.0-rc.1", - "resolved": "https://npm.jsr.io/~/11/@jsr/std__media-types/1.0.0-rc.1.tgz", - "integrity": "sha512-0UdhqMLempJi8rHXJ4xSsRs79gMIm+X8/jAcBxUa9sdbsRALM148t/c2JIRWN4S58LQVBRHNK3WYd/YgR4fLNA==" - }, - "node_modules/@std/path": { - "name": "@jsr/std__path", - "version": "1.0.0-rc.2", - "resolved": "https://npm.jsr.io/~/11/@jsr/std__path/1.0.0-rc.2.tgz", - "integrity": "sha512-qUHfVMqAtMGPFmdkOfmkCONqgpe4UFVAaD+wL6ZdlQxtvQiNxHQWsje+djri+0095s/kkoBOPH4CpYxBq+zh3A==" - }, - "node_modules/aes-js": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/aes-js/-/aes-js-3.1.2.tgz", - "integrity": "sha512-e5pEa2kBnBOgR4Y/p20pskXI74UEz7de8ZGVo58asOtvSVG5YAbJeELPZxOmt+Bnz3rX753YKhfIn4X4l1PPRQ==" + "node_modules/@rfetch/alias": { + "resolved": "backend/alias", + "link": true + }, + "node_modules/@rfetch/chunker": { + "resolved": "backend/chunker", + "link": true + }, + "node_modules/@rfetch/crypt": { + "resolved": "backend/crypt", + "link": true + }, + "node_modules/@rfetch/drive": { + "resolved": "backend/drive", + "link": true + }, + "node_modules/@rfetch/http": { + "resolved": "backend/http", + "link": true + }, + "node_modules/@rfetch/lib": { + "resolved": "lib", + "link": true + }, + "node_modules/@rfetch/local": { + "resolved": "backend/local", + "link": true + }, + "node_modules/@rfetch/memory": { + "resolved": "backend/memory", + "link": true + }, + "node_modules/@rfetch/s3": { + "resolved": "backend/s3", + "link": true + }, + "node_modules/@sntran/lol-html": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/@sntran/lol-html/-/lol-html-2.8.1.tgz", + "integrity": "sha512-OQj7nsvUNQ89dX+iXjPmIN+Y8oMyALwE85K3QK6lsOFY9mN3NZBGIu6QzBcq2m63Rtsk101A6N53ZZVoL8lzsw==", + "license": "BSD-3-Clause" }, "node_modules/base32-decode": { "version": "1.0.0", @@ -197,6 +210,18 @@ "node": "^12.20.0 || ^14.13.1 || >=16.0.0" } }, + "node_modules/entities": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-8.1.0.tgz", + "integrity": "sha512-kxL7msIffSuh9aaFAMD7rxAIuTRMAHMeBtgHW2yUdWw732ZNh4MehkF2gdjvtdmikkaIP9bFDDJOPlsvm7avrA==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=20.19.0" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/fast-check": { "version": "3.19.0", "resolved": "https://registry.npmjs.org/fast-check/-/fast-check-3.19.0.tgz", @@ -219,11 +244,6 @@ "node": ">=8.0.0" } }, - "node_modules/html-rewriter-wasm": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/html-rewriter-wasm/-/html-rewriter-wasm-0.4.1.tgz", - "integrity": "sha512-lNovG8CMCCmcVB1Q7xggMSf7tqPCijZXaH4gL6iE8BFghdQCbaY5Met9i1x2Ex8m/cZHDUtXK9H6/znKamRP8Q==" - }, "node_modules/pure-rand": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-6.1.0.tgz", diff --git a/package.json b/package.json index 8709463..73bd17b 100644 --- a/package.json +++ b/package.json @@ -3,14 +3,18 @@ "version": "0.1.0", "description": "Fetching between remotes, using Web APIs.", "engines": { - "node": ">=22.2.0", - "deno": ">=1.44.4" + "bun": ">=1.4.0", + "node": ">=22.2.0" }, "bin": { "rfetch": "main.js" }, "main": "mod.js", "type": "module", + "workspaces": [ + "lib", + "backend/*" + ], "repository": { "type": "git", "url": "git+https://github.com/sntran/rfetch.git" @@ -18,6 +22,7 @@ "scripts": { "start": "node main.js", "test": "node --test", + "bench": "node backend/chunker/main_bench.js && node cmd/lsjson/main_bench.js", "worker": "npx wrangler pages dev ." }, "author": "Trần Nguyễn Sơn", @@ -27,20 +32,15 @@ }, "homepage": "https://github.com/sntran/rfetch#readme", "dependencies": { - "@noble/ciphers": "^0.5.3", - "@noble/hashes": "^1.4.0", - "@sntran/html-rewriter": "^0.2.0", - "@sntran/serve": "^0.0.1", - "@std/crypto": "npm:@jsr/std__crypto@^1.0.0-rc.1", - "@std/encoding": "npm:@jsr/std__encoding@^1.0.0-rc.2", - "@std/fmt": "npm:@jsr/std__fmt@^0.225.4", - "@std/http": "npm:@jsr/std__http@^0.224.5", - "@std/ini": "npm:@jsr/std__ini@^0.225.1", - "@std/media-types": "npm:@jsr/std__media-types@^1.0.0-rc.1", - "@std/path": "npm:@jsr/std__path@^1.0.0-rc.2", - "aes-js": "^3.1.2", - "base32-decode": "^1.0.0", - "base32-encode": "^2.0.0" + "@rfetch/alias": "^0.1.0", + "@rfetch/chunker": "^0.1.0", + "@rfetch/crypt": "^0.1.0", + "@rfetch/drive": "^0.1.0", + "@rfetch/http": "^0.1.0", + "@rfetch/lib": "^0.1.0", + "@rfetch/local": "^0.1.0", + "@rfetch/memory": "^0.1.0", + "@rfetch/s3": "^0.1.0" }, "devDependencies": { "fast-check": "^3.19.0" diff --git a/worker_test.js b/worker_test.js new file mode 100644 index 0000000..2f028e9 --- /dev/null +++ b/worker_test.js @@ -0,0 +1,156 @@ +/** + * The tests of `_worker.js`: rfetch as a service, with no configuration in + * the Worker. + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import worker from "./_worker.js"; + +const PASSWORD = "4OSlj-ZYMGdEIGjcWg_gc-DL4O3TFAvr4ct_yV4"; + +/** The static files of the site: a stub that tells the path. */ +const env = { + ASSETS: { + /** @param {Request} request */ + fetch: (request) => + new Response(`asset ${new URL(request.url).pathname}`, { status: 200 }), + }, +}; + +/** + * Sends a request to the Worker. + * @param {string} path The path and the query, with percent-encoding. + * @param {RequestInit} [init] + */ +function send(path, init = {}) { + return worker.fetch( + new Request(`https://rfetch.test${path}`, { duplex: "half", ...init }), + env, + ); +} + +test("puts and gets a file of a remote on the fly", async () => { + const put = await send("/:memory:/worker/a%3F.txt", { + method: "PUT", + body: "text", + }); + assert.equal(put.status, 201); + assert.equal(await (await send("/:memory:/worker/a%3F.txt")).text(), "text"); + assert.equal((await send("/:memory:/worker/missing.txt")).status, 404); +}); + +test("reads a named remote of the headers", async () => { + const headers = { RFETCH_CONFIG_MEM_TYPE: "memory" }; + await send("/mem:worker/b.txt", { method: "PUT", body: "b", headers }); + const response = await send("/mem:worker/b.txt", { headers }); + assert.equal(await response.text(), "b"); + // With no headers, the remote does not exist. + assert.equal((await send("/mem:worker/b.txt")).status, 404); +}); + +test("the fields of the query are options of the remote", async () => { + await send("/:memory:/it's/c.txt", { method: "PUT", body: "c" }); + const list = await send( + `/:alias:/?remote=${encodeURIComponent(":memory:/it's")}`, + ); + assert.equal(list.status, 200); + const html = await list.text(); + // A link of the list works as it is. + const href = html.match(/href="(c\.txt[^"]*)"/)?.[1]?.replaceAll( + "&", + "&", + ); + assert.ok(href, html); + const file = await send(`/:alias:/${href}`); + assert.equal(await file.text(), "c"); +}); + +test("gives the list of lsjson for Accept: application/json", async () => { + await send("/:memory:/json/a.txt", { method: "PUT", body: "a" }); + await send("/:memory:/json/sub/b.txt", { method: "PUT", body: "bb" }); + const headers = { Accept: "application/json" }; + + const one = await send("/:memory:/json/", { headers }); + assert.equal(one.headers.get("Content-Type"), "application/json"); + assert.deepEqual( + (await one.json()).map(({ Path, IsDir }) => ({ Path, IsDir })), + [{ Path: "a.txt", IsDir: false }, { Path: "sub", IsDir: true }], + ); + + const all = await send("/:memory:/json/", { + headers: { ...headers, Depth: "infinity" }, + }); + assert.deepEqual( + (await all.json()).map(({ Path, Size }) => ({ Path, Size })).sort((a, b) => + a.Path < b.Path ? -1 : 1 + ), + [ + { Path: "a.txt", Size: 1 }, + { Path: "sub", Size: -1 }, + { Path: "sub/b.txt", Size: 2 }, + ], + ); + + const two = await send("/:memory:/json/", { + headers: { ...headers, Depth: "1" }, + }); + assert.equal((await two.json()).length, 2); + + for (const depth of ["0", "-1", "a", "99999"]) { + const bad = await send("/:memory:/json/", { + headers: { ...headers, Depth: depth }, + }); + assert.equal(bad.status, 400, depth); + } +}); + +test("lists crypt on a named remote, all from the headers", async () => { + const headers = { + RFETCH_CRYPT_REMOTE: "vault:crypt-worker", + RFETCH_CRYPT_PASSWORD: PASSWORD, + RFETCH_CONFIG_VAULT_TYPE: "memory", + }; + const put = await send("/:crypt:/x%20y.txt", { + method: "PUT", + body: "secret", + headers, + }); + assert.equal(put.status, 201); + const list = await send("/:crypt:/", { + headers: { ...headers, Accept: "application/json" }, + }); + const entries = await list.json(); + assert.deepEqual(entries.map(({ Name, Size }) => ({ Name, Size })), [ + { Name: "x y.txt", Size: 6 }, + ]); + const html = await (await send("/:crypt:/", { headers })).text(); + assert.doesNotMatch(html, new RegExp(PASSWORD)); + assert.doesNotMatch(html, /href="[^"]*\?/, "a link has options"); +}); + +test("gives a path with no remote to the static files", async () => { + assert.equal(await (await send("/")).text(), "asset /"); + assert.equal(await (await send("/README.md")).text(), "asset /README.md"); +}); + +test("gives 400 or 404 for a remote that is not correct", async () => { + assert.equal((await send("/:no-such-type:/a")).status, 404); + assert.equal((await send("/nobody:a")).status, 404); + assert.equal((await send("/:memory:/a?bad%20key=1")).status, 400); + assert.equal((await send("/:memory%:/a")).status, 400); +}); + +test("has no local backend: a client reads no file of the runtime", async () => { + for ( + const [path, headers] of [ + ["/:local:/etc/hostname", {}], + ["/:alias:hostname", { RFETCH_ALIAS_REMOTE: "/etc/" }], + ["/disk:etc/hostname", { RFETCH_CONFIG_DISK_TYPE: "local" }], + ] + ) { + const response = await send(path, { headers }); + assert.equal(response.status, 404, path); + assert.match(await response.text(), /Backend local not found/); + } +}); diff --git a/wrangler.toml b/wrangler.toml index 2d7af32..6f96998 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -1,3 +1,7 @@ -name = "denoclr" -compatibility_date = "2024-06-28" +name = "rfetch" +# This file is the configuration of the Pages project: the deploys from Git +# and `wrangler pages dev` read it. The repository root, with _worker.js, is +# the output of the build. +pages_build_output_dir = "." +compatibility_date = "2026-09-01" compatibility_flags = [ "nodejs_compat" ]