From 5acaa7af29b2210fbcf0e570047525ca9669e492 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C6=A1n=20Tr=E1=BA=A7n-Nguy=E1=BB=85n?= Date: Sat, 17 Oct 2026 01:02:38 +0000 Subject: [PATCH 01/87] refactor(crypt): EME with the steps and the names of the paper eme.js now follows the paper of Halevi and Rogaway and rfjakob/eme step by step, with the names of the paper (P, C, T, L, PPP, MP, MC, M, CCC), and a comment for each step. The doubling in GF(2^128) changes the block in place, with no branch on the data. The checks give a TypeError or a RangeError with a clear message, and the check of the sizes in xorBlocks, which used && in place of ||, goes away. The exports are encrypt and decrypt. The output is the same as before for 2000 random inputs, and the speed is the same. eme_test.js checks the two vectors of rfjakob/eme (16 and 2048 bytes) with the AES of aes-js and of node:crypto, the decrypt of each encrypt, the change of each block when one bit of the input or of the tweak changes, and the checks of the input. --- backend/crypt/PathCipher.js | 6 +- backend/crypt/eme.js | 298 +++++++++++++++--------------------- backend/crypt/eme_test.js | 202 ++++++++++++++++++++++++ dev_deps.js | 3 +- 4 files changed, 331 insertions(+), 178 deletions(-) create mode 100644 backend/crypt/eme_test.js diff --git a/backend/crypt/PathCipher.js b/backend/crypt/PathCipher.js index 6ca95af..282de4f 100644 --- a/backend/crypt/PathCipher.js +++ b/backend/crypt/PathCipher.js @@ -1,5 +1,5 @@ import { AES, decodeBase32, encodeBase32 } from "../../deps.js"; -import { Decrypt, Encrypt } from "./eme.js"; +import * as EME from "./eme.js"; import { pad, Padding, unpad } from "./padding.js"; const encoder = new TextEncoder(); @@ -41,7 +41,7 @@ export default function PathCipher({ nameKey, nameTweak } = {}) { if (name === "") return ""; const ciphertext = encoder.encode(name); const paddedCipherText = pad(ciphertext, Padding.PKCS7, 16); - const rawCipherText = Encrypt(nameCipher, nameTweak, paddedCipherText); + const rawCipherText = EME.encrypt(nameCipher, nameTweak, paddedCipherText); const encodedCipher = encodeBase32(rawCipherText, "RFC4648-HEX"); return encodedCipher.replace(/=+$/, "").toLowerCase(); @@ -68,7 +68,7 @@ export default function PathCipher({ nameKey, nameTweak } = {}) { const rawCipherText = new Uint8Array( decodeBase32(name.toUpperCase(), "RFC4648-HEX"), ); - const paddedPlaintext = Decrypt(nameCipher, nameTweak, rawCipherText); + const paddedPlaintext = EME.decrypt(nameCipher, nameTweak, rawCipherText); return decoder.decode(unpad(paddedPlaintext, Padding.PKCS7, 16)); } diff --git a/backend/crypt/eme.js b/backend/crypt/eme.js index aabfe50..c783acd 100644 --- a/backend/crypt/eme.js +++ b/backend/crypt/eme.js @@ -1,219 +1,169 @@ -// TypeScript port of https://github.com/rfjakob/eme/blob/master/eme.go by rfjakob - -// EME is a wide-block encryption mode developed by Halevi and Rogaway. +// EME (ECB-Mix-ECB) is the wide-block encryption mode of Halevi and Rogaway, +// "A Parallelizable Enciphering Mode" (2003): https://eprint.iacr.org/2003/147 +// +// EME makes one cipher for 1 to 128 blocks (16 to 2048 bytes) from a block +// cipher of 16 bytes, such as AES. Each bit of the output changes with each +// bit of the input and of the tweak. +// +// rclone encrypts each name of a file with EME and AES-256, through the +// library https://github.com/rfjakob/eme. This module does the same steps. +// The names are the names of the paper: P is the input, C is the output, T +// is the tweak, and E is the block cipher. Encryption and decryption have +// the same steps: only the direction of E changes. + +/** The size of one block, in bytes. */ +const BLOCK_SIZE = 16; + +/** The largest input, in blocks: 2048 bytes. */ +const MAX_BLOCKS = 128; /** - * @typedef {ArrayBuffer | Uint8Array | number[]} ByteSource + * A block cipher of 16 bytes with its key, for example `new AES(key)` of + * `aes-js`. Each function takes one block and gives a new block. It does not + * change its input. + * @typedef {Object} BlockCipher + * @property {(block: Uint8Array) => Uint8Array} encrypt + * @property {(block: Uint8Array) => Uint8Array} decrypt */ /** - * @enum { number } + * Encrypts the data with EME. + * @param {BlockCipher} cipher The block cipher, with the key. + * @param {Uint8Array} tweak 16 bytes. + * @param {Uint8Array} data 1 to 128 blocks of 16 bytes. + * @returns {Uint8Array} The encrypted data, a new array of the same size. */ -const Direction = { - Encrypt: 0, - Decrypt: 1, -}; +export function encrypt(cipher, tweak, data) { + return transform(cipher, tweak, data, (block) => cipher.encrypt(block)); +} /** - * @typedef {Object} BlockCipher - * @property {function(ByteSource): ByteSource} encrypt - * @property {function(ByteSource): ByteSource} decrypt + * Decrypts the data with EME. + * @param {BlockCipher} cipher The block cipher, with the key. + * @param {Uint8Array} tweak 16 bytes. + * @param {Uint8Array} data 1 to 128 blocks of 16 bytes. + * @returns {Uint8Array} The decrypted data, a new array of the same size. */ +export function decrypt(cipher, tweak, data) { + return transform(cipher, tweak, data, (block) => cipher.decrypt(block)); +} /** - * @param {Uint8Array} out - * @param {Uint8Array} input + * The steps of EME. + * @param {BlockCipher} cipher The block cipher. The table L uses its + * encryption in the two directions. + * @param {Uint8Array} T The tweak. + * @param {Uint8Array} P The input. + * @param {(block: Uint8Array) => Uint8Array} E The block cipher, in the + * direction of the transform. + * @returns {Uint8Array} C, the output. */ -function multByTwo(out, input) { - if (input.length !== 16) { - throw new Error("Invalid length"); +function transform(cipher, T, P, E) { + check(T, P); + const m = P.length / BLOCK_SIZE; + const L = table(cipher, m); + const C = new Uint8Array(P.length); + + // PPP_j = E(P_j xor L_j), and MP = T xor PPP_1 xor ... xor PPP_m. + const MP = T.slice(); + for (let j = 0; j < m; j++) { + const PPP = E(xor(block(P, j).slice(), L[j])); + C.set(PPP, j * BLOCK_SIZE); + xor(MP, PPP); } - let last = input[0]; - out[0] = (2 * input[0]) & 0xff; - if (input[15] >= 128) { - out[0] ^= 135; - } - for (let j = 1; j < 16; j++) { - const tmp = input[j]; - out[j] = (2 * input[j]) & 0xff; - if (last >= 128) { - out[j] = (out[j] + 1) & 0xff; - } - last = tmp; - } -} + // MC = E(MP), and the mask M = MP xor MC. + const MC = E(MP); + const M = xor(MP.slice(), MC); -/** - * @param {Uint8Array} out - * @param {Uint8Array} in1 - * @param {Uint8Array} in2 - */ -function xorBlocks(out, in1, in2) { - if (in1.length !== in2.length && in2.length !== out.length) { - throw new Error("Length must all match"); + // CCC_j = PPP_j xor 2^(j-1) M for j = 2 to m, in place in C. + // CCC_1 = MC xor T xor CCC_2 xor ... xor CCC_m. + const CCC1 = xor(MC.slice(), T); + for (let j = 1; j < m; j++) { + xor(CCC1, xor(block(C, j), double(M))); } + C.set(CCC1, 0); - for (let i = 0; i < in1.length; i++) { - out[i] = in1[i] ^ in2[i]; + // C_j = E(CCC_j) xor L_j. + for (let j = 0; j < m; j++) { + const Cj = block(C, j); + Cj.set(E(Cj)); + xor(Cj, L[j]); } + + return C; } /** - * @param {BlockCipher} bc + * The table of L for m blocks. The paper sets L = 2 E(0), and the block j + * (from 1) uses 2^(j-1) L. Thus L[0] is L, L[1] is 2 L, and so on. + * @param {BlockCipher} cipher * @param {number} m - * @returns + * @returns {Uint8Array[]} */ -function tabulateL(bc, m) { - /** - * @type {Uint8Array} - */ - const Li = bc.encrypt([ - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - ]); - const LTable = new Array(m); - for (let i = 0; i < m; i++) { - multByTwo(Li, Li); - LTable[i] = Li.slice(); - } - return LTable; +function table(cipher, m) { + const L = cipher.encrypt(new Uint8Array(BLOCK_SIZE)).slice(); + return Array.from({ length: m }, () => double(L).slice()); } /** - * @param {ByteSource} src - * @param {Direction} direction - * @param {BlockCipher} bc - * @returns {Uint8Array} + * Multiplies the block by 2 in GF(2^128), in place, as `multByTwo` of the + * draft EME-32: the bytes are little-endian, and a carry out of the last byte + * adds the polynomial 135 (x^7 + x^2 + x + 1) to the first byte. No branch + * depends on the data. + * @param {Uint8Array} block + * @returns {Uint8Array} The same block. */ -function aesTransform(src, direction, bc) { - if (direction === Direction.Encrypt) { - return bc.encrypt(src); - } else { - return bc.decrypt(src); +function double(block) { + const carry = block[BLOCK_SIZE - 1] >> 7; + for (let i = BLOCK_SIZE - 1; i > 0; i--) { + block[i] = (block[i] << 1) | (block[i - 1] >> 7); } + block[0] = (block[0] << 1) ^ (135 * carry); + return block; } /** - * @param {BlockCipher} bc - * @param {Uint8Array} tweak - * @param {Uint8Array} inputData - * @param {Direction} direction - * @returns {Uint8Array} + * Puts `a` xor `b` in `a`. + * @param {Uint8Array} a A block, which changes. + * @param {Uint8Array} b A block. + * @returns {Uint8Array} The block `a`. */ -function Transform(bc, tweak, inputData, direction) { - // In the paper, the tweak is just called "T". Call it the same here to - // make following the paper easy. - const T = tweak; // in bytes - - // In the paper, the plaintext data is called "P" and the ciphertext is - // called "C". Because encryption and decryption are virtually indentical, - // we share the code and always call the input data "P" and the output data - // "C", regardless of the direction. - const P = inputData; // in bytes - - if (T.length !== 16) { - throw new Error("Tweak must be 16 bytes long"); - } - if (P.length % 16 !== 0) { - throw new Error("Input Data must be a multiple of 16 long"); - } - - const m = P.length / 16; - if (m === 0 || m > 128) { - throw new Error( - "EME operates on 1 to 128 block-cipher blocks, you passed " + m, - ); - } - - // Result - const C = new Uint8Array(P.length); // in bytes - const LTable = tabulateL(bc, m); // in bytes - - const PPj = new Uint8Array(16); // in bytes - for (let j = 0; j < m; j++) { - const Pj = P.slice(j * 16, (j + 1) * 16); - /* PPj = 2**(j-1)*L xor Pj */ - xorBlocks(PPj, Pj, LTable[j]); - - /* PPPj = AESenc(K; PPj) */ - C.set(aesTransform(PPj, direction, bc), j * 16); - } - - /* MP =(xorSum PPPj) xor T */ - const MP = new Uint8Array(16); - xorBlocks(MP, C.subarray(0, 16), T); - for (let j = 1; j < m; j++) { - xorBlocks(MP, MP, C.subarray(j * 16, (j + 1) * 16)); - } - /* MC = AESenc(K; MP) */ - const MC = aesTransform(MP, direction, bc); - - /* M = MP xor MC */ - const M = new Uint8Array(16); - xorBlocks(M, MP, MC); - const CCCj = new Uint8Array(16); - for (let j = 1; j < m; j++) { - multByTwo(M, M); - /* CCCj = 2**(j-1)*M xor PPPj */ - xorBlocks(CCCj, C.subarray(j * 16, (j + 1) * 16), M); - C.set(CCCj, j * 16); - } - - /* CCC1 = (xorSum CCCj) xor T xor MC */ - const CCC1 = new Uint8Array(16); - xorBlocks(CCC1, MC, T); - for (let j = 1; j < m; j++) { - xorBlocks(CCC1, CCC1, C.subarray(j * 16, (j + 1) * 16)); - } - C.set(CCC1, 0); - - for (let j = 0; j < m; j++) { - /* CCj = AES-enc(K; CCCj) */ - C.set( - aesTransform(C.subarray(j * 16, (j + 1) * 16), direction, bc), - j * 16, - ); - - const tmp = C.subarray(j * 16, (j + 1) * 16); - /* Cj = 2**(j-1)*L xor CCj */ - xorBlocks(tmp, tmp, LTable[j]); - C.set(tmp, j * 16); +function xor(a, b) { + for (let i = 0; i < BLOCK_SIZE; i++) { + a[i] ^= b[i]; } - return C; + return a; } /** - * Encrypts the data using the EME mode. - * @param {BlockCipher} bc - * @param {Uint8Array} tweak + * Gives the block j (from 0) of the data, as a view: a change of the view + * changes the data. * @param {Uint8Array} data + * @param {number} j * @returns {Uint8Array} */ -export function Encrypt(bc, tweak, data) { - return Transform(bc, tweak, data, Direction.Encrypt); +function block(data, j) { + return data.subarray(j * BLOCK_SIZE, (j + 1) * BLOCK_SIZE); } + /** - * Decrypts the data using the EME mode. - * @param {BlockCipher} bc + * Makes sure that the tweak and the data have a size that EME accepts. * @param {Uint8Array} tweak * @param {Uint8Array} data - * @returns {Uint8Array} */ -export function Decrypt(bc, tweak, data) { - return Transform(bc, tweak, data, Direction.Decrypt); +function check(tweak, data) { + if (!(tweak instanceof Uint8Array) || tweak.length !== BLOCK_SIZE) { + throw new TypeError(`EME: the tweak must be ${BLOCK_SIZE} bytes.`); + } + if (!(data instanceof Uint8Array)) { + throw new TypeError("EME: the data must be a Uint8Array."); + } + const blocks = data.length / BLOCK_SIZE; + if (!Number.isInteger(blocks) || blocks < 1 || blocks > MAX_BLOCKS) { + throw new RangeError( + `EME: the data must be 1 to ${MAX_BLOCKS} blocks of ${BLOCK_SIZE} ` + + `bytes, not ${data.length} bytes.`, + ); + } } diff --git a/backend/crypt/eme_test.js b/backend/crypt/eme_test.js new file mode 100644 index 0000000..9fbe9be --- /dev/null +++ b/backend/crypt/eme_test.js @@ -0,0 +1,202 @@ +import { test } from "node:test"; +import { createCipheriv, createDecipheriv } from "node:crypto"; +import { AES } from "../../deps.js"; +import { + assertEquals, + assertNotEquals, + assertThrows, + fc, +} from "../../dev_deps.js"; + +import { decrypt, encrypt } from "./eme.js"; + +/** + * Makes the bytes of a text in hexadecimal. The text can have spaces and + * line breaks. + * @param {string} text + * @returns {Uint8Array} + */ +function hex(text) { + const digits = text.replace(/\s/g, ""); + return Uint8Array.from( + { length: digits.length / 2 }, + (_, i) => parseInt(digits.slice(i * 2, i * 2 + 2), 16), + ); +} + +/** + * AES with the key, as a block cipher of `node:crypto` (AES-ECB with no + * padding), to compare with the AES of `aes-js`. + * @param {Uint8Array} key 32 bytes. + * @returns {import("./eme.js").BlockCipher} + */ +function nodeAES(key) { + const ecb = (make) => (block) => { + const cipher = make("aes-256-ecb", key, null).setAutoPadding(false); + return new Uint8Array(cipher.update(block)); + }; + return { encrypt: ecb(createCipheriv), decrypt: ecb(createDecipheriv) }; +} + +/** + * Gives the blocks of 16 bytes of the data, as views. + * @param {Uint8Array} data + * @returns {Uint8Array[]} + */ +function blocks(data) { + return Array.from( + { length: data.length / 16 }, + (_, j) => data.subarray(j * 16, (j + 1) * 16), + ); +} + +const CIPHERS = { + "aes-js": (key) => new AES(key), + "node:crypto": nodeAES, +}; + +// The vectors of https://github.com/rfjakob/eme (eme_test.go), the library of +// rclone: the key, the tweak and the input are zeros. +const VECTORS = { + 16: hex("f1b9ce8ca15a4ba9fb476905434b9fd3"), + 2048: hex(` + 630500884158a8d41216aaa6351e92ea7ca540a949e73f1d6069afef372eae226d426d8f + 4dbbce505051fcd596e7b32cdd1e44a40bcce0887ebbc160f779dafbb68bc25a40040987 + cdb5aba19b956e11dd69d7ce74d1d8788903254217300f3afd9a92cea395a89fd842256d + 3919ac303c1d1a21ec44cb2898acabb3d0e04e05845e9183a64b4148f9614f62f79fa971 + 799c896c58cd2b285cb00e40720b5c118ef1c0dbf5e51f09cd10db672b9c22c53d4bf5da + bf16cfcbc382e3209e7ba27955b2afa055924d2155319205f964492e0e88172f4ffadca3 + 5cd3e694fd3d803f610edc1696113996749bdb2a2dfb8fbd92f0ef723146f93d969d1de4 + cc1c19efa33785f135a1203005ba2208fcd6b9f0d0e942993671b0c5baefe7ed0bbfe070 + a9e1d26eed37228ceda64b70c25a59fd040703acc385c6e4093a8e258984371c2d811c01 + 723b87c820a55470501b245d97a0c7564c44e1e31a89457c1125198d27ac2dec67f7237c + 781f9f622e420da471af577fa6ad8014bb1dad52dc450e37ebeeb1f1b7ba3c3d5796c664 + 6bd052fe66cf61d22f6e58efcbf0328d3b6e1088603e54bfd2da241c970fec58c7d21fc2 + b2f1b4b79552a92500a7e6848a22ec47ee362cef9787af3b26439470992c8d103997395a + efeff10802673d4630009e4e5ed26038d5f05c589e64046274d5ff133398e4bab08ca71c + 0ce93dbd0657785b150c9e418339e8e47d02ae7322b5b55364dfe10dd02d54dc5f2dbe82 + e55bcf0503b30d709301fc7f63cd15e7e1245189405b1364b6074c0e493f7beb27385986 + 4e95643c7d914279d95f220247618b7a5c896d604e9ce88e0a3ab434cfe97cabc520b20e + 2fffee958972e82e98b952cbf3567be560a8385e520b636829252cae16ac882d1d3dde44 + 7a90af52be7967f357a36382c9672a659ed0ffae9ee7b969c8c788f982fbd524efb6b852 + f65be5f371032691e9ef6e5a217607cf990e5351b251f841834045d9bd54633e8bb0d195 + 69377f805857a503a2642dfc06e2063dfd465dcae83b1e362394cdf20bd6a6d5dd0bacad + 7f2d407e597b004e90683bfc23b92796bb0cdf67752765dbba2d1332848adcb83e60b7f3 + f8c397a3aa5de08bf491676a18ee4a6ed9b342b33c081794365a72668f273a655e150fa3 + 6686aa5dfbc87ccda9cbe7982e531044cd9fcea68ca1a5034ee64e0558b7c6af56808c03 + 43506dccf39e4ca8634518e6420c107b7d5fe89943e9f40044b2ae7f047d97275ec4763b + 41416fb3faff897e6ee0047f97b215b02ee09a3f533cba2ee131a0aa68772860759325ea + 4dc7ab86528d0aff55f96d00a908ea32f9fa8bd0640d632bf0d812f43beb03e6c6d4a654 + b41f27143fde4aa7ed16ae4bba939fbed8ae28dace83fcb49fd3ca910512902fc6964199 + 68e260901796f3af75d059c6e93a8f53b445f921e60d9429007c78107309a86102b923db + b2f1fa04f84539c7c58ed45b40d4b267541d8f2f932ae75dfc13ef906089e1c847635aaa + da201e32301f326a733f4a99caa56fa00ccd6de2752a4495408bb4c930eaf84c97a28767 + a036f65627efa868adeee1cf47840e67ac4489381da1513dd3d1707a5ad91ac4ba1c3571 + 4b1cd77e05551612cdfb88187a5a39afd5b2a54f88aeb9605c1e01cd7a976a168c6e40ed + dcb20f09fe54a07a7c971eda08789b41e450f46a30c2b3494f30a06073631d151ae04879 + 3fe32ff08a08b8693d1da71451f7ff2438d7a521f8c97e4bc8ee744c1574d296c799d3f9 + 4e81522ce81def780660cee061d3110c1f18133dca1daabb2268aad76327f3cb6ae8c96f + c66b5506f5644f2652d022d7b616209970f78b8e5c9a74f5d1fc31037d01733bff097a66 + 1d4331adc087b84e32afb9d82a0614ea459022918f8ca60e74b7c6865d3e86e6ae4acf95 + 46985dc5ccf242dd9ab0486bde99a1e656499cbfceef8de5beafda89965bf208c70f4a0f + 8f677792962b09e0a355d53bbacb4a7acc16e50aa483994d85ca7e28a34ee52843d4f27a + cfb3b5347b89099fd01642b605f87f82283bb146aa319048462fe11a71d0fa064e825f69 + a502ae539240b13a2568383f80c30ebb9a29d2de4c45e6405bd8d1586fe5eef0477d3c21 + 434b3471c0f864dd4c2f135278645ef1b1ebb4d6954f92763d787ad3d0bb48409782cd80 + 3a877027ae105511cdaefe0110f9f6c65f2a34ad12532f4715c0341346dc06bcc4bb0e5c + 04ffdfd3f625e7e3811a760a2cfa9cbe1c2914882c192b793aeeb3ab8b03fd8be7e248fc + 9e5aa0c651d4bb5318c8a266b735ddc167d837499a3b38dcdfe188e25172eb974eb0eff2 + 0cc5c6e42e505fea7a3b6edfdf93b7062737ea00f72a415c4dfd3cd4fab2dad812d88d02 + 690f606b52dcfb0ab220dc65a7dee6f31e951324a1c442adfe9cf672646876bedc2c2cb7 + 1bac583f3cf4ea6769eb54d22bf22b0d908a297caa13767ed14c44b738799e821885b2b2 + 988a9b88a2f35bf8d307543af9007fadcfcc273f8029c2fe3d59449c5f193ca7f8986ca7 + 451d9a3ee14335b10868421b64ea1605d48d4fc22e378f82b7fe3e88aada30b4c545408b + 87b24511e371f25d1a94e7b78a6a7aec848e6f82614f96ef5ed8705e0adec0f889c1860c + f8211d716fddf319862290019a4387fb77c89a5ae7d0074fa72024f2e5096d39ebac8e8e + d02191879585394987fea3a07a720d6880910629cdcd83a02ffd98830d2f57ee6c53b5cf + edb42ec7c3d925cf3080c343f21711a90f117e9eaf91402d09b83e83bd18d1c4e3d165a8 + b9bac7adca12cb0147bfc4c6e2166c57b8182e63fbc698881ed5b329eb491eb98050a922 + c15804021013bf08942db9ee6d8f2a2c4eb93771340ed9e323d09e4256b7e5ac + `), +}; + +for (const [name, AESCipher] of Object.entries(CIPHERS)) { + test(`the vectors of rfjakob/eme, with AES of ${name}`, () => { + const cipher = AESCipher(new Uint8Array(32)); + const tweak = new Uint8Array(16); + for (const [size, expected] of Object.entries(VECTORS)) { + const zeros = new Uint8Array(Number(size)); + assertEquals(encrypt(cipher, tweak, zeros), expected); + assertEquals(decrypt(cipher, tweak, expected), zeros); + } + }); +} + +const KEY = fc.uint8Array({ minLength: 32, maxLength: 32 }); +const TWEAK = fc.uint8Array({ minLength: 16, maxLength: 16 }); +const DATA = fc.integer({ min: 1, max: 128 }).chain((blocks) => + fc.uint8Array({ minLength: blocks * 16, maxLength: blocks * 16 }) +); + +test("decrypt gives the data that encrypt took", () => { + fc.assert( + fc.property(KEY, TWEAK, DATA, (key, tweak, data) => { + const cipher = new AES(key); + const encrypted = encrypt(cipher, tweak, data); + assertEquals(encrypted.length, data.length); + assertEquals(decrypt(cipher, tweak, encrypted), data); + }), + ); +}); + +test("a change of one bit of the input or of the tweak changes each block", () => { + fc.assert( + fc.property(KEY, TWEAK, DATA, fc.nat(), (key, tweak, data, bit) => { + const cipher = new AES(key); + const before = blocks(encrypt(cipher, tweak, data)); + + const changed = data.slice(); + changed[(bit >> 3) % data.length] ^= 1 << (bit & 7); + blocks(encrypt(cipher, tweak, changed)).forEach((block, j) => + assertNotEquals(block, before[j]) + ); + + const otherTweak = tweak.slice(); + otherTweak[(bit >> 3) % 16] ^= 1 << (bit & 7); + blocks(encrypt(cipher, otherTweak, data)).forEach((block, j) => + assertNotEquals(block, before[j]) + ); + }), + ); +}); + +test("encrypt and decrypt do not change their input", () => { + const cipher = new AES(new Uint8Array(32)); + const tweak = new Uint8Array(16); + const data = new Uint8Array(64).fill(7); + decrypt(cipher, tweak, encrypt(cipher, tweak, data)); + assertEquals(tweak, new Uint8Array(16)); + assertEquals(data, new Uint8Array(64).fill(7)); +}); + +test("the tweak is 16 bytes, and the data are 1 to 128 blocks", () => { + const cipher = new AES(new Uint8Array(32)); + const tweak = new Uint8Array(16); + for (const f of [encrypt, decrypt]) { + assertThrows(() => f(cipher, new Uint8Array(15), new Uint8Array(16)), { + name: "TypeError", + message: "EME: the tweak must be 16 bytes.", + }); + assertThrows(() => f(cipher, tweak, [0, 0]), { + name: "TypeError", + message: "EME: the data must be a Uint8Array.", + }); + for (const size of [0, 15, 17, 2064]) { + assertThrows(() => f(cipher, tweak, new Uint8Array(size)), { + name: "RangeError", + message: + `EME: the data must be 1 to 128 blocks of 16 bytes, not ${size} bytes.`, + }); + } + } +}); diff --git a/dev_deps.js b/dev_deps.js index 0c396cc..e54c001 100644 --- a/dev_deps.js +++ b/dev_deps.js @@ -3,8 +3,9 @@ import { notDeepEqual as assertNotEquals, ok as assert, rejects as assertRejects, + throws as assertThrows, } from "node:assert/strict"; -export { assert, assertEquals, assertNotEquals, assertRejects }; +export { assert, assertEquals, assertNotEquals, assertRejects, assertThrows }; /** * Asserts a header From 763d82f3a085dd8d8094d129177bc304ab90dda5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C6=A1n=20Tr=E1=BA=A7n-Nguy=E1=BB=85n?= Date: Sat, 17 Oct 2026 01:02:44 +0000 Subject: [PATCH 02/87] fix(crypt): an empty name decrypts to an empty name decryptName gave 0 blocks to EME for an empty segment of a path, for example the end of a folder (x/), and EME threw. An empty segment now stays empty in the two directions, as in rclone. The test encode, with an empty name from fast-check, passes now. --- backend/crypt/PathCipher.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/crypt/PathCipher.js b/backend/crypt/PathCipher.js index 282de4f..0b88830 100644 --- a/backend/crypt/PathCipher.js +++ b/backend/crypt/PathCipher.js @@ -65,6 +65,8 @@ export default function PathCipher({ nameKey, nameTweak } = {}) { * @returns {string} */ function decryptName(name) { + // An empty segment of a path stays empty, as in rclone. + if (name === "") return ""; const rawCipherText = new Uint8Array( decodeBase32(name.toUpperCase(), "RFC4648-HEX"), ); From bb396508589ab38bf91b13196f66b918b81c80da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C6=A1n=20Tr=E1=BA=A7n-Nguy=E1=BB=85n?= Date: Sat, 17 Oct 2026 01:28:10 +0000 Subject: [PATCH 03/87] refactor(crypt): AES-256 of node:crypto in place of aes-js aes.js gives AES-256-ECB of node:crypto with no padding as the block cipher of EME, and the dependency aes-js goes away. It works in Node.js and in workerd with nodejs_compat at the compatibility date of wrangler.toml (a check with wrangler dev). workerd refuses null as the IV, so the IV is empty. A block cipher of EME now takes one or more blocks (ECB), so each ECB layer of EME is one call: 4 calls for each name. With node:crypto, 2048 bytes take 58 us (176 us before), and a short name about 9 us. EME copies with new Uint8Array in place of slice: the slice of a Buffer is a view, and node:crypto gives Buffers. A new test examines a Buffer as input. The output is the same as the first code for 2000 random inputs. --- backend/crypt/PathCipher.js | 7 ++- backend/crypt/aes.js | 21 +++++++++ backend/crypt/eme.js | 91 +++++++++++++++++++++---------------- backend/crypt/eme_test.js | 66 ++++++++++----------------- deps.js | 5 -- package-lock.json | 6 --- package.json | 1 - 7 files changed, 99 insertions(+), 98 deletions(-) create mode 100644 backend/crypt/aes.js diff --git a/backend/crypt/PathCipher.js b/backend/crypt/PathCipher.js index 0b88830..66e6fa6 100644 --- a/backend/crypt/PathCipher.js +++ b/backend/crypt/PathCipher.js @@ -1,4 +1,5 @@ -import { AES, decodeBase32, encodeBase32 } from "../../deps.js"; +import { decodeBase32, encodeBase32 } from "../../deps.js"; +import { aes256 } from "./aes.js"; import * as EME from "./eme.js"; import { pad, Padding, unpad } from "./padding.js"; @@ -28,9 +29,7 @@ export default function PathCipher({ nameKey, nameTweak } = {}) { if (nameKey === undefined || nameTweak === undefined) { throw new Error("nameKey and nameTweak must be specified"); } - // Name Cipher Fuctions - // TODO: Switch to https://github.com/invisal/god_crypto/blob/master/src/aes/aes_wc.ts - const nameCipher = new AES(nameKey); + const nameCipher = aes256(nameKey); /** * Encrypts a name diff --git a/backend/crypt/aes.js b/backend/crypt/aes.js new file mode 100644 index 0000000..8ea1070 --- /dev/null +++ b/backend/crypt/aes.js @@ -0,0 +1,21 @@ +import { createCipheriv, createDecipheriv } from "node:crypto"; + +/** + * AES-256 of `node:crypto` in ECB mode with no padding, the block cipher of + * EME. Each call takes one or more blocks of 16 bytes. One object does all + * the calls of one direction, because ECB keeps no state between blocks. + * @param {Uint8Array} key 32 bytes. + * @returns {import("./eme.js").BlockCipher} + */ +export function aes256(key) { + // ECB has no IV. workerd refuses null, so the IV is empty. + const iv = new Uint8Array(0); + const encryptor = createCipheriv("aes-256-ecb", key, iv); + const decryptor = createDecipheriv("aes-256-ecb", key, iv); + encryptor.setAutoPadding(false); + decryptor.setAutoPadding(false); + return { + encrypt: (block) => encryptor.update(block), + decrypt: (block) => decryptor.update(block), + }; +} diff --git a/backend/crypt/eme.js b/backend/crypt/eme.js index c783acd..e7908a8 100644 --- a/backend/crypt/eme.js +++ b/backend/crypt/eme.js @@ -18,12 +18,13 @@ const BLOCK_SIZE = 16; const MAX_BLOCKS = 128; /** - * A block cipher of 16 bytes with its key, for example `new AES(key)` of - * `aes-js`. Each function takes one block and gives a new block. It does not - * change its input. + * A block cipher of 16 bytes in ECB mode, with its key, for example + * AES-256-ECB of `node:crypto`. Each function takes one or more blocks, + * encrypts or decrypts each block alone, and gives the blocks in a new array. + * It does not change its input. * @typedef {Object} BlockCipher - * @property {(block: Uint8Array) => Uint8Array} encrypt - * @property {(block: Uint8Array) => Uint8Array} decrypt + * @property {(blocks: Uint8Array) => Uint8Array} encrypt + * @property {(blocks: Uint8Array) => Uint8Array} decrypt */ /** @@ -34,7 +35,7 @@ const MAX_BLOCKS = 128; * @returns {Uint8Array} The encrypted data, a new array of the same size. */ export function encrypt(cipher, tweak, data) { - return transform(cipher, tweak, data, (block) => cipher.encrypt(block)); + return transform(cipher, tweak, data, (blocks) => cipher.encrypt(blocks)); } /** @@ -45,16 +46,17 @@ export function encrypt(cipher, tweak, data) { * @returns {Uint8Array} The decrypted data, a new array of the same size. */ export function decrypt(cipher, tweak, data) { - return transform(cipher, tweak, data, (block) => cipher.decrypt(block)); + return transform(cipher, tweak, data, (blocks) => cipher.decrypt(blocks)); } /** - * The steps of EME. + * The steps of EME: an ECB layer, a mix, and a second ECB layer. Each ECB + * layer is one call of the block cipher for all the blocks. * @param {BlockCipher} cipher The block cipher. The table L uses its * encryption in the two directions. * @param {Uint8Array} T The tweak. * @param {Uint8Array} P The input. - * @param {(block: Uint8Array) => Uint8Array} E The block cipher, in the + * @param {(blocks: Uint8Array) => Uint8Array} E The block cipher, in the * direction of the transform. * @returns {Uint8Array} C, the output. */ @@ -62,48 +64,47 @@ function transform(cipher, T, P, E) { check(T, P); const m = P.length / BLOCK_SIZE; const L = table(cipher, m); - const C = new Uint8Array(P.length); - // PPP_j = E(P_j xor L_j), and MP = T xor PPP_1 xor ... xor PPP_m. - const MP = T.slice(); + // The first ECB layer: PPP_j = E(P_j xor L_j). + const PPP = E(xor(copy(P), L)); + + // The mix: MP = T xor PPP_1 xor ... xor PPP_m, MC = E(MP), and the mask + // M = MP xor MC. + const MP = copy(T); for (let j = 0; j < m; j++) { - const PPP = E(xor(block(P, j).slice(), L[j])); - C.set(PPP, j * BLOCK_SIZE); - xor(MP, PPP); + xor(MP, block(PPP, j)); } - - // MC = E(MP), and the mask M = MP xor MC. const MC = E(MP); - const M = xor(MP.slice(), MC); + const M = xor(copy(MP), MC); - // CCC_j = PPP_j xor 2^(j-1) M for j = 2 to m, in place in C. + // CCC_j = PPP_j xor 2^(j-1) M for j = 2 to m, and // CCC_1 = MC xor T xor CCC_2 xor ... xor CCC_m. - const CCC1 = xor(MC.slice(), T); + const CCC = copy(PPP); + const CCC1 = xor(copy(MC), T); for (let j = 1; j < m; j++) { - xor(CCC1, xor(block(C, j), double(M))); + xor(CCC1, xor(block(CCC, j), double(M))); } - C.set(CCC1, 0); + CCC.set(CCC1, 0); - // C_j = E(CCC_j) xor L_j. - for (let j = 0; j < m; j++) { - const Cj = block(C, j); - Cj.set(E(Cj)); - xor(Cj, L[j]); - } - - return C; + // The second ECB layer: C_j = E(CCC_j) xor L_j. + return xor(copy(E(CCC)), L); } /** - * The table of L for m blocks. The paper sets L = 2 E(0), and the block j - * (from 1) uses 2^(j-1) L. Thus L[0] is L, L[1] is 2 L, and so on. + * The table of L for m blocks, as m blocks in one array. The paper sets + * L = 2 E(0), and the block j (from 1) uses 2^(j-1) L. Thus the first block + * of the table is L, the second is 2 L, and so on. * @param {BlockCipher} cipher * @param {number} m - * @returns {Uint8Array[]} + * @returns {Uint8Array} */ function table(cipher, m) { - const L = cipher.encrypt(new Uint8Array(BLOCK_SIZE)).slice(); - return Array.from({ length: m }, () => double(L).slice()); + const L = new Uint8Array(m * BLOCK_SIZE); + const Lj = copy(cipher.encrypt(new Uint8Array(BLOCK_SIZE))); + for (let j = 0; j < m; j++) { + L.set(double(Lj), j * BLOCK_SIZE); + } + return L; } /** @@ -124,18 +125,28 @@ function double(block) { } /** - * Puts `a` xor `b` in `a`. - * @param {Uint8Array} a A block, which changes. - * @param {Uint8Array} b A block. - * @returns {Uint8Array} The block `a`. + * Puts `a` xor `b` in `a`. The two have the same size. + * @param {Uint8Array} a Blocks, which change. + * @param {Uint8Array} b Blocks. + * @returns {Uint8Array} The array `a`. */ function xor(a, b) { - for (let i = 0; i < BLOCK_SIZE; i++) { + for (let i = 0; i < a.length; i++) { a[i] ^= b[i]; } return a; } +/** + * Gives a copy of the block. `slice` does not do this for each array: the + * `slice` of a `Buffer` of Node.js is a view. + * @param {Uint8Array} block + * @returns {Uint8Array} + */ +function copy(block) { + return new Uint8Array(block); +} + /** * Gives the block j (from 0) of the data, as a view: a change of the view * changes the data. diff --git a/backend/crypt/eme_test.js b/backend/crypt/eme_test.js index 9fbe9be..b4aba11 100644 --- a/backend/crypt/eme_test.js +++ b/backend/crypt/eme_test.js @@ -1,6 +1,5 @@ import { test } from "node:test"; -import { createCipheriv, createDecipheriv } from "node:crypto"; -import { AES } from "../../deps.js"; +import { aes256 } from "./aes.js"; import { assertEquals, assertNotEquals, @@ -24,20 +23,6 @@ function hex(text) { ); } -/** - * AES with the key, as a block cipher of `node:crypto` (AES-ECB with no - * padding), to compare with the AES of `aes-js`. - * @param {Uint8Array} key 32 bytes. - * @returns {import("./eme.js").BlockCipher} - */ -function nodeAES(key) { - const ecb = (make) => (block) => { - const cipher = make("aes-256-ecb", key, null).setAutoPadding(false); - return new Uint8Array(cipher.update(block)); - }; - return { encrypt: ecb(createCipheriv), decrypt: ecb(createDecipheriv) }; -} - /** * Gives the blocks of 16 bytes of the data, as views. * @param {Uint8Array} data @@ -50,11 +35,6 @@ function blocks(data) { ); } -const CIPHERS = { - "aes-js": (key) => new AES(key), - "node:crypto": nodeAES, -}; - // The vectors of https://github.com/rfjakob/eme (eme_test.go), the library of // rclone: the key, the tweak and the input are zeros. const VECTORS = { @@ -120,17 +100,15 @@ const VECTORS = { `), }; -for (const [name, AESCipher] of Object.entries(CIPHERS)) { - test(`the vectors of rfjakob/eme, with AES of ${name}`, () => { - const cipher = AESCipher(new Uint8Array(32)); - const tweak = new Uint8Array(16); - for (const [size, expected] of Object.entries(VECTORS)) { - const zeros = new Uint8Array(Number(size)); - assertEquals(encrypt(cipher, tweak, zeros), expected); - assertEquals(decrypt(cipher, tweak, expected), zeros); - } - }); -} +test("the vectors of rfjakob/eme", () => { + const cipher = aes256(new Uint8Array(32)); + const tweak = new Uint8Array(16); + for (const [size, expected] of Object.entries(VECTORS)) { + const zeros = new Uint8Array(Number(size)); + assertEquals(encrypt(cipher, tweak, zeros), expected); + assertEquals(decrypt(cipher, tweak, expected), zeros); + } +}); const KEY = fc.uint8Array({ minLength: 32, maxLength: 32 }); const TWEAK = fc.uint8Array({ minLength: 16, maxLength: 16 }); @@ -141,7 +119,7 @@ const DATA = fc.integer({ min: 1, max: 128 }).chain((blocks) => test("decrypt gives the data that encrypt took", () => { fc.assert( fc.property(KEY, TWEAK, DATA, (key, tweak, data) => { - const cipher = new AES(key); + const cipher = aes256(key); const encrypted = encrypt(cipher, tweak, data); assertEquals(encrypted.length, data.length); assertEquals(decrypt(cipher, tweak, encrypted), data); @@ -152,7 +130,7 @@ test("decrypt gives the data that encrypt took", () => { test("a change of one bit of the input or of the tweak changes each block", () => { fc.assert( fc.property(KEY, TWEAK, DATA, fc.nat(), (key, tweak, data, bit) => { - const cipher = new AES(key); + const cipher = aes256(key); const before = blocks(encrypt(cipher, tweak, data)); const changed = data.slice(); @@ -170,17 +148,21 @@ test("a change of one bit of the input or of the tweak changes each block", () = ); }); -test("encrypt and decrypt do not change their input", () => { - const cipher = new AES(new Uint8Array(32)); - const tweak = new Uint8Array(16); - const data = new Uint8Array(64).fill(7); - decrypt(cipher, tweak, encrypt(cipher, tweak, data)); - assertEquals(tweak, new Uint8Array(16)); - assertEquals(data, new Uint8Array(64).fill(7)); +test("encrypt and decrypt do not change their input, also a Buffer", () => { + const cipher = aes256(new Uint8Array(32)); + // The slice of a Buffer is a view, not a copy. + for (const make of [(n) => new Uint8Array(n), (n) => Buffer.alloc(n)]) { + const tweak = make(16); + const data = make(64).fill(7); + const encrypted = encrypt(cipher, tweak, data); + assertEquals(decrypt(cipher, tweak, encrypted), new Uint8Array(64).fill(7)); + assertEquals(new Uint8Array(tweak), new Uint8Array(16)); + assertEquals(new Uint8Array(data), new Uint8Array(64).fill(7)); + } }); test("the tweak is 16 bytes, and the data are 1 to 128 blocks", () => { - const cipher = new AES(new Uint8Array(32)); + const cipher = aes256(new Uint8Array(32)); const tweak = new Uint8Array(16); for (const f of [encrypt, decrypt]) { assertThrows(() => f(cipher, new Uint8Array(15), new Uint8Array(16)), { diff --git a/deps.js b/deps.js index 79c4a83..b0914a8 100644 --- a/deps.js +++ b/deps.js @@ -13,11 +13,6 @@ import { crypto } from "@std/crypto"; export { default as decodeBase32 } from "base32-decode"; export { default as encodeBase32 } from "base32-encode"; -import aesjs from "aes-js"; -// `aes-js` is a CommonJS module, so we need to use `default` to import it. -const { AES } = aesjs; -export { AES }; - export { xsalsa20poly1305 } from "@noble/ciphers/salsa"; export { scryptAsync as scrypt } from "@noble/hashes/scrypt"; diff --git a/package-lock.json b/package-lock.json index 63eb940..3709f10 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,7 +20,6 @@ "@std/ini": "npm:@jsr/std__ini@^0.225.1", "@std/media-types": "npm:@jsr/std__media-types@^1.0.0-rc.1", "@std/path": "npm:@jsr/std__path@^1.0.0-rc.2", - "aes-js": "^3.1.2", "base32-decode": "^1.0.0", "base32-encode": "^2.0.0" }, @@ -176,11 +175,6 @@ "resolved": "https://npm.jsr.io/~/11/@jsr/std__path/1.0.0-rc.2.tgz", "integrity": "sha512-qUHfVMqAtMGPFmdkOfmkCONqgpe4UFVAaD+wL6ZdlQxtvQiNxHQWsje+djri+0095s/kkoBOPH4CpYxBq+zh3A==" }, - "node_modules/aes-js": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/aes-js/-/aes-js-3.1.2.tgz", - "integrity": "sha512-e5pEa2kBnBOgR4Y/p20pskXI74UEz7de8ZGVo58asOtvSVG5YAbJeELPZxOmt+Bnz3rX753YKhfIn4X4l1PPRQ==" - }, "node_modules/base32-decode": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/base32-decode/-/base32-decode-1.0.0.tgz", diff --git a/package.json b/package.json index 8709463..afddcd9 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,6 @@ "@std/ini": "npm:@jsr/std__ini@^0.225.1", "@std/media-types": "npm:@jsr/std__media-types@^1.0.0-rc.1", "@std/path": "npm:@jsr/std__path@^1.0.0-rc.2", - "aes-js": "^3.1.2", "base32-decode": "^1.0.0", "base32-encode": "^2.0.0" }, From 3d689cac2185b91cdc0580aa66a7bc131541e3e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C6=A1n=20Tr=E1=BA=A7n-Nguy=E1=BB=85n?= Date: Sat, 17 Oct 2026 01:31:02 +0000 Subject: [PATCH 04/87] build: Node.js only, with no Deno deno.json, the Deno settings of VS Code and the engine deno go away. main.js runs with node, and the backends have no shebang of deno serve: a backend is a module, not a program. The two benchmarks run with node and lib/bench.js, a small helper with the timer of Deno.bench, in place of Deno.bench and Deno.Command (npm run bench). The README and CONTRIBUTING.md tell the npm commands and node:test. The tests give the same result as before: 68 of 93 pass on Node.js 26. --- .vscode/settings.json | 4 -- CONTRIBUTING.md | 6 +- README.md | 14 +++-- backend/chunker/main.js | 2 - backend/chunker/main_bench.js | 101 +++++++++++++--------------------- backend/crypt/main.js | 2 - backend/drive/main.js | 2 - backend/http/main.js | 2 - backend/local/main.js | 2 - backend/memory/main.js | 2 - cmd/lsjson/main_bench.js | 62 ++++++--------------- deno.json | 6 -- lib/bench.js | 51 +++++++++++++++++ main.js | 2 +- package.json | 4 +- 15 files changed, 119 insertions(+), 143 deletions(-) delete mode 100644 .vscode/settings.json delete mode 100644 deno.json create mode 100644 lib/bench.js diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index aa1c94e..0000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "deno.enable": true, - "deno.unstable": true -} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 37b131e..87c3abc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -9,9 +9,9 @@ Choose a name. The docs here will use `remote` as an example. Note that in rfetch terminology a file system backend is called a remote or an fs. -- `deno init backend/remote` -- Edit "backend/remote/main.js" and "backend/remote/main_test.js" for the new - backend. +- Make the folder "backend/remote" with the files "main.js" and + "main_test.js" for the new backend. The tests use the test runner of + Node.js (`node:test`). - Implements a `fetch` default export function that handles "GET", "PUT" and "DELETE". - Uses `backend/local/main.js` as reference, or the boilerplate below: diff --git a/README.md b/README.md index 1a4e9b4..6cebbca 100644 --- a/README.md +++ b/README.md @@ -40,9 +40,11 @@ These backends adapt or modify other storage providers ## Development -- Clone the repository and navigate to the folder. -- `deno task start` to execute commands. -- Alternatively, `deno task install` to install a `rfetch` executable in PATH - that is symbolic linked to the `main.js` file and use `rfetch` in place of - `deno task start`. -- `deno task test` to run the test suites. +- Clone the repository and go to its folder. +- `npm install` to get the dependencies (Node.js 22.2 or later). +- `npm start -- COMMAND ...` to run a command. +- Alternatively, `npm link` to install a `rfetch` program in the PATH, with a + symbolic link to `main.js`, and use `rfetch` in place of `npm start --`. +- `npm test` to run the tests, with the test runner of Node.js. +- `npm run bench` to compare some commands with `rclone`, which must be in the + PATH. diff --git a/backend/chunker/main.js b/backend/chunker/main.js index 69fd307..4b90dd7 100644 --- a/backend/chunker/main.js +++ b/backend/chunker/main.js @@ -1,5 +1,3 @@ -#!/usr/bin/env -S deno serve --allow-all - /** * Chunker * diff --git a/backend/chunker/main_bench.js b/backend/chunker/main_bench.js index 7b5de63..f4ef1c2 100644 --- a/backend/chunker/main_bench.js +++ b/backend/chunker/main_bench.js @@ -1,4 +1,13 @@ +/** + * Benchmark of `PUT` of chunker against `rclone rcat` of a chunker remote. + * + * `rclone` must be in the PATH. Run with `node backend/chunker/main_bench.js`. + */ +import { spawn } from "node:child_process"; +import { once } from "node:events"; +import { Writable } from "node:stream"; import { mkBuffer } from "../../deps.js"; +import { bench } from "../../lib/bench.js"; import backend from "./main.js"; const buffer = mkBuffer(1024 * 1024 * 10); // 10M @@ -8,73 +17,39 @@ const file = new File([buffer], "10M.bin", { const chunkSize = 1024 * 1024 * 4; // 4M -//#region No chunking -Deno.bench( - "rclone", - { group: "backend/chunker: default chunk_size" }, - async () => { - const rclone = new Deno.Command("rclone", { - args: [ - "rcat", - `:chunker,remote=':memory:':${file.name}`, - ], - stdin: "piped", - }); - const rcat = rclone.spawn(); - file.stream().pipeTo(rcat.stdin); - await rcat.status; - }, -); - -Deno.bench("rfetch", { - group: "backend/chunker: default chunk_size", - baseline: true, -}, async () => { +/** + * Sends the file to `rclone rcat` of the remote. + * @param {string} remote + */ +async function rcat(remote) { + const rclone = spawn("rclone", ["rcat", `${remote}:${file.name}`], { + stdio: ["pipe", "ignore", "inherit"], + }); + await file.stream().pipeTo(Writable.toWeb(rclone.stdin)); + await once(rclone, "close"); +} + +/** + * Sends the file to chunker with `PUT`. + * @param {Record} params + */ +async function put(params) { const url = new URL(`/${file.name}`, "file:"); - url.searchParams.set("remote", ":memory:"); - + for (const [name, value] of Object.entries(params)) { + url.searchParams.set(name, value); + } const response = await backend.fetch( - new Request(url, { - method: "PUT", - body: file, - }), + new Request(url, { method: "PUT", body: file }), ); await response.arrayBuffer(); -}); -//#endregion No chunking +} -//#region Chunking -Deno.bench( - "rclone", - { group: "backend/chunker: small chunk_size" }, - async () => { - const rclone = new Deno.Command("rclone", { - args: [ - "rcat", - `:chunker,remote=':memory:',chunk_size=${chunkSize}:${file.name}`, - ], - stdin: "piped", - }); - const rcat = rclone.spawn(); - file.stream().pipeTo(rcat.stdin); - await rcat.status; - }, -); - -Deno.bench("rfetch", { - group: "backend/chunker: small chunk_size", - baseline: true, -}, async () => { - const url = new URL(`/${file.name}`, "file:"); - url.searchParams.set("remote", ":memory:"); - url.searchParams.set("chunk_size", `${chunkSize}`); +await bench("backend/chunker: default chunk_size", { + rfetch: () => put({ remote: ":memory:" }), + rclone: () => rcat(":chunker,remote=':memory:'"), +}); - const response = await backend.fetch( - new Request(url, { - method: "PUT", - body: file, - }), - ); - await response.arrayBuffer(); +await bench("backend/chunker: small chunk_size", { + rfetch: () => put({ remote: ":memory:", chunk_size: `${chunkSize}` }), + rclone: () => rcat(`:chunker,remote=':memory:',chunk_size=${chunkSize}`), }); -//#endregion Chunking diff --git a/backend/crypt/main.js b/backend/crypt/main.js index 94f48c1..3b43a22 100644 --- a/backend/crypt/main.js +++ b/backend/crypt/main.js @@ -1,5 +1,3 @@ -#!/usr/bin/env -S deno serve --allow-all - /** * Crypt remote * diff --git a/backend/drive/main.js b/backend/drive/main.js index 56c4615..1747f43 100644 --- a/backend/drive/main.js +++ b/backend/drive/main.js @@ -1,5 +1,3 @@ -#!/usr/bin/env -S deno serve --allow-all - import { formatBytes } from "../../deps.js"; import { auth } from "./auth.js"; diff --git a/backend/http/main.js b/backend/http/main.js index 44dda4c..96f1938 100644 --- a/backend/http/main.js +++ b/backend/http/main.js @@ -1,5 +1,3 @@ -#!/usr/bin/env -S deno serve --allow-all - /** * The HTTP remote is a read only remote for reading files of a webserver. The * webserver should provide file listings which rfetch will read and turn into diff --git a/backend/local/main.js b/backend/local/main.js index 06378d5..6ca9171 100644 --- a/backend/local/main.js +++ b/backend/local/main.js @@ -1,5 +1,3 @@ -#!/usr/bin/env -S deno serve --allow-all - import process from "node:process"; import { contentType, extname, formatBytes, join } from "../../deps.js"; diff --git a/backend/memory/main.js b/backend/memory/main.js index 518bdac..60c051c 100644 --- a/backend/memory/main.js +++ b/backend/memory/main.js @@ -1,5 +1,3 @@ -#!/usr/bin/env -S deno serve --allow-all - /** * Memory Backend * diff --git a/cmd/lsjson/main_bench.js b/cmd/lsjson/main_bench.js index 30f1a2c..1693341 100644 --- a/cmd/lsjson/main_bench.js +++ b/cmd/lsjson/main_bench.js @@ -1,61 +1,31 @@ /** - * Benchmark `lsjson` command vs `rclone lsjson` command. + * Benchmark of the `lsjson` command against `rclone lsjson`. * - * `rclone` must be installed and available in the PATH. + * `rclone` must be in the PATH. Run with `node cmd/lsjson/main_bench.js`. * - * This benchmark skips the time to spawn `rclone` process for more - * accurate comparison. + * This benchmark leaves out the time to start the `rclone` process, for a + * more accurate comparison. */ - +import { spawn } from "node:child_process"; +import { Readable } from "node:stream"; +import { bench } from "../../lib/bench.js"; import { lsjson } from "./main.js"; const remotes = ["./", "./backend/", "./cmd/"]; for (const remote of remotes) { - Deno.bench( - `rfetch lsjson ${remote} --recursive`, - { - baseline: true, - group: `timing ${remote} --recursive`, - permissions: { - env: true, - read: true, - }, - }, - async () => { + await bench(`timing ${remote} --recursive`, { + [`rfetch lsjson ${remote} --recursive`]: async () => { const response = await lsjson(remote, { recursive: true }); await response.arrayBuffer(); }, - ); - - Deno.bench( - `rclone lsjson ${remote} --recursive`, - { - group: `timing ${remote} --recursive`, - permissions: { - env: true, - read: true, - run: true, - }, - }, - async (b) => { - const command = new Deno.Command("rclone", { - args: [ - "lsjson", - remote, - "--recursive", - ], - stdout: "piped", + [`rclone lsjson ${remote} --recursive`]: async (timer) => { + const rclone = spawn("rclone", ["lsjson", remote, "--recursive"], { + stdio: ["ignore", "pipe", "inherit"], }); - - const process = command.spawn(); - - b.start(); - - const response = new Response(process.stdout); - await response.arrayBuffer(); - - b.end(); + timer.start(); + await new Response(Readable.toWeb(rclone.stdout)).arrayBuffer(); + timer.end(); }, - ); + }); } diff --git a/deno.json b/deno.json deleted file mode 100644 index 724bbe0..0000000 --- a/deno.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "tasks": { - "start": "deno run -A main.js", - "test": "deno test -A --fail-fast" - } -} diff --git a/lib/bench.js b/lib/bench.js new file mode 100644 index 0000000..500a080 --- /dev/null +++ b/lib/bench.js @@ -0,0 +1,51 @@ +/** + * A small benchmark for the scripts `*_bench.js`, in place of `Deno.bench`. + * Run a script with `node FILE`. + * + * Each case is a function that gets a timer. By default the time of a run is + * the time of the whole function. A case can call `timer.start()` and + * `timer.end()` to time only a part, for example to leave out the start of a + * process. + */ + +/** + * @typedef {Object} Timer + * @property {() => void} start + * @property {() => void} end + */ + +/** + * Runs each case of the group one time to warm up, then `runs` times, and + * prints the mean time of a run. + * @param {string} group + * @param {Record Promise>} cases + * @param {{ runs?: number }} [options] + */ +export async function bench(group, cases, { runs = 10 } = {}) { + console.log(group); + for (const [name, run] of Object.entries(cases)) { + await time(run); + let total = 0; + for (let i = 0; i < runs; i++) { + total += await time(run); + } + console.log(` ${name}: ${(total / runs).toFixed(1)} ms`); + } +} + +/** + * Gives the time of one run, in milliseconds. + * @param {(timer: Timer) => Promise} run + * @returns {Promise} + */ +async function time(run) { + let start, end; + const timer = { + start: () => (start = performance.now()), + end: () => (end = performance.now()), + }; + const before = performance.now(); + await run(timer); + const after = performance.now(); + return (end ?? after) - (start ?? before); +} diff --git a/main.js b/main.js index d26f471..0dd3257 100755 --- a/main.js +++ b/main.js @@ -1,4 +1,4 @@ -#!/usr/bin/env -S deno run --allow-all +#!/usr/bin/env node /** * rfetch - fetching between remotes diff --git a/package.json b/package.json index afddcd9..a399405 100644 --- a/package.json +++ b/package.json @@ -3,8 +3,7 @@ "version": "0.1.0", "description": "Fetching between remotes, using Web APIs.", "engines": { - "node": ">=22.2.0", - "deno": ">=1.44.4" + "node": ">=22.2.0" }, "bin": { "rfetch": "main.js" @@ -18,6 +17,7 @@ "scripts": { "start": "node main.js", "test": "node --test", + "bench": "node backend/chunker/main_bench.js && node cmd/lsjson/main_bench.js", "worker": "npx wrangler pages dev ." }, "author": "Trần Nguyễn Sơn", From c7246228fe5e1e88e6d101b5048bb01c05bc79ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C6=A1n=20Tr=E1=BA=A7n-Nguy=E1=BB=85n?= Date: Sat, 17 Oct 2026 01:38:02 +0000 Subject: [PATCH 05/87] fix(obscure): import the key at its first use The key of obscure came from a top-level await. With the circular imports of mod.js and the backends, esbuild puts each module in a lazy function, and a top-level await cannot be in such a function: wrangler pages dev 4.142 could not build _worker.js, also on the first commit. The key is now a promise that getKey makes at its first use. --- cmd/obscure/main.js | 98 +++++++++++++++++++++++++-------------------- deps.js | 93 ------------------------------------------ dev_deps.js | 42 ------------------- 3 files changed, 55 insertions(+), 178 deletions(-) delete mode 100644 deps.js delete mode 100644 dev_deps.js diff --git a/cmd/obscure/main.js b/cmd/obscure/main.js index 78faeb4..9d20f48 100644 --- a/cmd/obscure/main.js +++ b/cmd/obscure/main.js @@ -1,46 +1,58 @@ const ALGORITHM = "AES-CTR"; -const KEY = await crypto.subtle.importKey( - "raw", - // https://github.com/rclone/rclone/blob/10c884552caab3d3d8484ed3401e94f983a23f93/fs/config/obscure/obscure.go#L17-L22 - Uint8Array.from([ - 0x9c, - 0x93, - 0x5b, - 0x48, - 0x73, - 0x0a, - 0x55, - 0x4d, - 0x6b, - 0xfd, - 0x7c, - 0x63, - 0xc8, - 0x86, - 0xa9, - 0x2b, - 0xd3, - 0x90, - 0x19, - 0x8e, - 0xb8, - 0x12, - 0x8a, - 0xfb, - 0xf4, - 0xde, - 0x16, - 0x2b, - 0x8b, - 0x95, - 0xf6, - 0x38, - ]), - ALGORITHM, - false, - ["encrypt", "decrypt"], -); +/** @type {Promise | undefined} */ +let key; + +/** + * Gives the key of rclone for `obscure`. The import happens at the first use, + * not at the load of the module: a top-level await breaks the bundle of a + * Worker, and a Worker cannot use some APIs at the load. + * @returns {Promise} + */ +function getKey() { + key ??= crypto.subtle.importKey( + "raw", + // https://github.com/rclone/rclone/blob/10c884552caab3d3d8484ed3401e94f983a23f93/fs/config/obscure/obscure.go#L17-L22 + Uint8Array.from([ + 0x9c, + 0x93, + 0x5b, + 0x48, + 0x73, + 0x0a, + 0x55, + 0x4d, + 0x6b, + 0xfd, + 0x7c, + 0x63, + 0xc8, + 0x86, + 0xa9, + 0x2b, + 0xd3, + 0x90, + 0x19, + 0x8e, + 0xb8, + 0x12, + 0x8a, + 0xfb, + 0xf4, + 0xde, + 0x16, + 0x2b, + 0x8b, + 0x95, + 0xf6, + 0x38, + ]), + ALGORITHM, + false, + ["encrypt", "decrypt"], + ); + return key; +} const BlockSize = 16; @@ -64,7 +76,7 @@ export async function obscure(password) { counter: iv, length: 64, }, - KEY, + await getKey(), plaintext, ); // Copies the encrypted buffer into the ciphertext buffer. @@ -94,7 +106,7 @@ export async function reveal(cipherText) { counter: iv, length: 64, }, - KEY, + await getKey(), data, ); diff --git a/deps.js b/deps.js deleted file mode 100644 index b0914a8..0000000 --- a/deps.js +++ /dev/null @@ -1,93 +0,0 @@ -import { env } from "node:process"; - -export * as INI from "@std/ini"; -export { basename, extname, join, resolve } from "@std/path"; -export { getCookies } from "@std/http/cookie"; -export { contentType } from "@std/media-types"; -export { format as formatBytes } from "@std/fmt/bytes"; -export { format as formatDuration } from "@std/fmt/duration"; - -export { encodeBase64Url, encodeHex } from "@std/encoding"; -import { crypto } from "@std/crypto"; - -export { default as decodeBase32 } from "base32-decode"; -export { default as encodeBase32 } from "base32-encode"; - -export { xsalsa20poly1305 } from "@noble/ciphers/salsa"; -export { scryptAsync as scrypt } from "@noble/hashes/scrypt"; - -import { HTMLRewriter } from "@sntran/html-rewriter"; -globalThis.HTMLRewriter = HTMLRewriter; -export { HTMLRewriter }; - -/** - * Calculates the hash of a file. - * @param {string} path - * @param {import("@std/crypto").DigestAlgorithm} [algorithm=""] - * @returns {string} - */ -async function digest(path, algorithm = "MD5") { - const { readFile } = await import("node:fs/promises"); - const payload = await readFile(path); - const buffer = await crypto.subtle.digest(algorithm, payload); - const hashArray = Array.from(new Uint8Array(buffer)); - return hashArray.map((b) => b.toString(16).padStart(2, "0")).join(""); -} - -export { crypto, digest }; - -/** - * Retrieves the configuration directory for current platform. - * @returns {string | null} - */ -export function config_dir() { - let dir = env["XDG_CONFIG_HOME"]; - if (!dir) { - dir = env["APPDATA"]; // Windows - } - if (!dir) { - dir = env["HOME"]; - if (dir) { - dir += "/.config"; - } - } - - return dir; -} - -/** - * Converts a date string to a locale ISO string. - * @param {string | null} value - * @returns {string} - */ -export function toLocaleISOString(value) { - if (!value) return undefined; - - let date = new Date(value); - const off = date.getTimezoneOffset() * -1; - const del = date.getMilliseconds() ? "Z" : "."; // have milliseconds ? - date = new Date(date.getTime() + off * 60000); // add or subtract time zone - return date - .toISOString() - .split(del)[0] + - (off < 0 ? "-" : "+") + - ("0" + Math.abs(Math.floor(off / 60))).substring(-2) + - ":" + - ("0" + Math.abs(off % 60)).substring(-2); -} - -/** - * Makes a buffer filled with random values. - * @param {number} sizeInBytes - * @param {(Uint8Array) => Uint8Array} fill - * @returns {Uint8Array} - */ -export function mkBuffer(sizeInBytes, fill = crypto.getRandomValues) { - const buffer = new Uint8Array(sizeInBytes); - const chunkSize = 65536; // maximum chunk size allowed by crypto.getRandomValues() - for (let i = 0; i < buffer.length; i += chunkSize) { - const chunk = buffer.subarray(i, i + chunkSize); - fill(chunk); // fill the chunk with random values - } - return buffer; -} diff --git a/dev_deps.js b/dev_deps.js deleted file mode 100644 index e54c001..0000000 --- a/dev_deps.js +++ /dev/null @@ -1,42 +0,0 @@ -import { - deepEqual as assertEquals, - notDeepEqual as assertNotEquals, - ok as assert, - rejects as assertRejects, - throws as assertThrows, -} from "node:assert/strict"; -export { assert, assertEquals, assertNotEquals, assertRejects, assertThrows }; - -/** - * Asserts a header - * @param {Headers} headers - * @param {string} name - * @param {string} [value] - */ -export function assertHeader(headers, name, value) { - if (value !== undefined) { - assertEquals( - headers.get(name), - value, - `should have ${name} header of ${value} instead of ${headers.get(name)}`, - ); - } else { - assert(headers.get(name), `should have ${name} header`); - } -} - -/** - * Checks if two byte arrays are equal - * @param {Uint8Array} a - * @param {Uint8Array} b - * @returns {boolean} - */ -export function equalBytes(a, b) { - // Should not happen - if (a.length !== b.length) return false; - let isSame = true; - for (let i = 0; i < a.length; i++) isSame &&= a[i] === b[i]; // Lets hope JIT won't optimize away. - return isSame; -} - -export * as fc from "fast-check"; From 8b32b82a780e0b8a4bf678fd556c85a8526a4c22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C6=A1n=20Tr=E1=BA=A7n-Nguy=E1=BB=85n?= Date: Sat, 17 Oct 2026 01:38:02 +0000 Subject: [PATCH 06/87] build: each module imports its packages, with no deps.js deps.js and dev_deps.js re-exported the packages, as deps.ts in Deno. Now each module imports its packages, and package.json lists them. The modules that use HTMLRewriter import it: crypt used the global that deps.js set. Node.js has the same functions for four packages, so they go away: @std/path is node:path, @std/encoding is Buffer, the MD5 of @std/crypto is createHash of node:crypto (with no WebAssembly, which Workers block), and nobody used @std/http. The tests import node:assert/strict and fast-check. The helpers of deps.js go to their users: toLocaleISOString to lib/time.js, config_dir to cmd/config. The tests give the same result as before (68 of 93 pass). In wrangler pages dev, PUT, GET and the list of memory and of crypt over memory work, and rclone 1.75.1 decodes the name that crypt wrote. --- CONTRIBUTING.md | 2 +- backend/alias/main.js | 2 +- backend/alias/main_test.js | 4 +- backend/chunker/main.js | 19 +++++-- backend/chunker/main_bench.js | 4 +- backend/chunker/main_test.js | 59 +++++++++++--------- backend/crypt/PathCipher.js | 3 +- backend/crypt/eme_test.js | 35 ++++++------ backend/crypt/main.js | 5 +- backend/crypt/main_test.js | 39 +++++++++----- backend/crypt/secretbox.js | 2 +- backend/crypt/secretbox_test.js | 5 +- backend/drive/auth.js | 4 +- backend/drive/main.js | 2 +- backend/drive/main_test.js | 2 +- backend/http/main.js | 4 +- backend/http/main_test.js | 6 +-- backend/local/main.js | 4 +- backend/local/main_test.js | 63 +++++++++++++--------- backend/memory/main.js | 4 +- backend/memory/main_test.js | 34 ++++++++---- cmd/backend/main_test.js | 2 +- cmd/cat/main_test.js | 2 +- cmd/config/main.js | 22 +++++++- cmd/config/main_test.js | 43 +++++++-------- cmd/copy/main_test.js | 2 +- cmd/copyurl/main.js | 2 +- cmd/copyurl/main_test.js | 2 +- cmd/ls/main_test.js | 2 +- cmd/lsd/main_test.js | 2 +- cmd/lsf/main_test.js | 2 +- cmd/lsjson/main.js | 4 +- cmd/lsjson/main_test.js | 24 ++++----- cmd/lsl/main_test.js | 2 +- cmd/obscure/main_test.js | 5 +- cmd/rcat/main_test.js | 2 +- lib/streams/progress.js | 3 +- lib/time.js | 20 +++++++ main_test.js | 58 ++++++++++---------- package-lock.json | 96 --------------------------------- package.json | 4 -- 41 files changed, 309 insertions(+), 292 deletions(-) create mode 100644 lib/time.js diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 87c3abc..65d79d8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -17,7 +17,7 @@ fs. - Uses `backend/local/main.js` as reference, or the boilerplate below: ```ts -import {} from "../../deps.js"; +import { join } from "node:path"; function router(request: Request) { const { method, url } = request; diff --git a/backend/alias/main.js b/backend/alias/main.js index 5794266..6ead225 100644 --- a/backend/alias/main.js +++ b/backend/alias/main.js @@ -19,7 +19,7 @@ * allowed as a remote. To alias the current directory use `.` instead. */ -import { join } from "../../deps.js"; +import { join } from "node:path"; import { fetch } from "../../mod.js"; /** diff --git a/backend/alias/main_test.js b/backend/alias/main_test.js index b9d5d7c..f4ae74c 100644 --- a/backend/alias/main_test.js +++ b/backend/alias/main_test.js @@ -1,8 +1,8 @@ import { test } from "node:test"; import process from "node:process"; import { readdir } from "node:fs/promises"; -import { join } from "../../deps.js"; -import { assert } from "../../dev_deps.js"; +import { join } from "node:path"; +import assert from "node:assert/strict"; import backend from "./main.js"; diff --git a/backend/chunker/main.js b/backend/chunker/main.js index 4b90dd7..4494ecb 100644 --- a/backend/chunker/main.js +++ b/backend/chunker/main.js @@ -99,7 +99,7 @@ * to missing chunk errors (especially missing last chunk) than format with * metadata enabled. */ -import { crypto, encodeHex } from "../../deps.js"; +import { createHash } from "node:crypto"; import { Chunker } from "../../lib/streams/chunker.js"; import { fetch } from "../../mod.js"; @@ -246,7 +246,7 @@ async function chunker(request) { let fileSize = 0; const streams = metaFormat === "none" ? [request.body] : request.body.tee(); - const digestStream = streams[1] && crypto.subtle.digest("MD5", streams[1]); + const digestStream = streams[1] && md5(streams[1]); /** * @type {Uint8Array} @@ -303,7 +303,7 @@ async function chunker(request) { ver: METADATA_VERSION, size: fileSize, nchunks: chunkIndex - startFrom, - md5: encodeHex(await digestStream), + md5: await digestStream, }; // Adds metadata. await fetch(`${remote}/${fileName}`, { @@ -385,3 +385,16 @@ async function upload(url, init) { export default { fetch: chunker, }; + +/** + * Gives the MD5 of the stream, in hexadecimal. + * @param {ReadableStream} stream + * @returns {Promise} + */ +async function md5(stream) { + const hash = createHash("md5"); + for await (const chunk of stream) { + hash.update(chunk); + } + return hash.digest("hex"); +} diff --git a/backend/chunker/main_bench.js b/backend/chunker/main_bench.js index f4ef1c2..12947f1 100644 --- a/backend/chunker/main_bench.js +++ b/backend/chunker/main_bench.js @@ -4,13 +4,13 @@ * `rclone` must be in the PATH. Run with `node backend/chunker/main_bench.js`. */ import { spawn } from "node:child_process"; +import { randomBytes } from "node:crypto"; import { once } from "node:events"; import { Writable } from "node:stream"; -import { mkBuffer } from "../../deps.js"; import { bench } from "../../lib/bench.js"; import backend from "./main.js"; -const buffer = mkBuffer(1024 * 1024 * 10); // 10M +const buffer = randomBytes(1024 * 1024 * 10); // 10M const file = new File([buffer], "10M.bin", { type: "application/octet-stream", }); diff --git a/backend/chunker/main_test.js b/backend/chunker/main_test.js index f3e7ce2..0981406 100644 --- a/backend/chunker/main_test.js +++ b/backend/chunker/main_test.js @@ -1,14 +1,23 @@ import { test } from "node:test"; -import { crypto, encodeHex, mkBuffer } from "../../deps.js"; -import { assert, assertEquals, equalBytes } from "../../dev_deps.js"; +import { Buffer } from "node:buffer"; +import { createHash, randomBytes } from "node:crypto"; +import assert from "node:assert/strict"; import memory from "../memory/main.js"; import backend from "./main.js"; -const buffer = mkBuffer(1024 * 1024 * 10); // 10M +/** + * Tells if the two arrays have the same bytes. + * @param {Uint8Array} a + * @param {Uint8Array} b + * @returns {boolean} + */ +const sameBytes = (a, b) => Buffer.compare(a, b) === 0; + +const buffer = new Uint8Array(randomBytes(1024 * 1024 * 10)); // 10M const file = new File([buffer], "10M.bin", { type: "application/octet-stream", }); -const MD5 = encodeHex(await crypto.subtle.digest("MD5", buffer)); +const MD5 = createHash("md5").update(buffer).digest("hex"); let chunkSize = 1024 * 1024 * 4; // 4M @@ -48,16 +57,16 @@ test("PUT", async (t) => { const chunkName = `${ encodeURIComponent(file.name) }.rclone_chunk.00${startFrom}`; - assertEquals(link, `<${chunkName}>`, `should have ${chunkName}`); + assert.deepEqual(link, `<${chunkName}>`, `should have ${chunkName}`); request = new Request(new URL(`/${chunkName}`, "memory:/")); response = await memory.fetch(request); const chunk = await response.arrayBuffer(); if (startFrom < chunks) { - assertEquals(chunk.byteLength, chunkSize); + assert.deepEqual(chunk.byteLength, chunkSize); } else { - assertEquals(chunk.byteLength, file.size % chunkSize); + assert.deepEqual(chunk.byteLength, file.size % chunkSize); } } }); @@ -99,7 +108,7 @@ test("PUT", async (t) => { } else { chunkName = `${encodeURIComponent(file.name)}.part${startFrom}`; } - assertEquals( + assert.deepEqual( link, `<${chunkName}>`, `chunk ${startFrom} should have name of ${chunkName}, not ${link}`, @@ -110,9 +119,9 @@ test("PUT", async (t) => { const chunk = await response.arrayBuffer(); if (startFrom < chunks - 1) { - assertEquals(chunk.byteLength, chunkSize); + assert.deepEqual(chunk.byteLength, chunkSize); } else { - assertEquals(chunk.byteLength, file.size % chunkSize); + assert.deepEqual(chunk.byteLength, file.size % chunkSize); } } }); @@ -142,10 +151,10 @@ test("PUT", async (t) => { request = new Request(new URL(`/${file.name}`, "memory:/")); response = await memory.fetch(request); const metadata = await response.json(); - assertEquals(metadata.ver, 1); - assertEquals(metadata.size, file.size); - assertEquals(metadata.nchunks, Math.ceil(file.size / chunkSize)); - assertEquals(metadata.md5, MD5); + assert.deepEqual(metadata.ver, 1); + assert.deepEqual(metadata.size, file.size); + assert.deepEqual(metadata.nchunks, Math.ceil(file.size / chunkSize)); + assert.deepEqual(metadata.md5, MD5); }); await cleanup(); @@ -168,11 +177,11 @@ test("PUT", async (t) => { const links = response.headers.get("Link")?.split(/,\s*/); const chunks = Math.ceil(file.size / chunkSize); - assertEquals(links.length, chunks); + assert.deepEqual(links.length, chunks); request = new Request(new URL(`/${file.name}`, "memory:/")); response = await memory.fetch(request); - assertEquals(response.status, 404); + assert.deepEqual(response.status, 404); }); await cleanup(); @@ -196,8 +205,8 @@ test("PUT", async (t) => { response = await memory.fetch(request); const links = response.headers.get("Link")?.split(/,\s*/); - assertEquals(links.length, 1, "should only have 1 file"); - assertEquals( + assert.deepEqual(links.length, 1, "should only have 1 file"); + assert.deepEqual( links[0], `<${encodeURIComponent(file.name)}>`, "should have the original file name", @@ -207,7 +216,7 @@ test("PUT", async (t) => { response = await memory.fetch(request); const body = await response.arrayBuffer(); assert( - equalBytes(new Uint8Array(body), buffer), + sameBytes(new Uint8Array(body), buffer), "should be the original file", ); }); @@ -230,7 +239,7 @@ test("GET", async (t) => { request = new Request(url); response = await backend.fetch(request); const body = await response.arrayBuffer(); - assert(equalBytes(new Uint8Array(body), buffer)); + assert(sameBytes(new Uint8Array(body), buffer)); }); await cleanup(); @@ -249,7 +258,7 @@ test("GET", async (t) => { response = await backend.fetch(request); const body = await response.arrayBuffer(); assert( - equalBytes(new Uint8Array(body), buffer), + sameBytes(new Uint8Array(body), buffer), "should return original file", ); }); @@ -270,7 +279,7 @@ test("GET", async (t) => { response = await backend.fetch(request); const body = await response.arrayBuffer(); assert( - equalBytes(new Uint8Array(body), buffer), + sameBytes(new Uint8Array(body), buffer), "should return original file", ); }); @@ -291,7 +300,7 @@ test("GET", async (t) => { response = await fetch(request); const body = await response.arrayBuffer(); assert( - equalBytes(new Uint8Array(body), buffer), + sameBytes(new Uint8Array(body), buffer), "should return original file", ); }); @@ -313,7 +322,7 @@ test("GET", async (t) => { request = new Request(url); response = await backend.fetch(request); let body = await response.json(); - assertEquals(body, json, "should return original file"); + assert.deepEqual(body, json, "should return original file"); await cleanup(); @@ -328,7 +337,7 @@ test("GET", async (t) => { request = new Request(url); response = await backend.fetch(request); body = await response.json(); - assertEquals(body, json, "should return original file"); + assert.deepEqual(body, json, "should return original file"); }); }); diff --git a/backend/crypt/PathCipher.js b/backend/crypt/PathCipher.js index 66e6fa6..55f26ef 100644 --- a/backend/crypt/PathCipher.js +++ b/backend/crypt/PathCipher.js @@ -1,4 +1,5 @@ -import { decodeBase32, encodeBase32 } from "../../deps.js"; +import decodeBase32 from "base32-decode"; +import encodeBase32 from "base32-encode"; import { aes256 } from "./aes.js"; import * as EME from "./eme.js"; import { pad, Padding, unpad } from "./padding.js"; diff --git a/backend/crypt/eme_test.js b/backend/crypt/eme_test.js index b4aba11..0cd6546 100644 --- a/backend/crypt/eme_test.js +++ b/backend/crypt/eme_test.js @@ -1,11 +1,7 @@ import { test } from "node:test"; import { aes256 } from "./aes.js"; -import { - assertEquals, - assertNotEquals, - assertThrows, - fc, -} from "../../dev_deps.js"; +import assert from "node:assert/strict"; +import fc from "fast-check"; import { decrypt, encrypt } from "./eme.js"; @@ -105,8 +101,8 @@ test("the vectors of rfjakob/eme", () => { const tweak = new Uint8Array(16); for (const [size, expected] of Object.entries(VECTORS)) { const zeros = new Uint8Array(Number(size)); - assertEquals(encrypt(cipher, tweak, zeros), expected); - assertEquals(decrypt(cipher, tweak, expected), zeros); + assert.deepEqual(encrypt(cipher, tweak, zeros), expected); + assert.deepEqual(decrypt(cipher, tweak, expected), zeros); } }); @@ -121,8 +117,8 @@ test("decrypt gives the data that encrypt took", () => { fc.property(KEY, TWEAK, DATA, (key, tweak, data) => { const cipher = aes256(key); const encrypted = encrypt(cipher, tweak, data); - assertEquals(encrypted.length, data.length); - assertEquals(decrypt(cipher, tweak, encrypted), data); + assert.deepEqual(encrypted.length, data.length); + assert.deepEqual(decrypt(cipher, tweak, encrypted), data); }), ); }); @@ -136,13 +132,13 @@ test("a change of one bit of the input or of the tweak changes each block", () = const changed = data.slice(); changed[(bit >> 3) % data.length] ^= 1 << (bit & 7); blocks(encrypt(cipher, tweak, changed)).forEach((block, j) => - assertNotEquals(block, before[j]) + assert.notDeepEqual(block, before[j]) ); const otherTweak = tweak.slice(); otherTweak[(bit >> 3) % 16] ^= 1 << (bit & 7); blocks(encrypt(cipher, otherTweak, data)).forEach((block, j) => - assertNotEquals(block, before[j]) + assert.notDeepEqual(block, before[j]) ); }), ); @@ -155,9 +151,12 @@ test("encrypt and decrypt do not change their input, also a Buffer", () => { const tweak = make(16); const data = make(64).fill(7); const encrypted = encrypt(cipher, tweak, data); - assertEquals(decrypt(cipher, tweak, encrypted), new Uint8Array(64).fill(7)); - assertEquals(new Uint8Array(tweak), new Uint8Array(16)); - assertEquals(new Uint8Array(data), new Uint8Array(64).fill(7)); + assert.deepEqual( + decrypt(cipher, tweak, encrypted), + new Uint8Array(64).fill(7), + ); + assert.deepEqual(new Uint8Array(tweak), new Uint8Array(16)); + assert.deepEqual(new Uint8Array(data), new Uint8Array(64).fill(7)); } }); @@ -165,16 +164,16 @@ test("the tweak is 16 bytes, and the data are 1 to 128 blocks", () => { const cipher = aes256(new Uint8Array(32)); const tweak = new Uint8Array(16); for (const f of [encrypt, decrypt]) { - assertThrows(() => f(cipher, new Uint8Array(15), new Uint8Array(16)), { + assert.throws(() => f(cipher, new Uint8Array(15), new Uint8Array(16)), { name: "TypeError", message: "EME: the tweak must be 16 bytes.", }); - assertThrows(() => f(cipher, tweak, [0, 0]), { + assert.throws(() => f(cipher, tweak, [0, 0]), { name: "TypeError", message: "EME: the data must be a Uint8Array.", }); for (const size of [0, 15, 17, 2064]) { - assertThrows(() => f(cipher, tweak, new Uint8Array(size)), { + assert.throws(() => f(cipher, tweak, new Uint8Array(size)), { name: "RangeError", message: `EME: the data must be 1 to 128 blocks of 16 bytes, not ${size} bytes.`, diff --git a/backend/crypt/main.js b/backend/crypt/main.js index 3b43a22..e119bb2 100644 --- a/backend/crypt/main.js +++ b/backend/crypt/main.js @@ -93,7 +93,10 @@ * - 1049120 bytes total (a 0.05% overhead). This is the overhead for big files. */ -import { contentType, extname, scrypt } from "../../deps.js"; +import { extname } from "node:path"; +import { scryptAsync as scrypt } from "@noble/hashes/scrypt"; +import { HTMLRewriter } from "@sntran/html-rewriter"; +import { contentType } from "@std/media-types"; import { fetch } from "../../mod.js"; import { reveal } from "../../cmd/obscure/main.js"; import PathCipher from "./PathCipher.js"; diff --git a/backend/crypt/main_test.js b/backend/crypt/main_test.js index 7a84d68..e1b1d02 100644 --- a/backend/crypt/main_test.js +++ b/backend/crypt/main_test.js @@ -1,10 +1,21 @@ import { test } from "node:test"; import { readFile } from "node:fs/promises"; -import { contentType, extname, join } from "../../deps.js"; -import { assert, assertEquals, equalBytes, fc } from "../../dev_deps.js"; +import { Buffer } from "node:buffer"; +import { extname, join } from "node:path"; +import { contentType } from "@std/media-types"; +import assert from "node:assert/strict"; +import fc from "fast-check"; import { decode, default as crypt, encode } from "./main.js"; +/** + * Tells if the two arrays have the same bytes. + * @param {Uint8Array} a + * @param {Uint8Array} b + * @returns {boolean} + */ +const sameBytes = (a, b) => Buffer.compare(a, b) === 0; + const PASSWORD = "UmyLSdRHfew6aual28-ggx78qHqSfQ"; const SALT = "Cj3gLa5PVwc2aot0QpKiOZ3YEzs3Sw"; @@ -36,7 +47,7 @@ test("decode", async () => { for await (const originalFilename of Object.keys(CRYPT)) { const decoded = await getText(encode(options, originalFilename)); - assertEquals(decoded, CRYPT[originalFilename]); + assert.deepEqual(decoded, CRYPT[originalFilename]); } }); @@ -65,7 +76,7 @@ test("encode", async () => { const ciphertexts = await getText(encode(options, ...plaintexts)); // Decodes all ciphertexts into plaintexts separated by newlines. const decoded = await getText(decode(options, ...ciphertexts.split("\n"))); - assertEquals(decoded, plaintexts.join("\n")); + assert.deepEqual(decoded, plaintexts.join("\n")); }); test("GET", async (t) => { @@ -86,7 +97,7 @@ test("GET", async (t) => { response = await crypt.fetch(request); const decryptedFile = new Uint8Array(await response.arrayBuffer()); - assert(equalBytes(decryptedFile, originalFile)); + assert(sameBytes(decryptedFile, originalFile)); } }); @@ -104,10 +115,10 @@ test("GET", async (t) => { response = await crypt.fetch(request); const size = Number(response.headers.get("Content-Length")); - assertEquals(size, originalFile.byteLength); + assert.deepEqual(size, originalFile.byteLength); const type = response.headers.get("Content-Type"); - assertEquals(type, contentType(extname(originalFilename))); + assert.deepEqual(type, contentType(extname(originalFilename))); response.body?.cancel(); } @@ -132,10 +143,10 @@ test("PUT", async () => { // Asserts file does not exist in the original remote. response = await fetch(join(remote, originalFilename)); - assertEquals(response.status, 404); + assert.deepEqual(response.status, 404); response = await fetch(join(remote, encryptedFilename)); - assertEquals(response.status, 404); + assert.deepEqual(response.status, 404); // Uploads the file to the crypt remote. request = new Request(url, { @@ -146,21 +157,21 @@ test("PUT", async () => { // Asserts original file does not exist in the original remote. response = await fetch(join(remote, originalFilename)); - assertEquals(response.status, 404); + assert.deepEqual(response.status, 404); // Asserts encrypted file exists in the original remote. response = await fetch(join(remote, encryptedFilename)); - assertEquals(response.status, 200); + assert.deepEqual(response.status, 200); const encryptedBuffer = new Uint8Array(await response.arrayBuffer()); - assert(!equalBytes(encryptedBuffer, originalFile)); + assert(!sameBytes(encryptedBuffer, originalFile)); const magic = encryptedBuffer.subarray(0, 8); - assertEquals(magic, MAGIC, "should have magic header"); + assert.deepEqual(magic, MAGIC, "should have magic header"); // Gets the new file from the crypt remote. request = new Request(url); response = await crypt.fetch(request); const decryptedFile = new Uint8Array(await response.arrayBuffer()); - assert(equalBytes(decryptedFile, originalFile)); + assert(sameBytes(decryptedFile, originalFile)); } }); diff --git a/backend/crypt/secretbox.js b/backend/crypt/secretbox.js index 5fdde0d..e4acf67 100644 --- a/backend/crypt/secretbox.js +++ b/backend/crypt/secretbox.js @@ -1,7 +1,7 @@ /** * @module crypt/secretbox */ -import { xsalsa20poly1305 } from "../../deps.js"; +import { xsalsa20poly1305 } from "@noble/ciphers/salsa"; /** * Options for secretbox diff --git a/backend/crypt/secretbox_test.js b/backend/crypt/secretbox_test.js index 004b26a..1863ce0 100644 --- a/backend/crypt/secretbox_test.js +++ b/backend/crypt/secretbox_test.js @@ -1,5 +1,6 @@ import { test } from "node:test"; -import { assertEquals, fc } from "../../dev_deps.js"; +import assert from "node:assert/strict"; +import fc from "fast-check"; import { DecryptionStream, EncryptionStream } from "./secretbox.js"; @@ -17,7 +18,7 @@ test("decrypt/crypt", () => { const result = new Uint8Array( await new Response(decryptionStream).arrayBuffer(), ); - assertEquals(result, content); + assert.deepEqual(result, content); }, ), ); diff --git a/backend/drive/auth.js b/backend/drive/auth.js index eef0ae9..4d0c086 100644 --- a/backend/drive/auth.js +++ b/backend/drive/auth.js @@ -1,4 +1,4 @@ -import { encodeBase64Url } from "../../deps.js"; +import { Buffer } from "node:buffer"; import { reveal } from "../../cmd/obscure/main.js"; @@ -170,7 +170,7 @@ async function createJWT(serviceAccount, scopes) { key, encoder.encode(jwt), ); - const signatureEncoded = encodeBase64Url(new Uint8Array(signature)); + const signatureEncoded = Buffer.from(signature).toString("base64url"); return `${jwt}.${signatureEncoded}`; } diff --git a/backend/drive/main.js b/backend/drive/main.js index 1747f43..ff9b2f3 100644 --- a/backend/drive/main.js +++ b/backend/drive/main.js @@ -1,4 +1,4 @@ -import { formatBytes } from "../../deps.js"; +import { format as formatBytes } from "@std/fmt/bytes"; import { auth } from "./auth.js"; import { list } from "./list.js"; diff --git a/backend/drive/main_test.js b/backend/drive/main_test.js index 435db8a..9041d47 100644 --- a/backend/drive/main_test.js +++ b/backend/drive/main_test.js @@ -1,5 +1,5 @@ import { test } from "node:test"; -import { assert } from "../../dev_deps.js"; +import assert from "node:assert/strict"; import backend from "./main.js"; test("GET", () => { diff --git a/backend/http/main.js b/backend/http/main.js index 96f1938..b5bcb01 100644 --- a/backend/http/main.js +++ b/backend/http/main.js @@ -24,7 +24,9 @@ * To just download a single file it is easier to use [copyurl](../../cmd/copyurl/main.js). */ -import { formatBytes, HTMLRewriter, toLocaleISOString } from "../../deps.js"; +import { HTMLRewriter } from "@sntran/html-rewriter"; +import { format as formatBytes } from "@std/fmt/bytes"; +import { toLocaleISOString } from "../../lib/time.js"; export const options = { boolean: [ diff --git a/backend/http/main_test.js b/backend/http/main_test.js index 4974790..13f09cd 100644 --- a/backend/http/main_test.js +++ b/backend/http/main_test.js @@ -1,12 +1,12 @@ import { test } from "node:test"; -import { assertEquals } from "../../dev_deps.js"; +import assert from "node:assert/strict"; import backend from "./main.js"; test("GET", async (t) => { await t.test("error without url searchParam", async () => { const url = new URL("/", "file:"); const response = await backend.fetch(new Request(url)); - assertEquals(response.status, 400); + assert.deepEqual(response.status, 400); }); await t.test("a file", async () => { @@ -22,7 +22,7 @@ test("GET", async (t) => { // Then compares with result from the backend response = await backend.fetch(new Request(url)); const actual = await response.text(); - assertEquals( + assert.deepEqual( actual, expected, "should be the same as fetching the remote file directly", diff --git a/backend/local/main.js b/backend/local/main.js index 6ca9171..cab77f7 100644 --- a/backend/local/main.js +++ b/backend/local/main.js @@ -1,5 +1,7 @@ import process from "node:process"; -import { contentType, extname, formatBytes, join } from "../../deps.js"; +import { extname, join } from "node:path"; +import { format as formatBytes } from "@std/fmt/bytes"; +import { contentType } from "@std/media-types"; /** * Serves a local remote diff --git a/backend/local/main_test.js b/backend/local/main_test.js index 966f892..9f7adf6 100644 --- a/backend/local/main_test.js +++ b/backend/local/main_test.js @@ -1,13 +1,9 @@ import { test } from "node:test"; import process from "node:process"; import { readdir, readFile, stat } from "node:fs/promises"; -import { contentType, extname, join } from "../../deps.js"; -import { - assert, - assertEquals, - assertHeader, - assertRejects, -} from "../../dev_deps.js"; +import { extname, join } from "node:path"; +import { contentType } from "@std/media-types"; +import assert from "node:assert/strict"; import backend from "./main.js"; @@ -65,15 +61,21 @@ test("GET", async (t) => { const response = await fetch(request); const { headers, body } = response; - assertHeader(headers, "Content-Type", contentType(extname(name)) || ""); - assertHeader(headers, "Content-Length", `${file.size}`); - assertHeader(headers, "Last-Modified", file.mtime?.toUTCString() || ""); + assert.equal( + headers.get("Content-Type"), + contentType(extname(name)) || "", + ); + assert.equal(headers.get("Content-Length"), `${file.size}`); + assert.equal( + headers.get("Last-Modified"), + file.mtime?.toUTCString() || "", + ); assert( body instanceof ReadableStream, "should have body as a ReadableStream", ); - assertEquals( + assert.deepEqual( await response.text(), await readFile(path, "utf8"), "should have the same content", @@ -103,10 +105,10 @@ test("PUT", async (t) => { }); const { status, headers } = await fetch(request); - assertEquals(status, 201, "should respond with 201 Created"); - assertHeader(headers, "Content-Location", `/${newFile}`); + assert.deepEqual(status, 201, "should respond with 201 Created"); + assert.equal(headers.get("Content-Location"), `/${newFile}`); - assertEquals( + assert.deepEqual( await readFile(targetFile, "utf8"), body, "should have the same content", @@ -114,7 +116,11 @@ test("PUT", async (t) => { // Also checks the content with a GET request. const response = await fetch(new Request(url)); - assertEquals(await response.text(), body, "should have the same content"); + assert.deepEqual( + await response.text(), + body, + "should have the same content", + ); }); await t.test("a new folder", async () => { @@ -131,8 +137,8 @@ test("PUT", async (t) => { }); const { status, headers } = await fetch(request); - assertEquals(status, 201, "should respond with 201 Created"); - assertHeader(headers, "Content-Location", `/${newFolder}`); + assert.deepEqual(status, 201, "should respond with 201 Created"); + assert.equal(headers.get("Content-Location"), `/${newFolder}`); const stats = await stat(targetFolder); assert(stats.isDirectory, "should be a directory"); @@ -155,10 +161,13 @@ test("PUT", async (t) => { }); const { status, headers } = await fetch(request); - assertEquals(status, 201, "should respond with 201 Created"); - assertHeader(headers, "Content-Location", `/${join(newFolder, newFile)}`); + assert.deepEqual(status, 201, "should respond with 201 Created"); + assert.equal( + headers.get("Content-Location"), + `/${join(newFolder, newFile)}`, + ); - assertEquals( + assert.deepEqual( await readFile(targetFile, "utf8"), body, "should have the same content", @@ -166,7 +175,11 @@ test("PUT", async (t) => { // Also checks the content with a GET request. const response = await fetch(new Request(url)); - assertEquals(await response.text(), body, "should have the same content"); + assert.deepEqual( + await response.text(), + body, + "should have the same content", + ); }); }); @@ -179,9 +192,9 @@ test("DELETE", async (t) => { method: "DELETE", }); const { status } = await fetch(request); - assertEquals(status, 204, "should respond with 204 No Content"); + assert.deepEqual(status, 204, "should respond with 204 No Content"); - assertRejects(() => { + assert.rejects(() => { return stat(targetFile); }); }); @@ -194,9 +207,9 @@ test("DELETE", async (t) => { method: "DELETE", }); const { status } = await fetch(request); - assertEquals(status, 204, "should respond with 204 No Content"); + assert.deepEqual(status, 204, "should respond with 204 No Content"); - assertRejects(() => { + assert.rejects(() => { return stat(targetFolder); }); }); diff --git a/backend/memory/main.js b/backend/memory/main.js index 60c051c..a18b952 100644 --- a/backend/memory/main.js +++ b/backend/memory/main.js @@ -7,7 +7,9 @@ * Because it has no parameters you can just use it with the `:memory:` remote * name. */ -import { contentType, extname, formatBytes } from "../../deps.js"; +import { extname } from "node:path"; +import { format as formatBytes } from "@std/fmt/bytes"; +import { contentType } from "@std/media-types"; /** * @type {Map} diff --git a/backend/memory/main_test.js b/backend/memory/main_test.js index e357f07..9a46165 100644 --- a/backend/memory/main_test.js +++ b/backend/memory/main_test.js @@ -1,6 +1,6 @@ import { test } from "node:test"; -import { join } from "../../deps.js"; -import { assert, assertEquals, assertHeader } from "../../dev_deps.js"; +import { join } from "node:path"; +import assert from "node:assert/strict"; import backend from "./main.js"; @@ -64,8 +64,8 @@ test("PUT", async (t) => { }); const { status, headers } = await fetch(request); - assertEquals(status, 201, "should respond with 201 Created"); - assertHeader(headers, "Content-Location", fixture); + assert.deepEqual(status, 201, "should respond with 201 Created"); + assert.equal(headers.get("Content-Location"), fixture); } }); }); @@ -108,7 +108,7 @@ test("GET", async (t) => { const url = new URL(fixture, "memory:/"); const response = await fetch(new Request(url)); const responseText = await response.text(); - assertEquals(responseText, fixture, "should have the same content"); + assert.deepEqual(responseText, fixture, "should have the same content"); } }); }); @@ -123,10 +123,18 @@ test("DELETE", async (t) => { method: "DELETE", }), ); - assertEquals(response.status, 204, "should respond with 204 No Content"); + assert.deepEqual( + response.status, + 204, + "should respond with 204 No Content", + ); response = await fetch(new Request(url)); - assertEquals(response.status, 404, "should respond with 404 Not Found"); + assert.deepEqual( + response.status, + 404, + "should respond with 404 Not Found", + ); } }); @@ -138,12 +146,20 @@ test("DELETE", async (t) => { method: "DELETE", }), ); - assertEquals(response.status, 204, "should respond with 204 No Content"); + assert.deepEqual( + response.status, + 204, + "should respond with 204 No Content", + ); for await (const fixture of descendants(pathname)) { const url = new URL(fixture, "memory:/"); const response = await fetch(new Request(url)); - assertEquals(response.status, 404, "should respond with 404 Not Found"); + assert.deepEqual( + response.status, + 404, + "should respond with 404 Not Found", + ); } }); }); diff --git a/cmd/backend/main_test.js b/cmd/backend/main_test.js index 15ee5e6..f2aee2c 100644 --- a/cmd/backend/main_test.js +++ b/cmd/backend/main_test.js @@ -1,5 +1,5 @@ import { test } from "node:test"; -import { assert } from "../../dev_deps.js"; +import assert from "node:assert/strict"; import { backend } from "./main.js"; test("backend", () => { diff --git a/cmd/cat/main_test.js b/cmd/cat/main_test.js index 1d15a05..02994f8 100644 --- a/cmd/cat/main_test.js +++ b/cmd/cat/main_test.js @@ -1,5 +1,5 @@ import { test } from "node:test"; -import { assert } from "../../dev_deps.js"; +import assert from "node:assert/strict"; import { cat } from "./main.js"; test("cat", () => { diff --git a/cmd/config/main.js b/cmd/config/main.js index 015c261..9c9af68 100644 --- a/cmd/config/main.js +++ b/cmd/config/main.js @@ -1,6 +1,7 @@ import { env } from "node:process"; -import { config_dir, INI, join } from "../../deps.js"; +import { join } from "node:path"; +import * as INI from "@std/ini"; const NAME_REGEX = /^[\w.][\w.\s-]*$/; @@ -144,3 +145,22 @@ export async function config(subcommand, name, options, init) { return new Response(); } } + +/** + * Retrieves the configuration directory for current platform. + * @returns {string | null} + */ +export function config_dir() { + let dir = env["XDG_CONFIG_HOME"]; + if (!dir) { + dir = env["APPDATA"]; // Windows + } + if (!dir) { + dir = env["HOME"]; + if (dir) { + dir += "/.config"; + } + } + + return dir; +} diff --git a/cmd/config/main_test.js b/cmd/config/main_test.js index 93c3193..2d1adbd 100644 --- a/cmd/config/main_test.js +++ b/cmd/config/main_test.js @@ -1,9 +1,10 @@ import { test } from "node:test"; import { readFile, unlink, writeFile } from "node:fs/promises"; -import { config_dir, join } from "../../deps.js"; -import { assert, assertEquals, fc } from "../../dev_deps.js"; +import { join } from "node:path"; +import assert from "node:assert/strict"; +import fc from "fast-check"; -import { config as configure } from "./main.js"; +import { config as configure, config_dir } from "./main.js"; test("config", async (t) => { let command = "file"; @@ -11,7 +12,7 @@ test("config", async (t) => { const configDir = config_dir(); let path = await configure(command).then((res) => res.text()); - assertEquals( + assert.deepEqual( path, join(configDir, "rclone", "rclone.conf"), "should default to config directory", @@ -19,7 +20,7 @@ test("config", async (t) => { await writeFile("rclone.conf", new Uint8Array()); path = await configure(command).then((res) => res.text()); - assertEquals( + assert.deepEqual( path, "rclone.conf", "should use rclone.conf at current directory first", @@ -29,7 +30,7 @@ test("config", async (t) => { command = "create"; await t.test(command, async () => { let config = await readFile("rclone.conf", { encoding: "utf8" }); - assertEquals(config, "", "initial empty config"); + assert.deepEqual(config, "", "initial empty config"); await fc.assert( fc.asyncProperty(fc.string(), async (name) => { @@ -51,12 +52,12 @@ test("config", async (t) => { }); config = await readFile("rclone.conf", { encoding: "utf8" }); - assertEquals( + assert.deepEqual( config, "[source]\ntype = local", "should have the new remote added", ); - assertEquals( + assert.deepEqual( await response.text(), config, "should return the new remote in response", @@ -68,12 +69,12 @@ test("config", async (t) => { }); config = await readFile("rclone.conf", { encoding: "utf8" }); - assertEquals( + assert.deepEqual( config, "[source]\ntype = local\n\n[target]\ntype = local", "should have both remotes", ); - assertEquals( + assert.deepEqual( await response.text(), `[target]\ntype = local`, "should have only the new remote in response", @@ -84,28 +85,28 @@ test("config", async (t) => { await t.test(command, async () => { const config = await readFile("rclone.conf", { encoding: "utf8" }); let response = await configure(command); - assertEquals( + assert.deepEqual( await response.text(), config, "should return the full config when name is not specified", ); response = await configure(command, "source"); - assertEquals( + assert.deepEqual( await response.text(), "[source]\ntype = local", "should return only remote specified by name", ); response = await configure(command, "target"); - assertEquals( + assert.deepEqual( await response.text(), "[target]\ntype = local", "should return only remote specified by name", ); response = await configure(command, "404"); - assertEquals( + assert.deepEqual( response.status, 404, "should return 404 when remote not found", @@ -129,19 +130,19 @@ test("config", async (t) => { const remote = `[target]\ntype = alias\nremote = source:`; const config = await readFile("rclone.conf", { encoding: "utf8" }); - assertEquals( + assert.deepEqual( config, `[source]\ntype = local\n\n${remote}`, "should have the remote updated", ); - assertEquals( + assert.deepEqual( await response.text(), remote, "should return the updated remote in response", ); response = await configure(command, "404", {}); - assertEquals( + assert.deepEqual( response.status, 404, "should return 404 when remote not found", @@ -151,7 +152,7 @@ test("config", async (t) => { command = "dump"; await t.test(command, async () => { const response = await configure(command); - assertEquals(await response.json(), { + assert.deepEqual(await response.json(), { source: { type: "local", }, @@ -166,19 +167,19 @@ test("config", async (t) => { await t.test(command, async () => { let response = await configure(command, "target"); const config = await readFile("rclone.conf", { encoding: "utf8" }); - assertEquals( + assert.deepEqual( config, "[source]\ntype = local", "should have the remote deleted", ); - assertEquals( + assert.deepEqual( await response.text(), "", "should not return the deleted remote in response", ); response = await configure(command, "404"); - assertEquals( + assert.deepEqual( response.status, 200, "should still return 200 when remote not found", diff --git a/cmd/copy/main_test.js b/cmd/copy/main_test.js index 55c2303..cbfba36 100644 --- a/cmd/copy/main_test.js +++ b/cmd/copy/main_test.js @@ -1,5 +1,5 @@ import { test } from "node:test"; -import { assert } from "../../dev_deps.js"; +import assert from "node:assert/strict"; import { copy } from "./main.js"; test("copy", () => { diff --git a/cmd/copyurl/main.js b/cmd/copyurl/main.js index 5dc594b..bd4f7e9 100644 --- a/cmd/copyurl/main.js +++ b/cmd/copyurl/main.js @@ -1,4 +1,4 @@ -import { basename, resolve } from "../../deps.js"; +import { basename, resolve } from "node:path"; import { fetch } from "../../mod.js"; diff --git a/cmd/copyurl/main_test.js b/cmd/copyurl/main_test.js index 92bc62b..298ee45 100644 --- a/cmd/copyurl/main_test.js +++ b/cmd/copyurl/main_test.js @@ -1,5 +1,5 @@ import { test } from "node:test"; -import { assert } from "../../dev_deps.js"; +import assert from "node:assert/strict"; import { copyurl } from "./main.js"; test("copyurl", () => { diff --git a/cmd/ls/main_test.js b/cmd/ls/main_test.js index d751cc7..7effdd4 100644 --- a/cmd/ls/main_test.js +++ b/cmd/ls/main_test.js @@ -1,5 +1,5 @@ import { test } from "node:test"; -import { assert } from "../../dev_deps.js"; +import assert from "node:assert/strict"; import { ls } from "./main.js"; test("ls", () => { diff --git a/cmd/lsd/main_test.js b/cmd/lsd/main_test.js index 90e0ff0..7cb45d4 100644 --- a/cmd/lsd/main_test.js +++ b/cmd/lsd/main_test.js @@ -1,5 +1,5 @@ import { test } from "node:test"; -import { assert } from "../../dev_deps.js"; +import assert from "node:assert/strict"; import { lsd } from "./main.js"; test("lsd", () => { diff --git a/cmd/lsf/main_test.js b/cmd/lsf/main_test.js index 28bf17d..cd9a63d 100644 --- a/cmd/lsf/main_test.js +++ b/cmd/lsf/main_test.js @@ -1,5 +1,5 @@ import { test } from "node:test"; -import { assert } from "../../dev_deps.js"; +import assert from "node:assert/strict"; import { lsf } from "./main.js"; test("lsf", () => { diff --git a/cmd/lsjson/main.js b/cmd/lsjson/main.js index cc336d7..7f70b72 100644 --- a/cmd/lsjson/main.js +++ b/cmd/lsjson/main.js @@ -1,4 +1,6 @@ -import { basename, HTMLRewriter, toLocaleISOString } from "../../deps.js"; +import { basename } from "node:path"; +import { HTMLRewriter } from "@sntran/html-rewriter"; +import { toLocaleISOString } from "../../lib/time.js"; import { fetch } from "../../mod.js"; diff --git a/cmd/lsjson/main_test.js b/cmd/lsjson/main_test.js index a125098..599f398 100644 --- a/cmd/lsjson/main_test.js +++ b/cmd/lsjson/main_test.js @@ -1,5 +1,5 @@ import { test } from "node:test"; -import { assert, assertEquals } from "../../dev_deps.js"; +import assert from "node:assert/strict"; import { lsjson } from "./main.js"; const encoder = new TextEncoder(); @@ -55,44 +55,44 @@ test(`lsjson("${pathname}")`, async (t) => { const text = await response.clone().text(); const lines = text.trim().split("\n"); - assertEquals(lines.shift(), "["); + assert.deepEqual(lines.shift(), "["); for (const child of tree[pathname]) { const line = lines.shift().replace(/,$/, ""); const file = JSON.parse(line); - assertEquals(file.Name, child.replace("/", "")); + assert.deepEqual(file.Name, child.replace("/", "")); } - assertEquals(lines[0], "]"); + assert.deepEqual(lines[0], "]"); }); await t.test("Path field is same as Name", async () => { const result = await response.clone().json(); for (const file of result) { - assertEquals(file.Path, file.Name); + assert.deepEqual(file.Path, file.Name); } }); await t.test("each object is a chunk in body stream", async () => { const reader = response.body.getReader(); let result = await reader.read(); - assertEquals(result.done, false); - assertEquals(decoder.decode(result.value).trim(), "["); + assert.deepEqual(result.done, false); + assert.deepEqual(decoder.decode(result.value).trim(), "["); for (const child of tree[pathname]) { result = await reader?.read(); - assertEquals(result.done, false); + assert.deepEqual(result.done, false); const line = decoder.decode(result.value).trim().replace(/,$/, ""); const file = JSON.parse(line); - assertEquals(file.Name, child.replace("/", "")); + assert.deepEqual(file.Name, child.replace("/", "")); } result = await reader?.read(); - assertEquals(result.done, false); - assertEquals(decoder.decode(result.value).trim(), "]"); + assert.deepEqual(result.done, false); + assert.deepEqual(decoder.decode(result.value).trim(), "]"); result = await reader?.read(); - assertEquals(result.done, true); + assert.deepEqual(result.done, true); }); }); diff --git a/cmd/lsl/main_test.js b/cmd/lsl/main_test.js index 16454dc..f619fda 100644 --- a/cmd/lsl/main_test.js +++ b/cmd/lsl/main_test.js @@ -1,5 +1,5 @@ import { test } from "node:test"; -import { assert } from "../../dev_deps.js"; +import assert from "node:assert/strict"; import { lsl } from "./main.js"; test("lsl", () => { diff --git a/cmd/obscure/main_test.js b/cmd/obscure/main_test.js index 180e355..8a38ea6 100644 --- a/cmd/obscure/main_test.js +++ b/cmd/obscure/main_test.js @@ -1,5 +1,6 @@ import { test } from "node:test"; -import { assertEquals, fc } from "../../dev_deps.js"; +import assert from "node:assert/strict"; +import fc from "fast-check"; import { obscure, reveal } from "./main.js"; /** @@ -9,7 +10,7 @@ import { obscure, reveal } from "./main.js"; */ async function assertResponse(response, expected) { response = await response; - assertEquals(await response.text(), expected); + assert.deepEqual(await response.text(), expected); } test("reveal", () => { diff --git a/cmd/rcat/main_test.js b/cmd/rcat/main_test.js index 2f92719..6244f39 100644 --- a/cmd/rcat/main_test.js +++ b/cmd/rcat/main_test.js @@ -1,5 +1,5 @@ import { test } from "node:test"; -import { assert } from "../../dev_deps.js"; +import assert from "node:assert/strict"; import { rcat } from "./main.js"; test("rcat", () => { diff --git a/lib/streams/progress.js b/lib/streams/progress.js index 39095ee..1b8d5fe 100644 --- a/lib/streams/progress.js +++ b/lib/streams/progress.js @@ -1,4 +1,5 @@ -import { formatBytes, formatDuration } from "../../deps.js"; +import { format as formatBytes } from "@std/fmt/bytes"; +import { format as formatDuration } from "@std/fmt/duration"; export class Progress extends TransformStream { completed = 0; diff --git a/lib/time.js b/lib/time.js new file mode 100644 index 0000000..80320bb --- /dev/null +++ b/lib/time.js @@ -0,0 +1,20 @@ +/** + * Converts a date string to a locale ISO string. + * @param {string | null} value + * @returns {string} + */ +export function toLocaleISOString(value) { + if (!value) return undefined; + + let date = new Date(value); + const off = date.getTimezoneOffset() * -1; + const del = date.getMilliseconds() ? "Z" : "."; // have milliseconds ? + date = new Date(date.getTime() + off * 60000); // add or subtract time zone + return date + .toISOString() + .split(del)[0] + + (off < 0 ? "-" : "+") + + ("0" + Math.abs(Math.floor(off / 60))).substring(-2) + + ":" + + ("0" + Math.abs(off % 60)).substring(-2); +} diff --git a/main_test.js b/main_test.js index a14c261..eb2d909 100644 --- a/main_test.js +++ b/main_test.js @@ -2,7 +2,7 @@ import { test } from "node:test"; import process from "node:process"; import { unlink, writeFile } from "node:fs/promises"; import { serve } from "@sntran/serve"; -import { assert, assertEquals } from "./dev_deps.js"; +import assert from "node:assert/strict"; import { config } from "./mod.js"; @@ -39,16 +39,16 @@ test("fetch", async (t) => { }); const { status, headers } = response; - assertEquals(status, 200); - assertEquals(headers.get("Content-Type"), "message/http"); + assert.deepEqual(status, 200); + assert.deepEqual(headers.get("Content-Type"), "message/http"); - assertEquals(headers.get("Via"), "local/1.1"); + assert.deepEqual(headers.get("Via"), "local/1.1"); const body = await response.text(); const [, _method, url] = body.match(/^(TRACE) (.*) HTTP\/1.1$/m) || []; const { pathname } = new URL(url, "file:"); - assertEquals(pathname, cwd); + assert.deepEqual(pathname, cwd); }); // Sets up @@ -65,20 +65,20 @@ test("fetch", async (t) => { const { status, headers } = await fetch(":local:", { method: "TRACE", }); - assertEquals(status, 200); - assertEquals(headers.get("Content-Type"), "message/http"); + assert.deepEqual(status, 200); + assert.deepEqual(headers.get("Content-Type"), "message/http"); - assertEquals(headers.get("Via"), "local/1.1"); + assert.deepEqual(headers.get("Via"), "local/1.1"); }); await t.test("name:", async () => { const { status, headers } = await fetch("source:", { method: "TRACE", }); - assertEquals(status, 200); - assertEquals(headers.get("Content-Type"), "message/http"); + assert.deepEqual(status, 200); + assert.deepEqual(headers.get("Content-Type"), "message/http"); - assertEquals(headers.get("Via"), "local/1.1 source"); + assert.deepEqual(headers.get("Via"), "local/1.1 source"); }); /** Connection string */ @@ -91,17 +91,17 @@ test("fetch", async (t) => { }); const { status, headers } = response; - assertEquals(status, 200); - assertEquals(headers.get("Content-Type"), "message/http"); + assert.deepEqual(status, 200); + assert.deepEqual(headers.get("Content-Type"), "message/http"); - assertEquals(headers.get("Via"), "alias/1.1 target"); + assert.deepEqual(headers.get("Via"), "alias/1.1 target"); const body = await response.text(); const [, _method, url] = body.match(/^(TRACE) (.*) HTTP\/1.1$/m) || []; const { pathname, searchParams } = new URL(url, "file:"); - assertEquals(pathname, "/"); - assertEquals(searchParams.get("remote"), `source:`); + assert.deepEqual(pathname, "/"); + assert.deepEqual(searchParams.get("remote"), `source:`); }); await t.test( @@ -116,8 +116,8 @@ test("fetch", async (t) => { []; const { searchParams } = new URL(url, "file:"); - assertEquals(searchParams.get("remote"), `source:`); - assertEquals(searchParams.get("skip_links"), `true`); + assert.deepEqual(searchParams.get("remote"), `source:`); + assert.deepEqual(searchParams.get("skip_links"), `true`); }, ); @@ -133,7 +133,7 @@ test("fetch", async (t) => { []; const { searchParams } = new URL(url, "file:"); - assertEquals(searchParams.get("remote"), `/tmp/1`); + assert.deepEqual(searchParams.get("remote"), `/tmp/1`); assert( !searchParams.has("alias_remote"), "should not have param prefixed with backend type", @@ -153,7 +153,7 @@ test("fetch", async (t) => { []; const { searchParams } = new URL(url, "file:"); - assertEquals(searchParams.get("remote"), `/tmp/2`); + assert.deepEqual(searchParams.get("remote"), `/tmp/2`); assert( !searchParams.has("config_target_remote"), "should not have param prefixed with remote name", @@ -169,18 +169,18 @@ test("fetch", async (t) => { }); const { status, headers } = response; - assertEquals(status, 200); - assertEquals(headers.get("Content-Type"), "message/http"); + assert.deepEqual(status, 200); + assert.deepEqual(headers.get("Content-Type"), "message/http"); - assertEquals(headers.get("Via"), "alias/1.1 target"); + assert.deepEqual(headers.get("Via"), "alias/1.1 target"); const body = await response.text(); const [, _method, url] = body.match(/^(TRACE) (.*) HTTP\/1.1$/m) || []; const { pathname, searchParams } = new URL(url, "file:"); - assertEquals(pathname, "/"); - assertEquals(searchParams.get("remote"), `'/tmp/3'`); + assert.deepEqual(pathname, "/"); + assert.deepEqual(searchParams.get("remote"), `'/tmp/3'`); }, ); @@ -195,8 +195,8 @@ test("fetch", async (t) => { const [, _method, url] = body.match(/^(TRACE) (.*) HTTP\/1.1$/m) || []; const { pathname, searchParams } = new URL(url, "file:"); - assertEquals(pathname, "/"); - assertEquals(searchParams.get("remote"), `'${cwd}'`); + assert.deepEqual(pathname, "/"); + assert.deepEqual(searchParams.get("remote"), `'${cwd}'`); }); await t.test( @@ -213,8 +213,8 @@ test("fetch", async (t) => { []; const { pathname, searchParams } = new URL(url, "file:"); - assertEquals(pathname, "/"); - assertEquals(searchParams.get("remote"), `':local:${cwd}'`); + assert.deepEqual(pathname, "/"); + assert.deepEqual(searchParams.get("remote"), `':local:${cwd}'`); }, ); }); diff --git a/package-lock.json b/package-lock.json index 3709f10..401b78a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,13 +13,9 @@ "@noble/hashes": "^1.4.0", "@sntran/html-rewriter": "^0.2.0", "@sntran/serve": "^0.0.1", - "@std/crypto": "npm:@jsr/std__crypto@^1.0.0-rc.1", - "@std/encoding": "npm:@jsr/std__encoding@^1.0.0-rc.2", "@std/fmt": "npm:@jsr/std__fmt@^0.225.4", - "@std/http": "npm:@jsr/std__http@^0.224.5", "@std/ini": "npm:@jsr/std__ini@^0.225.1", "@std/media-types": "npm:@jsr/std__media-types@^1.0.0-rc.1", - "@std/path": "npm:@jsr/std__path@^1.0.0-rc.2", "base32-decode": "^1.0.0", "base32-encode": "^2.0.0" }, @@ -30,67 +26,9 @@ "fast-check": "^3.19.0" }, "engines": { - "deno": ">=1.44.4", "node": ">=22.2.0" } }, - "node_modules/@jsr/std__async": { - "version": "1.0.0-rc.2", - "resolved": "https://npm.jsr.io/~/11/@jsr/std__async/1.0.0-rc.2.tgz", - "integrity": "sha512-mW3WDco4WkcsVhuwWRRRMSOW2odyXb1O80TMnfFjJZbxmlVXZKZmR9RkJvSQv8BcxopdZ3JCigkHNEmtRFsTXQ==" - }, - "node_modules/@jsr/std__bytes": { - "version": "1.0.0-rc.3", - "resolved": "https://npm.jsr.io/~/11/@jsr/std__bytes/1.0.0-rc.3.tgz", - "integrity": "sha512-4cHQrAO+8e6YcJVNHiPvkX9LVPhy4ugiatYUZnZ5kO1JG2dZia6wu9dYEsL6pInUQuBx9tFcPIowc0YmeNP+CQ==" - }, - "node_modules/@jsr/std__cli": { - "version": "0.224.7", - "resolved": "https://npm.jsr.io/~/11/@jsr/std__cli/0.224.7.tgz", - "integrity": "sha512-SSE7AMT/nqTUKeQXB+PZcmD6L5IjsOFDQdm88Ik2o6P4gS+gXewLMPZJu/X/gaOH5ux2C4xZ26Dgp3vCAIuB4g==" - }, - "node_modules/@jsr/std__encoding": { - "version": "1.0.0-rc.2", - "resolved": "https://npm.jsr.io/~/11/@jsr/std__encoding/1.0.0-rc.2.tgz", - "integrity": "sha512-ISvxD+exWVg/aESN3G1Tr0oszHRgUEEK1JjiRcuyCtd1uqSGXXMAoswYlYvPwA6mxZ7FQbFSZEbP3tN2J+fIUQ==" - }, - "node_modules/@jsr/std__fmt": { - "version": "0.225.4", - "resolved": "https://npm.jsr.io/~/11/@jsr/std__fmt/0.225.4.tgz", - "integrity": "sha512-lb8QcV3YnAu3GsrG7wd51Fb56RxOBp4szMRS/TCLnQptmF0Rd0v8ksQeTHmGkuD6MY/V8gEy/Sj0qulRvR9lqQ==" - }, - "node_modules/@jsr/std__io": { - "version": "0.224.2", - "resolved": "https://npm.jsr.io/~/11/@jsr/std__io/0.224.2.tgz", - "integrity": "sha512-7IbtJWWM13TaXkZR3jL0LLkAbEaublLR6WSMZuzg58IACDGnzcmxKlI57KjEBTuawX4u6x7g8f6zRleuRpSXVA==", - "dependencies": { - "@jsr/std__bytes": "^1.0.0-rc.3" - } - }, - "node_modules/@jsr/std__media-types": { - "version": "1.0.0-rc.1", - "resolved": "https://npm.jsr.io/~/11/@jsr/std__media-types/1.0.0-rc.1.tgz", - "integrity": "sha512-0UdhqMLempJi8rHXJ4xSsRs79gMIm+X8/jAcBxUa9sdbsRALM148t/c2JIRWN4S58LQVBRHNK3WYd/YgR4fLNA==" - }, - "node_modules/@jsr/std__net": { - "version": "0.224.3", - "resolved": "https://npm.jsr.io/~/11/@jsr/std__net/0.224.3.tgz", - "integrity": "sha512-67B6bpkMjNSWtEPLy/k4OpuCb65w+L0mf9Amp1E2uUs4nt31vZZazW8IUykE9+1sv0tDVxa8nFZ2p9VwdiLXaA==" - }, - "node_modules/@jsr/std__path": { - "version": "1.0.0-rc.2", - "resolved": "https://npm.jsr.io/~/11/@jsr/std__path/1.0.0-rc.2.tgz", - "integrity": "sha512-qUHfVMqAtMGPFmdkOfmkCONqgpe4UFVAaD+wL6ZdlQxtvQiNxHQWsje+djri+0095s/kkoBOPH4CpYxBq+zh3A==" - }, - "node_modules/@jsr/std__streams": { - "version": "0.224.5", - "resolved": "https://npm.jsr.io/~/11/@jsr/std__streams/0.224.5.tgz", - "integrity": "sha512-LWs0LfBcyj4VzE3ff1cwoEyJ3AkoIT3CQubxeA0YQr3KEC/c6xB9Rq9nVit8gZ8RgtbMDlyfPtN1MkwTgPFj+g==", - "dependencies": { - "@jsr/std__bytes": "^1.0.0-rc.3", - "@jsr/std__io": "^0.224.1" - } - }, "node_modules/@noble/ciphers": { "version": "0.5.3", "resolved": "https://registry.npmjs.org/@noble/ciphers/-/ciphers-0.5.3.tgz", @@ -123,40 +61,12 @@ "resolved": "https://registry.npmjs.org/@sntran/serve/-/serve-0.0.1.tgz", "integrity": "sha512-yejSB/mTWeRg7fK++lLZcz5Av407HZ7VN4l84FxxpCoERpGRL1fP43wNeRC4xeup7xPtnHXsjinxz6xEByglNA==" }, - "node_modules/@std/crypto": { - "name": "@jsr/std__crypto", - "version": "1.0.0-rc.1", - "resolved": "https://npm.jsr.io/~/11/@jsr/std__crypto/1.0.0-rc.1.tgz", - "integrity": "sha512-HdqxgBx468H2ePsUUEYh4dyABELe1qxt4r2qK4UAR4gBrLxPRMBmAkZD0E7n/kUYA7+LTtLZ7OOhjZoQ9PPgrQ==" - }, - "node_modules/@std/encoding": { - "name": "@jsr/std__encoding", - "version": "1.0.0-rc.2", - "resolved": "https://npm.jsr.io/~/11/@jsr/std__encoding/1.0.0-rc.2.tgz", - "integrity": "sha512-ISvxD+exWVg/aESN3G1Tr0oszHRgUEEK1JjiRcuyCtd1uqSGXXMAoswYlYvPwA6mxZ7FQbFSZEbP3tN2J+fIUQ==" - }, "node_modules/@std/fmt": { "name": "@jsr/std__fmt", "version": "0.225.4", "resolved": "https://npm.jsr.io/~/11/@jsr/std__fmt/0.225.4.tgz", "integrity": "sha512-lb8QcV3YnAu3GsrG7wd51Fb56RxOBp4szMRS/TCLnQptmF0Rd0v8ksQeTHmGkuD6MY/V8gEy/Sj0qulRvR9lqQ==" }, - "node_modules/@std/http": { - "name": "@jsr/std__http", - "version": "0.224.5", - "resolved": "https://npm.jsr.io/~/11/@jsr/std__http/0.224.5.tgz", - "integrity": "sha512-PomDgsxwBoCkqTCG7e1Sz4QCAgNFhEGf71GWNercppQlgKtzp6zxsDgcJ4+vrqHIq5W8L1eqJWiceNkUUkMZjQ==", - "dependencies": { - "@jsr/std__async": "^1.0.0-rc.1", - "@jsr/std__cli": "^0.224.7", - "@jsr/std__encoding": "1.0.0-rc.2", - "@jsr/std__fmt": "^0.225.4", - "@jsr/std__media-types": "^1.0.0-rc.1", - "@jsr/std__net": "^0.224.3", - "@jsr/std__path": "1.0.0-rc.2", - "@jsr/std__streams": "^0.224.5" - } - }, "node_modules/@std/ini": { "name": "@jsr/std__ini", "version": "0.225.1", @@ -169,12 +79,6 @@ "resolved": "https://npm.jsr.io/~/11/@jsr/std__media-types/1.0.0-rc.1.tgz", "integrity": "sha512-0UdhqMLempJi8rHXJ4xSsRs79gMIm+X8/jAcBxUa9sdbsRALM148t/c2JIRWN4S58LQVBRHNK3WYd/YgR4fLNA==" }, - "node_modules/@std/path": { - "name": "@jsr/std__path", - "version": "1.0.0-rc.2", - "resolved": "https://npm.jsr.io/~/11/@jsr/std__path/1.0.0-rc.2.tgz", - "integrity": "sha512-qUHfVMqAtMGPFmdkOfmkCONqgpe4UFVAaD+wL6ZdlQxtvQiNxHQWsje+djri+0095s/kkoBOPH4CpYxBq+zh3A==" - }, "node_modules/base32-decode": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/base32-decode/-/base32-decode-1.0.0.tgz", diff --git a/package.json b/package.json index a399405..2b168fd 100644 --- a/package.json +++ b/package.json @@ -31,13 +31,9 @@ "@noble/hashes": "^1.4.0", "@sntran/html-rewriter": "^0.2.0", "@sntran/serve": "^0.0.1", - "@std/crypto": "npm:@jsr/std__crypto@^1.0.0-rc.1", - "@std/encoding": "npm:@jsr/std__encoding@^1.0.0-rc.2", "@std/fmt": "npm:@jsr/std__fmt@^0.225.4", - "@std/http": "npm:@jsr/std__http@^0.224.5", "@std/ini": "npm:@jsr/std__ini@^0.225.1", "@std/media-types": "npm:@jsr/std__media-types@^1.0.0-rc.1", - "@std/path": "npm:@jsr/std__path@^1.0.0-rc.2", "base32-decode": "^1.0.0", "base32-encode": "^2.0.0" }, From 49f284b73de5defd870be63fea93ec3acdd3ae6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C6=A1n=20Tr=E1=BA=A7n-Nguy=E1=BB=85n?= Date: Sat, 17 Oct 2026 01:40:44 +0000 Subject: [PATCH 07/87] build: compatibility date 2026-09-01 The date was 2024-06-28. 2026-09-01 is also the date of mkfifo.com, and workerd 1.20260926 has it. At this date nodejs_compat is on by default; the flag stays, so that the file tells it. In wrangler pages dev at this date, with no flag, PUT, GET and the list of memory and of crypt over memory work. workerd still refuses a null IV for AES-ECB, so aes.js keeps the empty IV. --- wrangler.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wrangler.toml b/wrangler.toml index 2d7af32..f8c3453 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -1,3 +1,3 @@ name = "denoclr" -compatibility_date = "2024-06-28" +compatibility_date = "2026-09-01" compatibility_flags = [ "nodejs_compat" ] From fb45d86ae6bbea84d7db9a5b9e7f984ffed6d6a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C6=A1n=20Tr=E1=BA=A7n-Nguy=E1=BB=85n?= Date: Sat, 17 Oct 2026 01:55:56 +0000 Subject: [PATCH 08/87] refactor: own code in place of @std/fmt, @std/ini and @std/media-types These three packages came from JSR through npm:@jsr aliases and the .npmrc of this repository, so a different project could not install rfetch. Now small modules of rfetch do the work, with the APIs of the web and of Node.js, and .npmrc goes away: - lib/format.js: formatBytes and formatDuration give the text of @std/fmt for the options of rfetch (no difference in 160,103 cases), but with KiB in place of kiB, the symbol of IEC, as rclone writes it. - lib/ini.js: parse and stringify in the format of the configuration file of rclone: [name], key = value, and an empty line between sections. @std/ini 0.225 did not use the options section and whitespace that cmd/config gave it. Two sections with the same name become one, as in rclone, and a name such as __proto__ does not change the prototype. - lib/mime.js: mimeType as MimeTypeFromName of rclone: the table of Go, the types that rclone adds, and .txt. An unknown extension gives application/octet-stream, as in rclone, in place of an empty Content-Type. The new tests of lib pass. --- .npmrc | 1 - backend/crypt/main.js | 7 ++-- backend/crypt/main_test.js | 6 +-- backend/drive/main.js | 2 +- backend/http/main.js | 2 +- backend/local/main.js | 8 ++-- backend/local/main_test.js | 12 +++--- backend/memory/main.js | 9 ++-- cmd/config/main.js | 7 +--- lib/format.js | 84 +++++++++++++++++++++++++++++++++++++ lib/format_test.js | 34 +++++++++++++++ lib/ini.js | 86 ++++++++++++++++++++++++++++++++++++++ lib/ini_test.js | 62 +++++++++++++++++++++++++++ lib/mime.js | 75 +++++++++++++++++++++++++++++++++ lib/mime_test.js | 19 +++++++++ lib/streams/progress.js | 7 ++-- package-lock.json | 21 ---------- package.json | 3 -- 18 files changed, 385 insertions(+), 60 deletions(-) delete mode 100644 .npmrc create mode 100644 lib/format.js create mode 100644 lib/format_test.js create mode 100644 lib/ini.js create mode 100644 lib/ini_test.js create mode 100644 lib/mime.js create mode 100644 lib/mime_test.js diff --git a/.npmrc b/.npmrc deleted file mode 100644 index 41583e3..0000000 --- a/.npmrc +++ /dev/null @@ -1 +0,0 @@ -@jsr:registry=https://npm.jsr.io diff --git a/backend/crypt/main.js b/backend/crypt/main.js index e119bb2..b12f192 100644 --- a/backend/crypt/main.js +++ b/backend/crypt/main.js @@ -93,10 +93,9 @@ * - 1049120 bytes total (a 0.05% overhead). This is the overhead for big files. */ -import { extname } from "node:path"; import { scryptAsync as scrypt } from "@noble/hashes/scrypt"; import { HTMLRewriter } from "@sntran/html-rewriter"; -import { contentType } from "@std/media-types"; +import { mimeType } from "../../lib/mime.js"; import { fetch } from "../../mod.js"; import { reveal } from "../../cmd/obscure/main.js"; import PathCipher from "./PathCipher.js"; @@ -186,7 +185,7 @@ async function crypt(request) { let { status, statusText, headers, body } = response; headers = new Headers(headers); - const mimetype = contentType(extname(pathname)) || ""; + const mimetype = mimeType(pathname); if (mimetype) { headers.set("Content-Type", mimetype); } @@ -217,7 +216,7 @@ async function crypt(request) { } if (type) { - type = contentType(extname(basename)) || type; + type = mimeType(basename); element.setAttribute("type", type); } }, diff --git a/backend/crypt/main_test.js b/backend/crypt/main_test.js index e1b1d02..a4d5b73 100644 --- a/backend/crypt/main_test.js +++ b/backend/crypt/main_test.js @@ -1,8 +1,8 @@ import { test } from "node:test"; import { readFile } from "node:fs/promises"; import { Buffer } from "node:buffer"; -import { extname, join } from "node:path"; -import { contentType } from "@std/media-types"; +import { join } from "node:path"; +import { mimeType } from "../../lib/mime.js"; import assert from "node:assert/strict"; import fc from "fast-check"; @@ -118,7 +118,7 @@ test("GET", async (t) => { assert.deepEqual(size, originalFile.byteLength); const type = response.headers.get("Content-Type"); - assert.deepEqual(type, contentType(extname(originalFilename))); + assert.deepEqual(type, mimeType(originalFilename)); response.body?.cancel(); } diff --git a/backend/drive/main.js b/backend/drive/main.js index ff9b2f3..ed5bd2d 100644 --- a/backend/drive/main.js +++ b/backend/drive/main.js @@ -1,4 +1,4 @@ -import { format as formatBytes } from "@std/fmt/bytes"; +import { formatBytes } from "../../lib/format.js"; import { auth } from "./auth.js"; import { list } from "./list.js"; diff --git a/backend/http/main.js b/backend/http/main.js index b5bcb01..cd7d4b9 100644 --- a/backend/http/main.js +++ b/backend/http/main.js @@ -25,7 +25,7 @@ */ import { HTMLRewriter } from "@sntran/html-rewriter"; -import { format as formatBytes } from "@std/fmt/bytes"; +import { formatBytes } from "../../lib/format.js"; import { toLocaleISOString } from "../../lib/time.js"; export const options = { diff --git a/backend/local/main.js b/backend/local/main.js index cab77f7..1258a3c 100644 --- a/backend/local/main.js +++ b/backend/local/main.js @@ -1,7 +1,7 @@ import process from "node:process"; -import { extname, join } from "node:path"; -import { format as formatBytes } from "@std/fmt/bytes"; -import { contentType } from "@std/media-types"; +import { join } from "node:path"; +import { formatBytes } from "../../lib/format.js"; +import { mimeType } from "../../lib/mime.js"; /** * Serves a local remote @@ -112,7 +112,7 @@ async function local(request) { headers.set("Content-Type", "text/html;charset=utf-8"); } else { - headers.append("Content-Type", contentType(extname(pathname)) || ""); + headers.append("Content-Type", mimeType(pathname)); headers.append("Content-Length", stats.size.toString()); if (method === "GET") { diff --git a/backend/local/main_test.js b/backend/local/main_test.js index 9f7adf6..99aaa86 100644 --- a/backend/local/main_test.js +++ b/backend/local/main_test.js @@ -1,8 +1,8 @@ import { test } from "node:test"; import process from "node:process"; import { readdir, readFile, stat } from "node:fs/promises"; -import { extname, join } from "node:path"; -import { contentType } from "@std/media-types"; +import { join } from "node:path"; +import { mimeType } from "../../lib/mime.js"; import assert from "node:assert/strict"; import backend from "./main.js"; @@ -63,7 +63,7 @@ test("GET", async (t) => { assert.equal( headers.get("Content-Type"), - contentType(extname(name)) || "", + mimeType(name), ); assert.equal(headers.get("Content-Length"), `${file.size}`); assert.equal( @@ -97,8 +97,7 @@ test("PUT", async (t) => { const request = new Request(url, { method: "PUT", headers: { - "Content-Type": contentType(extname(newFile)) || - "application/octet-stream", + "Content-Type": mimeType(newFile), "Content-Length": `${encoder.encode(body).byteLength}`, }, body, @@ -153,8 +152,7 @@ test("PUT", async (t) => { const request = new Request(url, { method: "PUT", headers: { - "Content-Type": contentType(extname(newFile)) || - "application/octet-stream", + "Content-Type": mimeType(newFile), "Content-Length": `${encoder.encode(body).byteLength}`, }, body, diff --git a/backend/memory/main.js b/backend/memory/main.js index a18b952..4b60962 100644 --- a/backend/memory/main.js +++ b/backend/memory/main.js @@ -7,9 +7,8 @@ * Because it has no parameters you can just use it with the `:memory:` remote * name. */ -import { extname } from "node:path"; -import { format as formatBytes } from "@std/fmt/bytes"; -import { contentType } from "@std/media-types"; +import { formatBytes } from "../../lib/format.js"; +import { mimeType } from "../../lib/mime.js"; /** * @type {Map} @@ -87,9 +86,7 @@ async function memory(request) { filePath += "/"; } - const type = isDirectory - ? "" - : contentType(extname(pathname)) || ""; + const type = isDirectory ? "" : mimeType(pathname); const size = Number(isDirectory ? "-1" : value.size); const lastModified = value?.lastModified ? new Date(value?.lastModified) diff --git a/cmd/config/main.js b/cmd/config/main.js index 9c9af68..05762e9 100644 --- a/cmd/config/main.js +++ b/cmd/config/main.js @@ -1,7 +1,7 @@ import { env } from "node:process"; import { join } from "node:path"; -import * as INI from "@std/ini"; +import * as INI from "../../lib/ini.js"; const NAME_REGEX = /^[\w.][\w.\s-]*$/; @@ -65,10 +65,7 @@ export async function config(subcommand, name, options, init) { * @type {Object} */ let config = INI.parse(ini); - const encodeOptions = { - section: "", - whitespace: true, - }; + const encodeOptions = { section: "" }; switch (subcommand) { case "create": { // Create a new remote with name, type and options. diff --git a/lib/format.js b/lib/format.js new file mode 100644 index 0000000..85e48be --- /dev/null +++ b/lib/format.js @@ -0,0 +1,84 @@ +/** + * Formats of numbers for people: a size in bytes and a duration. + * + * They give the same text as `format` of `@std/fmt/bytes` and of + * `@std/fmt/duration` for the options that rfetch uses, but with `KiB` in + * place of `kiB`: the symbol of IEC 80000-13, as rclone writes it. + */ + +/** The first letters of the units above the byte. */ +const PREFIXES = "kMGTPEZY"; + +/** + * Formats a size in bytes, for example `1.5 kB`, or `1.46 KiB` with `binary`. + * The number has 3 significant digits, and a size of less than 1 byte has no + * unit prefix. + * @param {number} bytes + * @param {{ binary?: boolean, locale?: boolean }} [options] `binary` uses + * units of 1024 (KiB, MiB, ...) in place of 1000 (kB, MB, ...). `locale` + * writes the number in the format of the locale of the runtime. + * @returns {string} + */ +export function formatBytes(bytes, { binary = false, locale = false } = {}) { + if (!Number.isFinite(bytes)) { + throw new TypeError( + `Expected a finite number, got ${typeof bytes}: ${bytes}`, + ); + } + const sign = bytes < 0 ? "-" : ""; + let value = Math.abs(bytes); + const text = (number) => locale ? number.toLocaleString() : number.toString(); + if (value < 1) { + return `${sign}${text(value)} B`; + } + + const exponent = Math.min( + Math.floor( + binary ? Math.log(value) / Math.log(1024) : Math.log10(value) / 3, + ), + PREFIXES.length, + ); + value /= Math.pow(binary ? 1024 : 1000, exponent); + const number = text(Number(value.toPrecision(3))); + return `${sign}${number} ${unit(exponent, binary)}`; +} + +/** + * Gives the unit of a power of 1000 or of 1024: B, kB, MB, ... or B, KiB, + * MiB, ... + * @param {number} exponent + * @param {boolean} binary + * @returns {string} + */ +function unit(exponent, binary) { + if (exponent === 0) { + return "B"; + } + const prefix = PREFIXES[exponent - 1]; + return binary ? `${prefix === "k" ? "K" : prefix}iB` : `${prefix}B`; +} + +/** + * Formats a duration in milliseconds with the units that are not zero, from + * days to nanoseconds, for example `1m 5s 300ms`. + * @param {number} milliseconds + * @returns {string} + */ +export function formatDuration(milliseconds) { + const ms = Math.abs(milliseconds); + // The digits of the microseconds and of the nanoseconds. + const fraction = ms.toFixed(7).slice(-7, -1); + const parts = [ + [Math.trunc(ms / 86400000), "d"], + [Math.trunc(ms / 3600000) % 24, "h"], + [Math.trunc(ms / 60000) % 60, "m"], + [Math.trunc(ms / 1000) % 60, "s"], + [Math.trunc(ms) % 1000, "ms"], + [Number(fraction.slice(0, 3)), "µs"], + [Number(fraction.slice(3, 6)), "ns"], + ]; + return parts.filter(([value]) => value).map(([value, unit]) => + `${value}${unit}` + ) + .join(" "); +} diff --git a/lib/format_test.js b/lib/format_test.js new file mode 100644 index 0000000..7dd5d78 --- /dev/null +++ b/lib/format_test.js @@ -0,0 +1,34 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { formatBytes, formatDuration } from "./format.js"; + +// The texts of `format` of @std/fmt 0.225, which these functions replace. +test("formatBytes", () => { + const cases = [ + [0, {}, "0 B"], + [0.5, {}, "0.5 B"], + [999, {}, "999 B"], + [1000, {}, "1 kB"], + [1536, {}, "1.54 kB"], + [1536, { binary: true }, "1.5 KiB"], + [1048576, { binary: true }, "1 MiB"], + [123456789, {}, "123 MB"], + [2 ** 40, { binary: true }, "1 TiB"], + [1e30, {}, "1000000 YB"], + [-1536, { binary: true }, "-1.5 KiB"], + [1234567, { locale: true }, `${(1.23).toLocaleString()} MB`], + ]; + for (const [bytes, options, text] of cases) { + assert.equal(formatBytes(bytes, options), text, `${bytes}`); + } + assert.throws(() => formatBytes(Infinity), TypeError); +}); + +test("formatDuration", () => { + assert.equal(formatDuration(0), ""); + assert.equal(formatDuration(61000), "1m 1s"); + assert.equal(formatDuration(90061001.5), "1d 1h 1m 1s 1ms 500µs"); + assert.equal(formatDuration(0.000001), "1ns"); + assert.equal(formatDuration(-3600000), "1h"); +}); diff --git a/lib/ini.js b/lib/ini.js new file mode 100644 index 0000000..55a8517 --- /dev/null +++ b/lib/ini.js @@ -0,0 +1,86 @@ +/** + * The INI format of the configuration file of rclone: + * + * ```ini + * [remote] + * type = s3 + * provider = Cloudflare + * + * [secret] + * type = crypt + * remote = remote:bucket + * ``` + * + * A `[name]` line starts a section, and each `key = value` line is a string + * in the current section. A line that starts with `#` or `;` is a comment. + */ + +/** + * Reads the text of a configuration file. The keys before the first section + * are at the top level. Two sections with the same name become one, as in + * rclone. + * @param {string} text + * @returns {Record>} + */ +export function parse(text) { + /** @type {Map>} */ + const config = new Map(); + let section = config; + for (const [index, line] of text.split(/\r?\n/).entries()) { + const content = line.trim(); + if (content === "" || content.startsWith("#") || content.startsWith(";")) { + continue; + } + const name = /^\[(.*)\]$/.exec(content)?.[1]; + if (name !== undefined) { + if (!(config.get(name) instanceof Map)) { + config.set(name, new Map()); + } + section = config.get(name); + continue; + } + const equal = content.indexOf("="); + if (equal === -1) { + throw new SyntaxError(`INI: line ${index + 1} has no "=": ${content}`); + } + section.set( + content.slice(0, equal).trim(), + content.slice(equal + 1).trim(), + ); + } + // Object.fromEntries makes own properties, so a name such as "__proto__" + // does not change the prototype. + return Object.fromEntries( + [...config].map(( + [key, value], + ) => [key, value instanceof Map ? Object.fromEntries(value) : value]), + ); +} + +/** + * Writes a configuration file: first the keys at the top level, then each + * section, with an empty line between the sections, as rclone writes them. + * A key with no value (undefined or null) is not written. + * @param {Record} config + * @param {{ section?: string }} [options] With a `section`, `config` is the + * keys of that one section. + * @returns {string} + */ +export function stringify(config, { section } = {}) { + if (section) { + return stringify({ [section]: config }); + } + const isSection = (value) => typeof value === "object" && value !== null; + const lines = (entries) => + entries + .filter(([, value]) => value !== undefined && value !== null) + .map(([key, value]) => `${key} = ${value}`); + + const entries = Object.entries(config); + const blocks = [lines(entries.filter(([, value]) => !isSection(value)))]; + for (const [name, keys] of entries.filter(([, value]) => isSection(value))) { + blocks.push([`[${name}]`, ...lines(Object.entries(keys))]); + } + return blocks.filter((block) => block.length).map((block) => block.join("\n")) + .join("\n\n"); +} diff --git a/lib/ini_test.js b/lib/ini_test.js new file mode 100644 index 0000000..7f89e00 --- /dev/null +++ b/lib/ini_test.js @@ -0,0 +1,62 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fc from "fast-check"; + +import { parse, stringify } from "./ini.js"; + +test("parse", () => { + assert.deepEqual(parse(""), {}); + assert.deepEqual( + parse( + "top = 1\n[a]\ntype = s3\n spaced key = a value \nurl=https://x/?a=b\n" + + "; a comment\n# a comment\n\n[b c]\nempty =\n", + ), + { + top: "1", + a: { type: "s3", "spaced key": "a value", url: "https://x/?a=b" }, + "b c": { empty: "" }, + }, + ); + // Two sections with the same name become one, as in rclone. + assert.deepEqual(parse("[a]\nx = 1\n[a]\ny = 2\r\n"), { + a: { x: "1", y: "2" }, + }); + assert.throws(() => parse("[a]\nno equal sign"), { + name: "SyntaxError", + message: 'INI: line 2 has no "=": no equal sign', + }); +}); + +test("parse does not change the prototype", () => { + const config = parse("[__proto__]\npolluted = yes\n"); + assert.deepEqual(Object.keys(config), ["__proto__"]); + assert.equal({}.polluted, undefined); +}); + +test("stringify writes the format of rclone", () => { + const config = { a: { type: "s3", key: "v=1" }, b: { type: "local" } }; + assert.equal( + stringify(config), + "[a]\ntype = s3\nkey = v=1\n\n[b]\ntype = local", + ); + assert.equal(stringify(config, { section: "" }), stringify(config)); + assert.equal(stringify(config.b, { section: "b" }), "[b]\ntype = local"); + assert.equal(stringify({ top: "1", a: { x: "2" } }), "top = 1\n\n[a]\nx = 2"); + assert.equal(stringify({ a: { x: undefined, y: null, z: 0 } }), "[a]\nz = 0"); + assert.equal(stringify({}), ""); +}); + +test("parse reads what stringify writes", () => { + const word = fc.stringMatching(/^[\w.][\w. -]*[\w.]$/); + const value = fc.stringMatching(/^\S(.*\S)?$/).filter((v) => + !/[\r\n]/.test(v) + ); + fc.assert( + fc.property( + fc.dictionary(word, fc.dictionary(word, value), { minKeys: 1 }), + (config) => { + assert.deepEqual(parse(stringify(config)), config); + }, + ), + ); +}); diff --git a/lib/mime.js b/lib/mime.js new file mode 100644 index 0000000..64f387e --- /dev/null +++ b/lib/mime.js @@ -0,0 +1,75 @@ +/** + * The media type of a file from its name, as `MimeTypeFromName` of rclone: + * the built-in table of the `mime` package of Go, the types that rclone adds + * (fs/mimetype.go of rclone 1.75.1), and `text/plain` for `.txt`, as the + * `mime.types` file of most systems gives it. + */ + +/** The types for each extension, in lowercase. */ +const TYPES = { + // The built-in table of Go. + ".avif": "image/avif", + ".css": "text/css; charset=utf-8", + ".gif": "image/gif", + ".htm": "text/html; charset=utf-8", + ".html": "text/html; charset=utf-8", + ".jpeg": "image/jpeg", + ".jpg": "image/jpeg", + ".js": "text/javascript; charset=utf-8", + ".json": "application/json", + ".mjs": "text/javascript; charset=utf-8", + ".pdf": "application/pdf", + ".png": "image/png", + ".svg": "image/svg+xml", + ".wasm": "application/wasm", + ".webp": "image/webp", + ".xml": "text/xml; charset=utf-8", + // The types that rclone adds. + ".flac": "audio/flac", + ".mpga": "audio/mpeg", + ".mpega": "audio/mpeg", + ".mp2": "audio/mpeg", + ".mp3": "audio/mpeg", + ".m4a": "audio/mpeg", + ".oga": "audio/ogg", + ".ogg": "audio/ogg", + ".opus": "audio/ogg", + ".spx": "audio/ogg", + ".wav": "audio/x-wav", + ".tiff": "image/tiff", + ".tif": "image/tiff", + ".dif": "video/dv", + ".dv": "video/dv", + ".fli": "video/fli", + ".mpeg": "video/mpeg", + ".mpg": "video/mpeg", + ".mpe": "video/mpeg", + ".ts": "video/MP2T", + ".mp4": "video/mp4", + ".qt": "video/quicktime", + ".mov": "video/quicktime", + ".ogv": "video/ogg", + ".webm": "video/webm", + ".avi": "video/x-msvideo", + ".mpv": "video/x-matroska", + ".mkv": "video/x-matroska", + ".srt": "application/x-subrip", + // The type of most systems. + ".txt": "text/plain; charset=utf-8", +}; + +/** + * Gives the media type of a file from the extension of its name, or + * `application/octet-stream` for an unknown extension. The extension can be + * in each case: `A.PNG` is an image. + * @param {string} name A name or a path of a file. + * @returns {string} + */ +export function mimeType(name) { + const base = name.slice(name.lastIndexOf("/") + 1); + const dot = base.lastIndexOf("."); + const extension = dot === -1 ? "" : base.slice(dot).toLowerCase(); + return Object.hasOwn(TYPES, extension) + ? TYPES[extension] + : "application/octet-stream"; +} diff --git a/lib/mime_test.js b/lib/mime_test.js new file mode 100644 index 0000000..0f38593 --- /dev/null +++ b/lib/mime_test.js @@ -0,0 +1,19 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { mimeType } from "./mime.js"; + +test("mimeType", () => { + // The table of Go, the additions of rclone, and .txt. + assert.equal(mimeType("index.html"), "text/html; charset=utf-8"); + assert.equal(mimeType("photo.JPG"), "image/jpeg"); + assert.equal(mimeType("a/b.c/song.mp3"), "audio/mpeg"); + assert.equal(mimeType("movie.mkv"), "video/x-matroska"); + assert.equal(mimeType("hello.txt"), "text/plain; charset=utf-8"); + // An unknown extension, no extension, and a dot in a folder only. + for ( + const name of ["1MB.bin", "README", "a.b/c", "toString", ".constructor"] + ) { + assert.equal(mimeType(name), "application/octet-stream", name); + } +}); diff --git a/lib/streams/progress.js b/lib/streams/progress.js index 1b8d5fe..8b4a078 100644 --- a/lib/streams/progress.js +++ b/lib/streams/progress.js @@ -1,5 +1,4 @@ -import { format as formatBytes } from "@std/fmt/bytes"; -import { format as formatDuration } from "@std/fmt/duration"; +import { formatBytes, formatDuration } from "../format.js"; export class Progress extends TransformStream { completed = 0; @@ -49,7 +48,7 @@ export class Progress extends TransformStream { } get etah() { - return formatDuration(this.eta * 1000, { ignoreZero: true }); + return formatDuration(this.eta * 1000); } get rateh() { @@ -57,7 +56,7 @@ export class Progress extends TransformStream { } get timeh() { - return formatDuration(this.time * 1000, { ignoreZero: true }); + return formatDuration(this.time * 1000); } get totalh() { diff --git a/package-lock.json b/package-lock.json index 401b78a..244f5db 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,9 +13,6 @@ "@noble/hashes": "^1.4.0", "@sntran/html-rewriter": "^0.2.0", "@sntran/serve": "^0.0.1", - "@std/fmt": "npm:@jsr/std__fmt@^0.225.4", - "@std/ini": "npm:@jsr/std__ini@^0.225.1", - "@std/media-types": "npm:@jsr/std__media-types@^1.0.0-rc.1", "base32-decode": "^1.0.0", "base32-encode": "^2.0.0" }, @@ -61,24 +58,6 @@ "resolved": "https://registry.npmjs.org/@sntran/serve/-/serve-0.0.1.tgz", "integrity": "sha512-yejSB/mTWeRg7fK++lLZcz5Av407HZ7VN4l84FxxpCoERpGRL1fP43wNeRC4xeup7xPtnHXsjinxz6xEByglNA==" }, - "node_modules/@std/fmt": { - "name": "@jsr/std__fmt", - "version": "0.225.4", - "resolved": "https://npm.jsr.io/~/11/@jsr/std__fmt/0.225.4.tgz", - "integrity": "sha512-lb8QcV3YnAu3GsrG7wd51Fb56RxOBp4szMRS/TCLnQptmF0Rd0v8ksQeTHmGkuD6MY/V8gEy/Sj0qulRvR9lqQ==" - }, - "node_modules/@std/ini": { - "name": "@jsr/std__ini", - "version": "0.225.1", - "resolved": "https://npm.jsr.io/~/11/@jsr/std__ini/0.225.1.tgz", - "integrity": "sha512-K23bS0WRbZ2riDg9jQMnaJW4uW/KaQpFJZRmv8cGDL+7JhmMkI+yC3wm8zGxctggPR1+pxIXH0WmK44CbGkORQ==" - }, - "node_modules/@std/media-types": { - "name": "@jsr/std__media-types", - "version": "1.0.0-rc.1", - "resolved": "https://npm.jsr.io/~/11/@jsr/std__media-types/1.0.0-rc.1.tgz", - "integrity": "sha512-0UdhqMLempJi8rHXJ4xSsRs79gMIm+X8/jAcBxUa9sdbsRALM148t/c2JIRWN4S58LQVBRHNK3WYd/YgR4fLNA==" - }, "node_modules/base32-decode": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/base32-decode/-/base32-decode-1.0.0.tgz", diff --git a/package.json b/package.json index 2b168fd..4224964 100644 --- a/package.json +++ b/package.json @@ -31,9 +31,6 @@ "@noble/hashes": "^1.4.0", "@sntran/html-rewriter": "^0.2.0", "@sntran/serve": "^0.0.1", - "@std/fmt": "npm:@jsr/std__fmt@^0.225.4", - "@std/ini": "npm:@jsr/std__ini@^0.225.1", - "@std/media-types": "npm:@jsr/std__media-types@^1.0.0-rc.1", "base32-decode": "^1.0.0", "base32-encode": "^2.0.0" }, From 4eda7400ae6b7e79ece6e1212ca5afa34d75473d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C6=A1n=20Tr=E1=BA=A7n-Nguy=E1=BB=85n?= Date: Sat, 17 Oct 2026 01:55:56 +0000 Subject: [PATCH 09/87] fix(config): the default file of rclone when no file exists With no configuration file, config file gave an empty path, and create wrote nowhere. Now the file is the default file of rclone, CONFIG_DIR/rclone/rclone.conf, as the test expects. With the new INI module, the 7 tests of cmd/config pass. --- cmd/config/main.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/config/main.js b/cmd/config/main.js index 05762e9..ef9fc66 100644 --- a/cmd/config/main.js +++ b/cmd/config/main.js @@ -41,7 +41,6 @@ const NAME_REGEX = /^[\w.][\w.\s-]*$/; export async function config(subcommand, name, options, init) { const { readFile, writeFile } = await import("node:fs/promises"); - let file = "", ini = ""; // Order as specified at https://rclone.org/docs/#config-config-file. const PATHS = [ "rfetch.conf", @@ -51,6 +50,8 @@ export async function config(subcommand, name, options, init) { join(config_dir(), "rclone", "rclone.conf"), join(env["HOME"], ".rclone.conf"), ]; + // With no file, the file is the default file of rclone. + let file = PATHS[4], ini = ""; for await (const path of PATHS) { try { ini = await readFile(path, { encoding: "utf8" }); From 48f4bda0009ce00b450c4cda87326963fa1da6e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C6=A1n=20Tr=E1=BA=A7n-Nguy=E1=BB=85n?= Date: Sat, 17 Oct 2026 01:56:50 +0000 Subject: [PATCH 10/87] build: wrangler.toml is the configuration of the Pages project rfetch pages_build_output_dir makes this file the configuration of the deploys from Git and of wrangler pages dev, in place of the settings of the dashboard. The output is the repository root, with _worker.js. The name is rfetch: denoclr was the earlier name, rcloned in reverse, with Deno in it. --- wrangler.toml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/wrangler.toml b/wrangler.toml index f8c3453..6f96998 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -1,3 +1,7 @@ -name = "denoclr" +name = "rfetch" +# This file is the configuration of the Pages project: the deploys from Git +# and `wrangler pages dev` read it. The repository root, with _worker.js, is +# the output of the build. +pages_build_output_dir = "." compatibility_date = "2026-09-01" compatibility_flags = [ "nodejs_compat" ] From 4ca38535b7d86d4b1c78dbcd4a2aab763811f6f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C6=A1n=20Tr=E1=BA=A7n-Nguy=E1=BB=85n?= Date: Sat, 17 Oct 2026 02:11:45 +0000 Subject: [PATCH 11/87] feat(lib): decodePath and escapeHTML for the backends decodePath gives the path of a URL with no percent-encoding. It gives null for a path that can go out of the root, for example /..%2Fsecret.txt, which the URL parser keeps. escapeHTML puts a character reference in place of each character that has a meaning in HTML. --- lib/html.js | 19 ++++++++++++++ lib/html_test.js | 17 +++++++++++++ lib/path.js | 36 +++++++++++++++++++++++++++ lib/path_test.js | 64 ++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 136 insertions(+) create mode 100644 lib/html.js create mode 100644 lib/html_test.js create mode 100644 lib/path.js create mode 100644 lib/path_test.js diff --git a/lib/html.js b/lib/html.js new file mode 100644 index 0000000..0eb6bd4 --- /dev/null +++ b/lib/html.js @@ -0,0 +1,19 @@ +/** The characters that have a meaning in HTML, and their references. */ +const REFERENCES = { + "&": "&", + "<": "<", + ">": ">", + '"': """, + "'": "'", +}; + +/** + * Puts a character reference in place of each character of the text that has + * a meaning in HTML. The result is safe as the text of an element, and as the + * value of an attribute in quotes. + * @param {string} text + * @returns {string} + */ +export function escapeHTML(text) { + return String(text).replace(/[&<>"']/g, (character) => REFERENCES[character]); +} diff --git a/lib/html_test.js b/lib/html_test.js new file mode 100644 index 0000000..cc1cb19 --- /dev/null +++ b/lib/html_test.js @@ -0,0 +1,17 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { escapeHTML } from "./html.js"; + +test("escapeHTML", async (t) => { + await t.test("puts a reference for each character of HTML", () => { + assert.equal( + escapeHTML(`Tom & Jerry's`), + "<a href="x">Tom & Jerry's</a>", + ); + }); + + await t.test("does not change other text", () => { + assert.equal(escapeHTML("Hello, 世界 ≠ ?+#"), "Hello, 世界 ≠ ?+#"); + }); +}); diff --git a/lib/path.js b/lib/path.js new file mode 100644 index 0000000..16c04bf --- /dev/null +++ b/lib/path.js @@ -0,0 +1,36 @@ +/** + * Gives the path of a URL with no percent-encoding, for a backend to use as + * the path of a file or a folder. The path keeps its `/` at the start, and at + * the end for a folder. + * + * It gives null when the path is not safe: when a name has bad + * percent-encoding, or when a name is empty, is `.` or `..`, or has a `/` or a + * NUL character after the decoding. The URL parser removes the segments `.` + * and `..`, but it keeps `..%2F`, which becomes `../` after the decoding. Thus + * a backend that uses only the result of this function cannot go out of its + * root folder. + * @param {string} pathname The path of a URL, for example `/a%20b/c.txt`. + * @returns {string | null} The path, for example `/a b/c.txt`, or null. + */ +export function decodePath(pathname) { + if (!pathname.startsWith("/")) return null; + + const names = pathname.slice(1).split("/"); + // The path of a folder ends with "/", thus its last name is empty. + const last = names.length - 1; + for (let i = 0; i < names.length; i++) { + let name; + try { + name = decodeURIComponent(names[i]); + } catch { + return null; + } + if (name === "" && i === last) continue; + if (name === "" || name === "." || name === ".." || /[/\0]/.test(name)) { + return null; + } + names[i] = name; + } + + return `/${names.join("/")}`; +} diff --git a/lib/path_test.js b/lib/path_test.js new file mode 100644 index 0000000..5889dd7 --- /dev/null +++ b/lib/path_test.js @@ -0,0 +1,64 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fc from "fast-check"; + +import { decodePath } from "./path.js"; + +test("decodePath", async (t) => { + await t.test("removes the percent-encoding of each name", () => { + assert.equal(decodePath("/"), "/"); + assert.equal(decodePath("/a%20b/c.txt"), "/a b/c.txt"); + assert.equal(decodePath("/a%20b/"), "/a b/"); + assert.equal(decodePath("/%E4%B8%96%E7%95%8C"), "/世界"); + assert.equal(decodePath("/a%3Fb%23c"), "/a?b#c"); + assert.equal(decodePath("/test%2546.txt"), "/test%46.txt"); + }); + + await t.test("gives the same name for each encoding of it", () => { + fc.assert( + fc.property(fc.string({ minLength: 1 }), (name) => { + fc.pre(!/[/\0]/.test(name) && name !== "." && name !== ".."); + let encoded; + try { + encoded = encodeURIComponent(name); + } catch { + fc.pre(false); // A string with a lone surrogate has no encoding. + } + assert.equal(decodePath(`/${encoded}`), `/${name}`); + assert.equal(decodePath(`/${encoded}/`), `/${name}/`); + }), + ); + }); + + await t.test("refuses a path that can go out of the root", () => { + for ( + const pathname of [ + "/..%2Fsecret.txt", + "/a/..%2F..%2Fsecret.txt", + "/%2E%2E%2Fsecret.txt", + "/..%2F", + "/%2E%2E", + "/.", + "/a/%2E/b", + ] + ) { + assert.equal(decodePath(pathname), null, pathname); + } + }); + + await t.test("refuses a path with a name that is not valid", () => { + for ( + const pathname of [ + "", + "a", + "//", + "/a//b", + "/a%2Fb", + "/a%00b", + "/%E0%A4%A", + ] + ) { + assert.equal(decodePath(pathname), null, pathname); + } + }); +}); From b95bcf4bbd704e0600627d6362d52f14f2cff28e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C6=A1n=20Tr=E1=BA=A7n-Nguy=E1=BB=85n?= Date: Sat, 17 Oct 2026 02:11:45 +0000 Subject: [PATCH 12/87] test: a shared test suite for the backends, as fstests of rclone backendSuite(name, backend) runs the same tests on each backend, in a new folder that it deletes at the end: - 404 for a file or a folder that does not exist; - PUT and DELETE of files and folders; - the names of the test FsEncoding of rclone; - a file of 0 bytes, and the two files of fstests; - the list of each folder; - the bytes and the headers of GET and HEAD. remote() is a client of a backend in the same process. parseListing reads the HTML list of a folder. --- lib/testing/listing.js | 79 ++++++++++ lib/testing/remote.js | 68 ++++++++ lib/testing/suite.js | 343 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 490 insertions(+) create mode 100644 lib/testing/listing.js create mode 100644 lib/testing/remote.js create mode 100644 lib/testing/suite.js diff --git a/lib/testing/listing.js b/lib/testing/listing.js new file mode 100644 index 0000000..19fd68f --- /dev/null +++ b/lib/testing/listing.js @@ -0,0 +1,79 @@ +import { HTMLRewriter } from "@sntran/html-rewriter"; + +/** + * An entry of the list of a folder, with the names of `rclone lsjson`. + * @typedef {Object} Entry + * @property {string} Name The name, with no percent-encoding. + * @property {boolean} IsDir + * @property {number} Size The value of ``, or -1 when there is none. + * @property {string} MimeType The `type` of the link, or "". + * @property {string} ModTime The `datetime` of `