From 2bb35a9b725974ec4d3b9aa7d0650f5767576954 Mon Sep 17 00:00:00 2001 From: Rolando Santamaria Maso Date: Sun, 8 Nov 2026 14:04:36 +0100 Subject: [PATCH 01/20] feat(cli): MLXRCLI parser kit with Ollama-style help, version and errors --- Package.swift | 4 +- Sources/MLXRCLI/CLI.swift | 75 ++++ Sources/MLXRCLI/Commands.swift | 66 ++++ Sources/MLXRCLI/Help.swift | 67 ++++ Sources/MLXRCLI/Legacy.swift | 507 +++++++++++++++++++++++++ Sources/MLXRCLI/Parser.swift | 121 ++++++ Sources/MLXRCLI/Parsing.swift | 126 +++++++ Sources/mlxr/main.swift | 528 +-------------------------- Tests/MLXRCLITests/ErrorTests.swift | 83 +++++ Tests/MLXRCLITests/HelpTests.swift | 90 +++++ Tests/MLXRCLITests/ParserTests.swift | 99 +++++ Tests/MLXRCLITests/TestSupport.swift | 47 +++ 12 files changed, 1286 insertions(+), 527 deletions(-) create mode 100644 Sources/MLXRCLI/CLI.swift create mode 100644 Sources/MLXRCLI/Commands.swift create mode 100644 Sources/MLXRCLI/Help.swift create mode 100644 Sources/MLXRCLI/Legacy.swift create mode 100644 Sources/MLXRCLI/Parser.swift create mode 100644 Sources/MLXRCLI/Parsing.swift create mode 100644 Tests/MLXRCLITests/ErrorTests.swift create mode 100644 Tests/MLXRCLITests/HelpTests.swift create mode 100644 Tests/MLXRCLITests/ParserTests.swift create mode 100644 Tests/MLXRCLITests/TestSupport.swift diff --git a/Package.swift b/Package.swift index afb851d..b5b40c5 100644 --- a/Package.swift +++ b/Package.swift @@ -17,8 +17,10 @@ let package = Package( swiftSettings: [.unsafeFlags(["-Ounchecked"], .when(configuration: .release))] ), .executableTarget(name: "MLXRunner", dependencies: ["MLXEngine"]), - .executableTarget(name: "mlxr", dependencies: ["MLXEngine"]), + .target(name: "MLXRCLI", dependencies: ["MLXEngine"]), + .executableTarget(name: "mlxr", dependencies: ["MLXRCLI"]), .testTarget(name: "MLXEngineTests", dependencies: ["MLXEngine"]), + .testTarget(name: "MLXRCLITests", dependencies: ["MLXRCLI", "MLXEngine"]), ], swiftLanguageModes: [.v5] ) diff --git a/Sources/MLXRCLI/CLI.swift b/Sources/MLXRCLI/CLI.swift new file mode 100644 index 0000000..854ac3c --- /dev/null +++ b/Sources/MLXRCLI/CLI.swift @@ -0,0 +1,75 @@ +import Foundation +import MLXEngine + +public enum CLI { + /// Parse a command line against a table without running anything. + public static func parse(_ args: [String], commands: [Command] = CLI.commands) throws -> Invocation { + guard let first = args.first else { throw CLIError.usage("accepts a command") } + guard let command = findCommand(first, in: commands) else { throw unknownCommand(first, commands) } + return try parse(args: Array(args.dropFirst()), command: command) + } + + static func unknownCommand(_ name: String, _ commands: [Command]) -> CLIError { + if name.hasPrefix("-") { return .usage("unknown flag: \(name)") } + var msg = "unknown command \"\(name)\" for \"\(rootName)\"" + let near = suggestions(for: name, in: commands) + if !near.isEmpty { msg += "\n\nDid you mean this?\n" + near.map { "\t\($0)\n" }.joined() } + return .usage(msg) + } + + /// Parse and run; returns the process exit code. Errors print `Error: ` on stderr. + public static func run(_ args: [String], console: Console, commands: [Command] = CLI.commands) async -> Int32 { + do { + if let first = args.first { + switch first { + case "-h", "--help": console.out(rootHelp(commands: commands)); return 0 + case "-v", "--version", "version": console.out("mlxr version is \(MLXRunnerVersion.current)\n"); return 0 + case "help": + guard args.count > 1 else { console.out(rootHelp(commands: commands)); return 0 } + guard let c = findCommand(args[1], in: commands) else { + throw CLIError.usage("unknown help topic \"\(args[1])\"") + } + console.out(commandHelp(c)); return 0 + default: break + } + } else { + console.out(rootHelp(commands: commands)); return 0 + } + let inv = try parse(args, commands: commands) + if inv.helpRequested { console.out(commandHelp(inv.command)); return 0 } + try await inv.command.run(inv, console) + return 0 + } catch CLIError.interrupted { + return 130 + } catch { + console.err("Error: \(message(for: error))\n") + return 1 + } + } + + /// The message printed for an error (Hugging Face 401/404 become one hint). + public static func message(for error: Error) -> String { + if let e = error as? CLIError { + switch e { + case .usage(let m), .failure(let m): return m + case .interrupted: return "interrupted" + } + } + let ns = error as NSError + if ns.domain == "HuggingFace", ns.code == 401 || ns.code == 404 { + return "not found, or private/gated (set HF_TOKEN)" + } + return error.localizedDescription + } + + /// Entry point of the `mlxr` executable. + public static func main(_ args: [String]) async -> Int32 { + setvbuf(stdout, nil, _IONBF, 0) + let console = Console( + out: { FileHandle.standardOutput.write(Data($0.utf8)) }, + err: { FileHandle.standardError.write(Data($0.utf8)) }, + stdoutIsTTY: isatty(1) != 0, stderrIsTTY: isatty(2) != 0, stdinIsTTY: isatty(0) != 0, + env: ProcessInfo.processInfo.environment) + return await run(args, console: console) + } +} diff --git a/Sources/MLXRCLI/Commands.swift b/Sources/MLXRCLI/Commands.swift new file mode 100644 index 0000000..899cbf3 --- /dev/null +++ b/Sources/MLXRCLI/Commands.swift @@ -0,0 +1,66 @@ +import Foundation +import MLXEngine + +extension CLI { + static let sparseHelp = "sparse decode attention: on, off or a number of long-range keys" + static let draftHelp = "draft model for speculative decoding: a model or auto" + + /// The command table of `mlxr`. + public static var commands: [Command] { + [ + Command(name: "run", summary: "Run a model", args: [Arg("MODEL", kind: .model), Arg("PROMPT", optional: true, variadic: true)], + options: [ + Opt(long: "prompt", short: "p", value: .text, help: "prompt"), + Opt(long: "max-tokens", aliases: ["max"], short: "n", value: .int(1...32768), defaultText: "256", help: "maximum tokens to generate"), + Opt(long: "temp", aliases: ["temperature"], value: .real(0...2), defaultText: "0", help: "sampling temperature"), + Opt(long: "ctx", value: .int(256...1_048_576), defaultText: "4096", help: "context length"), + Opt(long: "raw", help: "send the prompt without the chat template"), + Opt(long: "think", value: .choice(["true", "false", "high", "medium", "low"]), implicit: "true", help: "think before answering"), + Opt(long: "no-think", help: "answer without thinking"), + Opt(long: "ids", help: "print the prompt token ids"), + Opt(long: "draft", value: .model, help: draftHelp), + Opt(long: "sparse", value: .text, help: sparseHelp), + Opt(long: "spec", help: "speculative decoding with n-gram drafts"), + ], + examples: [" mlxr run mlx-community/Qwen3-4B-4bit -p \"Hello\" --temp 0"], + environment: [("MLXR_SPARSE_ATTN", "sparse decode attention default (0 turns it off)")], + run: { inv, _ in + let words = inv.positionals.dropFirst().joined(separator: " ") + let thinking: Bool? = inv.bool("no-think") ? false : inv.has("think") ? inv.string("think") != "false" : nil + try await legacyRun(LegacyRun( + model: inv.positionals[0], + prompt: inv.string("prompt") ?? (words.isEmpty ? "Write a haiku about Apple Silicon." : words), + maxTokens: inv.int("max-tokens") ?? 256, temp: Float(inv.double("temp") ?? 0), + ctx: inv.int("ctx") ?? 4096, raw: inv.bool("raw"), thinking: thinking, showIds: inv.bool("ids"), + draft: inv.string("draft"), sparse: inv.string("sparse"), spec: inv.bool("spec"))) + }), + Command(name: "serve", aliases: ["start"], summary: "Start the OpenAI-compatible server", args: [Arg("MODEL", kind: .model)], + options: [ + Opt(long: "port", value: .int(1...65535), defaultText: "1234", help: "port to listen on"), + Opt(long: "ctx", value: .int(256...1_048_576), defaultText: "8192", help: "context length"), + Opt(long: "lan", help: "listen on the local network (needs an API key)"), + Opt(long: "api-key", value: .text, help: "API key clients must send"), + Opt(long: "draft", value: .model, help: draftHelp), + Opt(long: "sparse", value: .text, help: sparseHelp), + ], + environment: [("MLXR_API_KEY", "API key, when --api-key is not given")], + run: { inv, _ in + try await legacyServe(model: inv.positionals[0], port: UInt16(inv.int("port") ?? 1234), + ctx: inv.int("ctx") ?? 8192, lan: inv.bool("lan"), apiKey: inv.string("api-key"), + draft: inv.string("draft"), sparse: inv.string("sparse")) + }), + Command(name: "download", aliases: ["pull"], summary: "Download a model from Hugging Face", args: [Arg("MODEL", kind: .model)], + run: { inv, _ in try await legacyDownload(inv.positionals[0]) }), + Command(name: "list", aliases: ["ls"], summary: "List installed models", run: { _, _ in legacyList() }), + Command(name: "show", summary: "Show information for a model", args: [Arg("MODEL", kind: .model)], + run: { _, _ in throw CLIError.failure("show is not available yet") }), + Command(name: "search", summary: "Search Hugging Face for MLX models", args: [Arg("QUERY", variadic: true)], + run: { inv, _ in try await legacySearch(inv.positionals.joined(separator: " ")) }), + Command(name: "tokenize", summary: "Tokenize text with a model's tokenizer", args: [Arg("MODEL", kind: .model), Arg("TEXT", variadic: true)], + group: "Developer", run: { inv, _ in try legacyTokenize(inv.positionals[0], inv.positionals.dropFirst().joined(separator: " ")) }), + Command(name: "bench", summary: "Benchmark a model", args: [Arg("MODEL", kind: .model)], + environment: [("MLXR_BENCH_DECODE", "decode benchmark only"), ("MLXR_BENCH_CTX", "long-context contexts, comma separated")], + group: "Developer", run: { inv, _ in try legacyBench(inv.positionals[0]) }), + ] + } +} diff --git a/Sources/MLXRCLI/Help.swift b/Sources/MLXRCLI/Help.swift new file mode 100644 index 0000000..39620e8 --- /dev/null +++ b/Sources/MLXRCLI/Help.swift @@ -0,0 +1,67 @@ +import Foundation + +/// Help text in the cobra layout. +extension CLI { + static let rootName = "mlxr" + static let rootSummary = "Run MLX models from Hugging Face on Apple Silicon, from the command line." + + static func rootHelp(commands: [Command]) -> String { + let shown = commands.filter { !$0.hidden } + let width = max(11, shown.map { $0.name.count }.max() ?? 0) + 1 + func rows(_ list: [Command]) -> String { + list.map { " " + $0.name.padding(toLength: width, withPad: " ", startingAt: 0) + " " + $0.summary + "\n" }.joined() + } + var out = rootSummary + "\n\nUsage:\n \(rootName) [flags]\n \(rootName) [command]\n\n" + out += "Available Commands:\n" + rows(shown.filter { $0.group == nil }) + "\n" + var groups: [String] = [] + for c in shown { if let g = c.group, !groups.contains(g) { groups.append(g) } } + for g in groups { out += "\(g) Commands:\n" + rows(shown.filter { $0.group == g }) + "\n" } + out += "Flags:\n -h, --help help for \(rootName)\n -v, --version show version information\n\n" + out += "Use \"\(rootName) [command] --help\" for more information about a command.\n" + return out + } + + static func commandHelp(_ c: Command) -> String { + var out = c.summary + "\n\nUsage:\n \(rootName) \(c.name)" + for a in c.args { + let n = a.variadic ? a.name + "..." : a.name + out += a.optional ? " [\(n)]" : " \(n)" + } + out += " [flags]\n" + if !c.aliases.isEmpty { out += "\nAliases:\n \(([c.name] + c.aliases).joined(separator: ", "))\n" } + if !c.examples.isEmpty { out += "\nExamples:\n" + c.examples.joined(separator: "\n") + "\n" } + var rows: [(String, String)] = c.options.filter { !$0.hidden }.map { o in + let short = o.short.map { "-\($0), " } ?? " " + var left = short + "--" + o.long + if let kind = o.value { left += " " + typeName(kind) } + var help = o.help + if let d = o.defaultText { help += " (default \(d))" } + return (left, help) + } + rows.append(("-h, --help", "help for \(c.name)")) + rows = rows.map { ($0.0.hasPrefix("-") && !$0.0.hasPrefix("--") ? $0.0 : " " + $0.0.trimmingCharacters(in: .init(charactersIn: " ")), $0.1) } + let sorted = rows.sorted { key($0.0) < key($1.0) } + let w = sorted.map { $0.0.count }.max() ?? 0 + out += "\nFlags:\n" + sorted.map { " " + $0.0.padding(toLength: w, withPad: " ", startingAt: 0) + " " + $0.1 + "\n" }.joined() + if !c.environment.isEmpty { + out += "\nEnvironment Variables:\n" + c.environment.map { + " " + $0.name.padding(toLength: 27, withPad: " ", startingAt: 0) + " " + $0.help + "\n" + }.joined() + } + return out + } + + private static func key(_ left: String) -> String { + left.trimmingCharacters(in: .whitespaces).replacingOccurrences(of: "-h, ", with: "").replacingOccurrences(of: "-", with: "") + .lowercased() + } + + static func typeName(_ k: ValueKind) -> String { + switch k { + case .int: return "int" + case .uint64: return "uint" + case .real: return "float" + case .text, .choice, .choiceOrInt, .model, .file: return "string" + } + } +} diff --git a/Sources/MLXRCLI/Legacy.swift b/Sources/MLXRCLI/Legacy.swift new file mode 100644 index 0000000..6129711 --- /dev/null +++ b/Sources/MLXRCLI/Legacy.swift @@ -0,0 +1,507 @@ +import Foundation +import MLXEngine + +// The original `mlxr` command bodies, kept as they were while the commands move onto the parser kit. + +func fail(_ msg: String) -> Never { + FileHandle.standardError.write((msg + "\n").data(using: .utf8)!) + exit(1) +} + + +func resolve(_ s: String) -> URL { + if FileManager.default.fileExists(atPath: s) { return URL(fileURLWithPath: s) } + if let m = ModelStore.scan().first(where: { $0.repo == s || $0.name == s }) { return m.directory } + fail("model not found: \(s) (try `mlxr download \(s)`)") +} + +/// `--draft `: the draft model for speculative decoding. +func draftDirectory(_ s: String?, target: URL) -> URL? { + guard let s else { return nil } + guard s == "auto" else { return resolve(s) } + guard let m = InferenceEngine.autoDraft(for: target) else { + print("no installed draft model pairs with \(target.lastPathComponent); running without one") + return nil + } + return m.directory +} + +/// `--sparse `: sparse decode attention for long contexts (N: long-range keys; the window, +/// minimum context and shared selection still come from MLXR_SPARSE_*). Without the +/// option: MLXR_SPARSE_ATTN, which defaults to on. +func sparseOption(_ s: String?) -> SparseAttention? { + guard let s else { return SparseAttention.fromEnvironment } + switch s.lowercased() { + case "on", "1": return .configured(topK: SparseAttention.standard.topK) + case "off", "0": return nil + default: + guard let n = Int(s), n > 0 else { fail("--sparse takes on, off or a number of long-range keys") } + return .configured(topK: n) + } +} + +func sparseSummary(_ info: LoadedModelInfo) -> String { + if info.sparseLayers > 0 { + return "sparse attention on \(info.sparseLayers) layers from \(info.sparseMinContext) tokens (\(info.sparseTopK) long-range keys)" + } + return "sparse attention off" + (info.sparseNote.map { ": \($0)" } ?? "") +} + +func formatBytes(_ b: Int) -> String { + ByteCountFormatter.string(fromByteCount: Int64(b), countStyle: .file) +} + +/// Long-context settings of `mlxr bench` (MLXR_BENCH_CTX turns the section on). +struct LongBench { + let contexts: [Int] + /// Teacher-forced positions of the text after the prompt (MLXR_BENCH_POSITIONS, default 512). + let positions: Int + /// Token offsets into the text where prompts start (MLXR_BENCH_TEXT_OFFSETS, default 0). + let offsets: [Int] + static let steps = 64 + /// Rows the model needs past the longest prompt: the measured positions plus a needle question. + var margin: Int { max(Self.steps, positions) + 96 } + + init(env: [String: String]) { + func ints(_ key: String) -> [Int] { + (env[key] ?? "").split(separator: ",").compactMap { Int($0.trimmingCharacters(in: .whitespaces)) } + } + contexts = ints("MLXR_BENCH_CTX").filter { $0 > 0 } + positions = min(max(ints("MLXR_BENCH_POSITIONS").first ?? 512, 0), 8192) + let o = ints("MLXR_BENCH_TEXT_OFFSETS").filter { $0 >= 0 } + offsets = o.isEmpty ? [0] : o + } +} + +/// `mlxr bench` with MLXR_BENCH_CTX: decode after long prompts, dense and (for a model loaded with +/// MLXR_SPARSE_ATTN) sparse. For each context: +/// - greedy decode speed, and how many greedy tokens sparse decode shares with dense decode; +/// - teacher-forced on dense decode's tokens: a control (dense forward against the GPU's greedy +/// tokens, must be all) and sparse argmax agreement; +/// - teacher-forced on the text after the prompt (MLXR_BENCH_TEXT only): KL(dense‖sparse), the +/// text's NLL under both, and top-1 agreement by dense decode's top-1 margin. Each position runs +/// sparse first and then dense, so the cache stays dense decode's; +/// - the cost of a dense verification pass of T tokens at that context; +/// - needles (MLXR_BENCH_TEXT only): five passkeys at 10–90% depth, recalled dense and sparse. +/// The prompt is MLXR_BENCH_TEXT from each of MLXR_BENCH_TEXT_OFFSETS, or pseudo-random tokens, on +/// which agreement says little. +func benchLongContext(_ t: Transformer, dir: URL, bench: LongBench) throws { + var greedy = SamplingParameters() + greedy.temperature = 0 + greedy.repetitionPenalty = 1 + let sp = SampleParams(greedy, vocab: t.vocabSize) + let V = t.vocabSize + let tokenizer = try Tokenizer(directory: dir) + var text: [Int32] = [] + if let path = ProcessInfo.processInfo.environment["MLXR_BENCH_TEXT"] { + guard let s = try? String(contentsOfFile: path, encoding: .utf8) else { fail("cannot read MLXR_BENCH_TEXT \(path)") } + text = tokenizer.encode(s).map { Int32($0) }.filter { $0 >= 0 && Int($0) < V } + guard !text.isEmpty else { fail("MLXR_BENCH_TEXT \(path) has no tokens") } + let bos = tokenizer.bosToken.flatMap { b in tokenizer.tokenId(b).map { " (prompts start with \(b), id \($0))" } } ?? "" + print("long-context prompt: \(path), \(text.count) tokens (repeated to length)\(bos)") + } else { + print("long-context prompt: pseudo-random tokens (MLXR_BENCH_TEXT= for real text)") + } + let original = t.sparse + defer { t.sparse = original } + // Hybrid models (recurrent state): every rerun of a position first restores the state there. + var base: Transformer.RecurrentSnapshot?, last: Transformer.RecurrentSnapshot? + func mark() { if t.config.isHybrid { last = t.snapshotRecurrent() } } + func seek(_ pos: Int) { + guard t.config.isHybrid, t.recurrentPosition != pos else { return } + guard let s = [base, last].compactMap({ $0 }).first(where: { $0.position == pos }) else { fail("no recurrent snapshot at \(pos)") } + t.restoreRecurrent(s) + } + let sparse = t.sparseAvailable ? original : nil + let steps = LongBench.steps + func logits(_ tokens: [Int32], offset: Int) throws -> [Float] { + Array(UnsafeBufferPointer(start: try t.forward(tokens, offset: offset), count: V)) + } + func argmax(_ x: [Float]) -> Int { + var best = 0 + for v in 1.. x[best] { best = v } + return best + } + func logSoftmax(_ x: [Float]) -> [Double] { + let m = Double(x.max() ?? 0) + var s = 0.0 + for v in x { s += exp(Double(v) - m) } + let lse = m + log(s) + return x.map { Double($0) - lse } + } + for ctx in bench.contexts { + guard ctx + bench.margin < t.maxSeq else { print("decode @\(ctx): skipped (the model's context is \(t.maxSeq))"); continue } + for (oi, off) in (text.isEmpty ? [0] : bench.offsets).enumerated() { + var rng: UInt64 = 0x9E3779B97F4A7C15 + let lo = min(1000, V / 2), span = UInt64(max(1, min(V, 30000) - lo)) + func token(_ i: Int) -> Int32 { + if !text.isEmpty { return text[(off + i) % text.count] } + rng ^= rng << 13; rng ^= rng >> 7; rng ^= rng << 17 + return Int32(lo + Int(rng % span)) + } + var prompt = (0..= 0, id < V { prompt[0] = Int32(id) } + } + let at = off > 0 ? " from text token \(off)" : "" + t.sparse = nil + t.seedSampler(1) + let s0 = Date() + let first = try t.prefill(prompt, offset: 0, sample: sp) + let prefillTime = Date().timeIntervalSince(s0) + if t.config.isHybrid { base = t.snapshotRecurrent() } + print(String(format: "prefill @%d%@: %.1f s (%.0f tok/s)", ctx, at as NSString, prefillTime, Double(ctx) / prefillTime)) + // Each run decodes from the same prompt (positions from ctx on are rewritten, the index with them). + var dense: [Int] = [] + var modes: [(String, SparseAttention?)] = [("dense", nil)] + if let sparse, oi == 0 { modes.append(("sparse", sparse)) } + for (label, mode) in modes { + t.sparse = mode + t.seedSampler(1) + seek(ctx) + var tokens: [Int] = [] + var gpu = 0.0 + let s = Date() + var cur = try t.beginDecodeStep(offset: ctx, sample: sp) + for i in 0.. 0 { + // Teacher-forced on the text after the prompt. + let N = bench.positions + var kl: [Double] = [], nllDense = 0.0, nllSparse = 0.0, agree = 0 + let edges: [Float] = [0.1, 0.5, 1] + var bucketTotal = [Int](repeating: 0, count: edges.count + 1), bucketAgree = bucketTotal + seek(ctx) + for i in 0.. -40 { k += exp(pd[v]) * (pd[v] - ps[v]) } + kl.append(max(0, k)) + nllDense -= pd[target] + nllSparse -= ps[target] + let top = argmax(ld) + var second = top == 0 ? 1 : 0 + for v in 0.. ld[second] { second = v } + let bucket = edges.firstIndex { ld[top] - ld[second] < $0 } ?? edges.count + bucketTotal[bucket] += 1 + if argmax(ls) == top { agree += 1; bucketAgree[bucket] += 1 } + } + kl.sort() + let meanKL = kl.reduce(0, +) / Double(N) + let dNLL = (nllSparse - nllDense) / Double(N) + print(String(format: " teacher-forced on the text (%d positions): KL(dense‖sparse) mean %.4f, median %.4f, p95 %.4f, max %.3f nats", + N, meanKL, kl[N / 2], kl[min(N - 1, N * 95 / 100)], kl[N - 1])) + print(String(format: " text NLL: dense %.4f, sparse %.4f nats/token (Δ %+.4f, perplexity ×%.4f)", + nllDense / Double(N), nllSparse / Double(N), dNLL, exp(dNLL))) + let labels = ["<0.1", "0.1–0.5", "0.5–1", "≥1"] + let byMargin = zip(labels, zip(bucketAgree, bucketTotal)).map { "\($0.0): \($0.1.0)/\($0.1.1)" }.joined(separator: ", ") + print(String(format: " top-1 agreement %d/%d (%.1f%%); by dense top-1 margin (nats) %@", agree, N, + 100 * Double(agree) / Double(N), byMargin as NSString)) + } + if oi == 0 && !t.config.isHybrid { + // A dense verification pass of T tokens at this context (speculative decoding). + t.sparse = nil + var line: [String] = [] + for T in [1, 2, 5, 8] where T <= Transformer.maxVerifyTokens { + var gpu = 0.0 + for i in 0..<4 { + _ = try t.forward((0.. Bool { + lock.lock(); defer { lock.unlock() } + guard Date().timeIntervalSince(date) >= s else { return false } + date = Date() + return true + } +} + +func legacySearch(_ query: String) async throws { + let results = try await HubClient.search(query, limit: 25) + for m in results { + print(String(format: "%-60@ %10d ↓ %@", m.id as NSString, m.downloads ?? 0, (m.modelTypeTag ?? "") as NSString)) + } + +} + +func legacyDownload(_ repo: String) async throws { + guard RepoID.isValid(repo) else { fail("invalid repo id: \(repo) (expected owner/name)") } + let details = try await HubClient.details(repo) + if !details.isSupported { print("warning: \(details.modelType ?? "unknown") architecture is not supported by the engine") } + print("Downloading \(repo) (\(formatBytes(details.downloadSize)))") + let dl = ModelDownload(details: details, destination: try ModelStore.directory(for: repo)) + let last = LockedDate() + dl.onProgress = { p, done, total, speed in + guard last.elapsed(0.25) || p >= 1 else { return } + print(String(format: "\r %5.1f%% %@ / %@ %@/s ", p * 100, formatBytes(Int(done)), formatBytes(Int(total)), formatBytes(Int(speed))), terminator: "") + } + dl.onEvent = { print("\n \($0)") } + do { + let dir = try await dl.start() + print("\nSaved to \(dir.path)") + } catch { + fail("\ndownload failed: \(error.localizedDescription)") + } + +} + +func legacyList() { + for m in ModelStore.scan() { + print(String(format: "%-60@ %10@ %@%@", m.repo as NSString, formatBytes(m.sizeBytes) as NSString, + (m.modelType ?? "?") as NSString, m.isSupported ? "" : " (unsupported)")) + } + +} + +func legacyTokenize(_ model: String, _ text: String) throws { + let tok = try Tokenizer(directory: resolve(model)) + let ids = tok.encode(text) + print(ids) + print(ids.map { tok.token($0) ?? "?" }) + print("roundtrip:", tok.decode(ids, includeSpecial: true)) + +} + +func legacyBench(_ model: String) throws { + let dir = resolve(model) + // Long-context decode (opt-in): MLXR_BENCH_CTX=8192,32768 adds a section at the end. + let longBench = LongBench(env: ProcessInfo.processInfo.environment) + let t = try Transformer(directory: dir, contextLength: max(4096, (longBench.contexts.max() ?? 0) + longBench.margin + 64)) + print("weights: \(formatBytes(t.weightBytes))") + if let sp = t.sparse { + print(t.sparseAvailable + ? "sparse decode attention: top-k \(sp.topK), window \(sp.window), sink \(sp.sink), from \(sp.minContext) tokens" + : "sparse decode attention: off (\(t.sparseOffReason ?? "unavailable"))") + } + print("attention: \(t.attentionSummary)") + let quick = ProcessInfo.processInfo.environment["MLXR_BENCH_DECODE"] != nil + let noProfile = quick || ProcessInfo.processInfo.environment["MLXR_BENCH_NOPROFILE"] != nil + if !noProfile { print(try t.profile()) } + if ProcessInfo.processInfo.environment["MLXR_BENCH_PROFILE_ONLY"] != nil { exit(0) } + _ = try t.forward([1], offset: 0) + for n in (quick ? [] : [128, 512, 2048]) { + let toks = (0.. 0 { + print(String(format: " sampler: %.2f attempts and pool %.0f per token on average, at most %d attempts", Double(diag.attempts) / Double(steps + 1), Double(diag.pool) / Double(steps + 1), diag.maxAttempts)) + } + } + } + // Speculative verification cost: one forward over T tokens with logits for every row. + // At a short context and after a 1900-token prompt (the attention part grows with the context). + // (A recurrent state cannot rewind to rerun a position: no verification on hybrid models.) + for ctx in t.config.isHybrid ? [] : [64, 1900] { + if ctx > 64 { _ = try t.forward((0.. 64 ? "verify @\(ctx)" : "verify" + print(String(format: "%@ T=%d: %6.2f ms per pass (%5.2f ms per token, gpu %.2f ms)", label as NSString, T, dt * 1000, dt * 1000 / Double(T), gpu / Double(steps) * 1000)) + } + } + for T in [5, 8, 12, 15] { + let s = Date() + for i in 0..<8 { _ = try t.forward((0.. 0, let dt = s.draftSeconds, let vt = s.verifySeconds { + print(String(format: "[draft model: %d cycles, %.2f tokens/cycle, draft %.1f ms + verify %.1f ms per cycle]", c, + Double(s.acceptedModelDraftTokens ?? 0) / Double(c) + 1, dt / Double(c) * 1000, vt / Double(c) * 1000)) + } + } + } +} + +func legacyServe(model: String, port: UInt16, ctx: Int, lan: Bool, apiKey: String?, draft: String?, sparse: String?) async throws { + let sparseArg = sparseOption(sparse) + // LAN mode always needs a key: use --api-key / MLXR_API_KEY, or generate one. + var key = apiKey ?? ProcessInfo.processInfo.environment["MLXR_API_KEY"] + if lan && key == nil { key = OpenAIServer.generateAPIKey() } + let dir = resolve(model) + let info = try await InferenceEngine.shared.load(directory: dir, draftDirectory: draftDirectory(draft, target: dir), contextLength: ctx, sparse: sparseArg) + print("Loaded \(info.name)\(info.draftName.map { " + draft \($0)" } ?? "") in \(String(format: "%.2f", info.loadSeconds))s") + print(sparseSummary(info)) + let server = OpenAIServer(port: port, allowLAN: lan, apiKey: key) + server.onLog = { print($0) } + do { try server.start() } catch { fail("cannot start server: \(error.localizedDescription)") } + print("OpenAI-compatible server on http://\(lan ? "0.0.0.0" : "127.0.0.1"):\(port)/v1 (Ctrl-C to stop)") + if let key { print("API key (send as 'Authorization: Bearer '): \(key)") } + while true { try await Task.sleep(nanoseconds: 3_600_000_000_000) } +} diff --git a/Sources/MLXRCLI/Parser.swift b/Sources/MLXRCLI/Parser.swift new file mode 100644 index 0000000..08c7fe2 --- /dev/null +++ b/Sources/MLXRCLI/Parser.swift @@ -0,0 +1,121 @@ +import Foundation + +/// What an option or positional argument accepts. Drives validation, help and completions. +public enum ValueKind { + case text + case int(ClosedRange) + case uint64 + case real(ClosedRange) + case choice([String]) + case choiceOrInt([String]) + case model + case file +} + +/// One option of a command. `value == nil` is a boolean flag (`--x`, `--x=true|false`). +public struct Opt { + public var long: String + public var aliases: [String] = [] + public var short: Character? = nil + public var value: ValueKind? = nil + /// Value taken when the option is given alone (cobra NoOptDefVal): `--think` means `--think=true`. + public var implicit: String? = nil + public var repeatable = false + public var env: String? = nil + public var defaultText: String? = nil + public var hidden = false + public var help: String = "" + + public init(long: String, aliases: [String] = [], short: Character? = nil, value: ValueKind? = nil, + implicit: String? = nil, repeatable: Bool = false, env: String? = nil, + defaultText: String? = nil, hidden: Bool = false, help: String = "") { + self.long = long; self.aliases = aliases; self.short = short; self.value = value + self.implicit = implicit; self.repeatable = repeatable; self.env = env + self.defaultText = defaultText; self.hidden = hidden; self.help = help + } +} + +/// One positional argument. A variadic argument takes all remaining positionals. +public struct Arg { + public var name: String + public var kind: ValueKind = .text + public var optional = false + public var variadic = false + + public init(_ name: String, kind: ValueKind = .text, optional: Bool = false, variadic: Bool = false) { + self.name = name; self.kind = kind; self.optional = optional; self.variadic = variadic + } +} + +/// Failures a command can report. `usage` and `failure` print `Error: ` and exit 1. +public enum CLIError: Error, Equatable { + case usage(String) + case failure(String) + case interrupted +} + +/// The streams and terminal facts a command sees; injected so commands are testable. +public final class Console: @unchecked Sendable { + public var out: (String) -> Void + public var err: (String) -> Void + public var stdoutIsTTY: Bool + public var stderrIsTTY: Bool + public var stdinIsTTY: Bool + public var width: Int + public var env: [String: String] + + public init(out: @escaping (String) -> Void, err: @escaping (String) -> Void, + stdoutIsTTY: Bool = false, stderrIsTTY: Bool = false, stdinIsTTY: Bool = false, + width: Int = 80, env: [String: String] = [:]) { + self.out = out; self.err = err + self.stdoutIsTTY = stdoutIsTTY; self.stderrIsTTY = stderrIsTTY; self.stdinIsTTY = stdinIsTTY + self.width = width; self.env = env + } +} + +/// The validated result of parsing one command line. +public struct Invocation { + public var command: Command + public var positionals: [String] + /// Raw validated values by long option name, in the order given. + public var values: [String: [String]] + + public init(command: Command, positionals: [String] = [], values: [String: [String]] = [:]) { + self.command = command; self.positionals = positionals; self.values = values + } + + /// True when `-h` / `--help` was given: the command does not run. + public var helpRequested = false + + /// Last value given for the option, or nil. + public func string(_ long: String) -> String? { values[long]?.last } + public func strings(_ long: String) -> [String] { values[long] ?? [] } + public func int(_ long: String) -> Int? { string(long).flatMap { Int($0) } } + public func double(_ long: String) -> Double? { string(long).flatMap { Double($0) } } + /// A boolean flag: true when given alone or as `=true`, false for `=false` or when absent. + public func bool(_ long: String) -> Bool { string(long).map { $0 != "false" } ?? false } + public func has(_ long: String) -> Bool { values[long] != nil } +} + +public struct Command { + public var name: String + public var aliases: [String] = [] + public var summary: String + public var args: [Arg] = [] + public var options: [Opt] = [] + public var examples: [String] = [] + public var environment: [(name: String, help: String)] = [] + /// Help heading in the root command list; nil means the main list. + public var group: String? = nil + public var hidden = false + public var run: (Invocation, Console) async throws -> Void + + public init(name: String, aliases: [String] = [], summary: String, args: [Arg] = [], + options: [Opt] = [], examples: [String] = [], + environment: [(name: String, help: String)] = [], group: String? = nil, + hidden: Bool = false, run: @escaping (Invocation, Console) async throws -> Void = { _, _ in }) { + self.name = name; self.aliases = aliases; self.summary = summary; self.args = args + self.options = options; self.examples = examples; self.environment = environment + self.group = group; self.hidden = hidden; self.run = run + } +} diff --git a/Sources/MLXRCLI/Parsing.swift b/Sources/MLXRCLI/Parsing.swift new file mode 100644 index 0000000..5d7013d --- /dev/null +++ b/Sources/MLXRCLI/Parsing.swift @@ -0,0 +1,126 @@ +import Foundation + +/// The command-line grammar of cobra/pflag: `--x v`, `--x=v`, `-x v`, `--` ends options, +/// options and positionals mix freely, the last value of an option wins. +extension CLI { + static func findCommand(_ name: String, in commands: [Command]) -> Command? { + commands.first { $0.name == name || $0.aliases.contains(name) } + } + + static func parse(args: [String], command: Command) throws -> Invocation { + var inv = Invocation(command: command) + var i = 0 + var onlyPositionals = false + func option(long name: String) -> Opt? { + command.options.first { $0.long == name || $0.aliases.contains(name) } + } + while i < args.count { + let a = args[i] + i += 1 + if onlyPositionals || a == "-" || !a.hasPrefix("-") { inv.positionals.append(a); continue } + if a == "--" { onlyPositionals = true; continue } + var opt: Opt? + var inline: String? + var shown: String + if a.hasPrefix("--") { + let body = String(a.dropFirst(2)) + let parts = body.split(separator: "=", maxSplits: 1, omittingEmptySubsequences: false) + let name = String(parts[0]) + if parts.count == 2 { inline = String(parts[1]) } + if name == "help" { inv.helpRequested = true; continue } + guard let o = option(long: name) else { throw CLIError.usage("unknown flag: --\(name)") } + opt = o; shown = "--\(o.long)" + } else { + let chars = Array(a.dropFirst()) + let c = chars[0] + if c == "h" && chars.count == 1 { inv.helpRequested = true; continue } + guard let o = command.options.first(where: { $0.short == c }) else { + throw CLIError.usage("unknown shorthand flag: '\(c)' in -\(String(chars))") + } + if chars.count > 1 { + // -pVALUE and -p=VALUE + var rest = String(chars.dropFirst()) + if rest.hasPrefix("=") { rest.removeFirst() } + inline = rest + } + opt = o; shown = "--\(o.long)" + } + let o = opt! + let raw: String + if let kind = o.value { + if let inline { raw = inline } + else if let implicit = o.implicit { raw = implicit } + else if i < args.count { raw = args[i]; i += 1 } + else { throw CLIError.usage(a.hasPrefix("--") ? "flag needs an argument: \(shown)" : "flag needs an argument: '\(o.short.map(String.init) ?? "")' in -\(o.short.map(String.init) ?? "")") } + try validate(raw, kind: kind, flag: shown) + } else { + if let inline { + guard ["true", "false", "1", "0", "t", "f", "TRUE", "FALSE", "True", "False", "T", "F"].contains(inline) else { + throw CLIError.usage("invalid argument \"\(inline)\" for \"\(shown)\" flag: strconv.ParseBool: parsing \"\(inline)\": invalid syntax") + } + raw = ["true", "1", "t", "TRUE", "True", "T"].contains(inline) ? "true" : "false" + } else { raw = "true" } + } + if o.repeatable { inv.values[o.long, default: []].append(raw) } else { inv.values[o.long] = [raw] } + } + if !inv.helpRequested { try checkArgCount(inv.positionals.count, command: command) } + return inv + } + + static func validate(_ raw: String, kind: ValueKind, flag: String) throws { + func bad(_ why: String) -> CLIError { .usage("invalid argument \"\(raw)\" for \"\(flag)\" flag: \(why)") } + switch kind { + case .text, .model, .file: break + case .int(let r): + guard let n = Int(raw) else { throw bad("strconv.ParseInt: parsing \"\(raw)\": invalid syntax") } + guard r.contains(n) else { throw bad("value \(n) is outside \(r.lowerBound)...\(r.upperBound)") } + case .uint64: + guard UInt64(raw) != nil else { throw bad("strconv.ParseUint: parsing \"\(raw)\": invalid syntax") } + case .real(let r): + guard let d = Double(raw), d.isFinite else { throw bad("strconv.ParseFloat: parsing \"\(raw)\": invalid syntax") } + guard r.contains(d) else { throw bad("value \(raw) is outside \(r.lowerBound)...\(r.upperBound)") } + case .choice(let c): + guard c.contains(raw) else { throw bad("must be one of \(c.joined(separator: ", "))") } + case .choiceOrInt(let c): + guard c.contains(raw) || (Int(raw).map { $0 > 0 } ?? false) else { + throw bad("must be one of \(c.joined(separator: ", ")) or a positive number") + } + } + } + + static func checkArgCount(_ n: Int, command: Command) throws { + let required = command.args.filter { !$0.optional }.count + let variadic = command.args.contains { $0.variadic } + if variadic { + if n < required { throw CLIError.usage("requires at least \(required) arg(s), only received \(n)") } + } else if required == command.args.count { + if n != required { throw CLIError.usage("accepts \(required) arg(s), received \(n)") } + } else if n < required || n > command.args.count { + throw CLIError.usage("accepts between \(required) and \(command.args.count) arg(s), received \(n)") + } + } + + /// Commands whose name is close to a mistyped one (cobra: edit distance <= 2 or a prefix). + static func suggestions(for name: String, in commands: [Command]) -> [String] { + let lower = name.lowercased() + return commands.filter { !$0.hidden }.compactMap { c -> String? in + let hit = ([c.name] + c.aliases).contains { n in + editDistance(lower, n.lowercased()) <= 2 || n.lowercased().hasPrefix(lower) + } + return hit ? c.name : nil + } + } + + static func editDistance(_ a: String, _ b: String) -> Int { + let a = Array(a), b = Array(b) + var prev = Array(0...b.count) + for i in 1...max(a.count, 1) where !a.isEmpty { + var cur = [i] + [Int](repeating: 0, count: b.count) + for j in 1...max(b.count, 1) where !b.isEmpty { + cur[j] = min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (a[i - 1] == b[j - 1] ? 0 : 1)) + } + prev = cur + } + return a.isEmpty ? b.count : prev[b.count] + } +} diff --git a/Sources/mlxr/main.swift b/Sources/mlxr/main.swift index 6030a85..9c3569d 100644 --- a/Sources/mlxr/main.swift +++ b/Sources/mlxr/main.swift @@ -1,528 +1,4 @@ import Foundation -import MLXEngine +import MLXRCLI -// mlxr — command-line companion to MLX Runner. -// mlxr search -// mlxr download -// mlxr list -// mlxr run [-p prompt] [--raw] [--max N] [--temp T] [--think|--no-think] [--ctx N] [--draft ] [--sparse ] -// mlxr tokenize -// mlxr serve [--port 1234] [--ctx N] [--lan] [--draft ] [--sparse ] - -setvbuf(stdout, nil, _IONBF, 0) -var args = Array(CommandLine.arguments.dropFirst()) - -func fail(_ msg: String) -> Never { - FileHandle.standardError.write((msg + "\n").data(using: .utf8)!) - exit(1) -} - -func option(_ name: String) -> String? { - guard let i = args.firstIndex(of: name), i + 1 < args.count else { return nil } - let v = args[i + 1] - args.removeSubrange(i...(i + 1)) - return v -} - -func flag(_ name: String) -> Bool { - guard let i = args.firstIndex(of: name) else { return false } - args.remove(at: i) - return true -} - -func resolve(_ s: String) -> URL { - if FileManager.default.fileExists(atPath: s) { return URL(fileURLWithPath: s) } - if let m = ModelStore.scan().first(where: { $0.repo == s || $0.name == s }) { return m.directory } - fail("model not found: \(s) (try `mlxr download \(s)`)") -} - -/// `--draft `: the draft model for speculative decoding. -func draftDirectory(_ s: String?, target: URL) -> URL? { - guard let s else { return nil } - guard s == "auto" else { return resolve(s) } - guard let m = InferenceEngine.autoDraft(for: target) else { - print("no installed draft model pairs with \(target.lastPathComponent); running without one") - return nil - } - return m.directory -} - -/// `--sparse `: sparse decode attention for long contexts (N: long-range keys; the window, -/// minimum context and shared selection still come from MLXR_SPARSE_*). Without the -/// option: MLXR_SPARSE_ATTN, which defaults to on. -func sparseOption(_ s: String?) -> SparseAttention? { - guard let s else { return SparseAttention.fromEnvironment } - switch s.lowercased() { - case "on", "1": return .configured(topK: SparseAttention.standard.topK) - case "off", "0": return nil - default: - guard let n = Int(s), n > 0 else { fail("--sparse takes on, off or a number of long-range keys") } - return .configured(topK: n) - } -} - -func sparseSummary(_ info: LoadedModelInfo) -> String { - if info.sparseLayers > 0 { - return "sparse attention on \(info.sparseLayers) layers from \(info.sparseMinContext) tokens (\(info.sparseTopK) long-range keys)" - } - return "sparse attention off" + (info.sparseNote.map { ": \($0)" } ?? "") -} - -func formatBytes(_ b: Int) -> String { - ByteCountFormatter.string(fromByteCount: Int64(b), countStyle: .file) -} - -/// Long-context settings of `mlxr bench` (MLXR_BENCH_CTX turns the section on). -struct LongBench { - let contexts: [Int] - /// Teacher-forced positions of the text after the prompt (MLXR_BENCH_POSITIONS, default 512). - let positions: Int - /// Token offsets into the text where prompts start (MLXR_BENCH_TEXT_OFFSETS, default 0). - let offsets: [Int] - static let steps = 64 - /// Rows the model needs past the longest prompt: the measured positions plus a needle question. - var margin: Int { max(Self.steps, positions) + 96 } - - init(env: [String: String]) { - func ints(_ key: String) -> [Int] { - (env[key] ?? "").split(separator: ",").compactMap { Int($0.trimmingCharacters(in: .whitespaces)) } - } - contexts = ints("MLXR_BENCH_CTX").filter { $0 > 0 } - positions = min(max(ints("MLXR_BENCH_POSITIONS").first ?? 512, 0), 8192) - let o = ints("MLXR_BENCH_TEXT_OFFSETS").filter { $0 >= 0 } - offsets = o.isEmpty ? [0] : o - } -} - -/// `mlxr bench` with MLXR_BENCH_CTX: decode after long prompts, dense and (for a model loaded with -/// MLXR_SPARSE_ATTN) sparse. For each context: -/// - greedy decode speed, and how many greedy tokens sparse decode shares with dense decode; -/// - teacher-forced on dense decode's tokens: a control (dense forward against the GPU's greedy -/// tokens, must be all) and sparse argmax agreement; -/// - teacher-forced on the text after the prompt (MLXR_BENCH_TEXT only): KL(dense‖sparse), the -/// text's NLL under both, and top-1 agreement by dense decode's top-1 margin. Each position runs -/// sparse first and then dense, so the cache stays dense decode's; -/// - the cost of a dense verification pass of T tokens at that context; -/// - needles (MLXR_BENCH_TEXT only): five passkeys at 10–90% depth, recalled dense and sparse. -/// The prompt is MLXR_BENCH_TEXT from each of MLXR_BENCH_TEXT_OFFSETS, or pseudo-random tokens, on -/// which agreement says little. -func benchLongContext(_ t: Transformer, dir: URL, bench: LongBench) throws { - var greedy = SamplingParameters() - greedy.temperature = 0 - greedy.repetitionPenalty = 1 - let sp = SampleParams(greedy, vocab: t.vocabSize) - let V = t.vocabSize - let tokenizer = try Tokenizer(directory: dir) - var text: [Int32] = [] - if let path = ProcessInfo.processInfo.environment["MLXR_BENCH_TEXT"] { - guard let s = try? String(contentsOfFile: path, encoding: .utf8) else { fail("cannot read MLXR_BENCH_TEXT \(path)") } - text = tokenizer.encode(s).map { Int32($0) }.filter { $0 >= 0 && Int($0) < V } - guard !text.isEmpty else { fail("MLXR_BENCH_TEXT \(path) has no tokens") } - let bos = tokenizer.bosToken.flatMap { b in tokenizer.tokenId(b).map { " (prompts start with \(b), id \($0))" } } ?? "" - print("long-context prompt: \(path), \(text.count) tokens (repeated to length)\(bos)") - } else { - print("long-context prompt: pseudo-random tokens (MLXR_BENCH_TEXT= for real text)") - } - let original = t.sparse - defer { t.sparse = original } - // Hybrid models (recurrent state): every rerun of a position first restores the state there. - var base: Transformer.RecurrentSnapshot?, last: Transformer.RecurrentSnapshot? - func mark() { if t.config.isHybrid { last = t.snapshotRecurrent() } } - func seek(_ pos: Int) { - guard t.config.isHybrid, t.recurrentPosition != pos else { return } - guard let s = [base, last].compactMap({ $0 }).first(where: { $0.position == pos }) else { fail("no recurrent snapshot at \(pos)") } - t.restoreRecurrent(s) - } - let sparse = t.sparseAvailable ? original : nil - let steps = LongBench.steps - func logits(_ tokens: [Int32], offset: Int) throws -> [Float] { - Array(UnsafeBufferPointer(start: try t.forward(tokens, offset: offset), count: V)) - } - func argmax(_ x: [Float]) -> Int { - var best = 0 - for v in 1.. x[best] { best = v } - return best - } - func logSoftmax(_ x: [Float]) -> [Double] { - let m = Double(x.max() ?? 0) - var s = 0.0 - for v in x { s += exp(Double(v) - m) } - let lse = m + log(s) - return x.map { Double($0) - lse } - } - for ctx in bench.contexts { - guard ctx + bench.margin < t.maxSeq else { print("decode @\(ctx): skipped (the model's context is \(t.maxSeq))"); continue } - for (oi, off) in (text.isEmpty ? [0] : bench.offsets).enumerated() { - var rng: UInt64 = 0x9E3779B97F4A7C15 - let lo = min(1000, V / 2), span = UInt64(max(1, min(V, 30000) - lo)) - func token(_ i: Int) -> Int32 { - if !text.isEmpty { return text[(off + i) % text.count] } - rng ^= rng << 13; rng ^= rng >> 7; rng ^= rng << 17 - return Int32(lo + Int(rng % span)) - } - var prompt = (0..= 0, id < V { prompt[0] = Int32(id) } - } - let at = off > 0 ? " from text token \(off)" : "" - t.sparse = nil - t.seedSampler(1) - let s0 = Date() - let first = try t.prefill(prompt, offset: 0, sample: sp) - let prefillTime = Date().timeIntervalSince(s0) - if t.config.isHybrid { base = t.snapshotRecurrent() } - print(String(format: "prefill @%d%@: %.1f s (%.0f tok/s)", ctx, at as NSString, prefillTime, Double(ctx) / prefillTime)) - // Each run decodes from the same prompt (positions from ctx on are rewritten, the index with them). - var dense: [Int] = [] - var modes: [(String, SparseAttention?)] = [("dense", nil)] - if let sparse, oi == 0 { modes.append(("sparse", sparse)) } - for (label, mode) in modes { - t.sparse = mode - t.seedSampler(1) - seek(ctx) - var tokens: [Int] = [] - var gpu = 0.0 - let s = Date() - var cur = try t.beginDecodeStep(offset: ctx, sample: sp) - for i in 0.. 0 { - // Teacher-forced on the text after the prompt. - let N = bench.positions - var kl: [Double] = [], nllDense = 0.0, nllSparse = 0.0, agree = 0 - let edges: [Float] = [0.1, 0.5, 1] - var bucketTotal = [Int](repeating: 0, count: edges.count + 1), bucketAgree = bucketTotal - seek(ctx) - for i in 0.. -40 { k += exp(pd[v]) * (pd[v] - ps[v]) } - kl.append(max(0, k)) - nllDense -= pd[target] - nllSparse -= ps[target] - let top = argmax(ld) - var second = top == 0 ? 1 : 0 - for v in 0.. ld[second] { second = v } - let bucket = edges.firstIndex { ld[top] - ld[second] < $0 } ?? edges.count - bucketTotal[bucket] += 1 - if argmax(ls) == top { agree += 1; bucketAgree[bucket] += 1 } - } - kl.sort() - let meanKL = kl.reduce(0, +) / Double(N) - let dNLL = (nllSparse - nllDense) / Double(N) - print(String(format: " teacher-forced on the text (%d positions): KL(dense‖sparse) mean %.4f, median %.4f, p95 %.4f, max %.3f nats", - N, meanKL, kl[N / 2], kl[min(N - 1, N * 95 / 100)], kl[N - 1])) - print(String(format: " text NLL: dense %.4f, sparse %.4f nats/token (Δ %+.4f, perplexity ×%.4f)", - nllDense / Double(N), nllSparse / Double(N), dNLL, exp(dNLL))) - let labels = ["<0.1", "0.1–0.5", "0.5–1", "≥1"] - let byMargin = zip(labels, zip(bucketAgree, bucketTotal)).map { "\($0.0): \($0.1.0)/\($0.1.1)" }.joined(separator: ", ") - print(String(format: " top-1 agreement %d/%d (%.1f%%); by dense top-1 margin (nats) %@", agree, N, - 100 * Double(agree) / Double(N), byMargin as NSString)) - } - if oi == 0 && !t.config.isHybrid { - // A dense verification pass of T tokens at this context (speculative decoding). - t.sparse = nil - var line: [String] = [] - for T in [1, 2, 5, 8] where T <= Transformer.maxVerifyTokens { - var gpu = 0.0 - for i in 0..<4 { - _ = try t.forward((0.. ...") - exit(0) -} -args.removeFirst() - -switch cmd { -case "search": - let results = try await HubClient.search(args.joined(separator: " "), limit: 25) - for m in results { - print(String(format: "%-60@ %10d ↓ %@", m.id as NSString, m.downloads ?? 0, (m.modelTypeTag ?? "") as NSString)) - } - -case "download": - guard let repo = args.first else { fail("usage: mlxr download ") } - guard RepoID.isValid(repo) else { fail("invalid repo id: \(repo) (expected owner/name)") } - let details = try await HubClient.details(repo) - if !details.isSupported { print("warning: \(details.modelType ?? "unknown") architecture is not supported by the engine") } - print("Downloading \(repo) (\(formatBytes(details.downloadSize)))") - let dl = ModelDownload(details: details, destination: try ModelStore.directory(for: repo)) - let last = LockedDate() - dl.onProgress = { p, done, total, speed in - guard last.elapsed(0.25) || p >= 1 else { return } - print(String(format: "\r %5.1f%% %@ / %@ %@/s ", p * 100, formatBytes(Int(done)), formatBytes(Int(total)), formatBytes(Int(speed))), terminator: "") - } - dl.onEvent = { print("\n \($0)") } - do { - let dir = try await dl.start() - print("\nSaved to \(dir.path)") - } catch { - fail("\ndownload failed: \(error.localizedDescription)") - } - -case "list": - for m in ModelStore.scan() { - print(String(format: "%-60@ %10@ %@%@", m.repo as NSString, formatBytes(m.sizeBytes) as NSString, - (m.modelType ?? "?") as NSString, m.isSupported ? "" : " (unsupported)")) - } - -case "tokenize": - guard args.count >= 2 else { fail("usage: mlxr tokenize ") } - let tok = try Tokenizer(directory: resolve(args[0])) - let ids = tok.encode(args[1...].joined(separator: " ")) - print(ids) - print(ids.map { tok.token($0) ?? "?" }) - print("roundtrip:", tok.decode(ids, includeSpecial: true)) - -case "run": - let prompt = option("-p") ?? "Write a haiku about Apple Silicon." - let maxTokens = Int(option("--max") ?? "256") ?? 256 - let temp = Float(option("--temp") ?? "0") ?? 0 - let ctx = Int(option("--ctx") ?? "4096") ?? 4096 - let raw = flag("--raw") - // Neither flag: the template's default (Qwen 3.5 0.8B and 2B answer directly, Qwen3 thinks). - let thinking: Bool? = flag("--no-think") ? false : flag("--think") ? true : nil - let showIds = flag("--ids") - let draftArg = option("--draft") - let sparseArg = sparseOption(option("--sparse")) - guard let model = args.first else { fail("usage: mlxr run [-p prompt]") } - let engine = InferenceEngine.shared - let dir = resolve(model) - let info = try await engine.load(directory: dir, draftDirectory: draftDirectory(draftArg, target: dir), contextLength: ctx, sparse: sparseArg) - print("Loaded \(info.name) [\(info.modelType), \(info.quantization), \(info.layers) layers, \(info.chatFormat)]\(info.draftName.map { " + draft \($0)" } ?? "") in \(String(format: "%.2f", info.loadSeconds))s") - print(sparseSummary(info)) - var params = SamplingParameters() - params.temperature = temp - params.maxTokens = maxTokens - params.repetitionPenalty = 1.0 - params.seed = 42 - if flag("--spec") { params.speculative = true } - let stream = raw ? engine.complete(prompt, parameters: params) - : engine.chat([ChatMessage(role: .user, content: prompt)], parameters: params, enableThinking: thinking) - if showIds { - let p = raw ? prompt : engine.renderChat([ChatMessage(role: .user, content: prompt)], enableThinking: thinking) - print("prompt ids:", engine.encode(p)) - } - for try await ev in stream { - switch ev { - case .promptProcessed(let s): - print(String(format: "[prompt: %d tokens, %.0f tok/s, ttft %.3fs]", s.promptTokens, s.promptTokensPerSecond, s.timeToFirstToken)) - case .text(let t): print(t, terminator: "") - case .finished(let s): - let model = s.modelDraftTokens.map { String(format: " (draft model %d/%d)", s.acceptedModelDraftTokens ?? 0, $0) } ?? "" - print(String(format: "\n[generated %d tokens at %.1f tok/s, stop: %@, drafts accepted %d/%d%@]", s.generatedTokens, s.tokensPerSecond, s.stopReason, s.acceptedDraftTokens ?? 0, s.draftTokens ?? 0, model)) - if let c = s.draftCycles, c > 0, let dt = s.draftSeconds, let vt = s.verifySeconds { - print(String(format: "[draft model: %d cycles, %.2f tokens/cycle, draft %.1f ms + verify %.1f ms per cycle]", c, - Double(s.acceptedModelDraftTokens ?? 0) / Double(c) + 1, dt / Double(c) * 1000, vt / Double(c) * 1000)) - } - } - } - -case "serve": - let port = UInt16(option("--port") ?? "1234") ?? 1234 - let ctx = Int(option("--ctx") ?? "8192") ?? 8192 - let lan = flag("--lan") - let draftArg = option("--draft") - let sparseArg = sparseOption(option("--sparse")) - guard let model = args.first else { fail("usage: mlxr serve [--port 1234] [--ctx N] [--lan] [--api-key KEY] [--draft ] [--sparse ]") } - // LAN mode always needs a key: use --api-key / MLXR_API_KEY, or generate one. - var key = option("--api-key") ?? ProcessInfo.processInfo.environment["MLXR_API_KEY"] - if lan && key == nil { key = OpenAIServer.generateAPIKey() } - let dir = resolve(model) - let info = try await InferenceEngine.shared.load(directory: dir, draftDirectory: draftDirectory(draftArg, target: dir), contextLength: ctx, sparse: sparseArg) - print("Loaded \(info.name)\(info.draftName.map { " + draft \($0)" } ?? "") in \(String(format: "%.2f", info.loadSeconds))s") - print(sparseSummary(info)) - let server = OpenAIServer(port: port, allowLAN: lan, apiKey: key) - server.onLog = { print($0) } - do { try server.start() } catch { fail("cannot start server: \(error.localizedDescription)") } - print("OpenAI-compatible server on http://\(lan ? "0.0.0.0" : "127.0.0.1"):\(port)/v1 (Ctrl-C to stop)") - if let key { print("API key (send as 'Authorization: Bearer '): \(key)") } - while true { try await Task.sleep(nanoseconds: 3_600_000_000_000) } - -case "bench": - guard let model = args.first else { fail("usage: mlxr bench ") } - let dir = resolve(model) - // Long-context decode (opt-in): MLXR_BENCH_CTX=8192,32768 adds a section at the end. - let longBench = LongBench(env: ProcessInfo.processInfo.environment) - let t = try Transformer(directory: dir, contextLength: max(4096, (longBench.contexts.max() ?? 0) + longBench.margin + 64)) - print("weights: \(formatBytes(t.weightBytes))") - if let sp = t.sparse { - print(t.sparseAvailable - ? "sparse decode attention: top-k \(sp.topK), window \(sp.window), sink \(sp.sink), from \(sp.minContext) tokens" - : "sparse decode attention: off (\(t.sparseOffReason ?? "unavailable"))") - } - print("attention: \(t.attentionSummary)") - let quick = ProcessInfo.processInfo.environment["MLXR_BENCH_DECODE"] != nil - let noProfile = quick || ProcessInfo.processInfo.environment["MLXR_BENCH_NOPROFILE"] != nil - if !noProfile { print(try t.profile()) } - if ProcessInfo.processInfo.environment["MLXR_BENCH_PROFILE_ONLY"] != nil { exit(0) } - _ = try t.forward([1], offset: 0) - for n in (quick ? [] : [128, 512, 2048]) { - let toks = (0.. 0 { - print(String(format: " sampler: %.2f attempts and pool %.0f per token on average, at most %d attempts", Double(diag.attempts) / Double(steps + 1), Double(diag.pool) / Double(steps + 1), diag.maxAttempts)) - } - } - } - // Speculative verification cost: one forward over T tokens with logits for every row. - // At a short context and after a 1900-token prompt (the attention part grows with the context). - // (A recurrent state cannot rewind to rerun a position: no verification on hybrid models.) - for ctx in t.config.isHybrid ? [] : [64, 1900] { - if ctx > 64 { _ = try t.forward((0.. 64 ? "verify @\(ctx)" : "verify" - print(String(format: "%@ T=%d: %6.2f ms per pass (%5.2f ms per token, gpu %.2f ms)", label as NSString, T, dt * 1000, dt * 1000 / Double(T), gpu / Double(steps) * 1000)) - } - } - for T in [5, 8, 12, 15] { - let s = Date() - for i in 0..<8 { _ = try t.forward((0.. Bool { - lock.lock(); defer { lock.unlock() } - guard Date().timeIntervalSince(date) >= s else { return false } - date = Date() - return true - } -} +exit(await CLI.main(Array(CommandLine.arguments.dropFirst()))) diff --git a/Tests/MLXRCLITests/ErrorTests.swift b/Tests/MLXRCLITests/ErrorTests.swift new file mode 100644 index 0000000..ae2be14 --- /dev/null +++ b/Tests/MLXRCLITests/ErrorTests.swift @@ -0,0 +1,83 @@ +import Foundation +import Testing +@testable import MLXRCLI + +@Suite("Errors and exit codes") +struct ErrorTests { + @Test func unknownCommand() async { + let cap = Capture() + let code = await CLI.run(["frobnicate"], console: cap.console()) + #expect(code == 1) + #expect(cap.stderr == "Error: unknown command \"frobnicate\" for \"mlxr\"\n") + #expect(cap.stdout.isEmpty) + } + + @Test func unknownCommandSuggestsTheClosest() async { + let cap = Capture() + let code = await CLI.run(["lst"], console: cap.console()) + #expect(code == 1) + #expect(cap.stderr.hasPrefix("Error: unknown command \"lst\" for \"mlxr\"\n\nDid you mean this?\n\tlist\n"), "\(cap.stderr)") + #expect(cap.stdout.isEmpty) + } + + @Test func unknownFlag() async { + let cap = Capture() + let code = await CLI.run(["list", "--x"], console: cap.console()) + #expect(code == 1) + #expect(cap.stderr == "Error: unknown flag: --x\n") + #expect(cap.stdout.isEmpty) + } + + @Test func wrongArgumentCountOnRealTable() async { + let cap = Capture() + let code = await CLI.run(["show"], console: cap.console()) + #expect(code == 1) + #expect(cap.stderr == "Error: accepts 1 arg(s), received 0\n") + #expect(cap.stdout.isEmpty) + } + + @Test func throwingCommandPrintsErrorAndExits1() async { + let cmd = Command(name: "boom", summary: "x", run: { _, _ in throw CLIError.failure("it broke") }) + let cap = Capture() + let code = await CLI.run(["boom"], console: cap.console(), commands: [cmd]) + #expect(code == 1) + #expect(cap.stderr == "Error: it broke\n") + #expect(cap.stdout.isEmpty) + } + + @Test func foreignErrorsUseTheirDescription() async { + struct Oops: LocalizedError { var errorDescription: String? { "disk on fire" } } + let cmd = Command(name: "boom", summary: "x", run: { _, _ in throw Oops() }) + let cap = Capture() + let code = await CLI.run(["boom"], console: cap.console(), commands: [cmd]) + #expect(code == 1) + #expect(cap.stderr == "Error: disk on fire\n") + } + + @Test func huggingFace401And404MapToOneHint() async { + let hint = "not found, or private/gated (set HF_TOKEN)" + for status in [401, 404] { + let err = NSError(domain: "HuggingFace", code: status, + userInfo: [NSLocalizedDescriptionKey: "Hugging Face returned HTTP \(status)"]) + #expect(CLI.message(for: err) == hint) + let cmd = Command(name: "boom", summary: "x", run: { _, _ in throw err }) + let cap = Capture() + #expect(await CLI.run(["boom"], console: cap.console(), commands: [cmd]) == 1) + #expect(cap.stderr == "Error: \(hint)\n") + } + } + + @Test func otherHuggingFaceErrorsKeepTheirMessage() { + let err = NSError(domain: "HuggingFace", code: 500, + userInfo: [NSLocalizedDescriptionKey: "Hugging Face returned HTTP 500"]) + #expect(CLI.message(for: err) == "Hugging Face returned HTTP 500") + #expect(CLI.message(for: CLIError.failure("plain")) == "plain") + } + + @Test func errorsNeverPrintUsage() async { + let cap = Capture() + _ = await CLI.run(["list", "--x"], console: cap.console()) + #expect(!cap.stderr.contains("Usage:")) + #expect(!cap.stderr.contains("Fatal error")) + } +} diff --git a/Tests/MLXRCLITests/HelpTests.swift b/Tests/MLXRCLITests/HelpTests.swift new file mode 100644 index 0000000..07d281c --- /dev/null +++ b/Tests/MLXRCLITests/HelpTests.swift @@ -0,0 +1,90 @@ +import Testing +@testable import MLXRCLI +import MLXEngine + +@Suite("Help and version") +struct HelpTests { + func output(_ argv: [String], commands: [Command] = CLI.commands) async -> (code: Int32, out: String, err: String) { + let cap = Capture() + let code = await CLI.run(argv, console: cap.console(), commands: commands) + return (code, cap.stdout, cap.stderr) + } + + @Test func bareAndDashHelpAreIdenticalRootHelp() async { + let bare = await output([]) + let flag = await output(["--help"]) + let short = await output(["-h"]) + #expect(bare.code == 0 && flag.code == 0 && short.code == 0) + #expect(bare.out == flag.out) + #expect(bare.out == short.out) + #expect(bare.err.isEmpty) + for needle in ["Usage:", "Available Commands:", "Flags:", + "Use \"mlxr [command] --help\" for more information about a command."] { + #expect(bare.out.contains(needle), "\(needle)") + } + for cmd in ["run", "serve", "download", "list", "show"] { + #expect(bare.out.contains("\n \(cmd) "), "\(cmd)") + } + } + + @Test func developerCommandsSitUnderTheirOwnHeading() async { + let out = await output([]).out + let dev = out.range(of: "Developer") + #expect(dev != nil) + for cmd in ["bench", "tokenize"] { + let at = out.range(of: "\n \(cmd) ") + #expect(at != nil, "\(cmd)") + if let dev, let at { #expect(dev.lowerBound < at.lowerBound, "\(cmd) below Developer") } + } + if let dev, let run = out.range(of: "\n run ") { #expect(run.lowerBound < dev.lowerBound) } + } + + @Test func commandHelpListsFlagsAndEnvironment() async { + let r = await output(["run", "-h"]) + #expect(r.code == 0) + #expect(r.out.contains("Usage:")) + #expect(r.out.contains("Flags:")) + #expect(r.out.contains("Environment Variables:")) + #expect(r.out.contains("--temp")) + #expect(r.err.isEmpty) + } + + @Test func helpCommandEqualsDashH() async { + let viaHelp = await output(["help", "run"]) + let viaFlag = await output(["run", "-h"]) + #expect(viaHelp.code == 0) + #expect(!viaHelp.out.isEmpty) + #expect(viaHelp.out == viaFlag.out) + let helpOnly = await output(["help"]) + let root = await output([]) + #expect(helpOnly.out == root.out) + } + + @Test func helpFlagNeverRunsTheCommand() async { + let log = RunLog() + let r = await output(["run", "-h"], commands: miniTable(log)) + #expect(r.code == 0) + #expect(log.ran.isEmpty) + #expect(r.out.contains("Usage:")) + let search = await output(["search", "-h"]) + #expect(search.out.contains("Usage:")) + #expect(search.code == 0) + } + + @Test func helpShowsAliasesAndExamplesWhenDeclared() async { + let cmd = Command(name: "list", aliases: ["ls"], summary: "List models", examples: [" mlxr list"]) + let r = await output(["list", "--help"], commands: [cmd]) + #expect(r.out.contains("Aliases:\n list, ls")) + #expect(r.out.contains("Examples:\n mlxr list")) + } + + @Test func versionInAllThreeForms() async { + let expected = "mlxr version is \(MLXRunnerVersion.current)\n" + for argv in [["-v"], ["--version"], ["version"]] { + let r = await output(argv) + #expect(r.code == 0, "\(argv)") + #expect(r.out == expected, "\(argv)") + #expect(r.err.isEmpty, "\(argv)") + } + } +} diff --git a/Tests/MLXRCLITests/ParserTests.swift b/Tests/MLXRCLITests/ParserTests.swift new file mode 100644 index 0000000..785dcd4 --- /dev/null +++ b/Tests/MLXRCLITests/ParserTests.swift @@ -0,0 +1,99 @@ +import Testing +@testable import MLXRCLI + +@Suite("Parser grammar") +struct ParserTests { + @Test func equalsFormDoubleDashAndMixedPositionals() throws { + let inv = try CLI.parse(["run", "m", "--temp=0.5", "-p", "hi", "--", "-x"], commands: miniTable()) + #expect(inv.command.name == "run") + #expect(inv.double("temp") == 0.5) + #expect(inv.string("prompt") == "hi") + #expect(inv.positionals == ["m", "-x"]) + } + + @Test func optionsMixFreelyWithPositionals() throws { + let inv = try CLI.parse(["run", "--temp", "1", "m", "word", "--verbose"], commands: miniTable()) + #expect(inv.positionals == ["m", "word"]) + #expect(inv.double("temp") == 1) + #expect(inv.bool("verbose")) + } + + @Test func thinkAloneIsTrueAndTakesValueOnlyViaEquals() throws { + let alone = try CLI.parse(["run", "m", "--think"], commands: miniTable()) + #expect(alone.string("think") == "true") + #expect(alone.bool("think")) + + let spaced = try CLI.parse(["run", "--think", "high", "m"], commands: miniTable()) + #expect(spaced.string("think") == "true") + #expect(spaced.positionals == ["high", "m"]) + + let equals = try CLI.parse(["run", "m", "--think=high"], commands: miniTable()) + #expect(equals.string("think") == "high") + #expect(equals.positionals == ["m"]) + } + + @Test func lastValueWins() throws { + let inv = try CLI.parse(["run", "m", "--temp", "0.1", "--temp", "0.7"], commands: miniTable()) + #expect(inv.double("temp") == 0.7) + } + + @Test func boolAcceptsEqualsTrueFalse() throws { + let off = try CLI.parse(["run", "m", "--verbose=false"], commands: miniTable()) + #expect(!off.bool("verbose")) + let on = try CLI.parse(["run", "m", "--verbose=true"], commands: miniTable()) + #expect(on.bool("verbose")) + #expect(!(try CLI.parse(["run", "m"], commands: miniTable())).bool("verbose")) + } + + @Test func aliasesResolveToTheirCommand() throws { + #expect(try CLI.parse(["ls"], commands: miniTable()).command.name == "list") + #expect(try CLI.parse(["pull", "a/b"], commands: miniTable()).command.name == "download") + #expect(try CLI.parse(["download", "a/b"], commands: miniTable()).command.name == "download") + } + + @Test func realTableResolvesLsAndPull() throws { + #expect(try CLI.parse(["ls"]).command.name == "list") + #expect(try CLI.parse(["pull", "a/b"]).command.name == "download") + } + + @Test func invalidValuesNameTheFlag() async { + for (argv, flag) in [(["run", "m", "--temp", "abc"], "--temp"), (["run", "m", "--count", "11"], "--count"), + (["run", "m", "--count", "0"], "--count"), (["run", "m", "--think=maybe"], "--think")] { + let cap = Capture() + let code = await CLI.run(argv, console: cap.console(), commands: miniTable()) + #expect(code == 1, "\(argv)") + #expect(cap.stderr.hasPrefix("Error: invalid argument \"\(argv.last!.split(separator: "=").last!)\" for \"\(flag)\" flag"), + "\(argv): \(cap.stderr)") + #expect(cap.stdout.isEmpty) + } + } + + @Test func missingValueIsAnError() async { + let cap = Capture() + let code = await CLI.run(["run", "m", "--temp"], console: cap.console(), commands: miniTable()) + #expect(code == 1) + #expect(cap.stderr == "Error: flag needs an argument: --temp\n") + } + + @Test func acceptsExactlyTheDeclaredArgumentCounts() async { + let log = RunLog() + let cap = Capture() + #expect(await CLI.run(["download"], console: cap.console(), commands: miniTable(log)) == 1) + #expect(cap.stderr == "Error: accepts 1 arg(s), received 0\n") + let cap2 = Capture() + #expect(await CLI.run(["download", "a/b", "c/d"], console: cap2.console(), commands: miniTable(log)) == 1) + #expect(cap2.stderr == "Error: accepts 1 arg(s), received 2\n") + let cap3 = Capture() + #expect(await CLI.run(["run"], console: cap3.console(), commands: miniTable(log)) == 1) + #expect(cap3.stderr == "Error: requires at least 1 arg(s), only received 0\n") + #expect(log.ran.isEmpty) + } + + @Test func aValidCommandRuns() async { + let log = RunLog() + let cap = Capture() + #expect(await CLI.run(["ls"], console: cap.console(), commands: miniTable(log)) == 0) + #expect(log.ran == ["list"]) + #expect(cap.stderr.isEmpty) + } +} diff --git a/Tests/MLXRCLITests/TestSupport.swift b/Tests/MLXRCLITests/TestSupport.swift new file mode 100644 index 0000000..d94e1ac --- /dev/null +++ b/Tests/MLXRCLITests/TestSupport.swift @@ -0,0 +1,47 @@ +import Foundation +@testable import MLXRCLI + +/// Collects what a command writes, per stream. +final class Capture: @unchecked Sendable { + private let lock = NSLock() + private var o = "", e = "" + var stdout: String { lock.withLock { o } } + var stderr: String { lock.withLock { e } } + + func console(stdoutTTY: Bool = false, stderrTTY: Bool = false, stdinTTY: Bool = false, + width: Int = 80, env: [String: String] = [:]) -> Console { + Console(out: { s in self.lock.withLock { self.o += s } }, + err: { s in self.lock.withLock { self.e += s } }, + stdoutIsTTY: stdoutTTY, stderrIsTTY: stderrTTY, stdinIsTTY: stdinTTY, width: width, env: env) + } +} + +/// Records which commands ran (and with what), so tests can prove help and errors never execute one. +final class RunLog: @unchecked Sendable { + private let lock = NSLock() + private var items: [String] = [] + var ran: [String] { lock.withLock { items } } + func add(_ s: String) { lock.withLock { items.append(s) } } +} + +/// A small table exercising the grammar independent of the real commands. +func miniTable(_ log: RunLog = RunLog()) -> [Command] { + func rec(_ n: String) -> (Invocation, Console) async throws -> Void { { _, _ in log.add(n) } } + return [ + Command(name: "run", summary: "Run a model", + args: [Arg("MODEL", kind: .model), Arg("PROMPT", optional: true, variadic: true)], + options: [ + Opt(long: "temp", value: .real(0...2), help: "temperature"), + Opt(long: "prompt", short: "p", value: .text, help: "prompt"), + Opt(long: "think", value: .choice(["true", "false", "high", "medium", "low"]), + implicit: "true", help: "think"), + Opt(long: "verbose", help: "stats"), + Opt(long: "count", value: .int(1...10), help: "count"), + ], run: rec("run")), + Command(name: "list", aliases: ["ls"], summary: "List models", run: rec("list")), + Command(name: "download", aliases: ["pull"], summary: "Download a model", + args: [Arg("MODEL", kind: .model)], run: rec("download")), + Command(name: "bench", summary: "Benchmark", args: [Arg("MODEL", kind: .model)], + group: "Developer", run: rec("bench")), + ] +} From d5b4d8df120d765c44b7961e0cded51f46b42ccc Mon Sep 17 00:00:00 2001 From: Rolando Santamaria Maso Date: Sun, 8 Nov 2026 14:15:39 +0100 Subject: [PATCH 02/20] test(cli): command table, aliases, ranges and parser edge cases --- Tests/MLXRCLITests/CommandTableTests.swift | 75 +++++++++++++++++ Tests/MLXRCLITests/ParsingEdgeTests.swift | 93 ++++++++++++++++++++++ 2 files changed, 168 insertions(+) create mode 100644 Tests/MLXRCLITests/CommandTableTests.swift create mode 100644 Tests/MLXRCLITests/ParsingEdgeTests.swift diff --git a/Tests/MLXRCLITests/CommandTableTests.swift b/Tests/MLXRCLITests/CommandTableTests.swift new file mode 100644 index 0000000..58653c8 --- /dev/null +++ b/Tests/MLXRCLITests/CommandTableTests.swift @@ -0,0 +1,75 @@ +import Foundation +import Testing +@testable import MLXRCLI + +@Suite("Command table") +struct CommandTableTests { + @Test func runOptionsParseWithAliasesAndShorts() throws { + let inv = try CLI.parse(["run", "m", "hello", "world", "--max", "10", "-n", "20", "--temperature", "0.5", + "--ctx", "512", "--raw", "--no-think", "--ids", "--spec", "--draft", "auto", + "--sparse", "off"]) + #expect(inv.command.name == "run") + #expect(inv.positionals == ["m", "hello", "world"]) + #expect(inv.int("max-tokens") == 20) + #expect(inv.double("temp") == 0.5) + #expect(inv.int("ctx") == 512) + #expect(inv.bool("raw") && inv.bool("no-think") && inv.bool("ids") && inv.bool("spec")) + #expect(inv.string("draft") == "auto") + #expect(inv.string("sparse") == "off") + #expect(!inv.has("think")) + } + + @Test func serveAndStartAlias() throws { + let inv = try CLI.parse(["start", "m", "--port", "8080", "--lan", "--api-key", "k", "--ctx", "1024"]) + #expect(inv.command.name == "serve") + #expect(inv.int("port") == 8080) + #expect(inv.bool("lan")) + #expect(inv.string("api-key") == "k") + #expect(inv.positionals == ["m"]) + } + + @Test func optionsMayPrecedeTheModel() throws { + let inv = try CLI.parse(["serve", "--api-key", "K", "m"]) + #expect(inv.positionals == ["m"]) + #expect(inv.string("api-key") == "K") + let spec = try CLI.parse(["run", "--spec", "m"]) + #expect(spec.positionals == ["m"]) + #expect(spec.bool("spec")) + } + + @Test func searchTokenizeBenchAcceptTheirArguments() throws { + #expect(try CLI.parse(["search", "qwen", "4bit"]).positionals == ["qwen", "4bit"]) + #expect(try CLI.parse(["tokenize", "m", "some", "text"]).positionals == ["m", "some", "text"]) + #expect(try CLI.parse(["bench", "m"]).command.name == "bench") + #expect(try CLI.parse(["download", "a/b"]).positionals == ["a/b"]) + #expect(try CLI.parse(["list"]).command.name == "list") + } + + @Test func showIsRegisteredAndFailsCleanlyForNow() async { + let cap = Capture() + #expect(await CLI.run(["show", "m"], console: cap.console()) == 1) + #expect(cap.stderr.hasPrefix("Error: ")) + } + + @Test func rejectsOutOfRangeValuesBeforeRunning() async { + for argv in [["run", "m", "--max", "0"], ["run", "m", "--max", "32769"], ["run", "m", "--temp", "nan"], + ["run", "m", "--temp", "2.1"], ["run", "m", "--ctx", "255"], ["serve", "m", "--port", "0"], + ["serve", "m", "--port", "70000"]] { + let cap = Capture() + #expect(await CLI.run(argv, console: cap.console()) == 1, "\(argv)") + #expect(cap.stderr.hasPrefix("Error: invalid argument"), "\(argv): \(cap.stderr)") + } + } + + @Test func everyCommandHasHelp() async { + for c in CLI.commands { + let cap = Capture() + #expect(await CLI.run([c.name, "--help"], console: cap.console()) == 0) + #expect(cap.stdout.contains("Usage:\n mlxr \(c.name)")) + } + let cap = Capture() + #expect(await CLI.run(["serve", "-h"], console: cap.console()) == 0) + #expect(cap.stdout.contains("Environment Variables:") && cap.stdout.contains("MLXR_API_KEY")) + #expect(cap.stdout.contains("--port int")) + } +} diff --git a/Tests/MLXRCLITests/ParsingEdgeTests.swift b/Tests/MLXRCLITests/ParsingEdgeTests.swift new file mode 100644 index 0000000..8d6b136 --- /dev/null +++ b/Tests/MLXRCLITests/ParsingEdgeTests.swift @@ -0,0 +1,93 @@ +import Foundation +import Testing +@testable import MLXRCLI + +@Suite("Parsing edge cases") +struct ParsingEdgeTests { + let table: [Command] = [ + Command(name: "t", summary: "t", args: [Arg("A"), Arg("B", optional: true)], options: [ + Opt(long: "name", short: "p", value: .text), + Opt(long: "yes", short: "y"), + Opt(long: "seed", value: .uint64), + Opt(long: "sparse", value: .choiceOrInt(["on", "off"])), + Opt(long: "tag", value: .text, repeatable: true), + Opt(long: "hidden", hidden: true), + ]), + Command(name: "none", summary: "n"), + Command(name: "secret", summary: "s", hidden: true), + ] + + func code(_ argv: [String]) async -> (Int32, String, String) { + let cap = Capture() + let c = await CLI.run(argv, console: cap.console(), commands: table) + return (c, cap.stdout, cap.stderr) + } + + @Test func shortFormsAndBooleans() throws { + let a = try CLI.parse(["t", "x", "-p", "v", "-y"], commands: table) + #expect(a.string("name") == "v" && a.bool("yes")) + let b = try CLI.parse(["t", "x", "-pv"], commands: table) + #expect(b.string("name") == "v") + let c = try CLI.parse(["t", "x", "-p=v"], commands: table) + #expect(c.string("name") == "v") + let d = try CLI.parse(["t", "x", "--yes=0"], commands: table) + #expect(!d.bool("yes")) + #expect(try CLI.parse(["t", "x", "-"], commands: table).positionals == ["x", "-"]) + } + + @Test func repeatableCollectsAllValues() throws { + let a = try CLI.parse(["t", "x", "--tag", "a", "--tag=b"], commands: table) + #expect(a.strings("tag") == ["a", "b"]) + } + + @Test func kindValidation() async { + #expect(try! CLI.parse(["t", "x", "--sparse", "on"], commands: table).string("sparse") == "on") + #expect(try! CLI.parse(["t", "x", "--sparse", "64"], commands: table).string("sparse") == "64") + #expect(try! CLI.parse(["t", "x", "--seed", "42"], commands: table).string("seed") == "42") + for argv in [["t", "x", "--sparse", "0"], ["t", "x", "--sparse", "zz"], ["t", "x", "--seed", "-1"], + ["t", "x", "--yes=maybe"]] { + let r = await code(argv) + #expect(r.0 == 1 && r.2.hasPrefix("Error: invalid argument"), "\(argv): \(r.2)") + } + } + + @Test func flagErrors() async { + #expect(await code(["t", "x", "-q"]).2 == "Error: unknown shorthand flag: 'q' in -q\n") + #expect(await code(["t", "x", "-p"]).2 == "Error: flag needs an argument: 'p' in -p\n") + #expect(await code(["t", "x", "--name"]).2 == "Error: flag needs an argument: --name\n") + #expect(await code(["--bogus"]).2 == "Error: unknown flag: --bogus\n") + #expect(await code(["t", "x", "y", "z"]).2 == "Error: accepts between 1 and 2 arg(s), received 3\n") + #expect(await code(["none", "extra"]).2 == "Error: accepts 0 arg(s), received 1\n") + #expect(await code(["help", "nope"]).2 == "Error: unknown help topic \"nope\"\n") + } + + @Test func parseWithoutCommandThrows() { + #expect(throws: CLIError.self) { try CLI.parse([], commands: table) } + } + + @Test func hiddenThingsStayOutOfHelpAndSuggestions() async { + let root = await code([]) + #expect(!root.1.contains("secret")) + let help = await code(["t", "-h"]) + #expect(!help.1.contains("--hidden")) + #expect(help.1.contains("-p, --name string")) + #expect(help.1.contains("Usage:\n mlxr t A [B] [flags]")) + let miss = await code(["secrex"]) + #expect(!miss.2.contains("Did you mean")) + } + + @Test func interruptedExitsWith130AndNoMessage() async { + let cmd = Command(name: "i", summary: "i", run: { _, _ in throw CLIError.interrupted }) + let cap = Capture() + #expect(await CLI.run(["i"], console: cap.console(), commands: [cmd]) == 130) + #expect(cap.stderr.isEmpty) + #expect(CLI.message(for: CLIError.interrupted) == "interrupted") + } + + @Test func editDistance() { + #expect(CLI.editDistance("", "abc") == 3) + #expect(CLI.editDistance("abc", "") == 3) + #expect(CLI.editDistance("lst", "list") == 1) + #expect(CLI.editDistance("kitten", "sitting") == 3) + } +} From 2e00d6c27fa52ca633ae509de782cd91a22d6f4d Mon Sep 17 00:00:00 2001 From: Rolando Santamaria Maso Date: Sun, 8 Nov 2026 14:30:46 +0100 Subject: [PATCH 03/20] fix(cli): route command errors through Error: output and the Hugging Face hint --- Sources/MLXRCLI/Legacy.swift | 78 +++++++-------------- Sources/MLXRCLI/Parsing.swift | 2 +- Sources/MLXRCLI/Seams.swift | 46 ++++++++++++ Tests/MLXRCLITests/LegacyErrorTests.swift | 85 +++++++++++++++++++++++ 4 files changed, 156 insertions(+), 55 deletions(-) create mode 100644 Sources/MLXRCLI/Seams.swift create mode 100644 Tests/MLXRCLITests/LegacyErrorTests.swift diff --git a/Sources/MLXRCLI/Legacy.swift b/Sources/MLXRCLI/Legacy.swift index 6129711..e1e0e37 100644 --- a/Sources/MLXRCLI/Legacy.swift +++ b/Sources/MLXRCLI/Legacy.swift @@ -3,22 +3,10 @@ import MLXEngine // The original `mlxr` command bodies, kept as they were while the commands move onto the parser kit. -func fail(_ msg: String) -> Never { - FileHandle.standardError.write((msg + "\n").data(using: .utf8)!) - exit(1) -} - - -func resolve(_ s: String) -> URL { - if FileManager.default.fileExists(atPath: s) { return URL(fileURLWithPath: s) } - if let m = ModelStore.scan().first(where: { $0.repo == s || $0.name == s }) { return m.directory } - fail("model not found: \(s) (try `mlxr download \(s)`)") -} - /// `--draft `: the draft model for speculative decoding. -func draftDirectory(_ s: String?, target: URL) -> URL? { +func draftDirectory(_ s: String?, target: URL) throws -> URL? { guard let s else { return nil } - guard s == "auto" else { return resolve(s) } + guard s == "auto" else { return try resolveModel(s) } guard let m = InferenceEngine.autoDraft(for: target) else { print("no installed draft model pairs with \(target.lastPathComponent); running without one") return nil @@ -26,20 +14,6 @@ func draftDirectory(_ s: String?, target: URL) -> URL? { return m.directory } -/// `--sparse `: sparse decode attention for long contexts (N: long-range keys; the window, -/// minimum context and shared selection still come from MLXR_SPARSE_*). Without the -/// option: MLXR_SPARSE_ATTN, which defaults to on. -func sparseOption(_ s: String?) -> SparseAttention? { - guard let s else { return SparseAttention.fromEnvironment } - switch s.lowercased() { - case "on", "1": return .configured(topK: SparseAttention.standard.topK) - case "off", "0": return nil - default: - guard let n = Int(s), n > 0 else { fail("--sparse takes on, off or a number of long-range keys") } - return .configured(topK: n) - } -} - func sparseSummary(_ info: LoadedModelInfo) -> String { if info.sparseLayers > 0 { return "sparse attention on \(info.sparseLayers) layers from \(info.sparseMinContext) tokens (\(info.sparseTopK) long-range keys)" @@ -94,9 +68,9 @@ func benchLongContext(_ t: Transformer, dir: URL, bench: LongBench) throws { let tokenizer = try Tokenizer(directory: dir) var text: [Int32] = [] if let path = ProcessInfo.processInfo.environment["MLXR_BENCH_TEXT"] { - guard let s = try? String(contentsOfFile: path, encoding: .utf8) else { fail("cannot read MLXR_BENCH_TEXT \(path)") } + guard let s = try? String(contentsOfFile: path, encoding: .utf8) else { throw CLIError.failure("cannot read MLXR_BENCH_TEXT \(path)") } text = tokenizer.encode(s).map { Int32($0) }.filter { $0 >= 0 && Int($0) < V } - guard !text.isEmpty else { fail("MLXR_BENCH_TEXT \(path) has no tokens") } + guard !text.isEmpty else { throw CLIError.failure("MLXR_BENCH_TEXT \(path) has no tokens") } let bos = tokenizer.bosToken.flatMap { b in tokenizer.tokenId(b).map { " (prompts start with \(b), id \($0))" } } ?? "" print("long-context prompt: \(path), \(text.count) tokens (repeated to length)\(bos)") } else { @@ -107,9 +81,9 @@ func benchLongContext(_ t: Transformer, dir: URL, bench: LongBench) throws { // Hybrid models (recurrent state): every rerun of a position first restores the state there. var base: Transformer.RecurrentSnapshot?, last: Transformer.RecurrentSnapshot? func mark() { if t.config.isHybrid { last = t.snapshotRecurrent() } } - func seek(_ pos: Int) { + func seek(_ pos: Int) throws { guard t.config.isHybrid, t.recurrentPosition != pos else { return } - guard let s = [base, last].compactMap({ $0 }).first(where: { $0.position == pos }) else { fail("no recurrent snapshot at \(pos)") } + guard let s = [base, last].compactMap({ $0 }).first(where: { $0.position == pos }) else { throw CLIError.failure("no recurrent snapshot at \(pos)") } t.restoreRecurrent(s) } let sparse = t.sparseAvailable ? original : nil @@ -170,7 +144,7 @@ func benchLongContext(_ t: Transformer, dir: URL, bench: LongBench) throws { for (label, mode) in modes { t.sparse = mode t.seedSampler(1) - seek(ctx) + try seek(ctx) var tokens: [Int] = [] var gpu = 0.0 let s = Date() @@ -198,12 +172,12 @@ func benchLongContext(_ t: Transformer, dir: URL, bench: LongBench) throws { // position, so the cache stays dense decode's). var match = 0, control = 0 var input = Int32(first) - seek(ctx) + try seek(ctx) for i in 0..'): \(key)") } while true { try await Task.sleep(nanoseconds: 3_600_000_000_000) } diff --git a/Sources/MLXRCLI/Parsing.swift b/Sources/MLXRCLI/Parsing.swift index 5d7013d..e9d52e0 100644 --- a/Sources/MLXRCLI/Parsing.swift +++ b/Sources/MLXRCLI/Parsing.swift @@ -43,7 +43,7 @@ extension CLI { if rest.hasPrefix("=") { rest.removeFirst() } inline = rest } - opt = o; shown = "--\(o.long)" + opt = o; shown = o.short.map { "-\($0), --\(o.long)" } ?? "--\(o.long)" } let o = opt! let raw: String diff --git a/Sources/MLXRCLI/Seams.swift b/Sources/MLXRCLI/Seams.swift new file mode 100644 index 0000000..ff39fa9 --- /dev/null +++ b/Sources/MLXRCLI/Seams.swift @@ -0,0 +1,46 @@ +import Foundation +import MLXEngine + +// Error-reporting seams for the legacy command bodies: they throw instead of calling exit(1), +// so every failure reaches CLI.run and prints `Error: `. + +/// A model by path, installed repo id or short name; throws when nothing matches. +func resolveModel(_ s: String) throws -> URL { + if FileManager.default.fileExists(atPath: s) { return URL(fileURLWithPath: s) } + if let m = ModelStore.scan().first(where: { $0.repo == s || $0.name == s }) { return m.directory } + throw CLIError.failure("model not found: \(s) (try `mlxr download \(s)`)") +} + +/// `--sparse `: sparse decode attention for long contexts (N: long-range keys; the window, +/// minimum context and shared selection still come from MLXR_SPARSE_*). Without the +/// option: MLXR_SPARSE_ATTN, which defaults to on. +func sparseSetting(_ s: String?) throws -> SparseAttention? { + guard let s else { return SparseAttention.fromEnvironment } + switch s.lowercased() { + case "on", "1": return .configured(topK: SparseAttention.standard.topK) + case "off", "0": return nil + default: + guard let n = Int(s), n > 0 else { throw CLIError.failure("--sparse takes on, off or a number of long-range keys") } + return .configured(topK: n) + } +} + +/// Repo ids must be `owner/name` and pass `RepoID.isValid`. +func validateRepoID(_ s: String) throws { + guard RepoID.isValid(s) else { throw CLIError.failure("invalid repo id: \(s) (expected owner/name)") } +} + +/// Runs a download. Hugging Face errors pass through (CLI.run maps 401/404 to the hint); others +/// become `download failed: `. +func performDownload(start: () async throws -> URL) async throws -> URL { + do { return try await start() } catch { + if (error as NSError).domain == "HuggingFace", [401, 404].contains((error as NSError).code) { throw error } + if error is CLIError { throw error } + throw CLIError.failure("download failed: \(error.localizedDescription)") + } +} + +/// Starts the server; a failure becomes `cannot start server: `. +func startServer(_ start: () throws -> Void) throws { + do { try start() } catch { throw CLIError.failure("cannot start server: \(error.localizedDescription)") } +} diff --git a/Tests/MLXRCLITests/LegacyErrorTests.swift b/Tests/MLXRCLITests/LegacyErrorTests.swift new file mode 100644 index 0000000..246f553 --- /dev/null +++ b/Tests/MLXRCLITests/LegacyErrorTests.swift @@ -0,0 +1,85 @@ +import Foundation +import Testing +import MLXEngine +@testable import MLXRCLI + +@Suite("Legacy command errors reach CLI.run") +struct LegacyErrorTests { + func message(_ body: () throws -> Void) -> String? { + do { try body(); return nil } catch { return CLI.message(for: error) } + } + + @Test func modelNotFound() { + let m = message { _ = try resolveModel("definitely-not-installed/model-xyz") } + #expect(m == "model not found: definitely-not-installed/model-xyz (try `mlxr download definitely-not-installed/model-xyz`)") + } + + @Test func existingPathResolves() throws { + let dir = FileManager.default.temporaryDirectory + #expect(try resolveModel(dir.path).path == dir.path) + } + + @Test func sparseValues() throws { + #expect(try sparseSetting("off") == nil) + #expect(try sparseSetting("0") == nil) + #expect(try sparseSetting("on") != nil) + #expect(try sparseSetting("12") != nil) + for bad in ["bogus", "-3", ""] { + #expect(message { _ = try sparseSetting(bad) } == "--sparse takes on, off or a number of long-range keys", "\(bad)") + } + } + + @Test func invalidRepoID() throws { + try validateRepoID("mlx-community/Qwen3-4B-4bit") + for bad in ["../x", "a/b/c", "noslash"] { + #expect(message { try validateRepoID(bad) } == "invalid repo id: \(bad) (expected owner/name)", "\(bad)") + } + } + + @Test func serverStartFailure() { + struct Busy: LocalizedError { var errorDescription: String? { "address in use" } } + #expect(message { try startServer { throw Busy() } } == "cannot start server: address in use") + #expect(message { try startServer {} } == nil) + } + + func downloadCommand(_ error: Error) -> Command { + Command(name: "dl", summary: "x", run: { _, _ in + _ = try await performDownload(start: { throw error }) + }) + } + + @Test func downloadHuggingFaceNotFoundGivesTheHint() async { + for status in [401, 404] { + let err = NSError(domain: "HuggingFace", code: status, userInfo: [NSLocalizedDescriptionKey: "HTTP \(status)"]) + let cap = Capture() + #expect(await CLI.run(["dl"], console: cap.console(), commands: [downloadCommand(err)]) == 1) + #expect(cap.stderr == "Error: not found, or private/gated (set HF_TOKEN)\n") + #expect(cap.stdout.isEmpty) + } + } + + @Test func otherDownloadFailuresSayDownloadFailed() async { + struct Disk: LocalizedError { var errorDescription: String? { "disk full" } } + let cap = Capture() + #expect(await CLI.run(["dl"], console: cap.console(), commands: [downloadCommand(Disk())]) == 1) + #expect(cap.stderr == "Error: download failed: disk full\n") + let other = NSError(domain: "HuggingFace", code: 500, userInfo: [NSLocalizedDescriptionKey: "HTTP 500"]) + let cap2 = Capture() + #expect(await CLI.run(["dl"], console: cap2.console(), commands: [downloadCommand(other)]) == 1) + #expect(cap2.stderr == "Error: download failed: HTTP 500\n") + } + + @Test func downloadSuccessReturnsTheDirectory() async throws { + let url = URL(fileURLWithPath: "/tmp/x") + #expect(try await performDownload(start: { url }) == url) + } + + @Test func shortFlagErrorsNameBothForms() async { + let cap = Capture() + #expect(await CLI.run(["run", "m", "-n", "abc"], console: cap.console()) == 1) + #expect(cap.stderr.hasPrefix("Error: invalid argument \"abc\" for \"-n, --max-tokens\" flag"), "\(cap.stderr)") + let cap2 = Capture() + #expect(await CLI.run(["run", "m", "--max", "abc"], console: cap2.console()) == 1) + #expect(cap2.stderr.hasPrefix("Error: invalid argument \"abc\" for \"--max-tokens\" flag"), "\(cap2.stderr)") + } +} From b9b65f73ad141a440d3340ef799d58be7fae6a46 Mon Sep 17 00:00:00 2001 From: Rolando Santamaria Maso Date: Sun, 8 Nov 2026 14:43:34 +0100 Subject: [PATCH 04/20] feat(cli): shared sampling limits, strict option validation and command tables --- Sources/MLXEngine/ModelConfig.swift | 39 +++++ Sources/MLXEngine/OpenAIServer.swift | 10 +- Sources/MLXEngine/SamplingLimits.swift | 26 ++++ Sources/MLXRCLI/Commands.swift | 74 +++++++-- Sources/MLXRCLI/Help.swift | 2 +- Sources/MLXRCLI/Legacy.swift | 22 ++- Sources/MLXRCLI/Parser.swift | 2 + Sources/MLXRCLI/Parsing.swift | 3 + Sources/MLXRCLI/Services.swift | 47 ++++++ .../MLXRCLITests/DeveloperCommandTests.swift | 144 ++++++++++++++++++ .../GenerationDefaultsTests.swift | 85 +++++++++++ .../MLXRCLITests/OptionValidationTests.swift | 130 ++++++++++++++++ Tests/MLXRCLITests/SamplingLimitsTests.swift | 50 ++++++ 13 files changed, 613 insertions(+), 21 deletions(-) create mode 100644 Sources/MLXEngine/SamplingLimits.swift create mode 100644 Sources/MLXRCLI/Services.swift create mode 100644 Tests/MLXRCLITests/DeveloperCommandTests.swift create mode 100644 Tests/MLXRCLITests/GenerationDefaultsTests.swift create mode 100644 Tests/MLXRCLITests/OptionValidationTests.swift create mode 100644 Tests/MLXRCLITests/SamplingLimitsTests.swift diff --git a/Sources/MLXEngine/ModelConfig.swift b/Sources/MLXEngine/ModelConfig.swift index 01c2200..562b31c 100644 --- a/Sources/MLXEngine/ModelConfig.swift +++ b/Sources/MLXEngine/ModelConfig.swift @@ -488,3 +488,42 @@ enum BFloat16Rounding { return Float(bitPattern: rounded) } } + +/// Sampling defaults from a model's generation_config.json. Every value passes through +/// `SamplingLimits`; a value outside them is dropped and described in `warnings`. +public struct GenerationDefaults: Equatable, Sendable { + public var temperature: Float? + public var topP: Float? + public var topK: Int? + public var minP: Float? + public var repetitionPenalty: Float? + /// One line per dropped value, each naming `generation_config.json` and the key. + public var warnings: [String] = [] + + public init() {} + + public static func parse(_ data: Data) -> GenerationDefaults { + var out = GenerationDefaults() + guard let g = (try? JSONSerialization.jsonObject(with: data)) as? [String: Any] else { return out } + func number(_ key: String, _ ok: (Double) -> Bool, _ rule: String) -> Double? { + guard let raw = g[key] else { return nil } + if let n = raw as? NSNumber, CFGetTypeID(n) != CFBooleanGetTypeID(), ok(n.doubleValue) { return n.doubleValue } + out.warnings.append("warning: generation_config.json: ignoring \(key) = \(raw) (\(rule))") + return nil + } + out.temperature = number("temperature", { SamplingLimits.temperature.contains($0) }, "must be between 0 and 2").map { Float($0) } + out.topP = number("top_p", SamplingLimits.isValidTopP, "must be greater than 0 and at most 1").map { Float($0) } + out.topK = number("top_k", { $0 == $0.rounded() && SamplingLimits.topK.contains(Int(exactly: $0) ?? -1) }, + "must be a whole number from 0 to \(SamplingLimits.topK.upperBound)").map { Int($0) } + out.minP = number("min_p", { SamplingLimits.minP.contains($0) }, "must be between 0 and 1").map { Float($0) } + out.repetitionPenalty = number("repetition_penalty", SamplingLimits.isValidRepeatPenalty, + "must be greater than 0 and at most 100").map { Float($0) } + return out + } + /// Reads `/generation_config.json`; absent or unreadable gives empty defaults. + public static func load(directory: URL) -> GenerationDefaults { + guard let data = try? Data(contentsOf: directory.appendingPathComponent("generation_config.json"), options: .mappedIfSafe), + data.count <= 1 << 20 else { return GenerationDefaults() } + return parse(data) + } +} diff --git a/Sources/MLXEngine/OpenAIServer.swift b/Sources/MLXEngine/OpenAIServer.swift index 6c005cc..00ba467 100644 --- a/Sources/MLXEngine/OpenAIServer.swift +++ b/Sources/MLXEngine/OpenAIServer.swift @@ -73,7 +73,7 @@ public final class OpenAIServer: @unchecked Sendable { public var maxBodyBytes = 4 * 1024 * 1024 public var maxMessages = 1000 /// Server-side ceiling for max_tokens / max_completion_tokens / max_output_tokens. - public var maxOutputTokens = 32_768 + public var maxOutputTokens = SamplingLimits.maxTokens public var headerTimeout: TimeInterval = 10 public var bodyTimeout: TimeInterval = 30 public var writeTimeout: TimeInterval = 30 @@ -102,7 +102,7 @@ public final class OpenAIServer: @unchecked Sendable { var maxConnections = 32 var maxQueued = 8 var maxMessages = 1000 - var maxOutputTokens = 32_768 + var maxOutputTokens = SamplingLimits.maxTokens var limits = HTTPConnection.Limits() } @@ -411,7 +411,7 @@ public final class OpenAIServer: @unchecked Sendable { } if let v = try b.number("top_k") { guard v >= -1 else { throw APIError.invalid("'top_k' must be >= 0 (or -1 to disable)", param: "top_k") } - p.topK = v <= 0 ? 0 : Int(min(v, 100_000)) + p.topK = SamplingLimits.clampTopK(v) } if let v = try b.number("min_p") { guard v >= 0 && v <= 1 else { throw APIError.invalid("'min_p' must be between 0 and 1", param: "min_p") } @@ -420,7 +420,7 @@ public final class OpenAIServer: @unchecked Sendable { let rpKey = b.has("repetition_penalty") ? "repetition_penalty" : "repeat_penalty" if let v = try b.number(rpKey) { guard v > 0 else { throw APIError.invalid("'\(rpKey)' must be greater than 0", param: rpKey) } - p.repetitionPenalty = Float(min(v, 100)) + p.repetitionPenalty = Float(SamplingLimits.clampRepeatPenalty(v)) } for k in ["frequency_penalty", "presence_penalty"] { // Accepted for compatibility and range-checked, but not applied (see type docs). @@ -430,7 +430,7 @@ public final class OpenAIServer: @unchecked Sendable { } if let k = maxKeys.first(where: { b.has($0) }), let v = try b.number(k) { guard v >= 1 else { throw APIError.invalid("'\(k)' must be at least 1", param: k) } - p.maxTokens = Int(min(v, Double(cfg.maxOutputTokens))) + p.maxTokens = SamplingLimits.clampMaxTokens(v, cap: cfg.maxOutputTokens) } else { p.maxTokens = min(defaultMax, cfg.maxOutputTokens) } diff --git a/Sources/MLXEngine/SamplingLimits.swift b/Sources/MLXEngine/SamplingLimits.swift new file mode 100644 index 0000000..edd38f1 --- /dev/null +++ b/Sources/MLXEngine/SamplingLimits.swift @@ -0,0 +1,26 @@ +import Foundation + +/// The bounds of every sampling and serving number, shared by the HTTP server and the `mlxr` CLI. +/// The server clamps some values to these bounds; the CLI rejects values outside them. +public enum SamplingLimits { + public static let temperature: ClosedRange = 0...2 + /// Top-p is valid in (0, topPMax]. + public static let topPMax: Double = 1 + public static let topK: ClosedRange = 0...100_000 + public static let minP: ClosedRange = 0...1 + /// Repeat penalty is valid in (0, repeatPenaltyMax]. + public static let repeatPenaltyMax: Double = 100 + public static let maxTokens: Int = 32_768 + public static let context: ClosedRange = 256...1_048_576 + public static let port: ClosedRange = 1...65_535 + public static let repeatLastN: ClosedRange = 0...4096 + + public static func isValidTopP(_ v: Double) -> Bool { v > 0 && v <= topPMax } + public static func isValidRepeatPenalty(_ v: Double) -> Bool { v > 0 && v <= repeatPenaltyMax } + + /// What the server does with out-of-range values it accepts: top_k and repeat penalty are + /// clamped to their ceiling, max_tokens to the given per-server cap. + public static func clampTopK(_ v: Double) -> Int { v <= 0 ? 0 : Int(min(v, Double(topK.upperBound))) } + public static func clampRepeatPenalty(_ v: Double) -> Double { min(v, repeatPenaltyMax) } + public static func clampMaxTokens(_ v: Double, cap: Int) -> Int { Int(min(v, Double(cap))) } +} diff --git a/Sources/MLXRCLI/Commands.swift b/Sources/MLXRCLI/Commands.swift index 899cbf3..f759f2c 100644 --- a/Sources/MLXRCLI/Commands.swift +++ b/Sources/MLXRCLI/Commands.swift @@ -6,14 +6,25 @@ extension CLI { static let draftHelp = "draft model for speculative decoding: a model or auto" /// The command table of `mlxr`. - public static var commands: [Command] { - [ + public static var commands: [Command] { commands(services: .live) } + + /// The command table with its outside world injected. + public static func commands(services: CLIServices) -> [Command] { + let ctxRange = SamplingLimits.context + return [ Command(name: "run", summary: "Run a model", args: [Arg("MODEL", kind: .model), Arg("PROMPT", optional: true, variadic: true)], options: [ Opt(long: "prompt", short: "p", value: .text, help: "prompt"), - Opt(long: "max-tokens", aliases: ["max"], short: "n", value: .int(1...32768), defaultText: "256", help: "maximum tokens to generate"), - Opt(long: "temp", aliases: ["temperature"], value: .real(0...2), defaultText: "0", help: "sampling temperature"), - Opt(long: "ctx", value: .int(256...1_048_576), defaultText: "4096", help: "context length"), + Opt(long: "max-tokens", aliases: ["max"], short: "n", value: .int(1...SamplingLimits.maxTokens), defaultText: "256", help: "maximum tokens to generate"), + Opt(long: "temp", aliases: ["temperature"], value: .real(SamplingLimits.temperature), defaultText: "0", help: "sampling temperature"), + Opt(long: "top-p", value: .realAboveZero(max: SamplingLimits.topPMax), help: "nucleus sampling probability"), + Opt(long: "top-k", value: .int(SamplingLimits.topK), help: "sample from the k likeliest tokens (0 turns it off)"), + Opt(long: "min-p", value: .real(SamplingLimits.minP), help: "minimum probability relative to the likeliest token"), + Opt(long: "repeat-penalty", value: .realAboveZero(max: SamplingLimits.repeatPenaltyMax), help: "repetition penalty (1 = off)"), + Opt(long: "repeat-last-n", value: .int(SamplingLimits.repeatLastN), help: "tokens the repetition penalty looks back over"), + Opt(long: "seed", value: .uint64, help: "random seed"), + Opt(long: "system", value: .text, help: "system prompt"), + Opt(long: "ctx", value: .int(ctxRange), defaultText: "4096", help: "context length"), Opt(long: "raw", help: "send the prompt without the chat template"), Opt(long: "think", value: .choice(["true", "false", "high", "medium", "low"]), implicit: "true", help: "think before answering"), Opt(long: "no-think", help: "answer without thinking"), @@ -32,12 +43,15 @@ extension CLI { prompt: inv.string("prompt") ?? (words.isEmpty ? "Write a haiku about Apple Silicon." : words), maxTokens: inv.int("max-tokens") ?? 256, temp: Float(inv.double("temp") ?? 0), ctx: inv.int("ctx") ?? 4096, raw: inv.bool("raw"), thinking: thinking, showIds: inv.bool("ids"), - draft: inv.string("draft"), sparse: inv.string("sparse"), spec: inv.bool("spec"))) + draft: inv.string("draft"), sparse: inv.string("sparse"), spec: inv.bool("spec"), + system: inv.string("system"), topP: inv.double("top-p").map { Float($0) }, topK: inv.int("top-k"), + minP: inv.double("min-p").map { Float($0) }, repeatPenalty: inv.double("repeat-penalty").map { Float($0) }, + repeatLastN: inv.int("repeat-last-n"), seed: inv.string("seed").flatMap { UInt64($0) })) }), Command(name: "serve", aliases: ["start"], summary: "Start the OpenAI-compatible server", args: [Arg("MODEL", kind: .model)], options: [ - Opt(long: "port", value: .int(1...65535), defaultText: "1234", help: "port to listen on"), - Opt(long: "ctx", value: .int(256...1_048_576), defaultText: "8192", help: "context length"), + Opt(long: "port", value: .int(SamplingLimits.port), defaultText: "1234", help: "port to listen on"), + Opt(long: "ctx", value: .int(ctxRange), defaultText: "8192", help: "context length"), Opt(long: "lan", help: "listen on the local network (needs an API key)"), Opt(long: "api-key", value: .text, help: "API key clients must send"), Opt(long: "draft", value: .model, help: draftHelp), @@ -55,12 +69,50 @@ extension CLI { Command(name: "show", summary: "Show information for a model", args: [Arg("MODEL", kind: .model)], run: { _, _ in throw CLIError.failure("show is not available yet") }), Command(name: "search", summary: "Search Hugging Face for MLX models", args: [Arg("QUERY", variadic: true)], - run: { inv, _ in try await legacySearch(inv.positionals.joined(separator: " ")) }), + options: [ + Opt(long: "limit", value: .int(1...100), defaultText: "25", help: "maximum number of results"), + Opt(long: "json", help: "print the results as JSON"), + ], + run: { inv, console in + let models = try await services.search(inv.positionals.joined(separator: " "), inv.int("limit") ?? 25) + console.out(renderSearch(models, json: inv.bool("json"))) + }), Command(name: "tokenize", summary: "Tokenize text with a model's tokenizer", args: [Arg("MODEL", kind: .model), Arg("TEXT", variadic: true)], - group: "Developer", run: { inv, _ in try legacyTokenize(inv.positionals[0], inv.positionals.dropFirst().joined(separator: " ")) }), + options: [Opt(long: "ids", help: "print only the token ids")], + group: "Developer", run: { inv, console in + try await services.tokenize(TokenizeOptions(model: inv.positionals[0], text: inv.positionals.dropFirst().joined(separator: " "), + idsOnly: inv.bool("ids")), console) + }), Command(name: "bench", summary: "Benchmark a model", args: [Arg("MODEL", kind: .model)], + options: [ + Opt(long: "decode", help: "decode benchmark only"), + Opt(long: "ctx", value: .text, help: "long-context contexts, comma separated"), + ], environment: [("MLXR_BENCH_DECODE", "decode benchmark only"), ("MLXR_BENCH_CTX", "long-context contexts, comma separated")], - group: "Developer", run: { inv, _ in try legacyBench(inv.positionals[0]) }), + group: "Developer", run: { inv, console in + try await services.bench(BenchOptions(model: inv.positionals[0], decodeOnly: inv.bool("decode"), contexts: inv.string("ctx")), console) + }), ] } } + +extension CLI { + /// Search results: one row per model, or a JSON array with `--json`. + static func renderSearch(_ models: [HubModel], json: Bool) -> String { + if json { + let rows: [[String: Any]] = models.map { m in + var r: [String: Any] = ["id": m.id] + if let d = m.downloads { r["downloads"] = d } + if let l = m.likes { r["likes"] = l } + if let t = m.modelTypeTag { r["model_type"] = t } + if let u = m.lastModified { r["last_modified"] = u } + return r + } + let data = (try? JSONSerialization.data(withJSONObject: rows, options: [.prettyPrinted, .sortedKeys])) ?? Data("[]".utf8) + return String(decoding: data, as: UTF8.self) + "\n" + } + return models.map { + String(format: "%-60@ %10d ↓ %@", $0.id as NSString, $0.downloads ?? 0, ($0.modelTypeTag ?? "") as NSString) + "\n" + }.joined() + } +} diff --git a/Sources/MLXRCLI/Help.swift b/Sources/MLXRCLI/Help.swift index 39620e8..c1093b2 100644 --- a/Sources/MLXRCLI/Help.swift +++ b/Sources/MLXRCLI/Help.swift @@ -60,7 +60,7 @@ extension CLI { switch k { case .int: return "int" case .uint64: return "uint" - case .real: return "float" + case .real, .realAboveZero: return "float" case .text, .choice, .choiceOrInt, .model, .file: return "string" } } diff --git a/Sources/MLXRCLI/Legacy.swift b/Sources/MLXRCLI/Legacy.swift index e1e0e37..5af6f31 100644 --- a/Sources/MLXRCLI/Legacy.swift +++ b/Sources/MLXRCLI/Legacy.swift @@ -314,10 +314,11 @@ func legacyList() { } -func legacyTokenize(_ model: String, _ text: String) throws { +func legacyTokenize(_ model: String, _ text: String, idsOnly: Bool = false) throws { let tok = try Tokenizer(directory: try resolveModel(model)) let ids = tok.encode(text) print(ids) + if idsOnly { return } print(ids.map { tok.token($0) ?? "?" }) print("roundtrip:", tok.decode(ids, includeSpecial: true)) @@ -421,6 +422,13 @@ struct LegacyRun { var draft: String? var sparse: String? var spec = false + var system: String? + var topP: Float? + var topK: Int? + var minP: Float? + var repeatPenalty: Float? + var repeatLastN: Int? + var seed: UInt64? } func legacyRun(_ o: LegacyRun) async throws { @@ -434,13 +442,19 @@ func legacyRun(_ o: LegacyRun) async throws { params.temperature = o.temp params.maxTokens = o.maxTokens params.repetitionPenalty = 1.0 - params.seed = 42 + params.seed = o.seed ?? 42 + if let v = o.topP { params.topP = v } + if let v = o.topK { params.topK = v } + if let v = o.minP { params.minP = v } + if let v = o.repeatPenalty { params.repetitionPenalty = v } + if let v = o.repeatLastN { params.repetitionWindow = v } + let system = o.system.map { [ChatMessage(role: .system, content: $0)] } ?? [] if o.spec { params.speculative = true } let prompt = o.prompt let stream = o.raw ? engine.complete(prompt, parameters: params) - : engine.chat([ChatMessage(role: .user, content: prompt)], parameters: params, enableThinking: o.thinking) + : engine.chat(system + [ChatMessage(role: .user, content: prompt)], parameters: params, enableThinking: o.thinking) if o.showIds { - let p = o.raw ? prompt : engine.renderChat([ChatMessage(role: .user, content: prompt)], enableThinking: o.thinking) + let p = o.raw ? prompt : engine.renderChat(system + [ChatMessage(role: .user, content: prompt)], enableThinking: o.thinking) print("prompt ids:", engine.encode(p)) } for try await ev in stream { diff --git a/Sources/MLXRCLI/Parser.swift b/Sources/MLXRCLI/Parser.swift index 08c7fe2..bfc5688 100644 --- a/Sources/MLXRCLI/Parser.swift +++ b/Sources/MLXRCLI/Parser.swift @@ -6,6 +6,8 @@ public enum ValueKind { case int(ClosedRange) case uint64 case real(ClosedRange) + /// A finite number greater than 0 and at most `max`. + case realAboveZero(max: Double) case choice([String]) case choiceOrInt([String]) case model diff --git a/Sources/MLXRCLI/Parsing.swift b/Sources/MLXRCLI/Parsing.swift index e9d52e0..e501d67 100644 --- a/Sources/MLXRCLI/Parsing.swift +++ b/Sources/MLXRCLI/Parsing.swift @@ -79,6 +79,9 @@ extension CLI { case .real(let r): guard let d = Double(raw), d.isFinite else { throw bad("strconv.ParseFloat: parsing \"\(raw)\": invalid syntax") } guard r.contains(d) else { throw bad("value \(raw) is outside \(r.lowerBound)...\(r.upperBound)") } + case .realAboveZero(let max): + guard let d = Double(raw), d.isFinite else { throw bad("strconv.ParseFloat: parsing \"\(raw)\": invalid syntax") } + guard d > 0 && d <= max else { throw bad("value \(raw) must be greater than 0 and at most \(max)") } case .choice(let c): guard c.contains(raw) else { throw bad("must be one of \(c.joined(separator: ", "))") } case .choiceOrInt(let c): diff --git a/Sources/MLXRCLI/Services.swift b/Sources/MLXRCLI/Services.swift new file mode 100644 index 0000000..69affa1 --- /dev/null +++ b/Sources/MLXRCLI/Services.swift @@ -0,0 +1,47 @@ +import Foundation +import MLXEngine + +/// Options of `mlxr bench`. +public struct BenchOptions: Equatable, Sendable { + public var model: String + /// `--decode`: decode benchmark only (MLXR_BENCH_DECODE). + public var decodeOnly = false + /// `--ctx 8192,32768`: long-context contexts (MLXR_BENCH_CTX). + public var contexts: String? = nil + public init(model: String, decodeOnly: Bool = false, contexts: String? = nil) { + self.model = model; self.decodeOnly = decodeOnly; self.contexts = contexts + } +} + +/// Options of `mlxr tokenize`. +public struct TokenizeOptions: Equatable, Sendable { + public var model: String + public var text: String + /// `--ids`: print only the token ids. + public var idsOnly = false + public init(model: String, text: String, idsOnly: Bool = false) { + self.model = model; self.text = text; self.idsOnly = idsOnly + } +} + +/// What the commands reach outside the process (network, models); tests inject fakes. +public struct CLIServices: Sendable { + public var search: @Sendable (_ query: String, _ limit: Int) async throws -> [HubModel] + public var bench: @Sendable (BenchOptions, Console) async throws -> Void + public var tokenize: @Sendable (TokenizeOptions, Console) async throws -> Void + + public init(search: @escaping @Sendable (String, Int) async throws -> [HubModel], + bench: @escaping @Sendable (BenchOptions, Console) async throws -> Void, + tokenize: @escaping @Sendable (TokenizeOptions, Console) async throws -> Void) { + self.search = search; self.bench = bench; self.tokenize = tokenize + } + + public static let live = CLIServices( + search: { q, limit in try await HubClient.search(q, limit: limit) }, + bench: { o, _ in + if o.decodeOnly { setenv("MLXR_BENCH_DECODE", "1", 1) } + if let c = o.contexts { setenv("MLXR_BENCH_CTX", c, 1) } + try legacyBench(o.model) + }, + tokenize: { o, _ in try legacyTokenize(o.model, o.text, idsOnly: o.idsOnly) }) +} diff --git a/Tests/MLXRCLITests/DeveloperCommandTests.swift b/Tests/MLXRCLITests/DeveloperCommandTests.swift new file mode 100644 index 0000000..5c3878c --- /dev/null +++ b/Tests/MLXRCLITests/DeveloperCommandTests.swift @@ -0,0 +1,144 @@ +import Foundation +import Testing +import MLXEngine +@testable import MLXRCLI + +/// Records what the injected services were asked; nothing touches the network or a model. +final class FakeServices: @unchecked Sendable { + private let lock = NSLock() + private var s: [(String, Int)] = [], b: [BenchOptions] = [], t: [TokenizeOptions] = [] + var searches: [(String, Int)] { lock.withLock { s } } + var benches: [BenchOptions] { lock.withLock { b } } + var tokenizes: [TokenizeOptions] { lock.withLock { t } } + let catalog: [HubModel] + + init(count: Int = 8) { + catalog = (0.. [Command] { CLI.commands(services: services) } +} + +@Suite("search, tokenize and bench through the table (AC-26)") +struct DeveloperCommandTests { + @Test func searchJSONHonorsTheLimit() async throws { + let fake = FakeServices() + let cap = Capture() + #expect(await CLI.run(["search", "qwen", "--limit", "5", "--json"], console: cap.console(), commands: fake.table()) == 0, "\(cap.stderr)") + let rows = try #require(try JSONSerialization.jsonObject(with: Data(cap.stdout.utf8)) as? [[String: Any]]) + #expect(rows.count == 5) + #expect(rows.first?["id"] as? String == "mlx-community/Model-0-4bit") + #expect(fake.searches.count == 1 && fake.searches[0].0 == "qwen" && fake.searches[0].1 == 5) + #expect(cap.stderr.isEmpty) + } + + @Test func searchPlainTextHasOneRowPerModel() async { + let fake = FakeServices(count: 3) + let cap = Capture() + #expect(await CLI.run(["search", "qwen"], console: cap.console(), commands: fake.table()) == 0, "\(cap.stderr)") + let lines = cap.stdout.split(separator: "\n").map(String.init) + #expect(lines.count == 3) + #expect(lines.allSatisfy { $0.contains("mlx-community/Model-") }) + #expect((try? JSONSerialization.jsonObject(with: Data(cap.stdout.utf8))) == nil) + #expect(fake.searches.first?.1 == 25, "default limit") + } + + @Test func searchMultiWordQueryAndFlagBeforeQuery() async { + let fake = FakeServices() + let cap = Capture() + #expect(await CLI.run(["search", "--limit=2", "qwen", "4b"], console: cap.console(), commands: fake.table()) == 0, "\(cap.stderr)") + #expect(fake.searches.first?.0 == "qwen 4b" && fake.searches.first?.1 == 2) + } + + @Test func badLimitsFailBeforeAnyRequest() async { + for bad in ["0", "abc", "101", "-1"] { + let fake = FakeServices() + let cap = Capture() + #expect(await CLI.run(["search", "q", "--limit", bad], console: cap.console(), commands: fake.table()) == 1, "\(bad)") + #expect(cap.stderr.hasPrefix("Error: invalid argument \"\(bad)\" for \"--limit\" flag"), "\(bad): \(cap.stderr)") + #expect(fake.searches.isEmpty, "\(bad)") + #expect(cap.stdout.isEmpty, "\(bad)") + } + let fake = FakeServices() + let cap = Capture() + #expect(await CLI.run(["search", "q", "--limit", "100"], console: cap.console(), commands: fake.table()) == 0) + #expect(fake.searches.first?.1 == 100) + } + + @Test func searchWithoutQueryIsAUsageError() async { + let fake = FakeServices() + let cap = Capture() + #expect(await CLI.run(["search"], console: cap.console(), commands: fake.table()) == 1) + #expect(cap.stderr == "Error: requires at least 1 arg(s), only received 0\n" || cap.stderr == "Error: accepts 1 arg(s), received 0\n", "\(cap.stderr)") + #expect(fake.searches.isEmpty) + } + + @Test func searchFailuresReachErrorOutput() async { + struct Down: LocalizedError { var errorDescription: String? { "offline" } } + let services = CLIServices(search: { _, _ in throw Down() }, bench: { _, _ in }, tokenize: { _, _ in }) + let cap = Capture() + #expect(await CLI.run(["search", "q"], console: cap.console(), commands: CLI.commands(services: services)) == 1) + #expect(cap.stderr == "Error: offline\n") + } + + @Test func tokenizeAcceptsFlagsBeforeAndAfterTheModel() async { + for args in [["tokenize", "--ids", "m", "hello", "world"], ["tokenize", "m", "hello", "world", "--ids"]] { + let fake = FakeServices() + let cap = Capture() + #expect(await CLI.run(args, console: cap.console(), commands: fake.table()) == 0, "\(args): \(cap.stderr)") + #expect(fake.tokenizes == [TokenizeOptions(model: "m", text: "hello world", idsOnly: true)], "\(args)") + #expect(cap.stdout == "[1, 2, 3]\n") + } + let fake = FakeServices() + #expect(await CLI.run(["tokenize", "m", "hi"], console: Capture().console(), commands: fake.table()) == 0) + #expect(fake.tokenizes == [TokenizeOptions(model: "m", text: "hi", idsOnly: false)]) + } + + @Test func tokenizeNeedsAModelAndText() async { + let fake = FakeServices() + let cap = Capture() + #expect(await CLI.run(["tokenize", "m"], console: cap.console(), commands: fake.table()) == 1) + #expect(cap.stderr.hasPrefix("Error: requires at least 2 arg(s), only received 1"), "\(cap.stderr)") + #expect(fake.tokenizes.isEmpty) + } + + @Test func benchReceivesParsedOptionsAndPassesOutputThrough() async { + let fake = FakeServices() + let cap = Capture() + #expect(await CLI.run(["bench", "--decode", "m", "--ctx", "8192,32768"], console: cap.console(), commands: fake.table()) == 0, "\(cap.stderr)") + #expect(fake.benches == [BenchOptions(model: "m", decodeOnly: true, contexts: "8192,32768")]) + #expect(cap.stdout == "weights: 1 B\nBENCH-OUTPUT-PASSED-THROUGH\n") + #expect(cap.stderr.isEmpty) + + let plain = FakeServices() + #expect(await CLI.run(["bench", "m"], console: Capture().console(), commands: plain.table()) == 0) + #expect(plain.benches == [BenchOptions(model: "m")]) + } + + @Test func benchAndTokenizeStayUnderDeveloperHeading() { + let table = FakeServices().table() + #expect(table.first { $0.name == "bench" }?.group == "Developer") + #expect(table.first { $0.name == "tokenize" }?.group == "Developer") + #expect(table.first { $0.name == "search" }?.options.contains { $0.long == "limit" } == true) + #expect(table.first { $0.name == "search" }?.options.contains { $0.long == "json" } == true) + } +} diff --git a/Tests/MLXRCLITests/GenerationDefaultsTests.swift b/Tests/MLXRCLITests/GenerationDefaultsTests.swift new file mode 100644 index 0000000..2fb47aa --- /dev/null +++ b/Tests/MLXRCLITests/GenerationDefaultsTests.swift @@ -0,0 +1,85 @@ +import Foundation +import Testing +import MLXEngine + +@Suite("generation_config.json sanitizing (AC-7)") +struct GenerationDefaultsTests { + func parse(_ json: String) -> GenerationDefaults { GenerationDefaults.parse(Data(json.utf8)) } + + @Test func validValuesAreKept() { + let g = parse(#"{"temperature":0.6,"top_p":0.95,"top_k":20,"min_p":0.05,"repetition_penalty":1.1,"eos_token_id":1}"#) + #expect(g.temperature == 0.6) + #expect(g.topP == 0.95) + #expect(g.topK == 20) + #expect(g.minP == 0.05) + #expect(g.repetitionPenalty == 1.1) + #expect(g.warnings.isEmpty) + } + + @Test func badTemperatureIsDroppedAndTheRestKept() { + let g = parse(#"{"temperature":99,"top_p":0.8,"top_k":20}"#) + #expect(g.temperature == nil) + #expect(g.topP == 0.8) + #expect(g.topK == 20) + #expect(g.warnings.count == 1) + #expect(g.warnings.first?.contains("generation_config.json") == true) + #expect(g.warnings.first?.contains("temperature") == true) + } + + @Test func everyOutOfRangeValueIsDropped() { + let cases: [(String, KeyPath)] = [ + (#"{"temperature":-0.1}"#, \.temperatureNil), + (#"{"top_p":0}"#, \.topPNil), + (#"{"top_p":1.5}"#, \.topPNil), + (#"{"top_k":100001}"#, \.topKNil), + (#"{"top_k":-1}"#, \.topKNil), + (#"{"min_p":2}"#, \.minPNil), + (#"{"repetition_penalty":0}"#, \.repetitionNil), + (#"{"repetition_penalty":500}"#, \.repetitionNil), + (#"{"temperature":"hot"}"#, \.temperatureNil), + (#"{"top_k":1.5}"#, \.topKNil), + ] + for (json, isNil) in cases { + let g = parse(json) + #expect(g[keyPath: isNil], "\(json)") + #expect(g.warnings.count == 1, "\(json): \(g.warnings)") + } + } + + @Test func boundaryValuesAreAccepted() { + let g = parse(#"{"temperature":2,"top_p":1,"top_k":100000,"min_p":1,"repetition_penalty":100}"#) + #expect(g.temperature == 2 && g.topP == 1 && g.topK == 100_000 && g.minP == 1 && g.repetitionPenalty == 100) + #expect(g.warnings.isEmpty) + let z = parse(#"{"temperature":0,"top_k":0,"min_p":0}"#) + #expect(z.temperature == 0 && z.topK == 0 && z.minP == 0) + #expect(z.warnings.isEmpty) + } + + @Test func garbageGivesEmptyDefaults() { + for bad in ["", "not json", "[1,2]", "null"] { + let g = parse(bad) + #expect(g.temperature == nil && g.topP == nil && g.topK == nil, "\(bad)") + } + } + + @Test func loadReadsTheFileAndToleratesItsAbsence() throws { + let dir = FileManager.default.temporaryDirectory.appendingPathComponent("gen-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: dir) } + let none = GenerationDefaults.load(directory: dir) + #expect(none.temperature == nil && none.warnings.isEmpty) + try Data(#"{"temperature":0.7,"top_k":99999999}"#.utf8).write(to: dir.appendingPathComponent("generation_config.json")) + let g = GenerationDefaults.load(directory: dir) + #expect(g.temperature == 0.7) + #expect(g.topK == nil) + #expect(g.warnings.count == 1) + } +} + +private extension GenerationDefaults { + var temperatureNil: Bool { temperature == nil } + var topPNil: Bool { topP == nil } + var topKNil: Bool { topK == nil } + var minPNil: Bool { minP == nil } + var repetitionNil: Bool { repetitionPenalty == nil } +} diff --git a/Tests/MLXRCLITests/OptionValidationTests.swift b/Tests/MLXRCLITests/OptionValidationTests.swift new file mode 100644 index 0000000..83f2bf1 --- /dev/null +++ b/Tests/MLXRCLITests/OptionValidationTests.swift @@ -0,0 +1,130 @@ +import Foundation +import Testing +import MLXEngine +@testable import MLXRCLI + +/// AC-6 and AC-8: strict option validation in the real command table, before any model loads. +@Suite("Option validation (AC-6, AC-8)") +struct OptionValidationTests { + /// The error message of a parse, or nil when it parsed. + func failure(_ args: [String]) -> String? { + do { _ = try CLI.parse(args); return nil } catch { return CLI.message(for: error) } + } + + func rejected(_ args: [String], flag: String, sourceLocation: SourceLocation = #_sourceLocation) { + let m = failure(args) + #expect(m?.hasPrefix("invalid argument") == true && m?.contains("\(flag)\" flag") == true, + "\(args): \(m ?? "accepted")", sourceLocation: sourceLocation) + } + + func accepted(_ args: [String], sourceLocation: SourceLocation = #_sourceLocation) { + let m = failure(args) + #expect(m == nil, "\(args): \(m ?? "")", sourceLocation: sourceLocation) + } + + @Test func maxTokens() { + rejected(["run", "m", "--max", "abc"], flag: "--max-tokens") + for bad in ["0", "-1", "32769"] { rejected(["run", "m", "--max", bad], flag: "--max-tokens") } + for ok in ["1", "32768"] { accepted(["run", "m", "--max", ok]) } + } + + @Test func port() { + for bad in ["0", "70000", "65536", "-1", "x"] { rejected(["serve", "m", "--port", bad], flag: "--port") } + for ok in ["1", "65535"] { accepted(["serve", "m", "--port", ok]) } + } + + @Test func temperature() { + for bad in ["nan", "inf", "-inf", "2.1", "-0.1"] { rejected(["run", "m", "--temp", bad], flag: "--temp") } + for ok in ["0", "2", "0.7"] { accepted(["run", "m", "--temp", ok]) } + } + + @Test func topP() { + for bad in ["0", "1.01", "nan", "-0.5"] { rejected(["run", "m", "--top-p", bad], flag: "--top-p") } + for ok in ["0.0001", "1", "0.95"] { accepted(["run", "m", "--top-p", ok]) } + } + + @Test func topK() { + for bad in ["100001", "-1", "1.5"] { rejected(["run", "m", "--top-k", bad], flag: "--top-k") } + for ok in ["0", "100000"] { accepted(["run", "m", "--top-k", ok]) } + } + + @Test func minP() { + for bad in ["-0.1", "1.1", "nan"] { rejected(["run", "m", "--min-p", bad], flag: "--min-p") } + for ok in ["0", "1"] { accepted(["run", "m", "--min-p", ok]) } + } + + @Test func repeatPenalty() { + for bad in ["0", "-1", "100.5", "inf"] { rejected(["run", "m", "--repeat-penalty", bad], flag: "--repeat-penalty") } + for ok in ["0.001", "1", "100"] { accepted(["run", "m", "--repeat-penalty", ok]) } + } + + @Test func repeatLastN() { + for bad in ["4097", "-1"] { rejected(["run", "m", "--repeat-last-n", bad], flag: "--repeat-last-n") } + for ok in ["0", "4096"] { accepted(["run", "m", "--repeat-last-n", ok]) } + } + + @Test func seedIsUnsigned64() { + rejected(["run", "m", "--seed", "-1"], flag: "--seed") + rejected(["run", "m", "--seed", "18446744073709551616"], flag: "--seed") + accepted(["run", "m", "--seed", "0"]) + accepted(["run", "m", "--seed", "18446744073709551615"]) + } + + @Test func context() { + for cmd in ["run", "serve"] { + for bad in ["255", "1048577", "x"] { rejected([cmd, "m", "--ctx", bad], flag: "--ctx") } + for ok in ["256", "1048576"] { accepted([cmd, "m", "--ctx", ok]) } + } + } + + @Test func systemPromptAndValuesAreRecorded() throws { + let inv = try CLI.parse(["run", "m", "--system", "Be brief", "--top-p", "0.9", "--top-k", "0", "--seed", "7", + "--min-p", "0.1", "--repeat-penalty", "1.2", "--repeat-last-n", "32"]) + #expect(inv.string("system") == "Be brief") + #expect(inv.double("top-p") == 0.9) + #expect(inv.int("top-k") == 0) + #expect(inv.string("seed") == "7") + #expect(inv.double("min-p") == 0.1) + #expect(inv.double("repeat-penalty") == 1.2) + #expect(inv.int("repeat-last-n") == 32) + } + + @Test func tableRangesComeFromSharedLimits() throws { + func range(_ name: String, _ cmd: String = "run") -> ValueKind? { + CLI.commands.first { $0.name == cmd }?.options.first { $0.long == name }?.value + } + if case .int(let r)? = range("max-tokens") { #expect(r.lowerBound == 1 && r.upperBound == SamplingLimits.maxTokens) } else { Issue.record("max-tokens is not an int option") } + if case .int(let r)? = range("ctx") { #expect(r == SamplingLimits.context) } else { Issue.record("ctx is not an int option") } + if case .int(let r)? = range("port", "serve") { #expect(r == SamplingLimits.port) } else { Issue.record("port is not an int option") } + if case .real(let r)? = range("temp") { #expect(r == SamplingLimits.temperature) } else { Issue.record("temp is not a real option") } + } + + // AC-8: options anywhere, the model stays the model. + @Test func specAndApiKeyDoNotSwallowTheModel() throws { + let a = try CLI.parse(["run", "--spec", "M"]) + #expect(a.positionals == ["M"] && a.bool("spec")) + let b = try CLI.parse(["serve", "--api-key", "K", "M"]) + #expect(b.positionals == ["M"] && b.string("api-key") == "K") + let c = try CLI.parse(["serve", "M", "--api-key=K", "--port", "8080"]) + #expect(c.positionals == ["M"] && c.string("api-key") == "K" && c.int("port") == 8080) + } + + @Test func helpFlagsShowHelpInsteadOfRunning() async { + for cmd in ["tokenize", "search", "bench"] { + let cap = Capture() + #expect(await CLI.run([cmd, "-h"], console: cap.console()) == 0, "\(cmd)") + #expect(cap.stdout.contains("Usage:") && cap.stdout.contains("Flags:"), "\(cmd)") + #expect(cap.stderr.isEmpty, "\(cmd)") + } + } + + @Test func invalidValuesExitOneBeforeRunning() async { + for args in [["run", "m", "--max", "0"], ["serve", "m", "--port", "70000"], ["run", "m", "--top-k", "100001"], + ["run", "m", "--temp", "nan"], ["run", "m", "--repeat-penalty", "0"]] { + let cap = Capture() + #expect(await CLI.run(args, console: cap.console()) == 1, "\(args)") + #expect(cap.stderr.hasPrefix("Error: invalid argument"), "\(args): \(cap.stderr)") + #expect(cap.stdout.isEmpty, "\(args)") + } + } +} diff --git a/Tests/MLXRCLITests/SamplingLimitsTests.swift b/Tests/MLXRCLITests/SamplingLimitsTests.swift new file mode 100644 index 0000000..108898e --- /dev/null +++ b/Tests/MLXRCLITests/SamplingLimitsTests.swift @@ -0,0 +1,50 @@ +import Foundation +import Testing +import MLXEngine +@testable import MLXRCLI + +@Suite("Shared sampling limits (AC-7)") +struct SamplingLimitsTests { + @Test func boundsMatchTheServerContract() { + #expect(SamplingLimits.temperature == 0...2) + #expect(SamplingLimits.topPMax == 1) + #expect(SamplingLimits.topK == 0...100_000) + #expect(SamplingLimits.minP == 0...1) + #expect(SamplingLimits.repeatPenaltyMax == 100) + #expect(SamplingLimits.maxTokens == 32_768) + #expect(SamplingLimits.context == 256...1_048_576) + #expect(SamplingLimits.port == 1...65_535) + #expect(SamplingLimits.repeatLastN == 0...4096) + } + + @Test func serverCeilingIsTheSharedConstant() { + #expect(OpenAIServer(port: 1).maxOutputTokens == SamplingLimits.maxTokens) + } + + @Test func topPAndRepeatPenaltyExcludeZero() { + #expect(!SamplingLimits.isValidTopP(0)) + #expect(SamplingLimits.isValidTopP(0.0001)) + #expect(SamplingLimits.isValidTopP(1)) + #expect(!SamplingLimits.isValidTopP(1.0001)) + #expect(!SamplingLimits.isValidTopP(.nan)) + #expect(!SamplingLimits.isValidRepeatPenalty(0)) + #expect(!SamplingLimits.isValidRepeatPenalty(-1)) + #expect(SamplingLimits.isValidRepeatPenalty(0.001)) + #expect(SamplingLimits.isValidRepeatPenalty(100)) + #expect(!SamplingLimits.isValidRepeatPenalty(100.5)) + #expect(!SamplingLimits.isValidRepeatPenalty(.infinity)) + } + + /// The server keeps clamping (not rejecting) what it accepts: exactly the behavior before this change. + @Test func serverClampsInsteadOfRejecting() { + #expect(SamplingLimits.clampTopK(200_000) == 100_000) + #expect(SamplingLimits.clampTopK(40) == 40) + #expect(SamplingLimits.clampTopK(0) == 0) + #expect(SamplingLimits.clampTopK(-1) == 0) + #expect(SamplingLimits.clampRepeatPenalty(500) == 100) + #expect(SamplingLimits.clampRepeatPenalty(1.05) == 1.05) + #expect(SamplingLimits.clampMaxTokens(1e18, cap: 32_768) == 32_768) + #expect(SamplingLimits.clampMaxTokens(100, cap: 32_768) == 100) + #expect(SamplingLimits.clampMaxTokens(1e18, cap: 1000) == 1000) + } +} From d781d549cd91fd252703567926202cfc548c4048 Mon Sep 17 00:00:00 2001 From: Rolando Santamaria Maso Date: Sun, 8 Nov 2026 15:07:33 +0100 Subject: [PATCH 05/20] feat(cli): model catalog with Ollama aliases, name resolution, presets and list/rm/cp --- Sources/MLXEngine/Catalog.swift | 46 ++++ Sources/MLXEngine/ModelStore.swift | 10 +- Sources/MLXRCLI/Commands.swift | 15 +- Sources/MLXRCLI/Legacy.swift | 8 - Sources/MLXRCLI/Library.swift | 203 ++++++++++++++++++ Sources/MLXRCLI/ModelCommands.swift | 128 +++++++++++ Sources/MLXRCLI/Parser.swift | 2 + Sources/MLXRunner/Recommended.swift | 28 ++- Tests/MLXRCLITests/CatalogTests.swift | 69 ++++++ Tests/MLXRCLITests/ModelFixtures.swift | 76 +++++++ Tests/MLXRCLITests/ModelListTests.swift | 85 ++++++++ Tests/MLXRCLITests/ModelManagementTests.swift | 157 ++++++++++++++ Tests/MLXRCLITests/ModelResolverTests.swift | 131 +++++++++++ Tests/MLXRCLITests/PresetStoreTests.swift | 59 +++++ .../UnsupportedCommandTests.swift | 58 +++++ 15 files changed, 1049 insertions(+), 26 deletions(-) create mode 100644 Sources/MLXEngine/Catalog.swift create mode 100644 Sources/MLXRCLI/Library.swift create mode 100644 Sources/MLXRCLI/ModelCommands.swift create mode 100644 Tests/MLXRCLITests/CatalogTests.swift create mode 100644 Tests/MLXRCLITests/ModelFixtures.swift create mode 100644 Tests/MLXRCLITests/ModelListTests.swift create mode 100644 Tests/MLXRCLITests/ModelManagementTests.swift create mode 100644 Tests/MLXRCLITests/ModelResolverTests.swift create mode 100644 Tests/MLXRCLITests/PresetStoreTests.swift create mode 100644 Tests/MLXRCLITests/UnsupportedCommandTests.swift diff --git a/Sources/MLXEngine/Catalog.swift b/Sources/MLXEngine/Catalog.swift new file mode 100644 index 0000000..9eb7053 --- /dev/null +++ b/Sources/MLXEngine/Catalog.swift @@ -0,0 +1,46 @@ +import Foundation + +/// One curated model with the Ollama-style names it answers to (`qwen3:8b`, `qwen3`, `qwen3:latest`). +public struct CatalogModel: Identifiable, Hashable, Sendable { + /// The Hugging Face repo id. + public let id: String + public let title: String + public let blurb: String + public let gigabytes: Double + /// Lowercase alias names, including the bare family name and `:latest` for the family default. + public let aliases: [String] + + public init(id: String, title: String, blurb: String, gigabytes: Double, aliases: [String] = []) { + self.id = id; self.title = title; self.blurb = blurb; self.gigabytes = gigabytes; self.aliases = aliases + } +} + +/// Curated models, shared by the app's recommendations and the CLI's name resolver. +public struct Catalog: Sendable { + public var models: [CatalogModel] + + public init(models: [CatalogModel]) { self.models = models } + + /// The built-in catalog. + public static let standard = Catalog(models: [ + .init(id: "mlx-community/Qwen3-0.6B-4bit", title: "Qwen3 0.6B", blurb: "Tiny and blazing fast, with thinking mode", gigabytes: 0.35, aliases: ["qwen3:0.6b"]), + .init(id: "mlx-community/Qwen3.5-0.8B-4bit", title: "Qwen3.5 0.8B", blurb: "Newest tiny Qwen: hybrid linear attention, quick direct answers", gigabytes: 0.65, aliases: ["qwen3.5:0.8b", "qwen3.5", "qwen3.5:latest"]), + .init(id: "mlx-community/Llama-3.2-3B-Instruct-4bit", title: "Llama 3.2 3B", blurb: "Meta's compact all-rounder", gigabytes: 1.8, aliases: ["llama3.2:3b", "llama3.2", "llama3.2:latest"]), + .init(id: "mlx-community/Qwen3-4B-4bit", title: "Qwen3 4B", blurb: "Strong reasoning in a small footprint", gigabytes: 2.3, aliases: ["qwen3:4b"]), + .init(id: "mlx-community/Phi-3.5-mini-instruct-4bit", title: "Phi-3.5 mini", blurb: "Microsoft's efficient 3.8B model", gigabytes: 2.2, aliases: ["phi3.5", "phi3.5:latest"]), + .init(id: "mlx-community/Mistral-7B-Instruct-v0.3-4bit", title: "Mistral 7B v0.3", blurb: "Classic, fluent 7B instruct model", gigabytes: 4.1, aliases: ["mistral:7b", "mistral", "mistral:latest"]), + .init(id: "mlx-community/Qwen2.5-Coder-7B-Instruct-4bit", title: "Qwen2.5 Coder 7B", blurb: "Code generation and review", gigabytes: 4.3, aliases: ["qwen2.5-coder:7b", "qwen2.5-coder", "qwen2.5-coder:latest"]), + .init(id: "mlx-community/Qwen3-8B-4bit", title: "Qwen3 8B", blurb: "Best quality under 5 GB", gigabytes: 4.6, aliases: ["qwen3:8b", "qwen3", "qwen3:latest"]), + .init(id: "mlx-community/Meta-Llama-3.1-8B-Instruct-4bit", title: "Llama 3.1 8B", blurb: "Popular 8B with 128K context", gigabytes: 4.5, aliases: ["llama3.1:8b", "llama3.1", "llama3.1:latest"]), + .init(id: "mlx-community/DeepSeek-R1-Distill-Qwen-7B-4bit", title: "DeepSeek R1 Distill 7B", blurb: "Step-by-step reasoning", gigabytes: 4.3, aliases: ["deepseek-r1:7b", "deepseek-r1", "deepseek-r1:latest"]), + ]) + + /// The repo id an alias names (case-insensitive), or nil. + public func repo(forAlias alias: String) -> String? { + let key = alias.lowercased() + return models.first { $0.aliases.contains(key) }?.id + } + + /// Every alias, for suggestions and completions. + public var aliases: [String] { models.flatMap(\.aliases) } +} diff --git a/Sources/MLXEngine/ModelStore.swift b/Sources/MLXEngine/ModelStore.swift index 8d67f9a..7c8a2bb 100644 --- a/Sources/MLXEngine/ModelStore.swift +++ b/Sources/MLXEngine/ModelStore.swift @@ -17,6 +17,14 @@ public struct LocalModel: Identifiable, Hashable, Sendable { public var isSupported: Bool { SupportedArchitectures.isSupported(modelType) && !isMoE } /// A mixture-of-experts variant of a supported type (Gemma 4 26B-A4B). public var isMoE: Bool = false + /// When the model was last changed on disk. + public var modified: Date? = nil + + public init(repo: String, directory: URL, sizeBytes: Int, modelType: String?, bits: Int?, source: Source, + isMoE: Bool = false, modified: Date? = nil) { + self.repo = repo; self.directory = directory; self.sizeBytes = sizeBytes; self.modelType = modelType + self.bits = bits; self.source = source; self.isMoE = isMoE; self.modified = modified + } } public enum ModelStore { @@ -103,7 +111,7 @@ public enum ModelStore { } var m = LocalModel(repo: repo, directory: dir, sizeBytes: size, modelType: (cfg["model_type"] as? String)?.lowercased(), - bits: (q?["bits"] as? NSNumber)?.intValue, source: source) + bits: (q?["bits"] as? NSNumber)?.intValue, source: source, modified: mtime(dir)) m.isMoE = SupportedArchitectures.isMoE(cfg) return m } diff --git a/Sources/MLXRCLI/Commands.swift b/Sources/MLXRCLI/Commands.swift index f759f2c..96585ad 100644 --- a/Sources/MLXRCLI/Commands.swift +++ b/Sources/MLXRCLI/Commands.swift @@ -6,10 +6,11 @@ extension CLI { static let draftHelp = "draft model for speculative decoding: a model or auto" /// The command table of `mlxr`. - public static var commands: [Command] { commands(services: .live) } + public static var commands: [Command] { commands() } /// The command table with its outside world injected. - public static func commands(services: CLIServices) -> [Command] { + public static func commands(services: CLIServices = .live, library: ModelLibrary = .live, presets: PresetStore = .live, + catalog: Catalog = .standard, now: @escaping @Sendable () -> Date = { Date() }) -> [Command] { let ctxRange = SamplingLimits.context return [ Command(name: "run", summary: "Run a model", args: [Arg("MODEL", kind: .model), Arg("PROMPT", optional: true, variadic: true)], @@ -48,7 +49,7 @@ extension CLI { minP: inv.double("min-p").map { Float($0) }, repeatPenalty: inv.double("repeat-penalty").map { Float($0) }, repeatLastN: inv.int("repeat-last-n"), seed: inv.string("seed").flatMap { UInt64($0) })) }), - Command(name: "serve", aliases: ["start"], summary: "Start the OpenAI-compatible server", args: [Arg("MODEL", kind: .model)], + Command(name: "serve", aliases: ["start"], summary: "Start the OpenAI-compatible server", args: [Arg("MODEL", kind: .model, optional: true)], options: [ Opt(long: "port", value: .int(SamplingLimits.port), defaultText: "1234", help: "port to listen on"), Opt(long: "ctx", value: .int(ctxRange), defaultText: "8192", help: "context length"), @@ -59,13 +60,15 @@ extension CLI { ], environment: [("MLXR_API_KEY", "API key, when --api-key is not given")], run: { inv, _ in - try await legacyServe(model: inv.positionals[0], port: UInt16(inv.int("port") ?? 1234), + guard let model = inv.positionals.first else { + throw CLIError.failure("serve needs a model for now: mlxr serve MODEL (serving without a model comes with the background server in a later release)") + } + try await legacyServe(model: model, port: UInt16(inv.int("port") ?? 1234), ctx: inv.int("ctx") ?? 8192, lan: inv.bool("lan"), apiKey: inv.string("api-key"), draft: inv.string("draft"), sparse: inv.string("sparse")) }), Command(name: "download", aliases: ["pull"], summary: "Download a model from Hugging Face", args: [Arg("MODEL", kind: .model)], run: { inv, _ in try await legacyDownload(inv.positionals[0]) }), - Command(name: "list", aliases: ["ls"], summary: "List installed models", run: { _, _ in legacyList() }), Command(name: "show", summary: "Show information for a model", args: [Arg("MODEL", kind: .model)], run: { _, _ in throw CLIError.failure("show is not available yet") }), Command(name: "search", summary: "Search Hugging Face for MLX models", args: [Arg("QUERY", variadic: true)], @@ -92,7 +95,7 @@ extension CLI { group: "Developer", run: { inv, console in try await services.bench(BenchOptions(model: inv.positionals[0], decodeOnly: inv.bool("decode"), contexts: inv.string("ctx")), console) }), - ] + ] + modelCommands(library: library, presets: presets, catalog: catalog, now: now) } } diff --git a/Sources/MLXRCLI/Legacy.swift b/Sources/MLXRCLI/Legacy.swift index 5af6f31..b4b6003 100644 --- a/Sources/MLXRCLI/Legacy.swift +++ b/Sources/MLXRCLI/Legacy.swift @@ -306,14 +306,6 @@ func legacyDownload(_ repo: String) async throws { } -func legacyList() { - for m in ModelStore.scan() { - print(String(format: "%-60@ %10@ %@%@", m.repo as NSString, formatBytes(m.sizeBytes) as NSString, - (m.modelType ?? "?") as NSString, m.isSupported ? "" : " (unsupported)")) - } - -} - func legacyTokenize(_ model: String, _ text: String, idsOnly: Bool = false) throws { let tok = try Tokenizer(directory: try resolveModel(model)) let ids = tok.encode(text) diff --git a/Sources/MLXRCLI/Library.swift b/Sources/MLXRCLI/Library.swift new file mode 100644 index 0000000..2a72294 --- /dev/null +++ b/Sources/MLXRCLI/Library.swift @@ -0,0 +1,203 @@ +import CryptoKit +import Foundation +import MLXEngine + +/// The installed models, injected so tests never scan or delete the real library. +public struct ModelLibrary: Sendable { + public var scan: @Sendable () -> [LocalModel] + public var delete: @Sendable (LocalModel) throws -> Void + + public init(scan: @escaping @Sendable () -> [LocalModel], delete: @escaping @Sendable (LocalModel) throws -> Void) { + self.scan = scan; self.delete = delete + } + + public static let live = ModelLibrary(scan: { ModelStore.scan() }, delete: { try ModelStore.delete($0) }) +} + +/// A named alias for a model, with the system prompt and parameters saved alongside it. +public struct Preset: Codable, Equatable, Sendable { + public var name: String + public var repo: String + public var system: String? + public var parameters: [String: String] + + public init(name: String, repo: String, system: String? = nil, parameters: [String: String] = [:]) { + self.name = name; self.repo = repo; self.system = system; self.parameters = parameters + } +} + +/// Presets as one `/.json` file each, read and written safely. +public struct PresetStore: Sendable { + public var directory: URL + + public init(directory: URL) { self.directory = directory } + + public static let live = PresetStore(directory: ModelStore.root.deletingLastPathComponent().appendingPathComponent("presets", isDirectory: true)) + + /// A preset name is 1...128 characters of letters, digits and `. _ - :`, not starting with a dot. + public static func validateName(_ name: String) throws { + let ok = !name.isEmpty && name.utf8.count <= 128 && !name.hasPrefix(".") && name.utf8.allSatisfy { + ($0 >= 48 && $0 <= 58) || ($0 >= 65 && $0 <= 90) || ($0 >= 97 && $0 <= 122) || $0 == 46 || $0 == 95 || $0 == 45 + } + guard ok else { throw CLIError.failure("invalid model name '\(name)' (use letters, digits and . _ - :)") } + } + + private func file(_ name: String) -> URL { directory.appendingPathComponent(name + ".json") } + + /// The preset, or nil when missing, unreadable (moved aside) or the name is unsafe. + public func load(_ name: String) -> Preset? { + guard (try? Self.validateName(name)) != nil else { return nil } + if case .loaded(let p) = SafeJSONFile.load(Preset.self, from: file(name)) { return p } + return nil + } + + public func list() -> [Preset] { + let names = ((try? FileManager.default.contentsOfDirectory(atPath: directory.path)) ?? []) + .filter { $0.hasSuffix(".json") && !$0.contains(".corrupt-") }.map { String($0.dropLast(5)) } + return names.compactMap { load($0) }.sorted { $0.name < $1.name } + } + + public func save(_ preset: Preset) throws { + try Self.validateName(preset.name) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + let encoder = JSONEncoder() + encoder.outputFormatting = [.prettyPrinted, .sortedKeys] + try encoder.encode(preset).write(to: file(preset.name), options: .atomic) + } + + public func delete(_ name: String) throws { + try Self.validateName(name) + let url = file(name) + if FileManager.default.fileExists(atPath: url.path) { try FileManager.default.removeItem(at: url) } + } +} + +/// What a model name resolved to. +public struct Resolution: Equatable, Sendable { + public enum Source: String, Sendable { case path, installed, huggingFace, catalog, preset, shortName } + public var source: Source + /// The Hugging Face repo id; nil for a plain path. + public var repo: String? + /// Where the model is on disk; nil when it is not installed. + public var directory: URL? + /// The preset a name resolved through. + public var preset: Preset? + + public init(source: Source, repo: String?, directory: URL?, preset: Preset? = nil) { + self.source = source; self.repo = repo; self.directory = directory; self.preset = preset + } +} + +/// Turns what the user typed into a model: path, installed repo id, `hf.co/org/repo`, catalog alias, +/// preset, installed short name, in that order. +public struct ModelResolver: Sendable { + public var library: ModelLibrary + public var presets: PresetStore + public var catalog: Catalog + + public init(library: ModelLibrary, presets: PresetStore, catalog: Catalog = .standard) { + self.library = library; self.presets = presets; self.catalog = catalog + } + + public func resolve(_ name: String) throws -> Resolution { + guard !name.isEmpty else { throw CLIError.failure("model name is empty") } + if FileManager.default.fileExists(atPath: name) { + return Resolution(source: .path, repo: nil, directory: URL(fileURLWithPath: name)) + } + let models = library.scan() + func installed(_ repo: String) -> LocalModel? { models.first { $0.repo.lowercased() == repo.lowercased() } } + func make(_ source: Resolution.Source, _ repo: String, preset: Preset? = nil) -> Resolution { + let m = installed(repo) + return Resolution(source: source, repo: m?.repo ?? repo, directory: m?.directory, preset: preset) + } + if let m = installed(name) { return make(.installed, m.repo) } + let lower = name.lowercased() + for prefix in ["hf.co/", "huggingface.co/"] where lower.hasPrefix(prefix) { + let repo = String(name.dropFirst(prefix.count)) + try validateRepo(repo) + return make(.huggingFace, repo) + } + if let repo = catalog.repo(forAlias: name) { return make(.catalog, repo) } + if let p = presets.load(name) { return make(.preset, p.repo, preset: p) } + let named = models.filter { $0.name.lowercased() == lower } + if named.count > 1 { + throw CLIError.failure("'\(name)' is ambiguous; matches: " + named.map(\.repo).joined(separator: ", ")) + } + if let m = named.first { return make(.shortName, m.repo) } + if name.contains("/") { + try validateRepo(name) + return make(.huggingFace, name) + } + throw CLIError.failure(missMessage(name, models: models)) + } + + private func validateRepo(_ repo: String) throws { + guard RepoID.isValid(repo) else { throw CLIError.failure("invalid repo id: \(repo) (expected owner/name)") } + } + + private func missMessage(_ name: String, models: [LocalModel]) -> String { + let lower = name.lowercased() + var candidates: [(shown: String, keys: [String])] = catalog.aliases.map { ($0, [$0]) } + candidates += models.map { ($0.repo, [$0.repo.lowercased(), $0.name.lowercased()]) } + candidates += presets.list().map { ($0.name, [$0.name.lowercased()]) } + var scored: [(String, Int)] = [] + for c in candidates where !scored.contains(where: { $0.0 == c.shown }) { + let d = c.keys.map { CLI.editDistance(lower, $0) }.min() ?? 99 + if d <= 2 { scored.append((c.shown, d)) } + } + var msg = "model '\(name)' not found" + let best = scored.sorted { ($0.1, $0.0) < ($1.1, $1.0) }.prefix(3).map { "'\($0.0)'" } + if !best.isEmpty { msg += ", did you mean " + best.joined(separator: " or ") + "?" } + return msg + } +} + +extension CLI { + /// Base-1000 size: `%d UNIT` from 10, else `%.1f UNIT` (`5.2 GB`, `350 MB`). + static func humanBytes(_ n: Int) -> String { + if n < 1000 { return "\(max(n, 0)) B" } + var value = Double(n) / 1000 + var unit = "KB" + for next in ["MB", "GB", "TB"] where value >= 1000 { value /= 1000; unit = next } + return (value >= 10 ? String(Int(value)) : String(format: "%.1f", value)) + " " + unit + } + + /// Ollama's relative time (`Less than a second`, `About a minute`, `5 minutes`, `3 weeks`) plus ` ago`. + static func relativeTime(_ date: Date, now: Date) -> String { + let delta = now.timeIntervalSince(date) + let seconds = Int(abs(delta)) + let hours = seconds / 3600 + let text: String + switch seconds { + case 0: text = "Less than a second" + case 1: text = "1 second" + case ..<60: text = "\(seconds) seconds" + case ..<120: text = "About a minute" + case ..<3600: text = "\(seconds / 60) minutes" + default: + if hours == 1 { text = "About an hour" } + else if hours < 48 { text = "\(hours) hours" } + else if hours < 24 * 14 { text = "\(hours / 24) days" } + else if hours < 24 * 60 { text = "\(hours / 24 / 7) weeks" } + else if hours < 24 * 365 * 2 { text = "\(hours / 24 / 30) months" } + else { text = "\(hours / 24 / 365) years" } + } + return text + (delta < 0 ? " from now" : " ago") + } + + /// Borderless table: every column but the last padded to its widest cell, then 4 spaces; the last + /// column's body cells followed by 4 spaces, its header by 1. + static func renderTable(header: [String], rows: [[String]]) -> String { + let last = header.count - 1 + let widths = header.indices.map { i in ([header[i]] + rows.map { $0[i] }).map(\.count).max() ?? 0 } + func line(_ cells: [String], tail: String) -> String { + cells.indices.map { i in i == last ? cells[i] + tail : cells[i].padding(toLength: widths[i], withPad: " ", startingAt: 0) + " " }.joined() + "\n" + } + return line(header, tail: " ") + rows.map { line($0, tail: " ") }.joined() + } + + /// The 12 hex characters shown as ID: the start of the SHA-256 of the repo id. + static func modelID(_ repo: String) -> String { + SHA256.hash(data: Data(repo.utf8)).prefix(6).map { String(format: "%02x", $0) }.joined() + } +} diff --git a/Sources/MLXRCLI/ModelCommands.swift b/Sources/MLXRCLI/ModelCommands.swift new file mode 100644 index 0000000..97c33d3 --- /dev/null +++ b/Sources/MLXRCLI/ModelCommands.swift @@ -0,0 +1,128 @@ +import Foundation +import MLXEngine + +extension CLI { + static let notSupported = ["create", "push", "signin", "signout", "launch"] + + /// list, rm, cp, ps, stop and the commands mlxr does not have. + static func modelCommands(library: ModelLibrary, presets: PresetStore, catalog: Catalog, + now: @escaping @Sendable () -> Date) -> [Command] { + let resolver = ModelResolver(library: library, presets: presets, catalog: catalog) + return [ + Command(name: "list", aliases: ["ls"], summary: "List installed models", args: [Arg("PREFIX", optional: true)], + options: [Opt(long: "json", help: "print the models as JSON")], + run: { inv, console in + console.out(renderList(library: library, presets: presets, prefix: inv.positionals.first, + json: inv.bool("json"), now: now())) + }), + Command(name: "rm", summary: "Remove a model", args: [Arg("MODEL", kind: .model, variadic: true)], + options: [Opt(long: "yes", short: "y", help: "do not ask before deleting from the Hugging Face cache")], + run: { inv, console in try remove(inv, console, library: library, presets: presets, resolver: resolver) }), + Command(name: "cp", summary: "Copy a model", args: [Arg("SOURCE", kind: .model), Arg("DESTINATION")], + run: { inv, console in + let dest = inv.positionals[1] + try PresetStore.validateName(dest) + let source = inv.positionals[0] + let r = try resolver.resolve(source) + guard let repo = r.repo else { throw CLIError.failure("cannot copy a model path; use a model name") } + try presets.save(Preset(name: dest, repo: repo, system: r.preset?.system, parameters: r.preset?.parameters ?? [:])) + console.out("copied '\(source)' to '\(dest)'\n") + console.err("success\n") + }), + Command(name: "ps", summary: "List running models", args: [Arg("PREFIX", optional: true)], + run: { _, console in + console.out(renderTable(header: ["NAME", "ID", "SIZE", "PROCESSOR", "CONTEXT", "RUNNER", "UNTIL"], rows: [])) + }), + Command(name: "stop", summary: "Stop a running model", args: [Arg("MODEL", kind: .model)], + run: { inv, _ in + throw CLIError.failure("couldn't find model \"\(inv.positionals[0])\" to stop\n" + + "mlxr runs a model only while a command uses it and keeps no background server; use 'mlxr serve MODEL' to start one") + }), + ] + unsupportedCommands + } + + /// create, push, signin, signout and launch exist in Ollama only. + static var unsupportedCommands: [Command] { + func stub(_ name: String, aliases: [String] = [], summary: String, options: [Opt] = []) -> Command { + let hint = name == "create" ? "; use 'mlxr pull '" : "" + return Command(name: name, aliases: aliases, summary: summary, args: [Arg("ARGS", optional: true, variadic: true)], + options: options, run: { _, _ in throw CLIError.failure("'\(name)' is not supported by mlxr\(hint)") }) + } + return [ + stub("create", summary: "Create a model (not supported)", options: [ + Opt(long: "file", short: "f", value: .text, help: "Modelfile"), Opt(long: "quantize", short: "q", value: .text, help: "quantization"), + Opt(long: "combine", help: "combine"), Opt(long: "draft-quantize", value: .text, help: "draft quantization"), + Opt(long: "force", help: "force")]), + stub("push", summary: "Push a model (not supported)", options: [Opt(long: "insecure", help: "use an insecure registry")]), + stub("signin", aliases: ["login"], summary: "Sign in (not supported)"), + stub("signout", aliases: ["logout"], summary: "Sign out (not supported)"), + stub("launch", summary: "Launch an integration (not supported)", options: [ + Opt(long: "model", short: "m", value: .text, help: "model"), Opt(long: "config", help: "config"), + Opt(long: "restore", help: "restore"), Opt(long: "yes", short: "y", help: "assume yes")]), + ] + } + + private static func renderList(library: ModelLibrary, presets: PresetStore, prefix: String?, json: Bool, now: Date) -> String { + struct Row { var name: String; var id: String; var size: Int?; var modified: Date? } + let models = library.scan() + var rows = models.map { Row(name: $0.repo, id: modelID($0.repo), size: $0.sizeBytes, modified: $0.modified) } + for p in presets.list() { + let base = models.first { $0.repo.lowercased() == p.repo.lowercased() } + rows.append(Row(name: p.name, id: modelID(p.repo), size: base?.sizeBytes, modified: nil)) + } + if let prefix = prefix?.lowercased(), !prefix.isEmpty { + rows = rows.filter { r in + r.name.lowercased().hasPrefix(prefix) || (r.name.split(separator: "/").last.map { $0.lowercased().hasPrefix(prefix) } ?? false) + } + } + if json { + let iso = ISO8601DateFormatter() + let items: [[String: Any]] = rows.map { r in + var d: [String: Any] = ["name": r.name, "id": r.id, "size": r.size ?? 0] + if let m = r.modified { d["modified"] = iso.string(from: m) } + return d + } + let data = (try? JSONSerialization.data(withJSONObject: items, options: [.prettyPrinted, .sortedKeys])) ?? Data("[]".utf8) + return String(decoding: data, as: UTF8.self) + "\n" + } + return renderTable(header: ["NAME", "ID", "SIZE", "MODIFIED"], rows: rows.map { + [$0.name, $0.id, $0.size.map(humanBytes) ?? "-", $0.modified.map { relativeTime($0, now: now) } ?? "-"] + }) + } + + private enum Removal { case preset(String), model(LocalModel) } + + private static func remove(_ inv: Invocation, _ console: Console, library: ModelLibrary, presets: PresetStore, + resolver: ModelResolver) throws { + let models = library.scan() + var plan: [(arg: String, what: Removal)] = [] + var seen = Set() + for arg in inv.positionals { + if presets.load(arg) != nil { + if seen.insert("preset:" + arg).inserted { plan.append((arg, .preset(arg))) } + continue + } + let r = try resolver.resolve(arg) + guard let repo = r.repo, let m = models.first(where: { $0.repo.lowercased() == repo.lowercased() }) else { + throw CLIError.failure("model '\(arg)' not found") + } + if seen.insert("model:" + m.repo).inserted { plan.append((arg, .model(m))) } + } + if !inv.bool("yes") && console.stdinIsTTY && console.stderrIsTTY { + for (arg, what) in plan { + guard case .model(let m) = what, m.source == .huggingFaceCache else { continue } + console.err("Delete '\(arg)' from the shared Hugging Face cache? [y/N] ") + guard let answer = console.readLine()?.trimmingCharacters(in: .whitespaces).lowercased(), ["y", "yes"].contains(answer) else { + throw CLIError.failure("aborted") + } + } + } + for (arg, what) in plan { + switch what { + case .preset(let name): try presets.delete(name) + case .model(let m): try library.delete(m) + } + console.out("deleted '\(arg)'\n") + } + } +} diff --git a/Sources/MLXRCLI/Parser.swift b/Sources/MLXRCLI/Parser.swift index bfc5688..825e8d2 100644 --- a/Sources/MLXRCLI/Parser.swift +++ b/Sources/MLXRCLI/Parser.swift @@ -65,6 +65,8 @@ public final class Console: @unchecked Sendable { public var stdinIsTTY: Bool public var width: Int public var env: [String: String] + /// Reads one line of input (a confirmation answer); nil at end of input. + public var readLine: () -> String? = { Swift.readLine() } public init(out: @escaping (String) -> Void, err: @escaping (String) -> Void, stdoutIsTTY: Bool = false, stderrIsTTY: Bool = false, stdinIsTTY: Bool = false, diff --git a/Sources/MLXRunner/Recommended.swift b/Sources/MLXRunner/Recommended.swift index b289433..6c4f182 100644 --- a/Sources/MLXRunner/Recommended.swift +++ b/Sources/MLXRunner/Recommended.swift @@ -1,4 +1,5 @@ import SwiftUI +import MLXEngine /// Hand-picked models that run well with the built-in engine. struct RecommendedModel: Identifiable, Hashable { @@ -8,19 +9,24 @@ struct RecommendedModel: Identifiable, Hashable { let gigabytes: Double let icon: String - static let all: [RecommendedModel] = [ - .init(id: "mlx-community/Qwen3-0.6B-4bit", title: "Qwen3 0.6B", blurb: "Tiny and blazing fast, with thinking mode", gigabytes: 0.35, icon: "hare.fill"), - .init(id: "mlx-community/Qwen3.5-0.8B-4bit", title: "Qwen3.5 0.8B", blurb: "Newest tiny Qwen: hybrid linear attention, quick direct answers", gigabytes: 0.65, icon: "bolt.horizontal.fill"), - .init(id: "mlx-community/Llama-3.2-3B-Instruct-4bit", title: "Llama 3.2 3B", blurb: "Meta's compact all-rounder", gigabytes: 1.8, icon: "bolt.fill"), - .init(id: "mlx-community/Qwen3-4B-4bit", title: "Qwen3 4B", blurb: "Strong reasoning in a small footprint", gigabytes: 2.3, icon: "brain.head.profile"), - .init(id: "mlx-community/Phi-3.5-mini-instruct-4bit", title: "Phi-3.5 mini", blurb: "Microsoft's efficient 3.8B model", gigabytes: 2.2, icon: "graduationcap.fill"), - .init(id: "mlx-community/Mistral-7B-Instruct-v0.3-4bit", title: "Mistral 7B v0.3", blurb: "Classic, fluent 7B instruct model", gigabytes: 4.1, icon: "wind"), - .init(id: "mlx-community/Qwen2.5-Coder-7B-Instruct-4bit", title: "Qwen2.5 Coder 7B", blurb: "Code generation and review", gigabytes: 4.3, icon: "chevron.left.forwardslash.chevron.right"), - .init(id: "mlx-community/Qwen3-8B-4bit", title: "Qwen3 8B", blurb: "Best quality under 5 GB", gigabytes: 4.6, icon: "sparkles"), - .init(id: "mlx-community/Meta-Llama-3.1-8B-Instruct-4bit", title: "Llama 3.1 8B", blurb: "Popular 8B with 128K context", gigabytes: 4.5, icon: "flame.fill"), - .init(id: "mlx-community/DeepSeek-R1-Distill-Qwen-7B-4bit", title: "DeepSeek R1 Distill 7B", blurb: "Step-by-step reasoning", gigabytes: 4.3, icon: "point.3.connected.trianglepath.dotted"), + private static let icons: [String: String] = [ + "mlx-community/Qwen3-0.6B-4bit": "hare.fill", + "mlx-community/Qwen3.5-0.8B-4bit": "bolt.horizontal.fill", + "mlx-community/Llama-3.2-3B-Instruct-4bit": "bolt.fill", + "mlx-community/Qwen3-4B-4bit": "brain.head.profile", + "mlx-community/Phi-3.5-mini-instruct-4bit": "graduationcap.fill", + "mlx-community/Mistral-7B-Instruct-v0.3-4bit": "wind", + "mlx-community/Qwen2.5-Coder-7B-Instruct-4bit": "chevron.left.forwardslash.chevron.right", + "mlx-community/Qwen3-8B-4bit": "sparkles", + "mlx-community/Meta-Llama-3.1-8B-Instruct-4bit": "flame.fill", + "mlx-community/DeepSeek-R1-Distill-Qwen-7B-4bit": "point.3.connected.trianglepath.dotted", ] + /// The shared engine catalog plus this app's icons. + static let all: [RecommendedModel] = Catalog.standard.models.map { + RecommendedModel(id: $0.id, title: $0.title, blurb: $0.blurb, gigabytes: $0.gigabytes, icon: icons[$0.id] ?? "cube.fill") + } + /// Picks that fit comfortably in this Mac's unified memory. static var forThisMac: [RecommendedModel] { let ram = Double(ProcessInfo.processInfo.physicalMemory) / 1_073_741_824 diff --git a/Tests/MLXRCLITests/CatalogTests.swift b/Tests/MLXRCLITests/CatalogTests.swift new file mode 100644 index 0000000..ed33f99 --- /dev/null +++ b/Tests/MLXRCLITests/CatalogTests.swift @@ -0,0 +1,69 @@ +import Foundation +import Testing +import MLXEngine + +@Suite("Model catalog (AC-10)") +struct CatalogTests { + @Test func ollamaStyleAliasesNameMLXRepos() { + let expected: [String: String] = [ + "qwen3:0.6b": "mlx-community/Qwen3-0.6B-4bit", + "qwen3:4b": "mlx-community/Qwen3-4B-4bit", + "qwen3:8b": "mlx-community/Qwen3-8B-4bit", + "qwen3.5:0.8b": "mlx-community/Qwen3.5-0.8B-4bit", + "llama3.2:3b": "mlx-community/Llama-3.2-3B-Instruct-4bit", + "phi3.5": "mlx-community/Phi-3.5-mini-instruct-4bit", + "mistral:7b": "mlx-community/Mistral-7B-Instruct-v0.3-4bit", + ] + for (alias, repo) in expected { #expect(Catalog.standard.repo(forAlias: alias) == repo, "\(alias)") } + } + + @Test func bareFamilyAndLatestMeanTheFamilyDefault() { + let c = Catalog.standard + #expect(c.repo(forAlias: "qwen3") == "mlx-community/Qwen3-8B-4bit") + #expect(c.repo(forAlias: "qwen3:latest") == c.repo(forAlias: "qwen3")) + #expect(c.repo(forAlias: "phi3.5:latest") == c.repo(forAlias: "phi3.5")) + #expect(c.repo(forAlias: "qwen3.5") == "mlx-community/Qwen3.5-0.8B-4bit") + } + + @Test func aliasesAreCaseInsensitiveAndUnknownOnesMiss() { + #expect(Catalog.standard.repo(forAlias: "Qwen3:8B") == "mlx-community/Qwen3-8B-4bit") + #expect(Catalog.standard.repo(forAlias: "nope:1b") == nil) + #expect(Catalog.standard.repo(forAlias: "") == nil) + } + + @Test func everyRepoIsAValidMLXCommunityIdAndAliasesAreUnique() { + let c = Catalog.standard + #expect(!c.models.isEmpty) + for m in c.models { + #expect(RepoID.isValid(m.id), "\(m.id)") + #expect(m.id.hasPrefix("mlx-community/"), "\(m.id)") + #expect(!m.aliases.isEmpty, "\(m.id) has no alias") + #expect(m.aliases.allSatisfy { $0 == $0.lowercased() && !$0.isEmpty }, "\(m.id)") + #expect(m.gigabytes > 0) + } + let all = c.models.flatMap(\.aliases) + #expect(Set(all).count == all.count, "an alias names two models") + #expect(Set(c.aliases) == Set(all)) + } + + @Test func keepsEveryModelTheAppRecommendedBefore() { + let previous = [ + "mlx-community/Qwen3-0.6B-4bit", "mlx-community/Qwen3.5-0.8B-4bit", "mlx-community/Llama-3.2-3B-Instruct-4bit", + "mlx-community/Qwen3-4B-4bit", "mlx-community/Phi-3.5-mini-instruct-4bit", + "mlx-community/Mistral-7B-Instruct-v0.3-4bit", "mlx-community/Qwen2.5-Coder-7B-Instruct-4bit", + "mlx-community/Qwen3-8B-4bit", "mlx-community/Meta-Llama-3.1-8B-Instruct-4bit", + "mlx-community/DeepSeek-R1-Distill-Qwen-7B-4bit", + ] + let ids = Set(Catalog.standard.models.map(\.id)) + for p in previous { #expect(ids.contains(p), "\(p) dropped from the catalog") } + } + + @Test func aCustomCatalogIsInjectable() { + let tiny = CatalogModel(id: "acme/Tiny-4bit", title: "Tiny", blurb: "b", gigabytes: 0.1, aliases: ["tiny", "tiny:1b", "tiny:latest"]) + let c = Catalog(models: [tiny]) + #expect(c.repo(forAlias: "tiny") == "acme/Tiny-4bit") + #expect(c.repo(forAlias: "TINY:1B") == "acme/Tiny-4bit") + #expect(c.repo(forAlias: "qwen3") == nil) + #expect(c.aliases.sorted() == ["tiny", "tiny:1b", "tiny:latest"]) + } +} diff --git a/Tests/MLXRCLITests/ModelFixtures.swift b/Tests/MLXRCLITests/ModelFixtures.swift new file mode 100644 index 0000000..6cd4666 --- /dev/null +++ b/Tests/MLXRCLITests/ModelFixtures.swift @@ -0,0 +1,76 @@ +import Foundation +@testable import MLXRCLI +import MLXEngine + +/// Feeds scripted answers to `Console.readLine`. +final class LineFeed: @unchecked Sendable { + private let lock = NSLock() + private var lines: [String] + private(set) var asked = 0 + init(_ lines: [String]) { self.lines = lines } + func next() -> String? { + lock.withLock { asked += 1; return lines.isEmpty ? nil : lines.removeFirst() } + } +} + +/// An in-memory model library and a presets directory under the temporary directory. Nothing touches +/// ~/Library or the Hugging Face cache. +final class LibraryFixture: @unchecked Sendable { + static let now = Date(timeIntervalSince1970: 1_800_000_000) + static let qwen8 = "mlx-community/Qwen3-8B-4bit" + static let qwen06 = "mlx-community/Qwen3-0.6B-4bit" + static let custom = "acme/Custom-Model-4bit" + + private let lock = NSLock() + private var current: [LocalModel] + private var removed: [String] = [] + let presetsDirectory: URL + var presets: PresetStore { PresetStore(directory: presetsDirectory) } + var deleted: [String] { lock.withLock { removed } } + + static func model(_ repo: String, bytes: Int, ageSeconds: TimeInterval, source: LocalModel.Source = .library) -> LocalModel { + LocalModel(repo: repo, directory: URL(fileURLWithPath: "/fixture/\(repo)"), sizeBytes: bytes, + modelType: "qwen3", bits: 4, source: source, modified: now.addingTimeInterval(-ageSeconds)) + } + + init(models: [LocalModel]) { + current = models + presetsDirectory = FileManager.default.temporaryDirectory.appendingPathComponent("mlxr-cli-tests-\(UUID().uuidString)", isDirectory: true) + try? FileManager.default.createDirectory(at: presetsDirectory, withIntermediateDirectories: true) + } + + deinit { try? FileManager.default.removeItem(at: presetsDirectory) } + + /// Two library models, 0.6B 21 days old and 8B 5 minutes old. + static func standard() -> LibraryFixture { + LibraryFixture(models: [ + model(qwen06, bytes: 350_000_000, ageSeconds: 21 * 86_400), + model(qwen8, bytes: 5_200_000_000, ageSeconds: 300), + ]) + } + + var library: ModelLibrary { + ModelLibrary(scan: { self.lock.withLock { self.current } }, + delete: { m in self.lock.withLock { self.removed.append(m.repo); self.current.removeAll { $0.repo == m.repo } } }) + } + + var resolver: ModelResolver { ModelResolver(library: library, presets: presets) } + + func table(catalog: Catalog = .standard) -> [Command] { + CLI.commands(services: FakeServices().services, library: library, presets: presets, catalog: catalog, now: { Self.now }) + } + + /// Runs a command line; returns the exit code and what it wrote. + func run(_ args: [String], tty: Bool = false, answers: LineFeed? = nil, catalog: Catalog = .standard) async -> (code: Int32, out: String, err: String) { + let cap = Capture() + let console = cap.console(stdoutTTY: tty, stderrTTY: tty, stdinTTY: tty) + if let answers { console.readLine = { answers.next() } } + let code = await CLI.run(args, console: console, commands: table(catalog: catalog)) + return (code, cap.stdout, cap.stderr) + } +} + +/// The message a throwing call fails with, or nil when it does not throw. +func failureMessage(_ body: () throws -> Any) -> String? { + do { _ = try body(); return nil } catch { return CLI.message(for: error) } +} diff --git a/Tests/MLXRCLITests/ModelListTests.swift b/Tests/MLXRCLITests/ModelListTests.swift new file mode 100644 index 0000000..3103ed2 --- /dev/null +++ b/Tests/MLXRCLITests/ModelListTests.swift @@ -0,0 +1,85 @@ +import Foundation +import Testing +import MLXEngine +@testable import MLXRCLI + +@Suite("list and its formatting (AC-9)") +struct ModelListTests { + static let header = "NAME ID SIZE MODIFIED \n" + static let row06 = "mlx-community/Qwen3-0.6B-4bit 51a1a9bb2c1e 350 MB 3 weeks ago \n" + static let row8 = "mlx-community/Qwen3-8B-4bit 9c29d1d07c97 5.2 GB 5 minutes ago \n" + + @Test func listRendersTheOllamaTableByteForByte() async { + let r = await LibraryFixture.standard().run(["list"]) + #expect(r.code == 0, "\(r.err)") + #expect(r.out == Self.header + Self.row06 + Self.row8) + #expect(r.err.isEmpty) + } + + @Test func lsIsAnAliasAndEmptyLibraryPrintsOnlyTheHeader() async { + let r = await LibraryFixture(models: []).run(["ls"]) + #expect(r.code == 0, "\(r.err)") + #expect(r.out == "NAME ID SIZE MODIFIED \n") + } + + @Test func prefixFilterIsCaseInsensitiveOnTheNameOrTheShortName() async { + let f = LibraryFixture.standard() + let short = await f.run(["list", "QWEN3-8"]) + #expect(short.out.contains("Qwen3-8B-4bit") && !short.out.contains("Qwen3-0.6B"), "\(short.out)") + let full = await f.run(["list", "MLX-COMMUNITY/qwen3-0"]) + #expect(full.out.contains("Qwen3-0.6B-4bit") && !full.out.contains("Qwen3-8B"), "\(full.out)") + let none = await f.run(["list", "zzz"]) + #expect(none.code == 0) + #expect(!none.out.contains("mlx-community")) + } + + @Test func jsonListsEveryModelAsValidJSON() async throws { + let r = await LibraryFixture.standard().run(["list", "--json"]) + #expect(r.code == 0, "\(r.err)") + let rows = try #require(try JSONSerialization.jsonObject(with: Data(r.out.utf8)) as? [[String: Any]]) + #expect(rows.count == 2) + #expect(rows[0]["name"] as? String == LibraryFixture.qwen06) + #expect(rows[0]["id"] as? String == "51a1a9bb2c1e") + #expect(rows[0]["size"] as? Int == 350_000_000) + #expect(rows[1]["name"] as? String == LibraryFixture.qwen8) + #expect(rows[1]["modified"] is String) + } + + @Test func presetsAreListedByName() async throws { + let f = LibraryFixture.standard() + try f.presets.save(Preset(name: "mychat", repo: LibraryFixture.qwen8)) + let r = await f.run(["list"]) + let lines = r.out.split(separator: "\n").map(String.init) + #expect(lines.contains { $0.hasPrefix("mychat") }, "\(r.out)") + #expect(lines.contains { $0.hasPrefix(LibraryFixture.qwen8) }) + } + + @Test func humanBytesUsesBase1000UnitsWithOneDecimalBelowTen() { + let cases: [(Int, String)] = [ + (0, "0 B"), (999, "999 B"), (1_500, "1.5 KB"), (350_000_000, "350 MB"), (512_000_000, "512 MB"), + (2_000_000_000, "2.0 GB"), (5_200_000_000, "5.2 GB"), (12_345_000_000, "12 GB"), + ] + for (n, want) in cases { #expect(CLI.humanBytes(n) == want, "\(n)") } + } + + @Test func relativeTimeReadsLikeOllama() { + let now = LibraryFixture.now + let cases: [(TimeInterval, String)] = [ + (0.2, "Less than a second ago"), (60, "About a minute ago"), (300, "5 minutes ago"), + (3 * 3_600, "3 hours ago"), (3 * 86_400, "3 days ago"), (21 * 86_400, "3 weeks ago"), + (90 * 86_400, "3 months ago"), (3 * 365 * 86_400, "3 years ago"), + ] + for (age, want) in cases { #expect(CLI.relativeTime(now.addingTimeInterval(-age), now: now) == want, "\(age)") } + } + + @Test func tableAlignsColumnsAndEndsEveryCellWithFourSpaces() { + let t = CLI.renderTable(header: ["A", "BB"], rows: [["xxx", "1"], ["y", "22"]]) + #expect(t == "A BB \nxxx 1 \ny 22 \n") + #expect(CLI.renderTable(header: ["A", "B"], rows: []) == "A B \n") + } + + @Test func modelIDIsTheFirstTwelveHexOfTheRepoIdDigest() { + #expect(CLI.modelID(LibraryFixture.qwen06) == "51a1a9bb2c1e") + #expect(CLI.modelID(LibraryFixture.qwen8) == "9c29d1d07c97") + } +} diff --git a/Tests/MLXRCLITests/ModelManagementTests.swift b/Tests/MLXRCLITests/ModelManagementTests.swift new file mode 100644 index 0000000..cae6175 --- /dev/null +++ b/Tests/MLXRCLITests/ModelManagementTests.swift @@ -0,0 +1,157 @@ +import Foundation +import Testing +import MLXEngine +@testable import MLXRCLI + +@Suite("rm and cp (AC-12)") +struct ModelManagementTests { + static let hf = "someone/Cached-4bit" + + func fixture() -> LibraryFixture { + LibraryFixture(models: [ + LibraryFixture.model(LibraryFixture.qwen8, bytes: 5_200_000_000, ageSeconds: 60), + LibraryFixture.model(Self.hf, bytes: 1_000_000_000, ageSeconds: 60, source: .huggingFaceCache), + ]) + } + + @Test func rmDeletesAndPrintsDeleted() async { + let f = fixture() + let r = await f.run(["rm", LibraryFixture.qwen8]) + #expect(r.code == 0, "\(r.err)") + #expect(r.out == "deleted '\(LibraryFixture.qwen8)'\n") + #expect(f.deleted == [LibraryFixture.qwen8]) + } + + @Test func rmTakesSeveralModelsByAliasOrShortName() async { + let f = fixture() + let r = await f.run(["rm", "qwen3:8b", "cached-4bit", "-y"]) + #expect(r.code == 0, "\(r.err)") + #expect(r.out == "deleted 'qwen3:8b'\ndeleted 'cached-4bit'\n") + #expect(Set(f.deleted) == [LibraryFixture.qwen8, Self.hf]) + } + + @Test func rmOnlyDeletesInstalledModels() async { + let f = fixture() + let r = await f.run(["rm", "qwen3:4b"]) + #expect(r.code == 1 && r.out.isEmpty) + #expect(r.err.hasPrefix("Error: model 'qwen3:4b' not found"), "\(r.err)") + let mixed = await f.run(["rm", LibraryFixture.qwen8, "nope"]) + #expect(mixed.code == 1) + #expect(f.deleted.isEmpty, "nothing is deleted when any name is unknown") + let none = await f.run(["rm"]) + #expect(none.err == "Error: requires at least 1 arg(s), only received 0\n") + } + + @Test func rmRefusesAnAmbiguousShortName() async { + let f = LibraryFixture(models: [ + LibraryFixture.model("a/Same-4bit", bytes: 1, ageSeconds: 1), + LibraryFixture.model("b/Same-4bit", bytes: 1, ageSeconds: 1), + ]) + let r = await f.run(["rm", "Same-4bit"]) + #expect(r.code == 1) + #expect(r.err.hasPrefix("Error: ") && r.err.contains("ambiguous") && r.err.contains("a/Same-4bit") && r.err.contains("b/Same-4bit"), "\(r.err)") + #expect(f.deleted.isEmpty) + } + + @Test func aTerminalAsksBeforeDeletingFromTheSharedHuggingFaceCache() async { + let confirmed = fixture() + let feed = LineFeed(["y"]) + let ok = await confirmed.run(["rm", Self.hf], tty: true, answers: feed) + #expect(ok.code == 0, "\(ok.err)") + #expect(ok.err.contains("Hugging Face") && ok.err.contains("[y/N]"), "\(ok.err)") + #expect(ok.out == "deleted '\(Self.hf)'\n") + #expect(confirmed.deleted == [Self.hf]) + + let yes = fixture() + #expect(await yes.run(["rm", Self.hf], tty: true, answers: LineFeed(["YES"])).code == 0) + #expect(yes.deleted == [Self.hf]) + + for answer in ["n", "", "maybe"] { + let declined = fixture() + let r = await declined.run(["rm", Self.hf], tty: true, answers: LineFeed([answer])) + #expect(r.code == 1, "\(answer)") + #expect(r.err.contains("aborted"), "\(r.err)") + #expect(declined.deleted.isEmpty, "\(answer) must not delete") + #expect(!r.out.contains("deleted")) + } + let eof = fixture() + #expect(await eof.run(["rm", Self.hf], tty: true, answers: LineFeed([])).code == 1) + #expect(eof.deleted.isEmpty) + } + + @Test func yesSkipsTheQuestionAndNoTerminalNeverAsks() async { + let f = fixture() + let feed = LineFeed(["n"]) + let r = await f.run(["rm", "-y", Self.hf], tty: true, answers: feed) + #expect(r.code == 0 && feed.asked == 0) + #expect(f.deleted == [Self.hf]) + + let long = fixture() + #expect(await long.run(["rm", "--yes", Self.hf], tty: true, answers: LineFeed([])).code == 0) + + let piped = fixture() + let p = await piped.run(["rm", Self.hf]) + #expect(p.code == 0 && p.err.isEmpty, "\(p.err)") + #expect(piped.deleted == [Self.hf]) + } + + @Test func libraryModelsAreDeletedWithoutAskingEvenOnATerminal() async { + let f = fixture() + let feed = LineFeed([]) + let r = await f.run(["rm", LibraryFixture.qwen8], tty: true, answers: feed) + #expect(r.code == 0 && feed.asked == 0 && r.err.isEmpty, "\(r.err)") + } + + @Test func rmOfAPresetNameRemovesOnlyThePreset() async throws { + let f = fixture() + try f.presets.save(Preset(name: "mychat", repo: LibraryFixture.qwen8)) + let r = await f.run(["rm", "mychat"]) + #expect(r.code == 0, "\(r.err)") + #expect(r.out == "deleted 'mychat'\n") + #expect(f.presets.load("mychat") == nil) + #expect(f.deleted.isEmpty) + } + + @Test func cpCreatesAPresetThatResolves() async throws { + let f = fixture() + let r = await f.run(["cp", LibraryFixture.qwen8, "mychat"]) + #expect(r.code == 0, "\(r.err)") + #expect(r.out == "copied '\(LibraryFixture.qwen8)' to 'mychat'\n") + #expect(r.err == "success\n") + #expect(f.presets.load("mychat")?.repo == LibraryFixture.qwen8) + let resolved = try f.resolver.resolve("mychat") + #expect(resolved.source == .preset && resolved.repo == LibraryFixture.qwen8) + } + + @Test func cpAcceptsAnyResolvableSourceAndCopiesPresetSettings() async throws { + let f = fixture() + let viaAlias = await f.run(["cp", "qwen3:8b", "short"]) + #expect(viaAlias.code == 0, "\(viaAlias.err)") + #expect(f.presets.load("short")?.repo == LibraryFixture.qwen8) + + try f.presets.save(Preset(name: "base", repo: LibraryFixture.qwen8, system: "be brief", parameters: ["temperature": "0.2"])) + let copy = await f.run(["cp", "base", "tuned:v2"]) + #expect(copy.code == 0, "\(copy.err)") + let saved = try #require(f.presets.load("tuned:v2")) + #expect(saved == Preset(name: "tuned:v2", repo: LibraryFixture.qwen8, system: "be brief", parameters: ["temperature": "0.2"])) + } + + @Test func cpRejectsMissingSourcesAndUnsafeNames() async throws { + let f = fixture() + let missing = await f.run(["cp", "nope", "x"]) + #expect(missing.code == 1 && missing.out.isEmpty) + #expect(missing.err.hasPrefix("Error: model 'nope' not found"), "\(missing.err)") + #expect(f.presets.list().isEmpty) + + for name in ["../evil", "a/b", ".hidden", "has space"] { + let r = await f.run(["cp", LibraryFixture.qwen8, name]) + #expect(r.code == 1 && r.out.isEmpty, "\(name)") + #expect(r.err.hasPrefix("Error: "), "\(name): \(r.err)") + } + #expect(f.presets.list().isEmpty) + let escaped = f.presetsDirectory.deletingLastPathComponent().appendingPathComponent("evil.json") + #expect(!FileManager.default.fileExists(atPath: escaped.path)) + let none = await f.run(["cp", "only-one"]) + #expect(none.err == "Error: accepts 2 arg(s), received 1\n") + } +} diff --git a/Tests/MLXRCLITests/ModelResolverTests.swift b/Tests/MLXRCLITests/ModelResolverTests.swift new file mode 100644 index 0000000..51c55e0 --- /dev/null +++ b/Tests/MLXRCLITests/ModelResolverTests.swift @@ -0,0 +1,131 @@ +import Foundation +import Testing +import MLXEngine +@testable import MLXRCLI + +@Suite("Model name resolver (AC-10)") +struct ModelResolverTests { + static let qwen8 = LibraryFixture.qwen8 + + func fixture() throws -> LibraryFixture { + let f = LibraryFixture(models: [ + LibraryFixture.model(Self.qwen8, bytes: 5_200_000_000, ageSeconds: 60), + LibraryFixture.model(LibraryFixture.custom, bytes: 1_000_000_000, ageSeconds: 60), + ]) + try f.presets.save(Preset(name: "mychat", repo: Self.qwen8, system: "be brief")) + return f + } + + @Test func existingPathWins() throws { + let f = try fixture() + let dir = FileManager.default.temporaryDirectory.appendingPathComponent("mlxr-model-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: dir) } + let r = try f.resolver.resolve(dir.path) + #expect(r.source == .path) + #expect(r.directory?.standardizedFileURL.path == dir.standardizedFileURL.path) + #expect(r.repo == nil) + } + + @Test func installedRepoIdIsCaseInsensitiveAndCanonical() throws { + let f = try fixture() + for typed in [Self.qwen8, "MLX-COMMUNITY/qwen3-8b-4bit"] { + let r = try f.resolver.resolve(typed) + #expect(r.source == .installed, "\(typed)") + #expect(r.repo == Self.qwen8) + #expect(r.directory?.path == "/fixture/\(Self.qwen8)") + } + } + + @Test func huggingFaceUrlsNameARepoWithoutPulling() throws { + let f = try fixture() + for prefix in ["hf.co/", "huggingface.co/"] { + let r = try f.resolver.resolve(prefix + "acme/Other-4bit") + #expect(r.source == .huggingFace) + #expect(r.repo == "acme/Other-4bit") + #expect(r.directory == nil, "not installed") + } + let installed = try f.resolver.resolve("hf.co/" + LibraryFixture.custom) + #expect(installed.repo == LibraryFixture.custom) + #expect(installed.directory?.path == "/fixture/\(LibraryFixture.custom)") + let bare = try f.resolver.resolve("acme/NotInstalled") + #expect(bare.source == .huggingFace && bare.repo == "acme/NotInstalled" && bare.directory == nil) + } + + @Test func catalogAliasesIncludeTheFamilyDefault() throws { + let f = try fixture() + let eight = try f.resolver.resolve("qwen3:8b") + #expect(eight.source == .catalog && eight.repo == Self.qwen8) + #expect(eight.directory?.path == "/fixture/\(Self.qwen8)", "installed, so a directory") + for family in ["qwen3", "qwen3:latest"] { + let r = try f.resolver.resolve(family) + #expect(r.source == .catalog && r.repo == Self.qwen8, "\(family)") + } + let four = try f.resolver.resolve("qwen3:4b") + #expect(four.repo == "mlx-community/Qwen3-4B-4bit") + #expect(four.directory == nil, "not installed") + } + + @Test func presetsResolveToTheirModelAndKeepTheirSettings() throws { + let f = try fixture() + let r = try f.resolver.resolve("mychat") + #expect(r.source == .preset && r.repo == Self.qwen8) + #expect(r.preset?.system == "be brief") + #expect(r.directory?.path == "/fixture/\(Self.qwen8)") + } + + @Test func installedShortNameComesLast() throws { + let f = try fixture() + for typed in ["Custom-Model-4bit", "custom-model-4bit"] { + let r = try f.resolver.resolve(typed) + #expect(r.source == .shortName && r.repo == LibraryFixture.custom, "\(typed)") + } + } + + @Test func orderIsCatalogBeforePresetBeforeShortName() throws { + let f = try fixture() + try f.presets.save(Preset(name: "qwen3:8b", repo: LibraryFixture.custom)) + try f.presets.save(Preset(name: "Custom-Model-4bit", repo: Self.qwen8)) + let catalogWins = try f.resolver.resolve("qwen3:8b") + #expect(catalogWins.source == .catalog && catalogWins.repo == Self.qwen8) + let presetWins = try f.resolver.resolve("Custom-Model-4bit") + #expect(presetWins.source == .preset && presetWins.repo == Self.qwen8) + } + + @Test func ambiguousShortNameListsTheCandidates() throws { + let f = LibraryFixture(models: [ + LibraryFixture.model("a/Same-4bit", bytes: 1, ageSeconds: 1), + LibraryFixture.model("b/Same-4bit", bytes: 1, ageSeconds: 1), + ]) + let m = try #require(failureMessage { try f.resolver.resolve("Same-4bit") }) + #expect(m.contains("ambiguous") && m.contains("a/Same-4bit") && m.contains("b/Same-4bit"), "\(m)") + } + + @Test func aMissSuggestsTheClosestNames() throws { + let f = try fixture() + let m = try #require(failureMessage { try f.resolver.resolve("qwen3:9b") }) + #expect(m.hasPrefix("model 'qwen3:9b' not found"), "\(m)") + #expect(m.contains("did you mean") && m.contains("qwen3:8b"), "\(m)") + let installed = try #require(failureMessage { try f.resolver.resolve("Custom-Model-4bi") }) + #expect(installed.contains("did you mean") && installed.contains(LibraryFixture.custom), "\(installed)") + let far = try #require(failureMessage { try f.resolver.resolve("zzzzzzzzzz") }) + #expect(far == "model 'zzzzzzzzzz' not found", "\(far)") + } + + @Test func derivedRepoIdsMustValidate() throws { + let f = try fixture() + for typed in ["../x", "a/b/c", "hf.co/a/b/c", "hf.co/org", "hf.co/../x", "huggingface.co//x"] { + let m = try #require(failureMessage { try f.resolver.resolve(typed) }, "\(typed) resolved") + #expect(m.contains("invalid repo id"), "\(typed): \(m)") + } + #expect(failureMessage { try f.resolver.resolve("") } != nil) + } + + @Test func aCustomCatalogIsUsed() throws { + let f = try fixture() + let tiny = CatalogModel(id: "acme/Tiny-4bit", title: "Tiny", blurb: "b", gigabytes: 0.1, aliases: ["tiny", "tiny:latest"]) + let resolver = ModelResolver(library: f.library, presets: f.presets, catalog: Catalog(models: [tiny])) + #expect(try resolver.resolve("tiny:latest").repo == "acme/Tiny-4bit") + #expect(failureMessage { try resolver.resolve("qwen3:8b") } != nil, "the built-in catalog is not consulted") + } +} diff --git a/Tests/MLXRCLITests/PresetStoreTests.swift b/Tests/MLXRCLITests/PresetStoreTests.swift new file mode 100644 index 0000000..24726a9 --- /dev/null +++ b/Tests/MLXRCLITests/PresetStoreTests.swift @@ -0,0 +1,59 @@ +import Foundation +import Testing +@testable import MLXRCLI + +@Suite("Preset store (AC-12)") +struct PresetStoreTests { + func store() -> (PresetStore, URL) { + let dir = FileManager.default.temporaryDirectory.appendingPathComponent("mlxr-presets-\(UUID().uuidString)", isDirectory: true) + return (PresetStore(directory: dir), dir) + } + + @Test func saveLoadListDeleteRoundTrip() throws { + let (s, dir) = store() + defer { try? FileManager.default.removeItem(at: dir) } + #expect(s.load("mychat") == nil) + #expect(s.list().isEmpty) + let p = Preset(name: "mychat", repo: "acme/X-4bit", system: "be brief", parameters: ["temperature": "0.2", "seed": "7"]) + try s.save(p) + try s.save(Preset(name: "alpha", repo: "acme/Y-4bit")) + #expect(s.load("mychat") == p) + #expect(FileManager.default.fileExists(atPath: dir.appendingPathComponent("mychat.json").path), "one .json file per preset") + #expect(s.list().map(\.name) == ["alpha", "mychat"], "sorted by name") + try s.save(Preset(name: "mychat", repo: "acme/Z-4bit")) + #expect(s.load("mychat")?.repo == "acme/Z-4bit", "saving again replaces") + try s.delete("mychat") + #expect(s.load("mychat") == nil) + #expect(s.list().map(\.name) == ["alpha"]) + } + + @Test func nameValidation() { + for ok in ["mychat", "qwen3-tuned:v2", "a.b_c", "A1"] { + #expect(failureMessage { try PresetStore.validateName(ok) } == nil, "\(ok)") + } + for bad in ["", "../x", "a/b", ".hidden", "a b", "x\0y", "back\\slash", String(repeating: "a", count: 129)] { + #expect(failureMessage { try PresetStore.validateName(bad) } != nil, "\(bad.debugDescription) accepted") + } + } + + @Test func saveRejectsAnUnsafeNameWithoutWritingAnything() throws { + let (s, dir) = store() + defer { try? FileManager.default.removeItem(at: dir) } + #expect(failureMessage { try s.save(Preset(name: "../evil", repo: "a/b")) } != nil) + #expect(!FileManager.default.fileExists(atPath: dir.deletingLastPathComponent().appendingPathComponent("evil.json").path)) + #expect(s.list().isEmpty) + #expect(s.load("../evil") == nil) + } + + @Test func aCorruptPresetIsMovedAsideNotOverwrittenOrFatal() throws { + let (s, dir) = store() + defer { try? FileManager.default.removeItem(at: dir) } + try s.save(Preset(name: "good", repo: "a/b")) + try Data("{ not json".utf8).write(to: dir.appendingPathComponent("bad.json")) + #expect(s.load("bad") == nil) + #expect(s.list().map(\.name) == ["good"]) + let files = (try? FileManager.default.contentsOfDirectory(atPath: dir.path)) ?? [] + #expect(files.contains { $0.hasPrefix("bad.corrupt-") }, "\(files)") + #expect(!files.contains("bad.json")) + } +} diff --git a/Tests/MLXRCLITests/UnsupportedCommandTests.swift b/Tests/MLXRCLITests/UnsupportedCommandTests.swift new file mode 100644 index 0000000..d24b8bd --- /dev/null +++ b/Tests/MLXRCLITests/UnsupportedCommandTests.swift @@ -0,0 +1,58 @@ +import Foundation +import Testing +@testable import MLXRCLI + +@Suite("ps, stop, serve without a model and commands mlxr does not have (AC-19)") +struct UnsupportedCommandTests { + func run(_ args: [String]) async -> (code: Int32, out: String, err: String) { + await LibraryFixture.standard().run(args) + } + + @Test func psPrintsOnlyTheHeader() async { + for args in [["ps"], ["ps", "qwen"]] { + let r = await run(args) + #expect(r.code == 0, "\(r.err)") + #expect(r.out == "NAME ID SIZE PROCESSOR CONTEXT RUNNER UNTIL \n") + #expect(r.err.isEmpty) + } + } + + @Test func stopSaysNothingIsRunning() async { + let r = await run(["stop", "qwen3:8b"]) + #expect(r.code == 1 && r.out.isEmpty) + #expect(r.err.hasPrefix("Error: couldn't find model \"qwen3:8b\" to stop"), "\(r.err)") + #expect(r.err.contains("mlxr serve"), "explains there is no background server: \(r.err)") + #expect(r.err.hasSuffix("\n")) + #expect(await run(["stop"]).err == "Error: accepts 1 arg(s), received 0\n") + } + + @Test func serveWithoutAModelPointsAtTheNextPhase() async { + for args in [["serve"], ["start"], ["serve", "--port", "8080"]] { + let r = await run(args) + #expect(r.code == 1 && r.out.isEmpty, "\(args)") + #expect(r.err.hasPrefix("Error: "), "\(args): \(r.err)") + #expect(!r.err.contains("accepts 1 arg"), "\(args): \(r.err)") + #expect(r.err.contains("mlxr serve MODEL"), "\(args): \(r.err)") + } + } + + @Test func commandsOllamaHasAndMlxrDoesNot() async { + let pull = "Error: 'create' is not supported by mlxr; use 'mlxr pull '\n" + let cases: [([String], String)] = [ + (["create", "m"], pull), + (["create", "m", "-f", "Modelfile"], pull), + (["push", "m"], "Error: 'push' is not supported by mlxr\n"), + (["signin"], "Error: 'signin' is not supported by mlxr\n"), + (["login"], "Error: 'signin' is not supported by mlxr\n"), + (["signout"], "Error: 'signout' is not supported by mlxr\n"), + (["logout"], "Error: 'signout' is not supported by mlxr\n"), + (["launch"], "Error: 'launch' is not supported by mlxr\n"), + (["launch", "claude", "-m", "x", "--yes"], "Error: 'launch' is not supported by mlxr\n"), + ] + for (args, want) in cases { + let r = await run(args) + #expect(r.code == 1 && r.out.isEmpty, "\(args)") + #expect(r.err == want, "\(args): \(r.err)") + } + } +} From 3f57be1caf198311aa18ce511c10dcb7ed1b0700 Mon Sep 17 00:00:00 2001 From: Rolando Santamaria Maso Date: Sun, 8 Nov 2026 15:15:03 +0100 Subject: [PATCH 06/20] fix(cli): validate preset repo ids before resolving or copying them --- Sources/MLXRCLI/Library.swift | 5 +- .../PresetRepoValidationTests.swift | 60 +++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 Tests/MLXRCLITests/PresetRepoValidationTests.swift diff --git a/Sources/MLXRCLI/Library.swift b/Sources/MLXRCLI/Library.swift index 2a72294..404e165 100644 --- a/Sources/MLXRCLI/Library.swift +++ b/Sources/MLXRCLI/Library.swift @@ -118,7 +118,10 @@ public struct ModelResolver: Sendable { return make(.huggingFace, repo) } if let repo = catalog.repo(forAlias: name) { return make(.catalog, repo) } - if let p = presets.load(name) { return make(.preset, p.repo, preset: p) } + if let p = presets.load(name) { + try validateRepo(p.repo) + return make(.preset, p.repo, preset: p) + } let named = models.filter { $0.name.lowercased() == lower } if named.count > 1 { throw CLIError.failure("'\(name)' is ambiguous; matches: " + named.map(\.repo).joined(separator: ", ")) diff --git a/Tests/MLXRCLITests/PresetRepoValidationTests.swift b/Tests/MLXRCLITests/PresetRepoValidationTests.swift new file mode 100644 index 0000000..855de76 --- /dev/null +++ b/Tests/MLXRCLITests/PresetRepoValidationTests.swift @@ -0,0 +1,60 @@ +import Foundation +import Testing +@testable import MLXRCLI + +@Suite("A preset's repo id is validated (SL03 review R-01)") +struct PresetRepoValidationTests { + static let badRepos = ["../x", "a/b/c", "org/../etc", "", "/abs", "no-slash"] + + /// Writes a preset file by hand, as a damaged or hostile file would be. + func plant(_ name: String, repo: String, in f: LibraryFixture) throws { + let json = try JSONSerialization.data(withJSONObject: ["name": name, "repo": repo, "parameters": [String: String]()]) + try json.write(to: f.presetsDirectory.appendingPathComponent("\(name).json")) + } + + func files(_ f: LibraryFixture) -> [String] { + ((try? FileManager.default.contentsOfDirectory(atPath: f.presetsDirectory.path)) ?? []).sorted() + } + + @Test func aPresetWithAnInvalidRepoDoesNotResolve() throws { + for (i, repo) in Self.badRepos.enumerated() { + let f = LibraryFixture.standard() + let name = "bad\(i)" + try plant(name, repo: repo, in: f) + let m = failureMessage { try f.resolver.resolve(name) } + #expect(m != nil, "preset repo \(repo.debugDescription) resolved") + #expect(m?.contains("invalid repo id") == true, "\(repo.debugDescription): \(m ?? "no error")") + } + } + + @Test func aGoodPresetStillResolves() throws { + let f = LibraryFixture.standard() + try plant("good", repo: LibraryFixture.qwen8, in: f) + let r = try f.resolver.resolve("good") + #expect(r.source == .preset && r.repo == LibraryFixture.qwen8) + } + + @Test func cpFromAPresetWithAnInvalidRepoFailsAndWritesNothing() async throws { + for (i, repo) in Self.badRepos.enumerated() { + let f = LibraryFixture.standard() + let name = "bad\(i)" + try plant(name, repo: repo, in: f) + let before = files(f) + let r = await f.run(["cp", name, "copy"]) + #expect(r.code == 1 && r.out.isEmpty, "\(repo.debugDescription)") + #expect(r.err.hasPrefix("Error: ") && r.err.contains("invalid repo id"), "\(repo.debugDescription): \(r.err)") + #expect(files(f) == before, "cp wrote a file for \(repo.debugDescription)") + #expect(f.presets.load("copy") == nil) + } + } + + @Test func rmOfABrokenPresetStillRemovesIt() async throws { + let f = LibraryFixture.standard() + try plant("broken", repo: "../x", in: f) + let r = await f.run(["rm", "broken"]) + #expect(r.code == 0, "\(r.err)") + #expect(r.out == "deleted 'broken'\n") + #expect(!files(f).contains("broken.json")) + #expect(f.deleted.isEmpty) + } +} From df3d3665582a9ca9abad3f98021d80855f5fd85f Mon Sep 17 00:00:00 2001 From: Rolando Santamaria Maso Date: Sun, 8 Nov 2026 15:31:25 +0100 Subject: [PATCH 07/20] feat(cli): show and pull with Ollama layout and progress, Hugging Face error mapping --- Sources/MLXEngine/Hub.swift | 6 + Sources/MLXRCLI/Commands.swift | 9 +- Sources/MLXRCLI/HubServices.swift | 36 ++++ Sources/MLXRCLI/PullProgress.swift | 76 +++++++ Sources/MLXRCLI/PullShowCommands.swift | 221 +++++++++++++++++++++ Tests/MLXRCLITests/HubFixtures.swift | 111 +++++++++++ Tests/MLXRCLITests/PullCommandTests.swift | 107 ++++++++++ Tests/MLXRCLITests/PullProgressTests.swift | 83 ++++++++ Tests/MLXRCLITests/ShowCommandTests.swift | 162 +++++++++++++++ 9 files changed, 806 insertions(+), 5 deletions(-) create mode 100644 Sources/MLXRCLI/HubServices.swift create mode 100644 Sources/MLXRCLI/PullProgress.swift create mode 100644 Sources/MLXRCLI/PullShowCommands.swift create mode 100644 Tests/MLXRCLITests/HubFixtures.swift create mode 100644 Tests/MLXRCLITests/PullCommandTests.swift create mode 100644 Tests/MLXRCLITests/PullProgressTests.swift create mode 100644 Tests/MLXRCLITests/ShowCommandTests.swift diff --git a/Sources/MLXEngine/Hub.swift b/Sources/MLXEngine/Hub.swift index a5f97f8..4bf5fa1 100644 --- a/Sources/MLXEngine/Hub.swift +++ b/Sources/MLXEngine/Hub.swift @@ -82,6 +82,12 @@ public struct HubModelDetails: Sendable { public let tags: [String] public let license: String? + public init(id: String, files: [HubFile], modelType: String?, bits: Int?, readmeURL: URL, downloads: Int?, likes: Int?, + tags: [String], license: String?, isMoE: Bool = false) { + self.id = id; self.files = files; self.modelType = modelType; self.bits = bits; self.readmeURL = readmeURL + self.downloads = downloads; self.likes = likes; self.tags = tags; self.license = license; self.isMoE = isMoE + } + public var isMoE: Bool = false public var isSupported: Bool { SupportedArchitectures.isSupported(modelType) && !isMoE && files.contains { $0.rfilename == "tokenizer.json" } diff --git a/Sources/MLXRCLI/Commands.swift b/Sources/MLXRCLI/Commands.swift index 96585ad..6377c97 100644 --- a/Sources/MLXRCLI/Commands.swift +++ b/Sources/MLXRCLI/Commands.swift @@ -10,7 +10,8 @@ extension CLI { /// The command table with its outside world injected. public static func commands(services: CLIServices = .live, library: ModelLibrary = .live, presets: PresetStore = .live, - catalog: Catalog = .standard, now: @escaping @Sendable () -> Date = { Date() }) -> [Command] { + catalog: Catalog = .standard, now: @escaping @Sendable () -> Date = { Date() }, + hub: HubServices = .live) -> [Command] { let ctxRange = SamplingLimits.context return [ Command(name: "run", summary: "Run a model", args: [Arg("MODEL", kind: .model), Arg("PROMPT", optional: true, variadic: true)], @@ -67,10 +68,8 @@ extension CLI { ctx: inv.int("ctx") ?? 8192, lan: inv.bool("lan"), apiKey: inv.string("api-key"), draft: inv.string("draft"), sparse: inv.string("sparse")) }), - Command(name: "download", aliases: ["pull"], summary: "Download a model from Hugging Face", args: [Arg("MODEL", kind: .model)], - run: { inv, _ in try await legacyDownload(inv.positionals[0]) }), - Command(name: "show", summary: "Show information for a model", args: [Arg("MODEL", kind: .model)], - run: { _, _ in throw CLIError.failure("show is not available yet") }), + pullCommand(library: library, presets: presets, catalog: catalog, hub: hub), + showCommand(library: library, presets: presets, catalog: catalog), Command(name: "search", summary: "Search Hugging Face for MLX models", args: [Arg("QUERY", variadic: true)], options: [ Opt(long: "limit", value: .int(1...100), defaultText: "25", help: "maximum number of results"), diff --git a/Sources/MLXRCLI/HubServices.swift b/Sources/MLXRCLI/HubServices.swift new file mode 100644 index 0000000..46146cb --- /dev/null +++ b/Sources/MLXRCLI/HubServices.swift @@ -0,0 +1,36 @@ +import Foundation +import MLXEngine + +/// What a running download reports: bytes done, bytes in total, bytes per second, and status lines. +public struct DownloadHooks: Sendable { + public var progress: @Sendable (_ done: Int64, _ total: Int64, _ bytesPerSecond: Double) -> Void + public var event: @Sendable (String) -> Void + + public init(progress: @escaping @Sendable (Int64, Int64, Double) -> Void, event: @escaping @Sendable (String) -> Void) { + self.progress = progress; self.event = event + } +} + +/// Hugging Face access for pull, show and auto-pull, injected so tests never touch the network or +/// the real model library. +public struct HubServices: Sendable { + public var details: @Sendable (_ repo: String) async throws -> HubModelDetails + public var destination: @Sendable (_ repo: String) throws -> URL + public var download: @Sendable (_ details: HubModelDetails, _ destination: URL, _ hooks: DownloadHooks) async throws -> URL + + public init(details: @escaping @Sendable (String) async throws -> HubModelDetails, + destination: @escaping @Sendable (String) throws -> URL, + download: @escaping @Sendable (HubModelDetails, URL, DownloadHooks) async throws -> URL) { + self.details = details; self.destination = destination; self.download = download + } + + public static let live = HubServices( + details: { try await HubClient.details($0) }, + destination: { try ModelStore.directory(for: $0) }, + download: { details, destination, hooks in + let dl = ModelDownload(details: details, destination: destination) + dl.onProgress = { _, done, total, speed in hooks.progress(done, total, speed) } + dl.onEvent = hooks.event + return try await dl.start() + }) +} diff --git a/Sources/MLXRCLI/PullProgress.swift b/Sources/MLXRCLI/PullProgress.swift new file mode 100644 index 0000000..80732c6 --- /dev/null +++ b/Sources/MLXRCLI/PullProgress.swift @@ -0,0 +1,76 @@ +import Foundation + +/// The progress line of `mlxr pull`, in Ollama's layout: +/// `"