diff --git a/controllers/crud.js b/controllers/crud.js index 93941a5c..dd9c4f82 100644 --- a/controllers/crud.js +++ b/controllers/crud.js @@ -86,8 +86,10 @@ const query = async function (req, res, next) { } const { limit, skip } = getPagination(req.query, { res }) try { - // One record past the page is read only to learn whether another page exists. It is never served. - let matches = await db.find(props).sort({ _id: 1 }).limit(limit + 1).skip(skip).toArray() + // Objects whose _id is not a string are bad data points, so they are not included. + // One record past the page is read only to learn whether another page exists. + let matches = await db.find({ $and: [props, { _id: { $type: "string" } }] }) + .sort({ _id: 1 }).limit(limit + 1).skip(skip).toArray() const hasNext = matches.length > limit matches = matches.slice(0, limit).map(o => idNegotiation(o)) res.set(utils.configureLDHeadersFor(matches)) diff --git a/controllers/search.js b/controllers/search.js index 16ee1009..4cc26bf6 100644 --- a/controllers/search.js +++ b/controllers/search.js @@ -6,111 +6,113 @@ */ import { db } from '../database/index.js' import utils from '../utils.js' -import { idNegotiation, getPagination } from './utils.js' +import { idNegotiation, getPagination, setNextPageLink } from './utils.js' /** - * Merges and deduplicates results from multiple MongoDB Atlas Search index queries. - * - * This function combines search results from both the IIIF Presentation API 3.0 index - * (presi3AnnotationText) and the IIIF Presentation API 2.1 index (presi2AnnotationText). - * - * @param {Array} results1 - Results from the first search index (typically IIIF 3.0) - * @param {Array} results2 - Results from the second search index (typically IIIF 2.1) - * @returns {Array} Merged array of unique results sorted by search score (descending) - * + * The MongoDB Atlas Search index every search operator queries. + * It covers both the IIIF Presentation API 3.0 paths and the IIIF Presentation API 2.1 paths. + */ +const SEARCH_INDEX = "annotationText" + +/** + * Completes an Atlas Search pipeline with the stages every search operator shares. + * + * @param {Object} searchQuery - The $search stage's query document, including its index + * @param {number} limit - Maximum number of results to serve + * @param {number} skip - Number of results to skip for pagination + * @returns {Array} The aggregation pipeline to hand to db.aggregate() + * * @description - * Process: - * 1. Combines both result arrays - * 2. Removes duplicates based on MongoDB _id (keeps first occurrence) - * 3. Sorts by search score in descending order (highest relevance first) + * Stages, in order: + * - $search, which returns matches in descending relevance score order + * - $match, to drop records a client could never resolve, before the page is measured + * - $addFields, to carry the relevance score onto each surviving record as __rerum.score + * - $skip and $limit, so paging is the database's work rather than this process's + */ +function searchPipelineFor(searchQuery, limit, skip) { + return [ + { $search: searchQuery }, + // Objects whose _id is not a string are bad data points, so they are not included. + { $match: { _id: { $type: "string" } } }, + { $addFields: { "__rerum.score": { $meta: "searchScore" } } }, + { $skip: skip }, + // One record past the page is read only to learn whether another page exists. It is never served. + { $limit: limit + 1 } + ] +} + +/** + * Serves one page of search results. The over-fetched record is the only evidence + * that another page exists. It is trimmed here and never serialized. * - * The function handles different _id formats: - * - ObjectId objects with $oid property - * - String-based _id values + * @param {Object} req - Express request, which the next-page link is built from + * @param {Object} res - Express response + * @param {Array} page - What the pipeline returned, including the over-fetched record + * @param {Object} pagination - The values getPagination() applied + * @param {number} pagination.limit - The applied limit + * @param {number} pagination.skip - The applied skip + */ +function serveSearchPage(req, res, page, { limit, skip }) { + const hasNext = page.length > limit + const results = page.slice(0, limit).map(o => idNegotiation(o)) + res.set(utils.configureLDHeadersFor(results)) + setNextPageLink(req, res, { limit, skip, hasNext }) + res.json(results) +} + +/** + * Reads the operator options a client sent in a JSON request body. + * Atlas rejects malformed operator options with an error that cannot be told apart from a server fault, + * so options that are not a JSON object are refused here, before they reach the $search stage. * + * @param {Object|string} body - The request body + * @param {Object} defaults - The options to use when the client sent none + * @returns {Object|null} The options to hand to the operator, or null when they are not a JSON object */ -function mergeSearchResults(results1, results2) { - const seen = new Set() - const merged = [] - - for (const result of [...results1, ...results2]) { - const id = result._id?.$oid || result._id?.toString() - if (!seen.has(id)) { - seen.add(id) - merged.push(result) - } - } - return merged.sort((a, b) => (b.__rerum?.score ?? 0) - (a.__rerum?.score ?? 0)) +function readSearchOptions(body, defaults) { + const options = body?.options ?? defaults + if (typeof options !== "object" || Array.isArray(options)) return null + return options } /** - * Builds parallel MongoDB Atlas Search aggregation pipelines for both IIIF 3.0 and 2.1 indexes. - * - * This function creates two separate search queries that will be executed in parallel: - * - One for IIIF Presentation API 3.0 resources (presi3AnnotationText index) - * - One for IIIF Presentation API 2.1 resources (presi2AnnotationText index) - * + * Builds the MongoDB Atlas Search aggregation pipeline for a text-like search operator. + * Every one of them is a "should" clause of a single compound query, so any one match qualifies and + * a document matching several of them scores higher for it. + * * @param {string} searchText - The text query to search for * @param {Object} operator - Search operator configuration * @param {string} operator.type - Type of search operator: "text", "wildcard", "phrase", etc. * @param {Object} operator.options - Additional options for the search operator (e.g., fuzzy options) - * @param {number} limit - Maximum number of results to return per index + * @param {number} limit - Maximum number of results to serve * @param {number} skip - Number of results to skip for pagination - * @returns {Array} Two-element array containing [presi3Pipeline, presi2Pipeline] - * + * @returns {Array} The aggregation pipeline to hand to db.aggregate() + * * @description - * IIIF 3.0 Query Structure (presi3AnnotationText index): - * - Searches direct text fields: body.value, bodyValue - * - Searches embedded items: items.annotations.items.body.value - * - Searches annotation items: annotations.items.body.value - * - Uses compound query with "should" clauses (any match qualifies) - * - * IIIF 2.1 Query Structure (presi2AnnotationText index): - * - Searches Open Annotation fields: resource.chars, resource.cnt:chars - * - Searches AnnotationList resources: resources[].resource.chars - * - Searches Canvas otherContent: otherContent[].resources[].resource.chars - * - Searches Manifest sequences: sequences[].canvases[].otherContent[].resources[].resource.chars - * - Uses nested embeddedDocument operators for multi-level array traversal - * - * Both queries use: - * - $search stage with the specified operator type (text, wildcard, phrase, etc.) - * - $addFields to include searchScore metadata - * - $limit to cap results (limit + skip to allow for pagination) + * The options are applied first in every clause, so they can tune the operator but never replace + * the validated searchText or the paths the clause searches. + * + * IIIF 3.0 clauses: + * - Web Annotation fields: body.value, bodyValue + * - Canvas annotations: annotations[].items[].body.value + * - AnnotationPage and Manifest items: items[].body.value, items[].annotations[].items[].body.value + * + * IIIF 2.1 clauses: + * - Open Annotation fields: resource.chars, resource.cnt:chars + * - AnnotationList resources: resources[].resource.chars + * - Canvas otherContent: otherContent[].resources[].resource.chars + * - Manifest sequences: sequences[].canvases[].otherContent[].resources[].resource.chars */ -function buildDualIndexQueries(searchText, operator, limit, skip) { - const presi3Query = { - index: "presi3AnnotationText", +function buildSearchPipeline(searchText, operator, limit, skip) { + const searchQuery = { + index: SEARCH_INDEX, compound: { should: [ { [operator.type]: { + ...operator.options, query: searchText, - path: ["body.value", "bodyValue"], - ...operator.options - } - }, - { - embeddedDocument: { - path: "items.annotations.items", - operator: { - [operator.type]: { - query: searchText, - path: ["items.annotations.items.body.value", "items.annotations.items.bodyValue"], - ...operator.options - } - } - } - }, - { - embeddedDocument: { - path: "annotations", - operator: { - [operator.type]: { - query: searchText, - path: ["annotations.items.body.value", "annotations.items.bodyValue"], - ...operator.options - } - } + path: ["body.value", "bodyValue"] } }, { @@ -118,9 +120,9 @@ function buildDualIndexQueries(searchText, operator, limit, skip) { path: "annotations.items", operator: { [operator.type]: { + ...operator.options, query: searchText, - path: ["annotations.items.body.value", "annotations.items.bodyValue"], - ...operator.options + path: ["annotations.items.body.value", "annotations.items.bodyValue"] } } } @@ -130,32 +132,23 @@ function buildDualIndexQueries(searchText, operator, limit, skip) { path: "items", operator: { [operator.type]: { + ...operator.options, query: searchText, path: [ "items.body.value", "items.bodyValue", "items.annotations.items.body.value", "items.annotations.items.bodyValue" - ], - ...operator.options + ] } } } - } - ], - minimumShouldMatch: 1 - } - } - - const presi2Query = { - index: "presi2AnnotationText", - compound: { - should: [ + }, { [operator.type]: { + ...operator.options, query: searchText, - path: ["resource.chars", "resource.cnt:chars"], - ...operator.options + path: ["resource.chars", "resource.cnt:chars"] } }, { @@ -163,9 +156,9 @@ function buildDualIndexQueries(searchText, operator, limit, skip) { path: "resources", operator: { [operator.type]: { + ...operator.options, query: searchText, - path: ["resources.resource.chars", "resources.resource.cnt:chars"], - ...operator.options + path: ["resources.resource.chars", "resources.resource.cnt:chars"] } } } @@ -175,9 +168,9 @@ function buildDualIndexQueries(searchText, operator, limit, skip) { path: "otherContent.resources", operator: { [operator.type]: { + ...operator.options, query: searchText, - path: ["otherContent.resources.resource.chars", "otherContent.resources.resource.cnt:chars"], - ...operator.options + path: ["otherContent.resources.resource.chars", "otherContent.resources.resource.cnt:chars"] } } } @@ -187,12 +180,12 @@ function buildDualIndexQueries(searchText, operator, limit, skip) { path: "sequences.canvases.otherContent.resources", operator: { [operator.type]: { + ...operator.options, query: searchText, path: [ "sequences.canvases.otherContent.resources.resource.chars", "sequences.canvases.otherContent.resources.resource.cnt:chars" - ], - ...operator.options + ] } } } @@ -201,28 +194,17 @@ function buildDualIndexQueries(searchText, operator, limit, skip) { minimumShouldMatch: 1 } } - - return [ - [ - { $search: presi3Query }, - { $addFields: { "__rerum.score": { $meta: "searchScore" } } }, - { $limit: limit + skip } - ], - [ - { $search: presi2Query }, - { $addFields: { "__rerum.score": { $meta: "searchScore" } } }, - { $limit: limit + skip } - ] - ] + return searchPipelineFor(searchQuery, limit, skip) } /** * Standard text search endpoint - searches for exact word matches across both IIIF 3.0 and 2.1 resources. * - * @route POST /search + * @route POST /v1/api/search * @param {Object} req.body - Request body containing search text * @param {string} req.body.searchText - The text to search for (can also be a plain string body) + * @param {Object} [req.body.options] - Text operator options, which must be a JSON object * @param {number} [req.query.limit=100] - Maximum number of results to return * @param {number} [req.query.skip=0] - Number of results to skip for pagination * @returns {Array} JSON array of matching annotation objects sorted by relevance score @@ -231,14 +213,16 @@ function buildDualIndexQueries(searchText, operator, limit, skip) { * Performs a standard MongoDB Atlas Search text query that: * - Tokenizes the search text into words * - Searches for exact word matches (case-insensitive) - * - Applies standard linguistic analysis (stemming, stop words, etc.) - * - Searches across both IIIF Presentation API 3.0 and 2.1 indexes in parallel + * - Uses the lucene.standard analyzer, which lowercases words but neither stems them nor removes stop words + * - Searches both IIIF Presentation API 3.0 and 2.1 fields in one query, against one index + * - Pages in the database, and carries a rel="next" Link header while another page exists * - Returns results sorted by relevance score (highest first) * * Search Behavior: - * - "Bryan Haberberger" → finds documents containing both "Bryan" AND "Haberberger" + * - "Bryan Haberberger" → finds documents containing "Bryan" OR "Haberberger", usually ranking documents that contain both higher * - Searches are case-insensitive - * - Standard analyzer removes common stop words + * - Words are not stemmed, so "transcribe" does not match "transcribing" + * - Stop words such as "the" are not removed, and are searched like any other word * - Partial word matches are NOT supported (use wildcardSearch for that) * * IIIF 3.0 Fields Searched: @@ -253,34 +237,33 @@ function buildDualIndexQueries(searchText, operator, limit, skip) { * - sequences[].canvases[].otherContent[].resources[].resource.chars (Manifest) * * @example - * POST /search + * POST /v1/api/search * Body: {"searchText": "Hello World"} - * Returns: All annotations containing "Hello" and "World" + * Returns: All annotations containing "Hello" or "World" * */ const searchAsWords = async function (req, res, next) { res.set("Content-Type", "application/json; charset=utf-8") let searchText = req.body?.searchText ?? req.body - const searchOptions = req.body?.options ?? {} - if (!searchText) { + const searchOptions = readSearchOptions(req.body, {}) + if (typeof searchText !== "string" || !searchText) { let err = { message: "You did not provide text to search for in the search request.", status: 400 } return next(utils.createExpressError(err)) } + if (!searchOptions) { + let err = { + message: "The 'options' property of the search request must be a JSON object.", + status: 400 + } + return next(utils.createExpressError(err)) + } const { limit, skip } = getPagination(req.query, { res }) - const [queryPresi3, queryPresi2] = buildDualIndexQueries(searchText, { type: "text", options: searchOptions }, limit, skip) + const pipeline = buildSearchPipeline(searchText, { type: "text", options: searchOptions }, limit, skip) try { - const [resultsPresi3, resultsPresi2] = await Promise.all([ - db.aggregate(queryPresi3).toArray().catch((err) => { console.error("Presi3 error:", err.message); return [] }), - db.aggregate(queryPresi2).toArray().catch((err) => { console.error("Presi2 error:", err.message); return [] }) - ]) - const merged = mergeSearchResults(resultsPresi3, resultsPresi2) - let results = merged.slice(skip, skip + limit) - results = results.map(o => idNegotiation(o)) - res.set(utils.configureLDHeadersFor(results)) - res.json(results) + serveSearchPage(req, res, await db.aggregate(pipeline).toArray(), { limit, skip }) } catch (error) { console.error(error) return next(utils.createExpressError(error)) @@ -290,9 +273,10 @@ const searchAsWords = async function (req, res, next) { /** * Phrase search endpoint - searches for multi-word phrases with words in proximity. * - * @route POST /phraseSearch + * @route POST /v1/api/search/phrase * @param {Object} req.body - Request body containing search phrase * @param {string} req.body.searchText - The phrase to search for (can also be a plain string body) + * @param {Object} [req.body.options] - Phrase operator options, which must be a JSON object * @param {number} [req.query.limit=100] - Maximum number of results to return * @param {number} [req.query.skip=0] - Number of results to skip for pagination * @returns {Array} JSON array of matching annotation objects sorted by relevance score @@ -302,10 +286,12 @@ const searchAsWords = async function (req, res, next) { * - Searches for terms in sequence or close proximity * - Allows up to 2 intervening words between search terms (slop: 2) * - More precise than standard text search for multi-word queries - * - Searches across both IIIF Presentation API 3.0 and 2.1 indexes in parallel + * - Searches both IIIF Presentation API 3.0 and 2.1 fields in one query, against one index + * - Pages in the database, and carries a rel="next" Link header while another page exists * * Phrase Options: * - slop: 2 (allows up to 2 words between search terms) + * - Options a client sends replace these defaults entirely. Without a slop of their own, the slop is 0. * * Phrase Matching Examples (with slop: 2): * - "Bryan Haberberger" → matches: @@ -317,8 +303,9 @@ const searchAsWords = async function (req, res, next) { * - "manuscript illumination" → matches: * ✓ "manuscript illumination" * ✓ "manuscript and illumination" - * ✓ "illumination of manuscript" (reversed order with slop) - * ✓ "illuminated manuscript" + * ✓ "illumination manuscript" (adjacent words in reverse order cost a slop of 2) + * ✗ "illumination of manuscript" (reverse order with a word between needs a slop of 3) + * ✗ "illuminated manuscript" (words are not stemmed) * * Use Cases: * - Finding exact or near-exact phrases @@ -327,7 +314,7 @@ const searchAsWords = async function (req, res, next) { * - More precise than standard search, more flexible than exact match * * Comparison with Other Search Types: - * - Standard search: Finds "Bryan" AND "Haberberger" anywhere in document + * - Standard search: Finds "Bryan" OR "Haberberger" anywhere in document * - Phrase search: Finds "Bryan" near "Haberberger" (within 2 words) * - Exact match: Would require "Bryan Haberberger" with no intervening words * @@ -337,36 +324,32 @@ const searchAsWords = async function (req, res, next) { * - Good balance of precision and recall * * @example - * POST /phraseSearch + * POST /v1/api/search/phrase * Body: "medieval manuscript" * Returns: Annotations with "medieval" and "manuscript" in proximity */ const searchAsPhrase = async function (req, res, next) { res.set("Content-Type", "application/json; charset=utf-8") let searchText = req.body?.searchText ?? req.body - const phraseOptions = req.body?.options ?? - { - slop: 2 - } - if (!searchText) { + const phraseOptions = readSearchOptions(req.body, { slop: 2 }) + if (typeof searchText !== "string" || !searchText) { let err = { message: "You did not provide text to search for in the search request.", status: 400 } return next(utils.createExpressError(err)) } + if (!phraseOptions) { + let err = { + message: "The 'options' property of the search request must be a JSON object.", + status: 400 + } + return next(utils.createExpressError(err)) + } const { limit, skip } = getPagination(req.query, { res }) - const [queryPresi3, queryPresi2] = buildDualIndexQueries(searchText, { type: "phrase", options: phraseOptions }, limit, skip) + const pipeline = buildSearchPipeline(searchText, { type: "phrase", options: phraseOptions }, limit, skip) try { - const [resultsPresi3, resultsPresi2] = await Promise.all([ - db.aggregate(queryPresi3).toArray().catch(() => []), - db.aggregate(queryPresi2).toArray().catch(() => []) - ]) - const merged = mergeSearchResults(resultsPresi3, resultsPresi2) - let results = merged.slice(skip, skip + limit) - results = results.map(o => idNegotiation(o)) - res.set(utils.configureLDHeadersFor(results)) - res.json(results) + serveSearchPage(req, res, await db.aggregate(pipeline).toArray(), { limit, skip }) } catch (error) { console.error(error) return next(utils.createExpressError(error)) @@ -376,9 +359,10 @@ const searchAsPhrase = async function (req, res, next) { /** * Fuzzy text search endpoint - searches for approximate matches allowing for typos and misspellings. * - * @route POST /fuzzySearch + * @route Not mounted. See the note at the end of routes/search.js. * @param {Object} req.body - Request body containing search text * @param {string} req.body.searchText - The text to search for (can also be a plain string body) + * @param {Object} [req.body.options] - Text operator options, which must be a JSON object * @param {number} [req.query.limit=100] - Maximum number of results to return * @param {number} [req.query.skip=0] - Number of results to skip for pagination * @returns {Array} JSON array of matching annotation objects sorted by relevance score @@ -388,7 +372,8 @@ const searchAsPhrase = async function (req, res, next) { * - Tolerates up to 1 character edit (insertion, deletion, substitution, transposition) * - Requires at least 2 characters to match exactly before fuzzy matching begins * - Expands to up to 50 similar terms - * - Searches across both IIIF Presentation API 3.0 and 2.1 indexes in parallel + * - Searches both IIIF Presentation API 3.0 and 2.1 fields in one query, against one index + * - Pages in the database, and carries a rel="next" Link header while another page exists * * Fuzzy Options: * - maxEdits: 1 (allows one character difference) @@ -418,33 +403,31 @@ const searchAsPhrase = async function (req, res, next) { const searchFuzzily = async function (req, res, next) { res.set("Content-Type", "application/json; charset=utf-8") let searchText = req.body?.searchText ?? req.body - const fuzzyOptions = req.body?.options ?? - { + const fuzzyOptions = readSearchOptions(req.body, { fuzzy: { maxEdits: 1, prefixLength: 2, maxExpansions: 50 } - } - if (!searchText) { + }) + if (typeof searchText !== "string" || !searchText) { let err = { message: "You did not provide text to search for in the search request.", status: 400 } return next(utils.createExpressError(err)) } + if (!fuzzyOptions) { + let err = { + message: "The 'options' property of the search request must be a JSON object.", + status: 400 + } + return next(utils.createExpressError(err)) + } const { limit, skip } = getPagination(req.query, { res }) - const [queryPresi3, queryPresi2] = buildDualIndexQueries(searchText, { type: "text", options: fuzzyOptions }, limit, skip) + const pipeline = buildSearchPipeline(searchText, { type: "text", options: fuzzyOptions }, limit, skip) try { - const [resultsPresi3, resultsPresi2] = await Promise.all([ - db.aggregate(queryPresi3).toArray().catch(() => []), - db.aggregate(queryPresi2).toArray().catch((error) => { console.error(error); return []; }) - ]) - const merged = mergeSearchResults(resultsPresi3, resultsPresi2) - let results = merged.slice(skip, skip + limit) - results = results.map(o => idNegotiation(o)) - res.set(utils.configureLDHeadersFor(results)) - res.json(results) + serveSearchPage(req, res, await db.aggregate(pipeline).toArray(), { limit, skip }) } catch (error) { console.error(error) return next(utils.createExpressError(error)) @@ -454,9 +437,10 @@ const searchFuzzily = async function (req, res, next) { /** * Wildcard pattern search endpoint - searches using wildcard patterns for partial matches. * - * @route POST /wildcardSearch + * @route Not mounted. See the note at the end of routes/search.js. * @param {Object} req.body - Request body containing search pattern * @param {string} req.body.searchText - The wildcard pattern to search for (must contain * or ?) + * @param {Object} [req.body.options] - Wildcard operator options, which must be a JSON object * @param {number} [req.query.limit=100] - Maximum number of results to return * @param {number} [req.query.skip=0] - Number of results to skip for pagination * @returns {Array} JSON array of matching annotation objects sorted by relevance score @@ -465,7 +449,8 @@ const searchFuzzily = async function (req, res, next) { * Performs a wildcard search using pattern matching: * - '*' matches zero or more characters (any length) * - '?' matches exactly one character - * - Searches across both IIIF Presentation API 3.0 and 2.1 indexes in parallel + * - Searches both IIIF Presentation API 3.0 and 2.1 fields in one query, against one index + * - Pages in the database, and carries a rel="next" Link header while another page exists * - Requires at least one wildcard character in the search pattern * * Wildcard Options: @@ -508,17 +493,21 @@ const searchFuzzily = async function (req, res, next) { const searchWildly = async function (req, res, next) { res.set("Content-Type", "application/json; charset=utf-8") let searchText = req.body?.searchText ?? req.body - const wildcardOptions = req.body?.options ?? - { - allowAnalyzedField: true - } - if (!searchText) { + const wildcardOptions = readSearchOptions(req.body, { allowAnalyzedField: true }) + if (typeof searchText !== "string" || !searchText) { let err = { message: "You did not provide text to search for in the search request.", status: 400 } return next(utils.createExpressError(err)) } + if (!wildcardOptions) { + let err = { + message: "The 'options' property of the search request must be a JSON object.", + status: 400 + } + return next(utils.createExpressError(err)) + } // Require wildcards in the search text if (!searchText.includes('*') && !searchText.includes('?')) { let err = { @@ -528,17 +517,9 @@ const searchWildly = async function (req, res, next) { return next(utils.createExpressError(err)) } const { limit, skip } = getPagination(req.query, { res }) - const [queryPresi3, queryPresi2] = buildDualIndexQueries(searchText, { type: "wildcard", options: wildcardOptions }, limit, skip) + const pipeline = buildSearchPipeline(searchText, { type: "wildcard", options: wildcardOptions }, limit, skip) try { - const [resultsPresi3, resultsPresi2] = await Promise.all([ - db.aggregate(queryPresi3).toArray().catch(() => []), - db.aggregate(queryPresi2).toArray().catch(() => []) - ]) - const merged = mergeSearchResults(resultsPresi3, resultsPresi2) - let results = merged.slice(skip, skip + limit) - results = results.map(o => idNegotiation(o)) - res.set(utils.configureLDHeadersFor(results)) - res.json(results) + serveSearchPage(req, res, await db.aggregate(pipeline).toArray(), { limit, skip }) } catch (error) { console.error(error) return next(utils.createExpressError(error)) @@ -548,7 +529,7 @@ const searchWildly = async function (req, res, next) { /** * "More Like This" search endpoint - finds documents similar to a provided example document. * - * @route POST /searchAlikes + * @route Not mounted. See the note at the end of routes/search.js. * @param {Object} req.body - A complete JSON document to use as the search example * @param {number} [req.query.limit=100] - Maximum number of results to return * @param {number} [req.query.skip=0] - Number of results to skip for pagination @@ -559,7 +540,8 @@ const searchWildly = async function (req, res, next) { * - Analyzes the provided document's text content * - Extracts significant terms and patterns * - Finds other documents with similar content - * - Uses both IIIF 3.0 (presi3AnnotationText) and IIIF 2.1 (presi2AnnotationText) indexes + * - Uses the one index that covers both the IIIF 3.0 and the IIIF 2.1 text fields + * - Pages in the database, and carries a rel="next" Link header while another page exists * - Great for discovery and finding related content * * How It Works: @@ -583,7 +565,7 @@ const searchWildly = async function (req, res, next) { * * Important Notes: * - Requires a full JSON document in request body (not just text) - * - Searches both IIIF 3.0 (presi3AnnotationText) and IIIF 2.1 (presi2AnnotationText) indexes + * - Searches the IIIF 3.0 and IIIF 2.1 text fields together, against one index * - Returns 400 error if body is empty or invalid * - More effective with documents containing substantial text content * @@ -621,56 +603,14 @@ const searchAlikes = async function (req, res, next) { return next(utils.createExpressError(err)) } const { limit, skip } = getPagination(req.query, { res }) - // Build moreLikeThis queries for both IIIF 3.0 and IIIF 2.1 indexes - const searchQuery_presi3 = [ - { - $search: { - index: "presi3AnnotationText", - moreLikeThis: { - like: Array.isArray(likeDocument) ? likeDocument : [likeDocument] - } - } - }, - { - $addFields: { - "__rerum.score": { $meta: "searchScore" } - } - }, - { - $limit: limit + skip // Get extra to handle deduplication + const pipeline = searchPipelineFor({ + index: SEARCH_INDEX, + moreLikeThis: { + like: Array.isArray(likeDocument) ? likeDocument : [likeDocument] } - ] - const searchQuery_presi2 = [ - { - $search: { - index: "presi2AnnotationText", - moreLikeThis: { - like: Array.isArray(likeDocument) ? likeDocument : [likeDocument] - } - } - }, - { - $addFields: { - "__rerum.score": { $meta: "searchScore" } - } - }, - { - $limit: limit + skip // Get extra to handle deduplication - } - ] + }, limit, skip) try { - // Execute both queries in parallel - const [results_presi3, results_presi2] = await Promise.all([ - db.aggregate(searchQuery_presi3).toArray(), - db.aggregate(searchQuery_presi2).toArray() - ]) - // Merge and deduplicate results - const merged = mergeSearchResults(results_presi3, results_presi2) - // Apply pagination after merging - let results = merged.slice(skip, skip + limit) - results = results.map(o => idNegotiation(o)) - res.set(utils.configureLDHeadersFor(results)) - res.json(results) + serveSearchPage(req, res, await db.aggregate(pipeline).toArray(), { limit, skip }) } catch (error) { console.error(error) return next(utils.createExpressError(error)) diff --git a/openapi/contracts/core-provider.openapi.yaml b/openapi/contracts/core-provider.openapi.yaml index fb37d317..6dec6e16 100644 --- a/openapi/contracts/core-provider.openapi.yaml +++ b/openapi/contracts/core-provider.openapi.yaml @@ -219,7 +219,9 @@ paths: $ref: '#/components/schemas/GenericObject' responses: '200': - description: Query results + description: >- + Query results, in ascending '_id' order. Objects whose '_id' is not a string are not + served. headers: Link: $ref: '#/components/headers/NextPageLink' @@ -248,7 +250,12 @@ paths: type: string responses: '200': - description: Search results + description: >- + Search results, in descending relevance score order. Objects whose '_id' is not a + string are not served. + headers: + Link: + $ref: '#/components/headers/NextPageLink' content: application/json: schema: @@ -274,7 +281,12 @@ paths: type: string responses: '200': - description: Phrase search results + description: >- + Phrase search results, in descending relevance score order. Objects whose '_id' is not + a string are not served. + headers: + Link: + $ref: '#/components/headers/NextPageLink' content: application/json: schema: diff --git a/public/API.html b/public/API.html index 5ff50dc6..d8a34f82 100644 --- a/public/API.html +++ b/public/API.html @@ -545,7 +545,7 @@

Custom Query

]

- All responses are a JSON Array, even if a single or zero records are matched. Results are in ascending _id order — see Pagination parameters. + All responses are a JSON Array, even if a single or zero records are matched. Results are in ascending _id order — see Pagination parameters. Objects whose _id is not a string are not served.

@@ -568,6 +568,8 @@

Custom Query

return allResults } if (!response.ok) throw new Error(await response.text()) + const applied = Number(response.headers.get("Pagination-Limit")) + if (applied < lim) console.warn(`Asked for ${lim} per page, served ${applied}. The rel="next" link already accounts for it.`) allResults.push(...await response.json()) url = response.headers.get("Link")?.match(/<([^>]+)>;\s*rel="next"/)?.[1] } @@ -619,12 +621,12 @@

Pagination parameters

Following rel="next"

- While another page exists, Custom Query responses carry a rel="next" link in the Link header (RFC 8288), next to the JSON-LD context link. The final page has no rel="next". Follow it until it is gone. + While another page exists, Custom Query, Text Search, and Phrase Search responses carry a rel="next" link in the Link header (RFC 8288), next to the JSON-LD context link. The final page has no rel="next". Follow it until it is gone.

rel="next" advances skip. It is still present when the next page would start past the skip maximum, and following it is answered with the 400 that names the maximum.

-

Custom Query takes a POST. The link only carries the paging URL parameters, so send the same request body to it that you sent to the first page.

+

All three endpoints take a POST. The link only carries the paging URL parameters, so send the same request body to it that you sent to the first page.

 
                 Link: <http://store.rerum.io/v1/context.json>; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json"
@@ -637,9 +639,8 @@ 
Result ordering

         

 
         

-            The search endpoints page in descending relevance score, which you can read back from __rerum.score on each result. That order is the same across both the IIIF 3.0 and IIIF 2.1 indexes, so the best match is the first record of the first page whichever shape it was written in.
+            Text Search and Phrase Search page in descending relevance score, which you can read back from __rerum.score on each result. Every object is scored once against one index that covers both the IIIF 3.0 and the IIIF 2.1 text fields, so the best match is the first record of the first page whichever shape it was written in.
         

-        
Descending score is an ordering, not a tiling guarantee.

 
         
Text Search

         
@@ -664,7 +665,7 @@ 
Text Search
  • Response: [{JSON}]—An array of annotation objects matching the search, sorted by relevance score
    • 
-            The Text Search endpoint performs a full-text search across Annotation text content. It searches for exact word matches and tokenizes the search text, finding documents that contain all the search terms anywhere in their text content.
+            The Text Search endpoint performs a full-text search across Annotation text content. It tokenizes the search text into words and matches each word exactly, finding documents that contain any of the search terms anywhere in their text content.
         
    
             The search covers multiple text fields depending on the syntax of objects.  In particular it covers the current Web Annotation syntax, IIIF Presentation API 3.0 syntax, and IIIF Presentation API 2.1 syntax⚠️.  See below for specific details.
@@ -674,7 +675,7 @@ 
    Text Search
  • IIIF Presentation API 2.1 fields⚠️: oa:Annotation resource.chars, oa:Annotation resource.cnt:chars, and nested structures in sc:AnnotationList resources, sc:Canvas otherContent, and sc:Manifest sequences.
    • 
-            To allow for more records in the response one can add the URL parameter limit to the search requests. If you expect the search request will have a very large response with many objects, your application should use a paged search by also using the skip URL parameter. You will see an example of this below.
+            To allow for more records in the response one can add the URL parameter limit to the search requests. If you expect the search request will have a very large response with many objects, your application should use a paged search that follows the rel="next" link in each response's Link header until there is none. You will see an example of this below.
             
    A limit above the maximum of 500 is reduced to 500, and the response says so — see Pagination parameters. Read the applied value from the Pagination-Limit response header rather than assuming you received everything you asked for.
    
@@ -684,9 +685,10 @@ 
    Text Search
  • Results will only include Web Annotation, IIIF Presentation API 3.0, and IIIF Presentation API 2.1⚠️ resource types that have the text embedded within their structure.
  • Searches are case-insensitive
  • Partial word matches and wildcards are NOT supported in this search
    • 
-            
  • Standard linguistic analysis is applied (stemming, stop words, etc.)
    • 
-            
  • Multi-word searches find documents containing all the words (AND logic)
    • 
+            
  • Words are not stemmed, so "transcribe" does not match "transcribing". Common words such as "the" are not removed, and are searched like any other word.
    • 
+            
  • Multi-word searches find documents containing any of the words (OR logic). Documents that contain more of the words usually score higher. To find words that appear together, use Phrase Search.
  • Results are sorted by relevance score (highest first)
    • 
+            
  • Objects whose _id is not a string are not served
  • A __rerum.score property is added to each result indicating match quality
    • 
@@ -696,7 +698,7 @@ 
    Text Search
  • General full-text search across annotation content
  • Finding all annotations containing specific keywords or concepts
  • Broad discovery when exact word order doesn't matter
    • 
-            
  • Best for single-word searches or when all search terms should appear anywhere in the document
    • 
+            
  • Best for single-word searches, or when a document may match on any one of the search terms
    • 
             
    Javascript Example (JSON Object)
    
@@ -759,6 +761,36 @@ 
    Text Search
    
                 Results are returned sorted by relevance score in descending order. The __rerum.score property indicates match quality.
             
    
+        
    
+            
    Paged Search Javascript Example
    
+            
     
+                const many_results = await pagedSearch("lorem ipsum")
+                
    
+                async function pagedSearch(searchText, lim = 100) {
+                    const request = {
+                        method: "POST",
+                        headers: { "Content-Type": "application/json; charset=utf-8" },
+                        body: JSON.stringify({ searchText })
+                    }
+                    const allResults = []
+                    let url = `https://devstore.rerum.io/v1/api/search?limit=${lim}`
+                    while (url) {
+                        const response = await fetch(url, request)
+                        if (response.status === 400 && allResults.length) {
+                            console.warn(`Stopped at the skip maximum of ${response.headers.get("Pagination-Skip-Max")}. These results are incomplete. Narrow the search.`)
+                            return allResults
+                        }
+                        if (!response.ok) throw new Error(await response.text())
+                        const applied = Number(response.headers.get("Pagination-Limit"))
+                        if (applied < lim) console.warn(`Asked for ${lim} per page, served ${applied}. The rel="next" link already accounts for it.`)
+                        allResults.push(...await response.json())
+                        url = response.headers.get("Link")?.match(/<([^>]+)>;\s*rel="next"/)?.[1]
+                    }
+                    return allResults
+                }
+            
    
+            
    Stop when there is no rel="next", not when a page looks short. /search/phrase pages the same way.
    
+        
    
             
         
         
         
             
         
         
         
         
             
             
             
             
             
         
         
             
             
             
         
         
         
         
         
    
@@ -777,7 +809,7 @@ 
    
             
             
         
    
         
      
-            
    • {JSON}—An object with a searchText property containing the phrase to search for, and an optional options property (default slop: 2)
      • 
+            
    • {JSON}—An object with a searchText property containing the phrase to search for, and an optional options property (default {"slop": 2})
      • 
             
    • "string"—Alternatively, a plain string phrase to search for
      • 
             
    • Response: [{JSON}]—An array of annotation objects matching the phrase search, sorted by relevance score
      • 
         
    
@@ -792,7 +824,7 @@ 
             
  • IIIF Presentation API 2.1 fields⚠️: oa:Annotation resource.chars, oa:Annotation resource.cnt:chars, and nested structures in sc:AnnotationList resources, sc:Canvas otherContent, and sc:Manifest sequences.
    • 
         
         
    
-            To allow for more records in the response one can add the URL parameter limit to the search requests. If you expect the search request will have a very large response with many objects, your application should use a paged search by also using the skip URL parameter. You will see an example of this below.
+            To allow for more records in the response one can add the URL parameter limit to the search requests. If you expect the search request will have a very large response with many objects, your application should use a paged search that follows the rel="next" link in each response's Link header until there is none. You will see an example of this below.
             
    A limit above the maximum of 500 is reduced to 500, and the response says so — see Pagination parameters. Read the applied value from the Pagination-Limit response header rather than assuming you received everything you asked for.
    
         
    
         
    
@@ -802,10 +834,11 @@ 
    
             
  • Results will only include Web Annotation, IIIF Presentation API 3.0, and IIIF Presentation API 2.1⚠️ resource types that have the text embedded within their structure.
    • 
             
  • Searches are case-insensitive
    • 
             
  • Partial word matches and wildcards are NOT supported in this search
    • 
-            
  • Uses a "slop" value (default: 2) that allows intervening words between search terms (up to the default or provided value).  You may supply your own slop option.
    • 
+            
  • Uses a "slop" value (default: 2) that allows intervening words between search terms (up to the default or provided value).  You may supply your own slop option.  Any options you send replace the default entirely, so include slop in them.  Without it the slop is 0, and the words must be adjacent and in order.
    • 
             
  • Words don't need to be directly adjacent, providing flexibility while maintaining phrase coherence
    • 
             
  • More precise than standard text search for multi-word queries
    • 
             
  • Results are sorted by relevance score (highest first)
    • 
+            
  • Objects whose _id is not a string are not served
    • 
             
  • A __rerum.score property is added to each result indicating match quality
    • 
         
         
    
@@ -816,7 +849,8 @@ 
    
                 
      
                     
    • ✓ "medieval manuscript"
      • 
                     
    • ✓ "medieval illuminated manuscript"
      • 
-                    
    • ✓ "manuscript from medieval times"
      • 
+                    
    • ✓ "manuscript medieval" (adjacent words in reverse order use a slop of 2)
      • 
+                    
    • ✗ "manuscript from medieval times" (reverse order with a word between needs a slop of 3)
      • 
                     
    • ✗ "medieval art with many beautiful decorated manuscripts" (too many words between)
      • 
                 
    
             
@@ -831,7 +865,7 @@ 
             
  • When you need more precision than standard search but more flexibility than exact matching
    • 
         
         
    
-            The limit and skip URL parameters work the same as in the standard text search endpoint for pagination support.
+            The limit and skip URL parameters, and the rel="next" link, work the same as in the standard text search endpoint — see Pagination parameters.
         
    
         
    
             
    Javascript Example with Custom Slop
    
diff --git a/routes/__tests__/query.test.js b/routes/__tests__/query.test.js
index a7970cfa..4d8e3af9 100644
--- a/routes/__tests__/query.test.js
+++ b/routes/__tests__/query.test.js
@@ -65,6 +65,50 @@ it("'/query' route functions", async () => {
   assert.strictEqual(response.body[0]._id, undefined)
 })
 
+describe('objects whose _id is not a string are not served by /query', () => {
+  /** The filter the controller handed to db.find on the most recent request. */
+  let filter = null
+
+  const post = (body) => {
+    db.find.mockImplementation((props) => {
+      filter = props
+      return {
+        sort() { return this },
+        limit() { return this },
+        skip() { return this },
+        async toArray() { return [] }
+      }
+    })
+    return request(routeTester)
+      .post("/query")
+      .set("Content-Type", "application/json")
+      .send(body)
+  }
+
+  it("adds the string _id condition to the client's filter", async () => {
+    const response = await post({ test: "item" })
+
+    assert.strictEqual(response.statusCode, 200)
+    assert.deepStrictEqual(filter, { $and: [{ test: "item" }, { _id: { $type: "string" } }] })
+  })
+
+  it("keeps a client's own _id condition, whether a literal or an operator", async () => {
+    await post({ _id: "testid123" })
+    assert.deepStrictEqual(
+      filter,
+      { $and: [{ _id: "testid123" }, { _id: { $type: "string" } }] },
+      "a merge into props would have replaced the client's _id outright"
+    )
+
+    await post({ _id: { $not: { $type: "string" } } })
+    assert.deepStrictEqual(
+      filter,
+      { $and: [{ _id: { $not: { $type: "string" } } }, { _id: { $type: "string" } }] },
+      "the two conditions contradict, which is how this query comes back empty"
+    )
+  })
+})
+
 describe('pagination parameters on /query', () => {
   const pagedTester = express()
   pagedTester.use(express.json({ type: ["application/json", "application/ld+json"] }))
diff --git a/routes/__tests__/search.test.js b/routes/__tests__/search.test.js
index d32f8bf8..6d841ccc 100644
--- a/routes/__tests__/search.test.js
+++ b/routes/__tests__/search.test.js
@@ -19,26 +19,43 @@ beforeEach(() => {
   resetMocks()
 })
 
+/** What the controller last handed to db.aggregate, and how many times, since the mock was installed. */
+const searchCalls = { count: 0, pipeline: null }
+
+/** The value a pipeline gives an aggregation operator, e.g. stageOf(pipeline, '$limit'). */
+const stageOf = (pipeline, operator) => pipeline.find(stage => operator in stage)?.[operator]
+
+/** Where an operator sits in a pipeline, so the order of the stages can be asserted. */
+const positionOf = (pipeline, operator) => pipeline.findIndex(stage => operator in stage)
+
 function mockAggregateResults(docs) {
-  // db.aggregate is called twice (presi3 + presi2 indexes) in parallel; mockReturnValue
-  // applies to every call until the next reset.
-  db.aggregate.mockReturnValue({
-    toArray: () => Promise.resolve(docs)
+  // One index means one aggregation per request, so there is a single pipeline to record.
+  searchCalls.count = 0
+  db.aggregate.mockImplementation((pipeline) => {
+    searchCalls.count++
+    searchCalls.pipeline = pipeline
+    return { toArray: () => Promise.resolve(docs) }
   })
 }
 
 /**
- * Answer the two branches with different documents, so cross-index behavior can be observed.
- *
- * The controllers build the Promise.all array literal presi3 first, and array elements evaluate
- * left to right, so the first queued result is the IIIF 3.0 branch.
+ * Answer the search from a fixed, score ordered set of documents, honouring the $skip and $limit
+ * the controller put in the pipeline so a walk pages the way Atlas would.
  */
-function mockBranchResults(presi3Docs, presi2Docs) {
-  db.aggregate.mockReturnValueOnce({ toArray: () => Promise.resolve(presi3Docs) })
-  db.aggregate.mockReturnValueOnce({ toArray: () => Promise.resolve(presi2Docs) })
+function mockSearchCollection(docs) {
+  searchCalls.count = 0
+  db.aggregate.mockImplementation((pipeline) => {
+    searchCalls.count++
+    searchCalls.pipeline = pipeline
+    const skip = stageOf(pipeline, '$skip') ?? 0
+    const limit = stageOf(pipeline, '$limit') ?? Infinity
+    return {
+      toArray: () => Promise.resolve(docs.slice(skip, skip + limit).map(doc => structuredClone(doc)))
+    }
+  })
 }
 
-/** A search hit carrying its relevance where the branch pipelines actually put it. */
+/** A search hit carrying its relevance where the search pipeline puts it. */
 const scoredDoc = (id, score) => ({
   _id: id,
   '@id': `https://store.rerum.io/v1/id/${id}`,
@@ -84,6 +101,77 @@ describe('search controllers', () => {
     assert.strictEqual(response.statusCode, 400)
   })
 
+  it("returns 400 without searching when the JSON body has no searchText string", async () => {
+    for (const path of ['/search', '/search/phrase']) {
+      for (const body of [{ text: 'line' }, { searchText: 5 }, { searchText: ['line'] }]) {
+        mockAggregateResults([])
+
+        const response = await request(routeTester)
+          .post(path)
+          .set('Content-Type', 'application/json')
+          .send(body)
+
+        assert.strictEqual(response.statusCode, 400, `${path} ${JSON.stringify(body)}`)
+        assert.strictEqual(searchCalls.count, 0, `${path} ${JSON.stringify(body)} must not reach the database`)
+      }
+    }
+  })
+
+  it("returns 400 without searching when options is not a JSON object", async () => {
+    for (const path of ['/search', '/search/phrase']) {
+      for (const options of ['x', ['x'], 5]) {
+        mockAggregateResults([])
+
+        const response = await request(routeTester)
+          .post(path)
+          .set('Content-Type', 'application/json')
+          .send({ searchText: 'a line', options })
+
+        assert.strictEqual(response.statusCode, 400, `${path} ${JSON.stringify(options)}`)
+        assert.strictEqual(searchCalls.count, 0, `${path} ${JSON.stringify(options)} must not reach the database`)
+      }
+    }
+  })
+
+  it("hands a client's options to the phrase operator, and the default slop when there are none", async () => {
+    for (const [options, slop] of [[{ slop: 5 }, 5], [null, 2]]) {
+      mockAggregateResults([])
+
+      const response = await request(routeTester)
+        .post('/search/phrase')
+        .set('Content-Type', 'application/json')
+        .send({ searchText: 'a line', options })
+
+      assert.strictEqual(response.statusCode, 200, JSON.stringify(options))
+      assert.strictEqual(searchCalls.pipeline[0].$search.compound.should[0].phrase.slop, slop)
+    }
+  })
+
+  it("never lets options replace the searchText or the paths a clause searches", async () => {
+    /** The operator document of every should clause, whether it is top level or inside embeddedDocument. */
+    const operatorsOf = (pipeline, type) => pipeline[0].$search.compound.should
+      .map(clause => (clause.embeddedDocument?.operator ?? clause)[type])
+    const search = async (path, options) => {
+      mockAggregateResults([])
+      const response = await request(routeTester)
+        .post(path)
+        .set('Content-Type', 'application/json')
+        .send({ searchText: 'a line', options })
+      assert.strictEqual(response.statusCode, 200, `${path} ${JSON.stringify(options)}`)
+      return searchCalls.pipeline
+    }
+
+    for (const [path, type] of [['/search', 'text'], ['/search/phrase', 'phrase']]) {
+      const expectedPaths = operatorsOf(await search(path, {}), type).map(op => op.path)
+      const operators = operatorsOf(await search(path, { query: '', path: 'x', slop: 3 }), type)
+
+      assert.ok(operators.length > 0, path)
+      for (const op of operators) assert.strictEqual(op.query, 'a line', `${path} must search the searchText`)
+      assert.deepStrictEqual(operators.map(op => op.path), expectedPaths, `${path} must search its own paths`)
+      assert.ok(operators.every(op => op.slop === 3), `${path} still applies the client's other options`)
+    }
+  })
+
   it("searchAsPhrase returns 200 and an array of results for a text body", async () => {
     const doc = {
       _id: 'doc-2',
@@ -103,16 +191,8 @@ describe('search controllers', () => {
     assert.strictEqual(response.body[0]['@id'], doc['@id'])
   })
 
-  // The two parallel db.aggregate calls (presi3 + presi2) can return overlapping documents.
-  // mergeSearchResults must dedupe by _id; a regression that drops the dedupe would return
-  // duplicates here.
-  it("searchAsWords dedupes when both indexes return the same document", async () => {
-    const doc = {
-      _id: 'shared-doc',
-      '@id': 'https://store.rerum.io/v1/id/shared-doc',
-      text: 'shared'
-    }
-    mockAggregateResults([doc])
+  it("searchAsWords runs one search, against the index that covers both vocabularies", async () => {
+    mockAggregateResults([])
 
     const response = await request(routeTester)
       .post('/search')
@@ -120,14 +200,28 @@ describe('search controllers', () => {
       .send('shared')
 
     assert.strictEqual(response.statusCode, 200)
-    assert.strictEqual(response.body.length, 1, 'duplicate _id across indexes should be deduped')
+    assert.strictEqual(searchCalls.count, 1)
+    assert.strictEqual(searchCalls.pipeline[0].$search.index, 'annotationText')
   })
 
-  it("searchAsWords ranks across both indexes by score, not by which index answered", async () => {
-    mockBranchResults(
-      [scoredDoc('presi3-weak', 1.69), scoredDoc('presi3-weaker', 1.24)],
-      [scoredDoc('presi2-best', 83.92)]
-    )
+  it("searches the IIIF 3.0 and IIIF 2.1 text fields in one compound query", async () => {
+    mockAggregateResults([])
+
+    await request(routeTester)
+      .post('/search')
+      .set('Content-Type', 'text/plain')
+      .send('line')
+
+    const compound = searchCalls.pipeline[0].$search.compound
+    const searched = JSON.stringify(compound.should)
+    for (const path of ['body.value', 'bodyValue', 'resource.chars', 'resource.cnt:chars']) {
+      assert.ok(searched.includes(path), `${path} must still be searched`)
+    }
+    assert.strictEqual(compound.minimumShouldMatch, 1, 'any one clause matching qualifies a document')
+  })
+
+  it("serves the relevance order the database returned, without resorting it", async () => {
+    mockAggregateResults([scoredDoc('best', 83.92), scoredDoc('weak', 1.69), scoredDoc('weaker', 1.24)])
 
     const response = await request(routeTester)
       .post('/search')
@@ -135,24 +229,154 @@ describe('search controllers', () => {
       .send('line')
 
     assert.strictEqual(response.statusCode, 200)
-    assert.deepStrictEqual(idsOf(response), ['presi2-best', 'presi3-weak', 'presi3-weaker'])
+    assert.deepStrictEqual(idsOf(response), ['best', 'weak', 'weaker'])
   })
 
-  it("pages the score order, so skip walks best-first across both indexes", async () => {
-    const branches = () => mockBranchResults(
-      [scoredDoc('p3-c', 3), scoredDoc('p3-d', 2)],
-      [scoredDoc('p2-a', 9), scoredDoc('p2-b', 5)]
-    )
-    const page = (queryString) => {
-      branches()
-      return request(routeTester)
-        .post(`/search${queryString}`)
+  it("hands skip and limit to the database rather than paging in memory", async () => {
+    mockSearchCollection([scoredDoc('a', 9), scoredDoc('b', 5), scoredDoc('c', 3), scoredDoc('d', 2)])
+    const page = (queryString) => request(routeTester)
+      .post(`/search${queryString}`)
+      .set('Content-Type', 'text/plain')
+      .send('line')
+
+    assert.deepStrictEqual(idsOf(await page('?limit=2&skip=0')), ['a', 'b'])
+    assert.strictEqual(stageOf(searchCalls.pipeline, '$skip'), 0)
+    assert.deepStrictEqual(idsOf(await page('?limit=2&skip=2')), ['c', 'd'])
+    assert.strictEqual(stageOf(searchCalls.pipeline, '$skip'), 2)
+    // limit + 1: the extra record is the only evidence that another page exists.
+    assert.strictEqual(stageOf(searchCalls.pipeline, '$limit'), 3)
+  })
+
+  it("excludes objects whose _id is not a string, before the page is measured", async () => {
+    for (const path of ['/search', '/search/phrase']) {
+      mockAggregateResults([])
+
+      await request(routeTester)
+        .post(path)
         .set('Content-Type', 'text/plain')
-        .send('line')
+        .send('manuscript')
+
+      const pipeline = searchCalls.pipeline
+      assert.deepStrictEqual(stageOf(pipeline, '$match'), { _id: { $type: 'string' } }, path)
+      assert.ok(
+        positionOf(pipeline, '$match') < positionOf(pipeline, '$limit'),
+        `${path} must exclude them before $limit, or the page under-fills`
+      )
+    }
+  })
+})
+
+describe('rel="next" links on /search', () => {
+  /** The Link header of a response as { rel: url }. */
+  const linksOf = (response) => Object.fromEntries(
+    [...(response.headers.link ?? '').matchAll(/<([^>]+)>;\s*rel="([^"]+)"/g)].map(([, url, rel]) => [rel, url])
+  )
+
+  /** The query string of a link target, which may be absolute or path-only. */
+  const paramsOf = (url) => new URL(url, 'http://localhost').searchParams
+
+  const post = (path) => request(routeTester)
+    .post(path)
+    .set('Content-Type', 'text/plain')
+    .send('line')
+
+  const scored = (ids) => ids.map((id, i) => scoredDoc(id, ids.length - i))
+
+  it("links the next page while more records exist, and serves only the page", async () => {
+    mockSearchCollection(scored(['a', 'b', 'c']))
+    const response = await post('/search?limit=2')
+    const links = linksOf(response)
+
+    assert.strictEqual(response.statusCode, 200)
+    assert.deepStrictEqual(idsOf(response), ['a', 'b'])
+    assert.ok(links.next, 'a next link is present while more records exist')
+    assert.strictEqual(paramsOf(links.next).get('limit'), '2')
+    assert.strictEqual(paramsOf(links.next).get('skip'), '2')
+  })
+
+  it("keeps the JSON-LD context link alongside the next link", async () => {
+    mockSearchCollection(scored(['a', 'b', 'c']))
+    const links = linksOf(await post('/search?limit=2'))
+    assert.ok(links['http://www.w3.org/ns/json-ld#context'])
+    assert.ok(links.next)
+  })
+
+  it("omits next on the final page", async () => {
+    mockSearchCollection(scored(['a', 'b']))
+    const response = await post('/search?limit=2')
+
+    assert.deepStrictEqual(idsOf(response), ['a', 'b'])
+    assert.deepStrictEqual(
+      Object.keys(linksOf(response)),
+      ['http://www.w3.org/ns/json-ld#context'],
+      'only the JSON-LD context link remains'
+    )
+  })
+
+  it("never serves the record read past the page, even at the maximum limit", async () => {
+    mockSearchCollection(scored(['a']))
+    const limitMax = Number((await post('/search?limit=1')).headers['pagination-limit-max'])
+    const ids = Array.from({ length: limitMax + 1 }, (_, i) => `id${String(i).padStart(6, '0')}`)
+    mockSearchCollection(scored(ids))
+
+    const response = await post(`/search?limit=${limitMax + 100}`)
+
+    assert.strictEqual(response.body.length, limitMax)
+    assert.ok(!idsOf(response).includes(ids.at(-1)), 'the extra record must not be served')
+    assert.ok(linksOf(response).next)
+  })
+
+  it("still links next past the skip maximum, so a walk too deep to finish ends in the 400 that names it", async () => {
+    mockSearchCollection(scored(['a']))
+    const skipMax = Number((await post('/search?limit=1')).headers['pagination-skip-max'])
+    // Every page this double serves has a record past it, however deep the skip.
+    db.aggregate.mockImplementation(() => ({
+      toArray: () => Promise.resolve(scored(['a', 'b', 'c']))
+    }))
+
+    const deepest = await post(`/search?limit=2&skip=${skipMax}`)
+    const next = linksOf(deepest).next
+    assert.strictEqual(deepest.statusCode, 200)
+    assert.strictEqual(
+      paramsOf(next).get('skip'),
+      String(skipMax + 2),
+      'absence would tell the client the walk was complete'
+    )
+
+    const beyond = await post(`/search?${paramsOf(next)}`)
+    assert.strictEqual(beyond.statusCode, 400)
+    assert.strictEqual(beyond.headers['pagination-skip-max'], String(skipMax))
+  })
+
+  it("walks every record exactly once by following only rel=\"next\"", async () => {
+    const ids = ['d01', 'd02', 'd03', 'd04', 'd05', 'd06', 'd07']
+    mockSearchCollection(scored(ids))
+
+    const walked = []
+    let path = '/search?limit=3'
+    for (let requests = 0; path; requests++) {
+      assert.ok(requests < ids.length, 'the walk must terminate')
+      const response = await post(path)
+      assert.strictEqual(response.statusCode, 200)
+      walked.push(...idsOf(response))
+      const next = linksOf(response).next
+      path = next && `/search?${paramsOf(next)}`
     }
 
-    assert.deepStrictEqual(idsOf(await page('?limit=2&skip=0')), ['p2-a', 'p2-b'])
-    assert.deepStrictEqual(idsOf(await page('?limit=2&skip=2')), ['p3-c', 'p3-d'])
+    assert.deepStrictEqual(walked, ids)
+  })
+
+  it("links the next page of a phrase search the same way", async () => {
+    mockSearchCollection(scored(['a', 'b', 'c']))
+    const response = await request(routeTester)
+      .post('/search/phrase?limit=2')
+      .set('Content-Type', 'text/plain')
+      .send('a line')
+    const links = linksOf(response)
+
+    assert.deepStrictEqual(idsOf(response), ['a', 'b'])
+    assert.ok(links['http://www.w3.org/ns/json-ld#context'])
+    assert.strictEqual(paramsOf(links.next).get('skip'), '2')
   })
 })