diff --git a/.claude/hooks/rubocop.sh b/.claude/hooks/rubocop.sh new file mode 100755 index 0000000..c83c6ce --- /dev/null +++ b/.claude/hooks/rubocop.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# PostToolUse hook: autocorrects the Ruby file Claude just wrote, then hands back what RuboCop cannot fix +# (exit 2 shows stderr to Claude). Files git ignores are left alone, such as vendored or cloned code. +set -u + +file=$(jq -r '.tool_input.file_path // empty') +case "$file" in + "$CLAUDE_PROJECT_DIR"/*) ;; + *) exit 0 ;; +esac +case "$file" in + *.rb | *.rake | */Gemfile | */Rakefile | *.gemspec) ;; + *) exit 0 ;; +esac +[ -f "$file" ] || exit 0 +cd "$CLAUDE_PROJECT_DIR" || exit 0 +git check-ignore --quiet "$file" && exit 0 + +# Autocorrect, except for removing assignments: the next edit may well be the one that reads them. +bundle exec rubocop --autocorrect --server --force-exclusion --format quiet --except Lint/UselessAssignment "$file" >/dev/null 2>&1 +if ! output=$(bundle exec rubocop --server --force-exclusion --no-color --format simple "$file" 2>&1); then + echo "$output" >&2 + exit 2 +fi diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 0000000..90e8fe0 --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,15 @@ +{ + "hooks": { + "PostToolUse": [ + { + "matcher": "Write|Edit|MultiEdit", + "hooks": [ + { + "type": "command", + "command": "\"$CLAUDE_PROJECT_DIR\"/.claude/hooks/rubocop.sh" + } + ] + } + ] + } +} diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..3983267 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,26 @@ +name: Bug report +description: A wrong sitemap, a file that should be rewritten and is not, a crash +labels: [bug] +body: + - type: textarea + id: code + attributes: + label: Source class and configuration + description: The smallest source class, configuration and records that reproduce it. + render: ruby + validations: + required: true + - type: textarea + id: expected + attributes: + label: Expected and actual output + description: The XML or the files you expected, what `Mapitaine.build` wrote instead, or the error and its backtrace. + validations: + required: true + - type: input + id: versions + attributes: + label: Versions + placeholder: Mapitaine 0.1.0 (or commit), Rails 8.1, Ruby 4.0.1, PostgreSQL 18 + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..feca4a5 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: false +contact_links: + - name: Report a security vulnerability + url: https://github.com/levups/mapitaine/security/advisories/new + about: Privately, never in a public issue. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..6668718 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,16 @@ +name: Feature request +description: A sitemap Mapitaine cannot write yet, a storage it cannot reach +body: + - type: textarea + id: need + attributes: + label: What you need + description: The pages and sitemap you are trying to produce, and what stops you today. + validations: + required: true + - type: textarea + id: api + attributes: + label: How you would write it + description: The source class or configuration you would like to write. + render: ruby diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..0b3d550 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,5 @@ + + +- [ ] Tests cover the change (a bug fix comes with a test that fails without it) +- [ ] Output unchanged, or every difference is intended and listed above diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..23ba887 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,13 @@ +# The actions are pinned by commit, not by tag, which their owner (or an attacker) can move: Dependabot proposes +# each new version, a week after its release, in one pull request. The same file in every Level UP Solutions gem. +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + cooldown: + default-days: 7 + groups: + actions: + patterns: ["*"] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..55ddbe7 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,112 @@ +name: CI + +# The blocks shared by the Level UP Solutions gems are kept identical across the family. + +on: + push: + branches: [main] + # A draft runs nothing: marking it ready for review starts the CI. + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + rubocop: + name: RuboCop + if: ${{ !github.event.pull_request.draft }} + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Set up Ruby + uses: ruby/setup-ruby@de492cf36cbab201753d720ce4f4e336cd688c94 # v1.336.0 + with: + ruby-version: "4.0" + bundler-cache: true + - name: Lint with RuboCop + run: bundle exec rubocop --format github + + test: + name: Ruby ${{ matrix.ruby }}${{ matrix.rails && format(' · Rails {0}', matrix.rails.version) || '' }} + if: ${{ !github.event.pull_request.draft }} + runs-on: ubuntu-latest + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + # Every Ruby the core team still maintains (https://endoflife.date/ruby); the -head ones are in head. + ruby: ["3.3", "3.4", "4.0"] + rails: [{version: "8.1", gemfile: Gemfile}, {version: "8.0", gemfile: gemfiles/rails_8_0.gemfile}] + env: + BUNDLE_GEMFILE: ${{ matrix.rails.gemfile != 'Gemfile' && matrix.rails.gemfile || '' }} + steps: &test-steps + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Set up Ruby + uses: ruby/setup-ruby@de492cf36cbab201753d720ce4f4e336cd688c94 # v1.336.0 + with: + ruby-version: ${{ matrix.ruby }} + bundler-cache: true + - name: Run the tests + run: bundle exec rake test + + # Allowed to fail, and truffleruby-head takes twice as long as the others: CI passed doesn't wait for them. + head: + name: Ruby ${{ matrix.ruby }}${{ matrix.rails && format(' · Rails {0}', matrix.rails.version) || '' }} + if: ${{ !github.event.pull_request.draft }} + runs-on: ubuntu-latest + timeout-minutes: 10 + continue-on-error: true + strategy: + fail-fast: false + matrix: + # ruby-head, then this gem's extra ones. + ruby: [ruby-head] + rails: [{version: "8.1", gemfile: Gemfile}, {version: "8.0", gemfile: gemfiles/rails_8_0.gemfile}] + env: + BUNDLE_GEMFILE: ${{ matrix.rails.gemfile != 'Gemfile' && matrix.rails.gemfile || '' }} + steps: *test-steps + + docs: + name: Highlighted docs + if: ${{ !github.event.pull_request.draft }} + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Set up Ruby + uses: ruby/setup-ruby@de492cf36cbab201753d720ce4f4e336cd688c94 # v1.336.0 + with: + ruby-version: "4.0" + bundler-cache: true + - name: Check the highlighted docs + run: bundle exec rake docs:highlight:check + + # The one check main's ruleset requires, so the Rubies, Rails and jobs above can change without touching it. + # Every other job belongs in needs: bin/x-gems-ci-check in levups/oss checks it. + passed: + name: CI passed + if: ${{ always() && !github.event.pull_request.draft }} + needs: [rubocop, test, docs] + runs-on: ubuntu-latest + timeout-minutes: 2 + steps: + - name: Check every job succeeded + run: echo "$NEEDS" | jq --exit-status 'all(.[]; .result == "success")' + env: + NEEDS: ${{ toJSON(needs) }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..0e4c54e --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,64 @@ +name: Release + +# bin/release pushes a vX.Y.Z tag; this publishes that version to RubyGems, then the GitHub release with its +# section of CHANGELOG.md. RubyGems trusts this workflow (trusted publishing): no API key is stored anywhere. +on: + push: + tags: ["v*"] + +permissions: + contents: read + +# One release at a time, never cancelled halfway through a publication. +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +jobs: + release: + name: RubyGems and GitHub release + runs-on: ubuntu-latest + # Settings > Environments > release: who must approve a publication. + environment: release + timeout-minutes: 10 + permissions: + contents: write # the GitHub release + id-token: write # trusted publishing to RubyGems + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Set up Ruby + uses: ruby/setup-ruby@de492cf36cbab201753d720ce4f4e336cd688c94 # v1.336.0 + with: + ruby-version: "4.0" + - name: Check the tag against the gemspec and the changelog + run: bin/release --check "$GITHUB_REF_NAME" + # A tag is all it takes to publish: only one signed with a key GitHub verifies, on a commit of main, does. + - name: Check the tag is signed and on main + env: + GH_TOKEN: ${{ github.token }} + run: | + tag=$(gh api "repos/$GITHUB_REPOSITORY/git/ref/tags/$GITHUB_REF_NAME" --jq 'select(.object.type == "tag") | .object.sha') + test -n "$tag" || { echo "::error::$GITHUB_REF_NAME is not an annotated tag"; exit 1; } + verified=$(gh api "repos/$GITHUB_REPOSITORY/git/tags/$tag" --jq "select(.object.sha == \"$GITHUB_SHA\") | .verification.verified") + test "$verified" = true || { echo "::error::$GITHUB_REF_NAME is not signed with a key GitHub verifies"; exit 1; } + status=$(gh api "repos/$GITHUB_REPOSITORY/compare/$GITHUB_SHA...main" --jq .status) + case "$status" in + identical | ahead) ;; + *) echo "::error::$GITHUB_SHA is not on main ($status)"; exit 1 ;; + esac + - name: Build the gem + run: gem build *.gemspec + - name: Get RubyGems credentials (trusted publishing) + uses: rubygems/configure-rubygems-credentials@dc5a8d8553e6ee01fc26761a49e99e733d17954a # v2.1.0 + - name: Publish to RubyGems + run: gem push *.gem + - name: Create the GitHub release + env: + GH_TOKEN: ${{ github.token }} + run: | + bin/release --notes "$GITHUB_REF_NAME" > "$RUNNER_TEMP/notes.md" + gh release create "$GITHUB_REF_NAME" *.gem --verify-tag \ + --title "$(bin/release --title "$GITHUB_REF_NAME")" --notes-file "$RUNNER_TEMP/notes.md" diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..96b8a28 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +/Gemfile.lock +/gemfiles/*.lock +/pkg/ +mapitaine-*.gem +/tmp/ diff --git a/.rubocop.yml b/.rubocop.yml new file mode 100644 index 0000000..8eb0c91 --- /dev/null +++ b/.rubocop.yml @@ -0,0 +1,126 @@ +# Standard's rules, run through RuboCop itself. +inherit_gem: + standard: config/base.yml + +plugins: + - rubocop-performance + +AllCops: + TargetRubyVersion: 3.3 + SuggestExtensions: false + Exclude: + - "tmp/**/*" + - "vendor/**/*" + +# A gem runs in other people's hot paths: every Performance cop (rubocop-performance 1.26.1), +# the opt-in ones included. Department-level `Enabled: true` leaves those off. +Performance/AncestorsInclude: + Enabled: true +Performance/ArraySemiInfiniteRangeSlice: + Enabled: true +Performance/BigDecimalWithNumericArgument: + Enabled: true +Performance/BindCall: + Enabled: true +Performance/BlockGivenWithExplicitBlock: + Enabled: true +Performance/Caller: + Enabled: true +Performance/CaseWhenSplat: + Enabled: true +Performance/Casecmp: + Enabled: true +Performance/ChainArrayAllocation: + Enabled: true + # Allocation counting only matters in code that ships: tests, benchmarks and scripts read better chained. + Exclude: ¬_shipped + - "bench/**/*" + - "script/**/*" + - "test/**/*" +Performance/CollectionLiteralInLoop: + Enabled: true + Exclude: *not_shipped +Performance/CompareWithBlock: + Enabled: true +Performance/ConcurrentMonotonicTime: + Enabled: true +Performance/ConstantRegexp: + Enabled: true +Performance/Count: + Enabled: true +Performance/DeletePrefix: + Enabled: true +Performance/DeleteSuffix: + Enabled: true +Performance/Detect: + Enabled: true +Performance/DoubleStartEndWith: + Enabled: true +Performance/EndWith: + Enabled: true +Performance/FixedSize: + Enabled: true +Performance/FlatMap: + Enabled: true +Performance/InefficientHashSearch: + Enabled: true +Performance/IoReadlines: + Enabled: true +Performance/MapCompact: + Enabled: true +Performance/MapMethodChain: + Enabled: true +Performance/MethodObjectAsBlock: + Enabled: true +Performance/OpenStruct: + Enabled: true +Performance/RangeInclude: + Enabled: true +Performance/RedundantBlockCall: + Enabled: true +Performance/RedundantEqualityComparisonBlock: + Enabled: true +Performance/RedundantMatch: + Enabled: true +Performance/RedundantMerge: + Enabled: true +Performance/RedundantSortBlock: + Enabled: true +Performance/RedundantSplitRegexpArgument: + Enabled: true +Performance/RedundantStringChars: + Enabled: true +Performance/RegexpMatch: + Enabled: true +Performance/ReverseEach: + Enabled: true +Performance/ReverseFirst: + Enabled: true +Performance/SelectMap: + Enabled: true +Performance/Size: + Enabled: true +Performance/SortReverse: + Enabled: true +Performance/Squeeze: + Enabled: true +Performance/StartWith: + Enabled: true +Performance/StringBytesize: + Enabled: true +Performance/StringIdentifierArgument: + Enabled: true +Performance/StringInclude: + Enabled: true +Performance/StringReplacement: + Enabled: true +Performance/Sum: + Enabled: true +Performance/TimesMap: + Enabled: true +Performance/UnfreezeString: + Enabled: true +Performance/UriDefaultParser: + Enabled: true +Performance/ZipWithoutBlock: + Enabled: true diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..dd8ce6d --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,88 @@ +# Mapitaine + +XML sitemaps for Rails sites in many languages: one `Mapitaine::Source` class per kind of page, hreflang links +written from the languages of each record, files cut by ranges of ids, and builds that rewrite only the ranges +whose count or latest change moved. + +`README.md` covers usage, `STYLE.md` code style, `CONTRIBUTING.md` contributions; this file adds what they leave out. + +## Commands + +```sh +bundle exec rake test # the newest Rails (Gemfile) +BUNDLE_GEMFILE=gemfiles/rails_8_0.gemfile bundle exec rake test # the oldest maintained Rails +bundle exec rubocop +bundle exec ruby [--yjit] bench/build.rb # 30,000 records in 24 languages +``` + +CI also runs `ruby-head`, and every Ruby with each Gemfile. A new Rails gets a `gemfiles/rails_X_Y.gemfile` and +joins CI's `rails` matrix. + +## Traps + +- Active Record, Action Pack and Active Storage stay out of the gemspec: a source duck-types its relation, the + Railtie brings the route helpers, and `Storage::ActiveStorage` looks its service up at the first call. Cloud + storage goes through Active Storage only: no SDK of our own. +- Mapitaine never requests a page of the site, and never hooks into models: a build reads the database and the + manifest of the last build, nothing else. +- A file name is a promise: the same ids land in the same file at every build. Anything that moves them (the shard + size, the slug, the file name) belongs in `Configuration#digest` or rewrites everything by itself. +- A build publishes the sitemap files, then the index, then deletes, then writes the manifest. Keep that order: it + is what lets the next build repair one that died midway. +- `priority`, `changefreq` and the ping endpoints are left out on purpose. +- The README is the user guide and `docs/` the GitHub Pages site: change them with the code. The README's table and + `docs/benchmarks.html` come from the same `bench/build.rb` run. + +## Rules for all our gems + +This section and the next two read the same in super-slim, gamba, crumbly, agentry and mapitaine: change all five +together. + +- Every maintained Ruby and Rails (https://endoflife.date), nothing older. The oldest sets `required_ruby_version`, + `TargetRubyVersion` and the Rails floor, and leaves in the release after its end of life. +- CI runs RuboCop and the tests on each maintained Ruby. No macOS job: we test on our Macs. +- `.claude/hooks/rubocop.sh` autocorrects each Ruby file you edit. +- Measure performance without JIT and with YJIT. A Performance cop the numbers contradict is disabled on its line, + with the reason. A pure performance change pins no internals in tests. +- Test and bench data are anonymized: `alice`, `Post`, `example.com`. +- Everything in the repository is in English (code, comments, commits, pull requests, issues, reviews, replies), + whatever the conversation's language. +- No AI attribution in anything people read; commits flag AI help with `Assisted-by`, never `Co-Authored-By`. + +## Pull requests and issues + +- `main` takes no direct push: every change, releases included, goes through a pull request that merges itself once + CI is green (`gh pr merge --auto --squash`). +- Drafts run no CI. Confident: open it ready, auto-merge on. Not yet: a draft, then `gh pr ready` when it needs CI. +- Never edit `CHANGELOG.md` in a pull request, they would all conflict: it is written at release time. +- Title: the change as a user sees it (an issue: the finding), in plain words, under 70 characters. +- Body, no headings: two short paragraphs, the problem then the change (an issue: what to do); 3 to 5 bullets, each + opening with a **bold 2–4 word lead-in**; the proof (reproduction, tests, benchmarks) folded in one `
`. + Nothing the diff or CI already says. +- Never hard-wrap a pull request, an issue or a comment: GitHub renders every line break. One line per paragraph and + per bullet. + +## Releases + +The user is needed twice: a working git signing key (the tag is signed) and one click approving the `release` +environment on GitHub. + +1. Another session may have just released: `git fetch --tags origin`, and `gh pr list --search "head:release/"` + must be empty. Unreleased changes: `git log --first-parent --oneline $(git describe --tags --abbrev=0 origin/main)..origin/main`, + then `gh pr view ` for each. +2. Write the release under `## [Unreleased]` in `CHANGELOG.md`, uncommitted: a one-sentence lede, then **Breaking**, + **New**, **Faster** (measured), **Fixed**, only those with entries. One entry per change users see, written for + them; CI, refactoring, docs and tests get none. No version, date or link: `bin/release` adds them. +3. Show it to the user, wait for their go. From `main` level with `origin/main`, `CHANGELOG.md` its only change: + `bin/release minor --dry-run`, then `bin/release minor --yes` (or `patch`, `major`). It opens `release/vX.Y.Z`, + waits for it to merge itself, pushes the signed tag. Signing fails: the user fixes it, never an unsigned tag. + Interrupted after the merge: `bin/release --tag X.Y.Z`. +4. The tag runs `release.yml`, which waits for the user to approve the `release` environment, then publishes to + RubyGems and GitHub. Open that run in their browser, where they click Review deployments, `release`, Approve: + `gh run view --web $(gh run list --workflow release.yml --branch vX.Y.Z --json databaseId -q '.[0].databaseId')`, + once the run exists: it starts a few seconds after the tag, and without `--branch` the previous release's run + opens. + Then check RubyGems. + +RubyGems trusts only that workflow: never `gem push`, except a new gem name's first version. `bin/release` and +`release.yml` are identical in all five gems. diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..1a58814 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,3 @@ +# Changelog + +## [Unreleased] diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..43c994c --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..d1ebb17 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,89 @@ +# Contributor Covenant 3.0 Code of Conduct + +## Our Pledge + +We pledge to make our community welcoming, safe, and equitable for all. + +We are committed to fostering an environment that respects and promotes the dignity, rights, and contributions of all individuals, regardless of characteristics including race, ethnicity, caste, color, age, physical characteristics, neurodiversity, disability, sex or gender, gender identity or expression, sexual orientation, language, philosophy or religion, national or social origin, socio-economic position, level of education, or other status. The same privileges of participation are extended to everyone who participates in good faith and in accordance with this Covenant. + + +## Encouraged Behaviors + +While acknowledging differences in social norms, we all strive to meet our community's expectations for positive behavior. We also understand that our words and actions may be interpreted differently than we intend based on culture, background, or native language. + +With these considerations in mind, we agree to behave mindfully toward each other and act in ways that center our shared values, including: + +1. Respecting the **purpose of our community**, our activities, and our ways of gathering. +2. Engaging **kindly and honestly** with others. +3. Respecting **different viewpoints** and experiences. +4. **Taking responsibility** for our actions and contributions. +5. Gracefully giving and accepting **constructive feedback**. +6. Committing to **repairing harm** when it occurs. +7. Behaving in other ways that promote and sustain the **well-being of our community**. + + +## Restricted Behaviors + +We agree to restrict the following behaviors in our community. Instances, threats, and promotion of these behaviors are violations of this Code of Conduct. + +1. **Harassment.** Violating explicitly expressed boundaries or engaging in unnecessary personal attention after any clear request to stop. +2. **Character attacks.** Making insulting, demeaning, or pejorative comments directed at a community member or group of people. +3. **Stereotyping or discrimination.** Characterizing anyone’s personality or behavior on the basis of immutable identities or traits. +4. **Sexualization.** Behaving in a way that would generally be considered inappropriately intimate in the context or purpose of the community. +5. **Violating confidentiality**. Sharing or acting on someone's personal or private information without their permission. +6. **Endangerment.** Causing, encouraging, or threatening violence or other harm toward any person or group. +7. Behaving in other ways that **threaten the well-being** of our community. + +### Other Restrictions + +1. **Misleading identity.** Impersonating someone else for any reason, or pretending to be someone else to evade enforcement actions. +2. **Failing to credit sources.** Not properly crediting the sources of content you contribute. +3. **Promotional materials**. Sharing marketing or other commercial content in a way that is outside the norms of the community. +4. **Irresponsible communication.** Failing to responsibly present content which includes, links or describes any other restricted behaviors. + + +## Reporting an Issue + +Tensions can occur between community members even when they are trying their best to collaborate. Not every conflict represents a code of conduct violation, and this Code of Conduct reinforces encouraged behaviors and norms that can help avoid conflicts and minimize harm. + +When an incident does occur, it is important to report it promptly. To report a possible violation, email clement@levups.com. The Community Moderators are the two maintainers, Clément Joubert and Robert Maerten. + +Community Moderators take reports of violations seriously and will make every effort to respond in a timely manner. They will investigate all reports of code of conduct violations, reviewing messages, logs, and recordings, or interviewing witnesses and other participants. Community Moderators will keep investigation and enforcement actions as transparent as possible while prioritizing safety and confidentiality. In order to honor these values, enforcement actions are carried out in private with the involved parties, but communicating to the whole community may be part of a mutually agreed upon resolution. + + +## Addressing and Repairing Harm + +If an investigation by the Community Moderators finds that this Code of Conduct has been violated, the following enforcement ladder may be used to determine how best to repair harm, based on the incident's impact on the individuals involved and the community as a whole. Depending on the severity of a violation, lower rungs on the ladder may be skipped. + +1) Warning + 1) Event: A violation involving a single incident or series of incidents. + 2) Consequence: A private, written warning from the Community Moderators. + 3) Repair: Examples of repair include a private written apology, acknowledgement of responsibility, and seeking clarification on expectations. +2) Temporarily Limited Activities + 1) Event: A repeated incidence of a violation that previously resulted in a warning, or the first incidence of a more serious violation. + 2) Consequence: A private, written warning with a time-limited cooldown period designed to underscore the seriousness of the situation and give the community members involved time to process the incident. The cooldown period may be limited to particular communication channels or interactions with particular community members. + 3) Repair: Examples of repair may include making an apology, using the cooldown period to reflect on actions and impact, and being thoughtful about re-entering community spaces after the period is over. +3) Temporary Suspension + 1) Event: A pattern of repeated violation which the Community Moderators have tried to address with warnings, or a single serious violation. + 2) Consequence: A private written warning with conditions for return from suspension. In general, temporary suspensions give the person being suspended time to reflect upon their behavior and possible corrective actions. + 3) Repair: Examples of repair include respecting the spirit of the suspension, meeting the specified conditions for return, and being thoughtful about how to reintegrate with the community when the suspension is lifted. +4) Permanent Ban + 1) Event: A pattern of repeated code of conduct violations that other steps on the ladder have failed to resolve, or a violation so serious that the Community Moderators determine there is no way to keep the community safe with this person as a member. + 2) Consequence: Access to all community spaces, tools, and communication channels is removed. In general, permanent bans should be rarely used, should have strong reasoning behind them, and should only be resorted to if working through other remedies has failed to change the behavior. + 3) Repair: There is no possible repair in cases of this severity. + +This enforcement ladder is intended as a guideline. It does not limit the ability of Community Managers to use their discretion and judgment, in keeping with the best interests of our community. + + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public or other spaces. Examples of representing our community include using an official email address, posting via an official social media account, or acting as an appointed representative at an online or offline event. + + +## Attribution + +This Code of Conduct is adapted from the Contributor Covenant, version 3.0, permanently available at [https://www.contributor-covenant.org/version/3/0/](https://www.contributor-covenant.org/version/3/0/). + +Contributor Covenant is stewarded by the Organization for Ethical Source and licensed under CC BY-SA 4.0. To view a copy of this license, visit [https://creativecommons.org/licenses/by-sa/4.0/](https://creativecommons.org/licenses/by-sa/4.0/) + +For answers to common questions about Contributor Covenant, see the FAQ at [https://www.contributor-covenant.org/faq](https://www.contributor-covenant.org/faq). Translations are provided at [https://www.contributor-covenant.org/translations](https://www.contributor-covenant.org/translations). Additional enforcement and community guideline resources can be found at [https://www.contributor-covenant.org/resources](https://www.contributor-covenant.org/resources). The enforcement ladder was inspired by the work of [Mozilla’s code of conduct team](https://github.com/mozilla/inclusion). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..7f49ae8 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,34 @@ +# Contributing + +Thank you for helping. Mapitaine is maintained by Level UP Solutions, two developers who use it in production +every day: small, focused pull requests get reviewed fastest. + +## Before you start + +- **Bugs**: open an issue with the smallest source class and records that reproduce it, and the XML you expected. +- **Features**: open an issue first, so we can agree on the scope before you write the code. Mapitaine stays small: + it writes what search engines read today, and leaves out what they ignore. +- **Security issues** never go in a public issue: see [SECURITY.md](SECURITY.md). + +## Supported Rubies + +Every CRuby version the Ruby core team still maintains, down to the gem's floor (`required_ruby_version`), runs +in CI and must pass. A version leaves CI and the floor in the release after its end of life. `ruby-head`, and +JRuby or TruffleRuby where the gem already runs on them, are tested too, but a failure there does not block a +release. A pull request that makes a gem run on JRuby or TruffleRuby without slowing CRuby down is welcome. + +## Pull requests + +- The repository's README says how to run its tests, its corpus and its benchmarks. CI runs the same. +- A bug fix comes with a test that fails without it; a new behaviour with tests of its own. +- A performance change states its numbers (before, after, Ruby version, with and without YJIT) and proves the + output did not change. +- Code follows [STYLE.md](STYLE.md) and the RuboCop config (`bundle exec rubocop`). +- One change per pull request, titled for users: the changelog is written from it when releasing. Leave + `CHANGELOG.md` alone: pull requests editing it conflict with each other. + +Using an AI assistant is fine, as long as you have read and understood every line you submit and answer the review +yourself. Say so in the description. + +By contributing, you agree that your work is released under the repository's license, and to follow our +[code of conduct](CODE_OF_CONDUCT.md). diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000..a56528e --- /dev/null +++ b/Gemfile @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +gemspec + +group :development, :test do + gem "actionpack" + gem "activerecord" + gem "activestorage" + gem "activesupport" + gem "minitest" + gem "rake" + gem "rubocop", require: false + gem "rubocop-performance", require: false + gem "sqlite3" + gem "standard", require: false +end + +group :development do + gem "rouge", "5.1.0" # pinned: docs:highlight:check compares its output with the committed pages +end diff --git a/README.md b/README.md new file mode 100644 index 0000000..231e6b0 --- /dev/null +++ b/README.md @@ -0,0 +1,239 @@ +# Mapitaine + +Mapitaine writes the XML sitemaps of a Rails site in many languages, and rewrites only the files whose pages +changed since the last build. + +You describe each kind of page once, as a small class that names its records, their URLs and their languages: + +```ruby +# app/sitemaps/product_sitemap.rb +class ProductSitemap < Mapitaine::Source + scope { Product.published } + url { |product, locale| product_url(product, locale:) } +end +``` + +Mapitaine does the rest: one `` per language, each listing every language of the page and the default one, +files that never outgrow what the protocol allows, the sitemap index, and a build you can run every hour. + +The guide and the benchmarks are also on the website: . + +## Why Mapitaine + +Mapitaine was written for a site of 30,000 pages in 24 languages, 720,000 URLs, where regenerating everything +on each run is time spent on pages that did not change. It works differently from older generators in a few ways: + +- A page is one record in several languages, not a list of unrelated URLs. The `hreflang` links come from that: + every language lists all the others and itself, which search engines require before they trust any of them. +- Each file holds a fixed range of ids, so a record stays in the same file from one build to the next. A build + asks the database one question per source, the count and the latest change of each range, and rewrites only + the files whose answer moved. No callback in your models: `update_all`, imports and deletions are seen too. +- `lastmod` is the only hint search engines still read, so it is the date your content changed, in the files + and in the index. `priority` and `changefreq`, which Google ignores, are not written. +- Search engines are told through [IndexNow](https://www.indexnow.org) which URLs changed. The ping endpoints + older gems call have been closed. +- It never requests your pages. What it checks, it checks on the data it writes. + +On 30,000 records in 24 languages (SQLite in memory, Ruby 3.3.6 without YJIT, `bench/build.rb`): + +| Build | Time | Files written | +|---|---|---| +| Everything, 720,000 URLs | 31.9 s | 55 (17 MiB gzipped) | +| Nothing new | 0.01 s | 0 | +| 3 records changed | 1.0 s | 2 | + +Its only runtime dependency is Active Support. + +## Installation + +```ruby +gem "mapitaine" +``` + +Mapitaine needs Ruby 3.3 and Rails 8.0, or newer. + +## Configuration + +```ruby +# config/initializers/mapitaine.rb +Mapitaine.configure do |config| + config.host = "https://www.example.com" + config.locales = %i[en fr de] # default: I18n.available_locales + config.x_default = :en # the language of hreflang="x-default"; none when left out +end +``` + +Then tell crawlers where the index is, in `public/robots.txt`: + +``` +Sitemap: https://www.example.com/sitemap.xml +``` + +## Sources + +A source is a class in `app/sitemaps`. Its blocks run in an object that has your route helpers, already set to +`config.host`. + +```ruby +class ProductSitemap < Mapitaine::Source + # The records: an Active Record relation. Add the `includes` your other blocks need. + scope { Product.published.includes(:photos) } + + # The URL of a record in a language. A path gets the host; nil leaves that language out. + url { |product, locale| product_url(product, locale:) } + + # The languages a record exists in. Without it, every configured language. + locales { |product| product.translated_locales } + + # The column that holds the date the content changed. Default: updated_at. + lastmod :content_updated_at + + # Image URLs, for Google Images. + images { |product| product.photos.map(&:url) } +end +``` + +A page in one language gets no `hreflang` link. A page that does not exist in the `x_default` language gets +no `x-default` link. + +### When the date depends on more than one column + +`lastmod` also takes a block. The build then needs to know what to watch in the database, as an SQL expression: + +```ruby +class ProductSitemap < Mapitaine::Source + scope { Product.published } + url { |product, locale| product_url(product, locale:) } + lastmod { |product| [product.updated_at, product.reviewed_at].compact.max } + fingerprint "GREATEST(products.updated_at, COALESCE(products.reviewed_at, products.updated_at))" +end +``` + +A column that moves when the content does not, such as an `updated_at` touched by a view counter, makes every +build rewrite files for nothing and tells search engines a date that is not true. Give such a model a column of +its own for content changes. + +### Pages that are not records + +The scope can be any list. Such a source is one file, rendered at each build and published when it differs: + +```ruby +class PageSitemap < Mapitaine::Source + scope { %i[root about contact] } + url { |page, locale| public_send(:"#{page}_url", locale:) } +end +``` + +### Files + +A source named `ProductSitemap` writes `sitemap-product-00000.xml.gz`, `sitemap-product-00001.xml.gz` and so +on, one per range of ids; `slug "products"` changes the middle part. The size of a range comes from the number +of languages, so that a file stays under 50,000 URLs and 50 MiB: 50,000 ids with one language, 554 with 24. +`shard_size 1_000` sets it yourself. A range that still outgrows a file continues in +`sitemap-product-00000-2.xml.gz`. + +The model needs an integer primary key. + +## Building + +```sh +bin/rails mapitaine:build # rewrite the files whose records changed +bin/rails mapitaine:rebuild # rewrite everything +``` + +`Mapitaine.build` does the same from Ruby and returns what it did (`written`, `kept`, `deleted`, `url_count`, +`submitted`, `warnings`). Run it on a schedule, every night or every hour. With Solid Queue: + +```yaml +# config/recurring.yml +production: + sitemap: + command: "Mapitaine.build" + schedule: every hour +``` + +A build writes the sitemap files first, then the index, then removes the files no range fills any more, then +writes `sitemap-manifest.json`, its memory of what each file held. A build that fails midway is repaired by the +next one. Changing the host, the languages or the version of Mapitaine rewrites everything once. + +## Storage + +Files go to `public/` by default. + +### Through Active Storage + +An app that already uses Active Storage can send the sitemaps through one of its services, with the credentials +it already has. Give them a service of their own in `config/storage.yml`, in the bucket you want: + +```yaml +sitemaps: + service: S3 + bucket: example-sitemaps + region: eu-west-3 + public: true + access_key_id: <%= Rails.application.credentials.dig(:aws, :access_key_id) %> + secret_access_key: <%= Rails.application.credentials.dig(:aws, :secret_access_key) %> +``` + +```ruby +Mapitaine.configure do |config| + config.storage = Mapitaine::Storage::ActiveStorage.new(:sitemaps, prefix: "sitemaps") + config.sitemaps_url = "https://cdn.example.com/sitemaps" # where those files are served from +end +``` + +No blob is created: the files keep their names as keys, and nothing is written to the database. Any service +works that serves a key at a stable URL: S3, Google Cloud Storage, Azure, a mirror of them. The Disk service +does not, it scatters keys in subfolders: keep the default for files on disk. + +A storage is any object with `read(name)`, `write(name, path)` and `delete(name)`. + +## IndexNow + +```ruby +config.index_now_key = "c1f0e2b4a6d84f3e9b7a5c3d1e0f2a4b" # yours: 8 to 128 characters, a-z, A-Z, 0-9 and - +``` + +Each build then publishes the key as `.txt` next to the sitemaps, and sends the URLs of the rewritten files +whose `lastmod` is later than the previous build. The first build sends nothing. When the key cannot live at +the root of the site, host it yourself and set `config.index_now_key_location` to its URL. An endpoint that +cannot be reached is a warning, never a failed build. + +## Checking + +```sh +bin/rails mapitaine:check +``` + +It renders every sitemap without publishing or submitting anything, and fails on a URL outside `config.host`, +a URL listed twice in a source, or a `lastmod` in the future. A build reports the same, for the files it +rewrites, in `warnings`. + +## Instrumentation + +`build.mapitaine` wraps a build, with `full` and `result` in its payload; `shard.mapitaine` wraps each file +range it renders, with `source` and `shard`. + +```ruby +ActiveSupport::Notifications.subscribe("build.mapitaine") do |event| + built = event.payload[:result] + Rails.logger.info "Sitemap: #{built.written.size} files written in #{event.duration.round} ms" +end +``` + +## Not there yet + +Video and news sitemaps, models without an integer primary key, and building several ranges in parallel. + +## Development + +```sh +bundle exec rake test +bundle exec rubocop +bundle exec ruby bench/build.rb +bundle exec rake docs:highlight # after editing a code block in docs/ +``` + +## License + +MIT. See [LICENSE](LICENSE). diff --git a/Rakefile b/Rakefile new file mode 100644 index 0000000..82d4040 --- /dev/null +++ b/Rakefile @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +require "bundler/gem_tasks" +require "rake/testtask" + +Rake::TestTask.new do |task| + task.libs << "lib" << "test" + task.test_files = FileList["test/*_test.rb"] +end + +task default: :test + +namespace :docs do + desc "Highlight the code blocks of docs/*.html with Rouge" + task :highlight do + ruby "script/highlight_docs.rb" + end + + desc "Fail if a code block of docs/*.html is not highlighted" + task "highlight:check" do + ruby "script/highlight_docs.rb", "--check" + end +end diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..7f9e4ec --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,14 @@ +# Security policy + +## Supported versions + +Only the latest release of each gem gets security fixes. Upgrade before reporting if you can. + +## Reporting a vulnerability + +Please do not open a public issue. Report it privately from the repository's **Security** tab +(_Report a vulnerability_), or email clement@levups.com if you cannot. + +Tell us what an attacker can do, which versions are affected, and how to reproduce it: a template, a PDF, a user +agent string, a few lines of Ruby. We are two maintainers: expect an answer within a week, and a fixed release as +soon as we have one. We credit you in the advisory and the changelog, unless you would rather we did not. diff --git a/STYLE.md b/STYLE.md new file mode 100644 index 0000000..0481edb --- /dev/null +++ b/STYLE.md @@ -0,0 +1,49 @@ +# Code style + +RuboCop (Standard's rules plus every rubocop-performance cop) settles the layout. This page covers what it can't: +how we name, cut and comment code. Upstream code copied verbatim keeps its own style. + +## Keep it simple + +The best code is the code you don't write. Stop at the first rung that holds: not needed, don't write it; the +standard library does it; Rails or an installed dependency does it; one line; only then, the minimum that works. +Input validation at the gem's boundary, data loss and security are never cut. + +## Naming + +- Spell identifiers in full. Blocks name their parameter; `_1` only in a one-expression block where a name adds + nothing (no bare `it` while Ruby 3.3 is supported). +- In `rescue`, the variable is `exception`, never `e`. +- A hash keyed by X is `_by_` (`width_by_glyph`, `count_by_state`). +- Predicates end in `?` and read affirmatively (`bot?`, `current?`), boolean keywords too (`strict: true`). +- Constants carry their unit (`READ_TIMEOUT_SECONDS`, `MAX_GLYPHS_PER_RUN`). +- No placeholder names (`data`, `result`, `tmp`, `obj`): say what it holds. + +## Methods + +- Memoize with `@name`, not `@_name`. +- Keep a private method called once when its name says something its body doesn't; inline it when the name only + paraphrases the body. +- New methods use `def`/`end`, not the endless form; existing ones stay until you touch them. +- Guard clauses for early exits, the modifier form for short conditions, but a balanced two-branch `if/else` stays. +- `map`, `filter_map`, `flat_map` first; an explicit loop when a chain needs nil-filtering or `.then` gymnastics. +- Don't abstract until the pattern repeats; reserve `yield` for genuine around-style wrappers. + +## Comments + +None by default: make the code say it. A comment earns its place only if removing it would make a careful reader +break the code. Three things a name cannot carry: + +- **Outside vocabulary**: a spec's field order, a format's magic number, quoted verbatim from its source. +- **The reason for a number**: why 65,535 and not 65,536. +- **The trap that looks like a mistake**: a workaround for another library's bug, a rewrite we deliberately refused. + +One line on the surprising line, four at most, dated when it may expire (`# 2026-11: Ruby 3.3's strscan lacks +peek_byte`). Never a header restating a name, narration of the next line, history of the change (that goes in the +commit), or commented-out code. + +## Dead code + +Leave code cleaner than you found it: remove dead code, unused files and guards against impossible states +(`nil` checks on values that can't be `nil`, bare `rescue`, unreachable fallbacks). A guard stays when the state is +reachable, or when it protects against corrupt input: say which on its line. diff --git a/bench/build.rb b/bench/build.rb new file mode 100644 index 0000000..d8d590e --- /dev/null +++ b/bench/build.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +# bundle exec ruby [--yjit] bench/build.rb: a full build of 30,000 records in 24 languages, a build with nothing +# new, then one after three records changed. + +require "bundler/setup" +require "active_record" +require "benchmark" +require "mapitaine" + +RECORD_COUNT = 30_000 +LOCALES = %i[ar bg cs da de el en es fi fr he hu it ja ko nb nl pl pt ro sk sv tr zh].freeze + +ActiveRecord::Base.establish_connection(adapter: "sqlite3", database: ":memory:") +ActiveRecord::Schema.verbose = false +ActiveRecord::Schema.define do + create_table :posts do |table| + table.string :slug, null: false + table.timestamps + end +end + +class Post < ActiveRecord::Base +end + +class PostSitemap < Mapitaine::Source + scope { Post.all } + url { |post, locale| "/#{locale}/posts/#{post.slug}" } +end + +yesterday = Time.now - 86_400 +Post.insert_all(Array.new(RECORD_COUNT) do |number| + {slug: "a-fairly-long-post-slug-number-#{number}", created_at: yesterday, updated_at: yesterday} +end) + +Dir.mktmpdir do |directory| + Mapitaine.configure do |config| + config.host = "https://www.example.com" + config.locales = LOCALES + config.x_default = :en + config.storage = Mapitaine::Storage::Local.new(directory) + config.sources = [PostSitemap] + end + + built = nil + puts format("Full build %8.3f s", Benchmark.realtime { built = Mapitaine.build }) + sizes = Dir[File.join(directory, "*.gz")].map { |path| [File.size(path), Zlib.gunzip(File.binread(path)).bytesize] } + puts format(" %d URLs in %d files, the largest %.1f MiB, %.1f MiB gzipped in all", + built.url_count, sizes.size, sizes.map(&:last).max / 1_048_576.0, sizes.sum(&:first) / 1_048_576.0) + puts format("Nothing new %8.3f s", Benchmark.realtime { Mapitaine.build }) + Post.where(id: [5, 17_000, 17_001]).update_all(updated_at: Time.now) + puts format("3 records changed %8.3f s", Benchmark.realtime { built = Mapitaine.build }) + puts format(" %d files written, %d kept", built.written.size, built.kept.size) +end diff --git a/bin/release b/bin/release new file mode 100755 index 0000000..8ac1419 --- /dev/null +++ b/bin/release @@ -0,0 +1,232 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# Releases the gem through a pull request, as main takes no direct push. CHANGELOG.md's Unreleased section +# becomes the version's and the version constant is bumped on a branch release/vX.Y.Z, whose pull request merges +# itself once CI is green. The merge commit then gets a signed tag vX.Y.Z, which starts +# .github/workflows/release.yml: it publishes the gem to RubyGems and the GitHub release. The same file in every +# Level UP Solutions gem. +# +# bin/release minor # or patch, major, or an exact version such as 1.2.0 +# bin/release minor --dry-run # shows the notes and the changes, touches nothing +# bin/release --tag 1.2.0 # waits for the release pull request, then tags it (after a Ctrl-C) +# +# Used by the release workflow too: +# +# bin/release --check v1.2.0 # fails unless the tag matches the gemspec's version and has notes +# bin/release --notes v1.2.0 # the GitHub release notes: the version's section of CHANGELOG.md +# bin/release --title v1.2.0 # the GitHub release title, "Mapitaine 1.2.0" + +require "date" +require "json" +require "open3" +require "optparse" +require "rubygems" +require "tempfile" + +CHANGELOG = "CHANGELOG.md" +REFERENCE = /^\[([^\]]+)\]: (\S+)$/ + +def run(*command) + output, status = Open3.capture2e(*command) + abort "#{command.join(" ")} failed:\n#{output}" unless status.success? + output +end + +# A version's section: its heading, then everything up to the next heading or the reference links. +def section(version) + /^## \[#{Regexp.escape(version)}\][^\n]*\n(.*?)(?=^## |^\[[^\]]+\]: |\z)/m +end + +spec = Gem::Specification.load(Dir["*.gemspec"].first) or abort "No gemspec here" +current = spec.version +repo_url = spec.metadata["source_code_uri"] || spec.homepage +display_name = File.read("README.md", encoding: "UTF-8")[/\A# (\S+)/, 1] || spec.name +changelog = File.read(CHANGELOG, encoding: "UTF-8") + +# Waits for the release pull request to merge, then tags its merge commit and pushes the tag alone. Stops when +# the pull request can't merge by itself: closed, auto-merge turned off, or a required check failed. +publish = lambda do |target| + tag = "v#{target}" + branch = "release/#{tag}" + resume = "bin/release --tag #{target} tags it once merged" + puts "Waiting for #{branch} to merge (Ctrl-C stops waiting; #{resume})" + commit = loop do + output, status = Open3.capture2("gh", "pr", "view", branch, "--json", "state,url,mergeCommit,autoMergeRequest") + if status.success? # otherwise GitHub didn't answer: ask again + pull_request = JSON.parse(output) + url = pull_request["url"] + oid = pull_request.dig("mergeCommit", "oid") + break oid if pull_request["state"] == "MERGED" && oid + abort "#{url} was closed unmerged: #{tag} not pushed" if pull_request["state"] == "CLOSED" + end + if status.success? && pull_request["state"] == "OPEN" + abort "#{url} has auto-merge off: merge it, then #{resume}" if pull_request["autoMergeRequest"].nil? + # Exits non-zero while checks are pending or failing; prints nothing as long as none reported. + checks = Open3.capture2("gh", "pr", "checks", branch, "--required", "--json", "name,bucket").first + checks = checks.start_with?("[") ? JSON.parse(checks) : [] + failed = checks.filter_map { |check| check["name"] if check["bucket"] == "fail" } + abort "#{url}: #{failed.join(", ")} failed. Fix it on #{branch}; #{resume}" if failed.any? + end + sleep 20 + rescue Interrupt + abort "\nStopped waiting: #{resume}" + end + run("git", "fetch", "--quiet", "--tags", "origin", "main") + notes = run("git", "show", "#{commit}:#{CHANGELOG}") + abort "#{commit[0, 7]} has no #{target} section in #{CHANGELOG}" unless notes.match?(section(target.to_s)) + tagged = Open3.capture2("git", "rev-parse", "--verify", "--quiet", "#{tag}^{commit}").first.strip + if tagged.empty? + run("git", "tag", "--annotate", "--sign", tag, "-m", "#{display_name} #{target}", commit) + elsif tagged != commit + abort "#{tag} already tags #{tagged[0, 7]}, not the merge commit #{commit[0, 7]}" + end + run("git", "push", "origin", tag) + puts "Pushed #{tag}: #{repo_url}/actions/workflows/release.yml publishes it to RubyGems, then #{repo_url}/releases/tag/#{tag}" + # Local housekeeping, after the release is out: a failure here only warns. + if run("git", "branch", "--show-current").strip == "main" && run("git", "status", "--porcelain", "--untracked-files=no").empty? + output, status = Open3.capture2e("git", "merge", "--quiet", "--ff-only", "origin/main") + warn "main not fast-forwarded to origin/main:\n#{output}" unless status.success? + end + Open3.capture2e("git", "branch", "--quiet", "--delete", "--force", branch) +end + +mode = :release +dry_run = false +assume_yes = false +OptionParser.new do |parser| + parser.banner = "Usage: bin/release (patch | minor | major | X.Y.Z) [--dry-run] [--yes] | --tag X.Y.Z" + parser.on("--dry-run", "Show the notes and the changes, touch nothing") { dry_run = true } + parser.on("--yes", "Do not ask before opening the release pull request") { assume_yes = true } + parser.on("--tag", "Wait for the release pull request of the version given, then tag it") { mode = :tag } + parser.on("--check", "Fail unless the tag given matches the gemspec's version and has notes") { mode = :check } + parser.on("--notes", "Print the release notes of the tag given") { mode = :notes } + parser.on("--title", "Print the release title of the tag given") { mode = :title } +end.parse! +argument = ARGV.shift or abort "Which version? patch, minor, major or X.Y.Z (bin/release --help)" + +if mode != :release + version = argument.delete_prefix("v") + notes = changelog[section(version), 1].to_s.strip + case mode + when :check + abort "Tag #{argument}, but #{spec.name}.gemspec says #{current}" unless version == current.to_s + abort "#{CHANGELOG} has no notes for #{version}" if notes.empty? + puts "#{argument} matches #{spec.name} #{current}" + when :notes + abort "#{CHANGELOG} has no notes for #{version}" if notes.empty? + compare = changelog.scan(REFERENCE).to_h[version] + puts notes + puts "", "**Full diff**: #{compare}" if compare&.include?("/compare/") + when :title + puts "#{display_name} #{version}" + when :tag + publish.call(Gem::Version.new(version)) + end + exit +end + +major, minor, patch = current.segments.first(3).fill(0, current.segments.size...3) +target = Gem::Version.new( + case argument + when "major" then "#{major + 1}.0.0" + when "minor" then "#{major}.#{minor + 1}.0" + when "patch" then "#{major}.#{minor}.#{patch + 1}" + else argument.delete_prefix("v") + end +) +tag = "v#{target}" + +# The tree, the branch, the tags and CI. +# CHANGELOG.md alone may be uncommitted: the notes written for this release, committed with the version. +uncommitted = run("git", "status", "--short", "--", ".", ":(exclude)#{CHANGELOG}") +abort "Uncommitted changes:\n#{uncommitted}" unless uncommitted.empty? +branch = run("git", "branch", "--show-current").strip +abort "On #{branch}: releases are cut from main" unless branch == "main" +run("git", "fetch", "--quiet", "--tags", "origin", "main") +head = run("git", "rev-parse", "HEAD").strip +abort "main is not origin/main: pull or push first" unless head == run("git", "rev-parse", "origin/main").strip +released = run("git", "tag", "--list", "v*").split.map { |name| Gem::Version.new(name.delete_prefix("v")) } +abort "#{target} is not above #{released.max}, the last release" if released.any? && target <= released.max +abort "#{target} is below #{current}, the gemspec's version" if target < current +previous = released.max + +runs = JSON.parse(run("gh", "run", "list", "--commit", head, "--workflow", "ci.yml", "--json", "status,conclusion,url")) +unless runs.first&.fetch("conclusion") == "success" + state = runs.first ? runs.first.values_at("status", "conclusion", "url").join(" ").squeeze(" ") : "no run" + abort "CI is not green on #{head[0, 7]} (#{state}): wait for it, or fix it" +end + +# CHANGELOG.md: the Unreleased entries become the version's section, unless a commit already wrote it. +unreleased = changelog[section("Unreleased"), 1].to_s.strip +if changelog.match?(section(target.to_s)) + abort "#{CHANGELOG} has a #{target} section and Unreleased entries: move them" unless unreleased.empty? +else + abort "#{CHANGELOG}'s Unreleased section is empty: nothing to release" if unreleased.empty? + changelog = changelog.sub(section("Unreleased")) { "## [Unreleased]\n\n## [#{target}] — #{Date.today.iso8601}\n\n#{unreleased}\n\n" } +end + +# Reference links at the end: Unreleased compares with the new tag, then each version newest first. +references = changelog.scan(REFERENCE).to_h +references["Unreleased"] = "#{repo_url}/compare/#{tag}...HEAD" +references[target.to_s] = previous ? "#{repo_url}/compare/v#{previous}...#{tag}" : "#{repo_url}/releases/tag/#{tag}" +unreleased_link = references.delete("Unreleased") +versions = references.sort_by { |name, _url| Gem::Version.new(name) }.reverse +links = [["Unreleased", unreleased_link], *versions].map { |name, url| "[#{name}]: #{url}" } +changelog = "#{changelog.gsub(/^#{REFERENCE.source}\n?/, "").rstrip}\n\n#{links.join("\n")}\n" + +# The version constant, and Gemfile.lock where it is committed. +changes = {CHANGELOG => changelog} +if target != current + constant = /^(\s*VERSION = )(["'])#{Regexp.escape(current.to_s)}\2/ + holders = Dir["lib/**/version.rb"].select { |path| File.read(path).match?(constant) } + abort "Expected one VERSION = \"#{current}\" in lib/**/version.rb, found #{holders}" unless holders.one? && File.read(holders.first).scan(constant).one? + changes[holders.first] = File.read(holders.first).sub(constant) { "#{$1}#{$2}#{target}#{$2}" } + unless run("git", "ls-files", "Gemfile.lock").empty? + lock = File.read("Gemfile.lock").sub(/^( #{Regexp.escape(spec.name)} \()#{Regexp.escape(current.to_s)}\)/) { "#{$1}#{target})" } + changes["Gemfile.lock"] = lock + end +end +changes.reject! { |path, text| File.read(path, encoding: "UTF-8") == text } + +puts "#{display_name} #{current} → #{target}, through a pull request, then tagged #{tag}", "" +puts changelog[section(target.to_s), 1].strip, "" +if dry_run + changes.each do |path, text| + Tempfile.create do |file| + file.write(text) + file.flush + diff = Open3.capture2("git", "diff", "--no-index", "--src-prefix=", "--dst-prefix=", path, file.path).first + puts diff.gsub(/\S*#{Regexp.escape(File.basename(file.path))}/, path) + end + end + puts "Dry run: nothing written" + exit +end + +abort "Nothing to change for #{target}: if its pull request merged, bin/release --tag #{target} tags it" if changes.empty? +unless assume_yes + print "Open the release pull request, then push #{tag} and publish #{spec.name} #{target} once it merges? [y/N] " + abort "Stopped" unless $stdin.gets.to_s.strip.casecmp?("y") +end + +branch = "release/#{tag}" +title = "Release #{target}" +body = "#{changelog[section(target.to_s), 1].strip}\n\nOnce merged, `bin/release` tags the merge commit #{tag}, " \ + "which publishes the gem." +begin + run("git", "switch", "--quiet", "--create", branch) + changes.each { |path, text| File.write(path, text) } + run("git", "commit", "--quiet", "-m", title, "--", *changes.keys) + run("git", "push", "--quiet", "origin", branch) + run("git", "switch", "--quiet", "main") + run("gh", "pr", "create", "--base", "main", "--head", branch, "--title", title, "--body", body) + run("gh", "pr", "merge", branch, "--auto", "--squash") +rescue SystemExit + warn "Stopped before #{branch}'s pull request merges by itself. Either finish by hand (open it, merge it, then " \ + "bin/release --tag #{target}), or undo and start again: git switch main && git branch -D #{branch} && " \ + "git push origin :#{branch} (the notes are in #{branch}'s commit)" + raise +end +puts "Opened #{JSON.parse(run("gh", "pr", "view", branch, "--json", "url"))["url"]}: it merges itself once CI is green" +publish.call(target) diff --git a/docs/.nojekyll b/docs/.nojekyll new file mode 100644 index 0000000..e69de29 diff --git a/docs/benchmarks.html b/docs/benchmarks.html new file mode 100644 index 0000000..b08c104 --- /dev/null +++ b/docs/benchmarks.html @@ -0,0 +1,81 @@ + + + + + +Mapitaine Benchmarks + + + + + + + + + + + + + + + + + +
+ + +
+
+

Benchmarks

+

A site of 30,000 pages in 24 languages, built in full, then again.

+
+ +
+

Building 720,000 URLs

+
+

bench/build.rb creates 30,000 records, builds their sitemap in 24 languages, builds it again with nothing new, then once more after three records changed. Every URL carries the 25 hreflang links of its page.

+ + + + + + + + +
Ruby 3.3.6 without YJIT, Rails 8.1.4, SQLite in memory, Linux x86_64, one run
BuildTimeFiles written
Everything, 720,000 URLs31.9 s55 (17 MiB gzipped)
Nothing new0.01 s0
3 records changed1.0 s2
+

The largest file is 39.8 MiB before compression, under the 50 MiB the protocol allows. The URLs here are built from a string: with route helpers and a real database, the full build takes longer, and the other two change little.

+
+
+ +
+

Running it again

+
+

From a checkout of the gem, once with YJIT off and once with it on:

+
bundle exec ruby bench/build.rb
+bundle exec ruby --yjit bench/build.rb
+

The numbers depend on the machine and the database: compare builds on the same one rather than with this table.

+
+
+
+ + +
+ + diff --git a/docs/favicon.svg b/docs/favicon.svg new file mode 100644 index 0000000..ef4677c --- /dev/null +++ b/docs/favicon.svg @@ -0,0 +1 @@ + diff --git a/docs/guide.html b/docs/guide.html new file mode 100644 index 0000000..fe0f027 --- /dev/null +++ b/docs/guide.html @@ -0,0 +1,219 @@ + + + + + +Mapitaine Guide + + + + + + + + + + + + + + + + + +
+ + +
+
+

Guide

+

How to describe your pages, build the sitemaps on a schedule, store them, and tell search engines what changed.

+ +
+ +
+

Installation

+
+
# Gemfile
+gem "mapitaine"
+

Mapitaine needs Ruby 3.3 and Rails 8.0, or newer. Its only runtime dependency is Active Support. Set the host of the site in an initializer:

+
# config/initializers/mapitaine.rb
+Mapitaine.configure do |config|
+  config.host = "https://www.example.com"
+  config.locales = %i[en fr de]   # default: I18n.available_locales
+  config.x_default = :en          # for hreflang="x-default"
+end
+

Then tell crawlers where the index is, in public/robots.txt:

+
Sitemap: https://www.example.com/sitemap.xml
+
+
+ +
+

Sources

+
+

A source is a class in app/sitemaps that inherits from Mapitaine::Source, one per kind of page. Its blocks run in an object that has your route helpers, already set to config.host.

+
# app/sitemaps/product_sitemap.rb
+class ProductSitemap < Mapitaine::Source
+  # The records: an Active Record relation. Add the includes your other blocks need.
+  scope { Product.published.includes(:photos) }
+
+  # The URL of a record in a language. A path gets the host; nil leaves that language out.
+  url { |product, locale| product_url(product, locale:) }
+
+  # Image URLs, for Google Images.
+  images { |product| product.photos.map(&:url) }
+end
+

The model needs an integer primary key: files are cut by ranges of ids.

+

Pages that are not records

+

The scope can be any list. Such a source is one file, rendered at each build and published when it differs from the last one.

+
class PageSitemap < Mapitaine::Source
+  scope { %i[root about contact] }
+  url { |page, locale| public_send(:"#{page}_url", locale:) }
+end
+
+
+ +
+

Languages

+
+

Every record gets one URL per language of config.locales. When records are not all translated, locales names the languages each one exists in:

+
class ProductSitemap < Mapitaine::Source
+  scope { Product.published }
+  url { |product, locale| product_url(product, locale:) }
+  locales { |product| product.translated_locales }
+end
+

Each <url> then lists every language of its page, itself included, as hreflang links. A page in one language gets no link. A page that does not exist in the x_default language gets no x-default link.

+
+
+ +
+

The date of a page

+
+

lastmod is read from updated_at. Name another column when the content has its own:

+
lastmod :content_updated_at
+

A column that moves when the content does not, such as an updated_at touched by a view counter, makes every build rewrite files for nothing and tells search engines a date that is not true. Give such a model a column for content changes.

+

When the date depends on more than one column

+

lastmod also takes a block. The build then needs to know what to watch in the database, as an SQL expression:

+
class ProductSitemap < Mapitaine::Source
+  scope { Product.published }
+  url { |product, locale| product_url(product, locale:) }
+  lastmod { |product| [product.updated_at, product.reviewed_at].compact.max }
+  fingerprint "GREATEST(products.updated_at, COALESCE(products.reviewed_at, products.updated_at))"
+end
+
+
+ +
+

Files

+
+

A source named ProductSitemap writes sitemap-product-00000.xml.gz, sitemap-product-00001.xml.gz and so on, one per range of ids. slug "products" changes the middle part, config.filename the first.

+

The size of a range comes from the number of languages, so that a file stays under the 50,000 URLs and 50 MiB the protocol allows: 50,000 ids with one language, 554 with 24. shard_size 1_000 sets it yourself. A range that still outgrows a file continues in sitemap-product-00000-2.xml.gz.

+

sitemap.xml is the index. It lists every file with the latest lastmod it holds, so a crawler fetches again only the files that changed.

+
+
+ +
+

Building

+
+
bin/rails mapitaine:build     # rewrite the files whose records changed
+bin/rails mapitaine:rebuild   # rewrite everything
+

Mapitaine.build does the same from Ruby and returns what it did: written, kept, deleted, url_count, submitted and warnings. Run it on a schedule, every night or every hour. With Solid Queue:

+
# config/recurring.yml
+production:
+  sitemap:
+    command: "Mapitaine.build"
+    schedule: every hour
+

How a build decides

+

For each source, one query groups the records by range of ids and returns the count and the latest change of each. A range whose answer is the one the last build stored keeps its file. The others are rendered again. Nothing hooks into your models, so update_all, imports and deletions are seen too.

+

A build writes the sitemap files first, then the index, then removes the files no range fills any more, then writes sitemap-manifest.json, its memory of what each file held. A build that fails midway is repaired by the next one. Changing the host, the languages or the version of Mapitaine rewrites everything once.

+
+
+ +
+

Storage

+
+

Files go to public/ by default.

+

Through Active Storage

+

An app that already uses Active Storage can send the sitemaps through one of its services, with the credentials it already has. Give them a service of their own in config/storage.yml, in the bucket you want:

+
sitemaps:
+  service: S3
+  bucket: example-sitemaps
+  region: eu-west-3
+  public: true
+  access_key_id: <%= Rails.application.credentials.dig(:aws, :access_key_id) %>
+  secret_access_key: <%= Rails.application.credentials.dig(:aws, :secret_access_key) %>
+
Mapitaine.configure do |config|
+  config.storage = Mapitaine::Storage::ActiveStorage.new(:sitemaps, prefix: "sitemaps")
+  config.sitemaps_url = "https://cdn.example.com/sitemaps"   # where those files are served from
+end
+

No blob is created: the files keep their names as keys, and nothing is written to the database. Any service works that serves a key at a stable URL: S3, Google Cloud Storage, Azure, a mirror of them. The Disk service does not, it scatters keys in subfolders: keep the default for files on disk.

+

Sitemaps served from another host than the pages they list are accepted when robots.txt names the index.

+

Your own

+

A storage is any object with read(name), write(name, path) and delete(name).

+
+
+ +
+

IndexNow

+
+

IndexNow tells Bing, Yandex, Seznam and Naver that a URL changed. Google does not take part. Choose a key of 8 to 128 characters among a-z, A-Z, 0-9 and -:

+
config.index_now_key = "c1f0e2b4a6d84f3e9b7a5c3d1e0f2a4b"
+

Each build then publishes the key as <key>.txt next to the sitemaps, and sends the URLs of the rewritten files whose lastmod is later than the previous build. The first build sends nothing.

+

When the key cannot live at the root of the site, host it yourself and set config.index_now_key_location to its URL. An endpoint that cannot be reached is a warning, never a failed build.

+
+
+ +
+

Checking

+
+
bin/rails mapitaine:check
+

It renders every sitemap without publishing or submitting anything, and fails on a URL outside config.host, a URL listed twice in a source, or a lastmod in the future. A build reports the same in warnings, for the files it rewrites.

+

Mapitaine never requests a page of the site. Whether a URL answers 200 or carries a noindex is what Search Console reports, from the visits crawlers make anyway.

+
+
+ +
+

Instrumentation

+
+

build.mapitaine wraps a build, with full and result in its payload. shard.mapitaine wraps each range it renders, with source and shard.

+
ActiveSupport::Notifications.subscribe("build.mapitaine") do |event|
+  built = event.payload[:result]
+  Rails.logger.info "Sitemap: #{built.written.size} files written in #{event.duration.round} ms"
+end
+
+
+ +
+ + +
+ + diff --git a/docs/index.html b/docs/index.html new file mode 100644 index 0000000..7730555 --- /dev/null +++ b/docs/index.html @@ -0,0 +1,152 @@ + + + + + +Mapitaine: sitemaps for Rails sites in many languages + + + + + + + + + + + + + + + + + + +
+ + +
+
+

Mapitaine

+

Sitemaps for Rails.The XML files that tell search engines which pages a site has, in which languages, and when each one changed.

+
+

Each kind of page is a small class that names its records, their URLs and their languages. Mapitaine writes the hreflang links itself, cuts the sitemap into files that keep the same records from one build to the next, and rewrites only the files whose records changed.

+ gem "mapitaine" +

Get startedBenchmarks

+
+
+ +
+

How it works

+
+

Add the gem (it needs Ruby 3.3 or later and Rails 8.0 or later), set the host of the site, then write one class per kind of page. Its blocks have your route helpers. bin/rails mapitaine:build writes the files and the index, and the next build rewrites only what moved.

+
+
+

app/sitemaps/product_sitemap.rb

+
class ProductSitemap < Mapitaine::Source
+  scope { Product.published }
+  url { |product, locale| product_url(product, locale:) }
+end
+

sitemap-product-00000.xml.gz, for a site in English and French

+
<url>
+  <loc>https://www.example.com/en/products/42</loc>
+  <lastmod>2026-11-03T09:12:44Z</lastmod>
+  <xhtml:link rel="alternate" hreflang="en" href="https://www.example.com/en/products/42"/>
+  <xhtml:link rel="alternate" hreflang="fr" href="https://www.example.com/fr/products/42"/>
+  <xhtml:link rel="alternate" hreflang="x-default" href="https://www.example.com/en/products/42"/>
+</url>
+<url>
+  <loc>https://www.example.com/fr/products/42</loc>
+  <lastmod>2026-11-03T09:12:44Z</lastmod>
+  <xhtml:link rel="alternate" hreflang="en" href="https://www.example.com/en/products/42"/>
+  <xhtml:link rel="alternate" hreflang="fr" href="https://www.example.com/fr/products/42"/>
+  <xhtml:link rel="alternate" hreflang="x-default" href="https://www.example.com/en/products/42"/>
+</url>
+
+
+

A page is one record in several languages, so every language of it lists all the others and itself. Search engines ignore hreflang links that the page they point to does not return: here they cannot be written wrong. The guide covers the rest.

+
+
+ +
+

What you get

+
+
+

Languages built in

+

One <url> per language, each with the hreflang links of all of them and x-default. A record that exists in three languages out of twenty-four lists those three.

+

Languages

+
+
+

Files that stay put

+

Each file holds a fixed range of ids, so a record is in the same file at every build. The range is sized from the number of languages, to stay under 50,000 URLs and 50 MiB a file.

+

Files

+
+
+

Builds that skip what did not move

+

A build asks the database for the count and the latest change of each range, and compares them with the last build. One query per source, and only the files that differ are rewritten.

+

Building

+
+
+

No callback in your models

+

Nothing hooks into Active Record. update_all, imports and deletions are seen like any other change, because the build reads the table, not a list of events.

+

How a build decides

+
+
+

Dates that are true

+

lastmod is the only hint search engines still read. It is the date the content changed, in the files and in the index. priority and changefreq are not written.

+

The date of a page

+
+
+

Any bucket Active Storage reaches

+

Files go to public/, or through a service of config/storage.yml to S3, Google Cloud Storage or Azure, with the credentials the app already has.

+

Storage

+
+
+

IndexNow

+

Each build sends Bing, Yandex, Seznam and Naver the URLs that changed since the previous one. The first build sends nothing.

+

IndexNow

+
+
+

Checks without a request

+

mapitaine:check fails on a URL outside the host, a URL listed twice or a date in the future. It reads what it would write, and never requests a page of the site.

+

Checking

+
+
+

720,000 URLs in half a minute

+

30,000 records in 24 languages build in 32 s. A build with nothing new takes 10 ms, and three changed records rewrite 2 files out of 55 in a second.

+

Benchmarks

+
+
+
+ +
+

Where it comes from

+
+

Mapitaine was written for a site of about 30,000 pages in 24 languages, 720,000 URLs. At that size, a script that lists every URL again at each run spends nearly all its time on pages that did not change, and the links between languages are easy to get wrong by hand.

+

Sitemaps changed in the meantime. Google stopped reading priority and changefreq, the ping endpoints closed, IndexNow appeared, and an honest lastmod became the one signal worth sending. Mapitaine writes what search engines read today, and nothing else.

+
+
+ +
+ + +
+ + diff --git a/docs/style.css b/docs/style.css new file mode 100644 index 0000000..bdf9936 --- /dev/null +++ b/docs/style.css @@ -0,0 +1,169 @@ +:root { + --ground: #eceff3; + --surface: #f6f7f9; + --ink: #1a1e27; + --muted: #5d6573; + --accent: #6b3fa0; + --rail: #c3c9d3; + --flagged: #b3261e; + --clean: #1e6b3a; + --measure: 44rem; +} + +* { box-sizing: border-box; } +html { -webkit-text-size-adjust: 100%; } +body { margin: 0; background: var(--ground); color: var(--ink); font: 400 1.0625rem/1.6 "Inter Tight", system-ui, sans-serif; } +code, pre { font-family: "Fira Code", ui-monospace, monospace; font-size: 0.9em; font-variant-ligatures: none; } +p code, li code, td code, .rule-message code { background: var(--surface); padding: 0.05em 0.3em; border-radius: 3px; } +a { color: var(--accent); text-underline-offset: 0.2em; } +a:focus-visible, summary:focus-visible { outline: 2px solid var(--accent); outline-offset: 2px; } +pre { margin: 0; padding: 0.9rem 1rem; background: var(--surface); border-radius: 4px; overflow-x: auto; line-height: 1.55; } +/* Rouge tokens (script/highlight_docs.rb): the accent's OKLCH hue turned 120°, 240° and 180°, each at least 4.5:1 on --surface. */ +pre code { --token-string: #077b71; --token-constant: #825da8; --token-number: #2172ad; } +pre .k, pre .kd, pre .kn, pre .kp, pre .kr, pre .kv, pre .nt, pre .nd, pre .cp, pre .gp { color: var(--accent); } +pre .s, pre .s1, pre .s2, pre .sb, pre .sc, pre .sd, pre .se, pre .sh, pre .si, pre .sx, pre .sr, pre .dl { color: var(--token-string); } +pre .c, pre .c1, pre .cm, pre .cs, pre .ch { color: var(--muted); } +pre .kc, pre .no, pre .nc, pre .nn, pre .ss, pre .vi, pre .na, pre .nv { color: var(--token-constant); } +pre .m, pre .mi, pre .mf, pre .mh, pre .mo, pre .mb, pre .il { color: var(--token-number); } +pre .gi { color: var(--clean); } +pre .gd { color: var(--flagged); } + +.page { max-width: 72rem; margin: 0 auto; padding: 2rem 1rem 4rem; } + +.hero { display: grid; grid-template-columns: 1fr; gap: 1.5rem 3rem; align-items: start; border-bottom: 1px solid var(--rail); padding-bottom: 2.5rem; } +@media (min-width: 52rem) { .hero { grid-template-columns: 1fr 1fr; } .hero .wordmark { grid-column: 1 / -1; } } +/* The name is the brief: one line across the page, the tagline and the install block side by side below it. Big Shoulders sets a name of up to 9 letters within 3.5 em, so 24vw fits a phone and 19rem the 70rem page; the bottom margin clears the descenders. */ +.wordmark { font: 800 clamp(4.5rem, 24vw, 19rem)/0.8 "Big Shoulders Display", sans-serif; letter-spacing: -0.02em; margin: 0 0 0.12em; color: var(--accent); } +.byline { font-size: clamp(1.4rem, 2.6vw, 2rem); line-height: 1.2; font-weight: 600; margin: 0; } +.byline span { display: block; } +.byline-sub { color: var(--muted); font-weight: 400; } +.summary { color: var(--muted); margin: 0 0 1.5rem; max-width: 40ch; } +.install { display: inline-block; font-size: 1rem; padding: 0.55rem 0.9rem; border: 1.5px solid var(--ink); border-radius: 3px; } +.links { margin: 1rem 0 0; display: flex; gap: 1.25rem; flex-wrap: wrap; } + +section { margin-top: 4rem; } +h2 { font: 800 clamp(2.2rem, 5vw, 3rem)/1 "Big Shoulders Display", sans-serif; margin: 0 0 1rem; } +h3 { font-size: 1.05rem; margin: 0 0 0.75rem; } +.prose { max-width: var(--measure); } +.prose p { margin: 0 0 1rem; } +.note { color: var(--muted); max-width: var(--measure); margin: 0 0 2rem; } + +.proof { max-width: var(--measure); border: 1px solid var(--rail); border-radius: 4px; background: var(--surface); } +.proof-row { display: grid; grid-template-columns: 6.5rem 1fr; gap: 0.75rem; padding: 0.75rem 1rem; font-size: 0.85rem; align-items: baseline; } +.proof-row + .proof-row { border-top: 1px solid var(--rail); } +.proof-row code { overflow-wrap: anywhere; color: var(--accent); } +.proof-verdict { margin: 0; padding: 0.75rem 1rem; border-top: 1px solid var(--rail); font-weight: 600; } + +.charts { display: grid; gap: 2.5rem; grid-template-columns: repeat(auto-fit, minmax(14rem, 1fr)); } +.chart { margin: 0; } +.chart small { color: var(--muted); font-weight: 400; } +.columns { display: grid; grid-template-columns: repeat(var(--count, 5), 1fr); gap: 2px; height: 14rem; align-items: end; border-bottom: 1px solid var(--ink); margin-bottom: 2rem; } +.column { position: relative; display: flex; flex-direction: column; justify-content: flex-end; align-items: center; height: 100%; } +.column-bar { width: 0.9rem; min-height: 2px; background: var(--rail); border-radius: 4px 4px 0 0; } +.column-bar.crumbly { background: var(--accent); } +.column-value { font-size: 0.75rem; color: var(--muted); margin-bottom: 0.3rem; white-space: nowrap; } +.column.crumbly .column-value { color: var(--ink); font-weight: 600; } +.column-name { position: absolute; bottom: -1.7rem; font-size: 0.8125rem; } +.column:hover .column-bar, .column:focus-visible .column-bar { outline: 2px solid var(--ink); outline-offset: 1px; } +.column:focus-visible { outline: none; } +.chart details { font-size: 0.875rem; } +.chart summary { cursor: pointer; color: var(--muted); } +table { border-collapse: collapse; margin-top: 0.5rem; width: 100%; } +th, td { text-align: left; padding: 0.3rem 0.5rem 0.3rem 0; border-bottom: 1px solid var(--rail); } +td.number, th.number { text-align: right; font-variant-numeric: tabular-nums; } +.corpus { margin-top: 3rem; } + +.steps { padding-left: 1.25rem; max-width: var(--measure); } +.steps li { margin-bottom: 0.75rem; } + +.rule-group { margin-top: 2rem; max-width: 56rem; } +.rule { border-top: 1px solid var(--rail); } +.rule:last-child { border-bottom: 1px solid var(--rail); } +.rule summary { cursor: pointer; display: grid; grid-template-columns: 1fr; gap: 0.15rem 1rem; padding: 0.6rem 0; list-style: none; } +.rule summary::-webkit-details-marker { display: none; } +@media (min-width: 44rem) { .rule summary { grid-template-columns: 15rem 1fr; } } +.rule-name { font-weight: 600; } +.rule-default { font-size: 0.8125rem; color: var(--muted); margin-left: 0.5rem; font-weight: 400; } +@media (min-width: 44rem) { .rule-default { grid-row: 2; grid-column: 1; margin-left: 0; } } +.rule-message { color: var(--muted); } +.rule[open] .rule-name { color: var(--accent); } +.rule-examples { display: grid; gap: 1rem; grid-template-columns: repeat(auto-fit, minmax(14rem, 1fr)); padding: 0.25rem 0 1rem; } +.example-label { margin: 0 0 0.3rem; font-size: 0.8125rem; font-weight: 600; } +.rule-examples div:first-child .example-label { color: var(--flagged); } +.rule-examples div:last-child .example-label { color: var(--clean); } + +.left-out { max-width: var(--measure); padding-left: 1.25rem; } + +footer { margin-top: 5rem; padding-top: 1.5rem; border-top: 1px solid var(--rail); color: var(--muted); font-size: 0.9375rem; display: flex; gap: 0.5rem 1.5rem; flex-wrap: wrap; align-items: baseline; } +footer p { margin: 0; } +footer ul { display: flex; gap: 0.25rem 1rem; flex-wrap: wrap; margin: 0; padding: 0; list-style: none; } +footer a[aria-current="page"] { font-weight: 600; } + +@media (prefers-reduced-motion: no-preference) { + .column-bar { transition: height 0.6s cubic-bezier(0.2, 0.7, 0.2, 1); } +} + +/* Grid tracks default to their content's width: long code would push the page wider than a phone. */ +.hero > *, .charts > *, .rule-examples > *, .proof-row > *, .rule summary > * { min-width: 0; } +p code, li code, .rule-message code { overflow-wrap: anywhere; } + +/* Navigation shared by every page. */ +.site-nav { display: flex; align-items: baseline; gap: 0.25rem 1.25rem; flex-wrap: wrap; padding-bottom: 1rem; margin-bottom: 2rem; border-bottom: 1px solid var(--rail); } +.site-nav .brand { font: 800 1.9rem/1 "Big Shoulders Display", sans-serif; color: var(--accent); text-decoration: none; margin-right: auto; } +.site-nav a { color: var(--ink); text-decoration: none; } +.site-nav a:hover { color: var(--accent); } +.site-nav a[aria-current="page"] { color: var(--accent); font-weight: 600; } +.home .site-nav { border-bottom: 0; margin-bottom: 0; } +.home .site-nav .brand { visibility: hidden; } + +.page-title { font: 800 clamp(3.2rem, 9vw, 5.5rem)/0.9 "Big Shoulders Display", sans-serif; margin: 0 0 1rem; } +.lede { font-size: 1.25rem; color: var(--muted); max-width: var(--measure); margin: 0 0 1rem; } +.toc { display: flex; gap: 0.25rem 1rem; flex-wrap: wrap; margin: 0 0 1rem; padding: 0; list-style: none; font-size: 0.9375rem; } + +/* Headline numbers. */ +.tiles { display: grid; gap: 1px; background: var(--rail); border: 1px solid var(--rail); border-radius: 4px; overflow: hidden; grid-template-columns: repeat(auto-fit, minmax(13rem, 1fr)); } +.tile { background: var(--surface); padding: 1.25rem 1.25rem 1.1rem; } +.tile-value { display: block; font: 800 3.4rem/1 "Big Shoulders Display", sans-serif; color: var(--accent); } +.tile-label { display: block; font-weight: 600; margin-top: 0.4rem; } +.tile-detail { display: block; color: var(--muted); font-size: 0.875rem; margin-top: 0.2rem; } +.more { margin: 1rem 0 0; } + +.cards { display: grid; gap: 1.5rem 2.5rem; grid-template-columns: repeat(auto-fit, minmax(17rem, 1fr)); } +.card h3 { font-size: 1.15rem; margin-bottom: 0.4rem; } +.card p { margin: 0 0 0.5rem; color: var(--muted); } + +.split { display: grid; gap: 1rem 2.5rem; grid-template-columns: repeat(auto-fit, minmax(18rem, 1fr)); max-width: 60rem; } +.split > * { min-width: 0; } +.code-label { margin: 0 0 0.3rem; font-size: 0.8125rem; font-weight: 600; color: var(--muted); } +.prose pre, .split pre { margin: 0 0 1rem; } +.prose h3 { margin-top: 2rem; } +.prose ul { padding-left: 1.25rem; } + +.options { max-width: 60rem; } +.options td:first-child { white-space: nowrap; padding-right: 1.5rem; vertical-align: top; } +.options td { vertical-align: top; } +.options th { font-size: 0.875rem; color: var(--muted); font-weight: 600; } + +.projects { columns: 2 18rem; column-gap: 2.5rem; padding: 0; list-style: none; max-width: 60rem; font-size: 0.9375rem; } +.projects li { break-inside: avoid; padding: 0.35rem 0; border-bottom: 1px solid var(--rail); display: flex; gap: 0.75rem; justify-content: space-between; } +.projects span { color: var(--muted); font-variant-numeric: tabular-nums; white-space: nowrap; } +.charts + .prose { margin-top: 2.5rem; } +@media (max-width: 40rem) { .options td:first-child { white-space: normal; padding-right: 0.75rem; } .options td { overflow-wrap: anywhere; } } +.options + .prose { margin-top: 2rem; } + +/* Skip link, shown on focus. */ +.skip-link { position: absolute; left: 1rem; top: -3rem; background: var(--ink); color: var(--ground); padding: 0.5rem 0.9rem; border-radius: 3px; } +.skip-link:focus { top: 1rem; } + + +/* Pages of reference tables and side-by-side cases. */ +.prose ol { padding-left: 1.25rem; } +.prose li { margin-bottom: 0.4rem; } +td code, th code { background: var(--surface); padding: 0.05em 0.3em; border-radius: 3px; overflow-wrap: anywhere; } +table { margin-bottom: 1.5rem; } +th, td { vertical-align: top; } +th { font-size: 0.875rem; color: var(--muted); font-weight: 600; } +tbody th { font-size: inherit; color: var(--ink); font-weight: 400; } +caption { caption-side: top; text-align: left; padding-bottom: 0.4rem; font-size: 0.8125rem; font-weight: 600; color: var(--muted); } +.case + .case { margin-top: 2.5rem; } +.case p { max-width: var(--measure); } diff --git a/gemfiles/rails_8_0.gemfile b/gemfiles/rails_8_0.gemfile new file mode 100644 index 0000000..af1daa9 --- /dev/null +++ b/gemfiles/rails_8_0.gemfile @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +gemspec path: ".." + +gem "actionpack", "~> 8.0.0" +gem "activerecord", "~> 8.0.0" +gem "activestorage", "~> 8.0.0" +gem "activesupport", "~> 8.0.0" +gem "minitest" +gem "rake" +gem "sqlite3" diff --git a/lib/mapitaine.rb b/lib/mapitaine.rb new file mode 100644 index 0000000..696b9a5 --- /dev/null +++ b/lib/mapitaine.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +require "active_support" +require "active_support/core_ext/object/blank" +require "active_support/core_ext/string/inflections" +require "active_support/notifications" +require "digest" +require "fileutils" +require "json" +require "net/http" +require "tmpdir" +require "uri" +require "zlib" + +require_relative "mapitaine/version" +require_relative "mapitaine/errors" +require_relative "mapitaine/storage/local" +require_relative "mapitaine/storage/active_storage" +require_relative "mapitaine/storage/null" +require_relative "mapitaine/configuration" +require_relative "mapitaine/page" +require_relative "mapitaine/xml" +require_relative "mapitaine/source" +require_relative "mapitaine/shard_writer" +require_relative "mapitaine/manifest" +require_relative "mapitaine/index_now" +require_relative "mapitaine/builder" +require_relative "mapitaine/railtie" if defined?(Rails::Railtie) + +module Mapitaine + class << self + def configuration + @configuration ||= Configuration.new + end + + def configure + yield configuration + end + + def reset_configuration + @configuration = nil + end + + # full: true rewrites every file, whatever the manifest of the last build says. + def build(full: false) + Builder.new(configuration).build(full:) + end + + # Renders every sitemap without writing or submitting anything, and returns what looks wrong in them. + def check + Builder.new(configuration, storage: Storage::Null.new, submit: false).build(full: true).warnings + end + end +end diff --git a/lib/mapitaine/builder.rb b/lib/mapitaine/builder.rb new file mode 100644 index 0000000..4b66dea --- /dev/null +++ b/lib/mapitaine/builder.rb @@ -0,0 +1,148 @@ +# frozen_string_literal: true + +module Mapitaine + class Builder + Result = Data.define(:written, :kept, :deleted, :url_count, :submitted, :warnings) do + def changed? + written.any? || deleted.any? + end + end + + Warning = Data.define(:source, :url, :message) do + def to_s + "#{source}: #{url} #{message}" + end + end + + def initialize(configuration, storage: configuration.storage, submit: true) + @configuration = configuration + @storage = storage + @submit = submit + end + + def build(full: false) + ActiveSupport::Notifications.instrument("build.mapitaine", full:) do |payload| + payload[:result] = Dir.mktmpdir("mapitaine") { |directory| build_in(directory, full:) } + end + end + + private + + def build_in(directory, full:) + @directory = directory + @previous = Manifest.read(@storage, manifest_name) + @reusable = !full && @previous.digest == @configuration.digest + @manifest = Manifest.new(digest: @configuration.digest, built_at: Time.now) + @written = [] + @kept = [] + @warnings = [] + @changed_urls = [] + + @configuration.sources.each { |source_class| build_source(source_class.new(@configuration)) } + publish_index + deleted = @previous.file_names - @manifest.file_names + deleted.each { |name| @storage.delete(name) } + upload(manifest_name, @manifest.to_json) + + Result.new(written: @written, kept: @kept, deleted:, url_count: @manifest.files.sum { |file| file["urls"] }, + submitted: submit_changed_urls, warnings: @warnings) + end + + def build_source(source) + seen_urls = Set.new + source.fingerprint_by_shard.sort_by { |shard, _fingerprint| shard.to_i }.each do |shard, fingerprint| + stored = @previous.shard(source.slug, shard) if @reusable + if fingerprint && stored && stored["fingerprint"] == fingerprint + keep(source, shard, stored) + else + render(source, shard, fingerprint, stored, seen_urls) + end + end + end + + def keep(source, shard, stored) + @manifest.record(source.slug, shard, fingerprint: stored["fingerprint"], files: stored["files"]) + @kept.concat(stored["files"].map { |file| file["name"] }) + end + + def render(source, shard, fingerprint, stored, seen_urls) + writer = ShardWriter.new(directory: @directory, basename: source.basename(shard)) + changed_urls = [] + ActiveSupport::Notifications.instrument("shard.mapitaine", source: source.slug, shard:) do + source.each_page(shard) do |page| + inspect_page(source, page, seen_urls) + writer.add(Xml.url_entries(page, x_default: @configuration.x_default), Xml.timestamp(page.lastmod)) + changed_urls.concat(page.urls) if changed_since_last_build?(page) + end + writer.close + end + + fingerprint ||= writer.content_digest + return keep(source, shard, stored) if stored && stored["fingerprint"] == fingerprint + + publish(source, shard, fingerprint, writer.files) + @changed_urls.concat(changed_urls) + end + + def publish(source, shard, fingerprint, files) + files.each do |file| + @storage.write(file.name, file.path) + File.delete(file.path) + end + @written.concat(files.map(&:name)) + @manifest.record(source.slug, shard, fingerprint:, files: files.map do |file| + {"name" => file.name, "urls" => file.url_count, "lastmod" => file.lastmod} + end) + end + + def publish_index + entries = @manifest.files.map do |file| + Xml.sitemap_entry("#{@configuration.sitemaps_url}/#{file["name"]}", file["lastmod"]) + end + upload("#{@configuration.filename}.xml", "#{Xml::INDEX_OPEN}#{entries.join}#{Xml::INDEX_CLOSE}") + end + + def upload(name, content) + path = File.join(@directory, name) + File.write(path, content) + @storage.write(name, path) + end + + def manifest_name + "#{@configuration.filename}-manifest.json" + end + + def inspect_page(source, page, seen_urls) + page.urls.each do |url| + warn(source, url, "is outside #{@configuration.host!}") unless url.start_with?("#{@configuration.host!}/") || url == @configuration.host! + warn(source, url, "is listed twice") unless seen_urls.add?(url) + end + modified_at = page.modified_at + warn(source, page.urls.first, "has a lastmod in the future (#{Xml.timestamp(page.lastmod)})") if modified_at && modified_at > @manifest.built_at + end + + def warn(source, url, message) + @warnings << Warning.new(source: source.slug, url:, message:) + end + + # The first build has no "before": submitting a whole site at once is not what IndexNow is for. + def changed_since_last_build?(page) + modified_at = page.modified_at + @previous.built_at && modified_at && modified_at >= @previous.built_at + end + + def submit_changed_urls + key = @configuration.index_now_key + return 0 unless @submit && key + + upload("#{key}.txt", key) unless @configuration.index_now_key_location + return 0 if @changed_urls.empty? + + IndexNow.new(host: @configuration.host!, key:, key_location: @configuration.index_now_key_location).submit(@changed_urls) + rescue *IndexNow::NETWORK_ERRORS => exception + # The sitemaps are published: a search engine that cannot be reached must not fail the build. + @warnings << Warning.new(source: "indexnow", url: IndexNow::ENDPOINT.to_s, message: "could not be reached (#{exception.class})") + 0 + end + end +end diff --git a/lib/mapitaine/configuration.rb b/lib/mapitaine/configuration.rb new file mode 100644 index 0000000..129656f --- /dev/null +++ b/lib/mapitaine/configuration.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Mapitaine + class Configuration + attr_accessor :filename, :index_now_key, :index_now_key_location, :load_sources, :storage, :x_default + attr_reader :host + attr_writer :locales, :sitemaps_url, :sources + + def initialize + @filename = "sitemap" + @load_sources = -> {} + @storage = Storage::Local.new("public") + end + + def host=(url) + @host = absolute_url(url, "host") + end + + def host! + host || raise(ConfigurationError, "Mapitaine.configuration.host is not set") + end + + def locales + Array(@locales || I18n.available_locales) + end + + # Where the sitemap files are served from, when that is not the site itself (a bucket, a CDN). + def sitemaps_url + @sitemaps_url ? absolute_url(@sitemaps_url, "sitemaps_url") : host! + end + + def sources + return @sources if @sources + + load_sources.call + Source.descendants.select(&:scope?).sort_by(&:slug) + end + + # Whatever changes every file: a build under another digest starts over. + def digest + Digest::SHA256.hexdigest([VERSION, host!, sitemaps_url, locales, x_default, filename].to_json) + end + + private + + def absolute_url(url, setting) + uri = URI.parse(url.to_s) + raise ConfigurationError, "#{setting} must be an http(s) URL, got #{url.inspect}" unless uri.is_a?(URI::HTTP) && uri.host.present? + + url.to_s.delete_suffix("/") + rescue URI::InvalidURIError + raise ConfigurationError, "#{setting} must be an http(s) URL, got #{url.inspect}" + end + end +end diff --git a/lib/mapitaine/errors.rb b/lib/mapitaine/errors.rb new file mode 100644 index 0000000..8090946 --- /dev/null +++ b/lib/mapitaine/errors.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Mapitaine + class Error < StandardError; end + + class ConfigurationError < Error; end + + class SourceError < Error; end +end diff --git a/lib/mapitaine/index_now.rb b/lib/mapitaine/index_now.rb new file mode 100644 index 0000000..55390d8 --- /dev/null +++ b/lib/mapitaine/index_now.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Mapitaine + # Tells Bing, Yandex, Seznam and Naver which URLs changed: https://www.indexnow.org/documentation + class IndexNow + ENDPOINT = URI("https://api.indexnow.org/indexnow") + MAX_URLS_PER_REQUEST = 10_000 + HEADERS = {"Content-Type" => "application/json; charset=utf-8"}.freeze + NETWORK_ERRORS = [SystemCallError, SocketError, Timeout::Error, OpenSSL::SSL::SSLError, EOFError].freeze + + def initialize(host:, key:, key_location: nil, transport: Net::HTTP.method(:post)) + @host = URI.parse(host).host + @key = key + @key_location = key_location + @transport = transport + end + + # Returns how many URLs the endpoint took. + def submit(urls) + urls.each_slice(MAX_URLS_PER_REQUEST).sum do |batch| + body = {host: @host, key: @key, keyLocation: @key_location, urlList: batch}.compact + response = @transport.call(ENDPOINT, body.to_json, HEADERS) + response.code.to_i.between?(200, 299) ? batch.size : 0 + end + end + end +end diff --git a/lib/mapitaine/manifest.rb b/lib/mapitaine/manifest.rb new file mode 100644 index 0000000..d85cf4d --- /dev/null +++ b/lib/mapitaine/manifest.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Mapitaine + # What the last build wrote: for each shard of each source, its fingerprint and its files. + class Manifest + attr_reader :built_at, :digest, :shards_by_source + + def self.read(storage, name) + json = storage.read(name) + return new unless json + + stored = JSON.parse(json) + new(digest: stored["digest"], built_at: Time.iso8601(stored["built_at"]), shards_by_source: stored["sources"]) + rescue JSON::ParserError, ArgumentError, TypeError + # A manifest we cannot read is a manifest we do not have: the build starts over and replaces it. + new + end + + def initialize(digest: nil, built_at: nil, shards_by_source: {}) + @digest = digest + @built_at = built_at + @shards_by_source = shards_by_source + end + + def shard(source, key) + shards_by_source.dig(source, key) + end + + def record(source, key, fingerprint:, files:) + (shards_by_source[source] ||= {})[key] = {"fingerprint" => fingerprint, "files" => files} + end + + def files + shards_by_source.each_value.flat_map { |shards| shards.each_value.flat_map { |shard| shard["files"] } } + end + + def file_names + files.map { |file| file["name"] } + end + + def to_json(*) + JSON.pretty_generate("digest" => digest, "built_at" => built_at.utc.iso8601, "sources" => shards_by_source) + end + end +end diff --git a/lib/mapitaine/page.rb b/lib/mapitaine/page.rb new file mode 100644 index 0000000..3e9962c --- /dev/null +++ b/lib/mapitaine/page.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Mapitaine + # One page of the site: the same content at one URL per language. + Page = Data.define(:url_by_locale, :lastmod, :images) do + def urls + url_by_locale.values + end + + def modified_at + lastmod.to_time if lastmod.respond_to?(:to_time) + end + end +end diff --git a/lib/mapitaine/railtie.rb b/lib/mapitaine/railtie.rb new file mode 100644 index 0000000..ede43d1 --- /dev/null +++ b/lib/mapitaine/railtie.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Mapitaine + class Railtie < Rails::Railtie + SOURCES_DIRECTORY = "app/sitemaps" + + rake_tasks { load File.expand_path("tasks.rake", __dir__) } + + config.after_initialize do |app| + Mapitaine::Source.include(app.routes.url_helpers) + Mapitaine.configuration.load_sources = lambda do + directory = app.root.join(SOURCES_DIRECTORY) + Rails.autoloaders.main.eager_load_dir(directory) if directory.directory? + end + end + end +end diff --git a/lib/mapitaine/shard_writer.rb b/lib/mapitaine/shard_writer.rb new file mode 100644 index 0000000..da387f4 --- /dev/null +++ b/lib/mapitaine/shard_writer.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +module Mapitaine + # Writes the pages of one shard as gzipped sitemap files, opening a second file when the first is full. + class ShardWriter + # https://www.sitemaps.org/protocol.html: 50,000 URLs and 50MB (52,428,800 bytes) a file, uncompressed. + MAX_URLS_PER_FILE = 50_000 + MAX_BYTES_PER_FILE = 52_428_800 + + Written = Data.define(:name, :path, :url_count, :lastmod) + + attr_reader :files + + def initialize(directory:, basename:, max_urls: MAX_URLS_PER_FILE, max_bytes: MAX_BYTES_PER_FILE) + @directory = directory + @basename = basename + @max_urls = max_urls + @max_bytes = max_bytes + @files = [] + @digest = Digest::SHA256.new + end + + # The entries of one page stay in the same file. + def add(entries, lastmod) + bytes = entries.sum(&:bytesize) + roll if @gzip && (@url_count + entries.size > @max_urls || @bytes + bytes > @max_bytes) + open_file unless @gzip + entries.each { |entry| write(entry) } + @url_count += entries.size + @lastmod = [@lastmod, lastmod].compact.max + end + + def close + roll if @gzip + self + end + + def content_digest + @digest.hexdigest + end + + private + + def open_file + suffix = @files.empty? ? "" : "-#{@files.size + 1}" + @name = "#{@basename}#{suffix}.xml.gz" + @path = File.join(@directory, @name) + @gzip = Zlib::GzipWriter.open(@path) + @url_count = 0 + @bytes = Xml::URLSET_CLOSE.bytesize + @lastmod = nil + write(Xml::URLSET_OPEN) + end + + def roll + @gzip.write(Xml::URLSET_CLOSE) + @digest << Xml::URLSET_CLOSE + @gzip.close + @gzip = nil + @files << Written.new(name: @name, path: @path, url_count: @url_count, lastmod: @lastmod) + end + + def write(text) + @gzip.write(text) + @digest << text + @bytes += text.bytesize + end + end +end diff --git a/lib/mapitaine/source.rb b/lib/mapitaine/source.rb new file mode 100644 index 0000000..75dd4b2 --- /dev/null +++ b/lib/mapitaine/source.rb @@ -0,0 +1,202 @@ +# frozen_string_literal: true + +module Mapitaine + # One kind of page. A subclass names its records, their URLs and their languages: + # + # class ProductSitemap < Mapitaine::Source + # scope { Product.published } + # url { |product, locale| product_url(product, locale:) } + # end + class Source + # The protocol allows 50 MiB a file, uncompressed. Shards aim lower: the sizes below are estimates. + TARGET_BYTES_PER_FILE = 40 * 1024 * 1024 + BYTES_PER_URL_ESTIMATE = 150 + BYTES_PER_ALTERNATE_ESTIMATE = 120 + MYSQL_ADAPTERS = %w[mysql2 trilogy].freeze + SLUG_FORMAT = /\A[a-z0-9][a-z0-9_-]*\z/ + # A source that is not an Active Record relation is one shard, rendered at each build. + WHOLE = "all" + + class << self + def inherited(subclass) + super + subclass.instance_variable_set(:@definitions, definitions.dup) + end + + def definitions + @definitions ||= {lastmod: :updated_at} + end + + # The records, as an Active Record relation or any enumerable. + def scope(&block) + definitions[:scope] = block + end + + def scope? + definitions.key?(:scope) + end + + # The URL of a record in a language, or nil when it has none. A path is completed with the host. + def url(&block) + definitions[:url] = block + end + + # The languages a record exists in. Without it, every configured language. + def locales(&block) + definitions[:locales] = block + end + + # A column, or a block for anything else. With a block on a relation, name what changes with it in + # `fingerprint`: the column alone no longer tells a build what moved. + def lastmod(column = nil, &block) + definitions[:lastmod] = block || column + end + + def images(&block) + definitions[:images] = block + end + + # The SQL expression whose greatest value changes whenever a record's sitemap entry does. + def fingerprint(sql) + definitions[:fingerprint] = sql + end + + # How many consecutive ids share a file. + def shard_size(count) + definitions[:shard_size] = Integer(count) + end + + def slug(name = nil) + definitions[:slug] = name.to_s if name + return definitions[:slug] if definitions[:slug] + + self.name ? self.name.demodulize.underscore.delete_suffix("_sitemap") : superclass.slug + end + end + + attr_reader :configuration + + def initialize(configuration) + @configuration = configuration + return if self.class.slug.match?(SLUG_FORMAT) + + raise SourceError, "#{self.class.name}'s slug #{self.class.slug.inspect} must be lowercase letters, digits, - and _" + end + + def slug + self.class.slug + end + + # What `url` blocks read when they call a route helper. + def default_url_options + uri = URI.parse(configuration.host!) + {host: uri.host, protocol: uri.scheme, port: (uri.port unless uri.port == uri.default_port)}.compact + end + + # A nil fingerprint means the shard has to be rendered to know whether it changed. + def fingerprint_by_shard + return {WHOLE => nil} unless relation? + + relation.reorder(nil).group(shard_sql).pluck(shard_sql, Arel.sql("COUNT(*)"), fingerprint_sql) + .to_h { |shard, count, latest| [shard.to_i.to_s, "#{count}/#{precise(latest)}"] } + end + + def each_page(shard) + return to_enum(:each_page, shard) unless block_given? + + records_of(shard) do |record| + page = page_for(record) + yield page unless page.url_by_locale.empty? + end + end + + def basename(shard) + prefix = "#{configuration.filename}-#{slug}" + (shard == WHOLE) ? prefix : "#{prefix}-#{shard.rjust(5, "0")}" + end + + private + + def definitions + self.class.definitions + end + + def relation + @relation ||= instance_exec(&definitions.fetch(:scope)) + end + + def relation? + relation.respond_to?(:find_each) && relation.respond_to?(:group) + end + + def model + relation.klass + end + + def records_of(shard, &) + return relation.each(&) unless relation? + + first_id = shard.to_i * shard_size + relation.where(model.primary_key => first_id...(first_id + shard_size)).find_each(&) + end + + def page_for(record) + locales = definitions[:locales] ? Array(instance_exec(record, &definitions[:locales])) : configuration.locales + url_by_locale = locales.filter_map do |locale| + location = instance_exec(record, locale, &definitions.fetch(:url)) + [locale, absolute(location)] if location + end + Page.new(url_by_locale: url_by_locale.to_h, lastmod: lastmod_of(record), images: images_of(record)) + end + + def absolute(location) + location.start_with?("/") ? "#{configuration.host!}#{location}" : location + end + + def lastmod_of(record) + lastmod = definitions[:lastmod] + return instance_exec(record, &lastmod) if lastmod.respond_to?(:call) + + record.public_send(lastmod) if lastmod && record.respond_to?(lastmod) + end + + def images_of(record) + definitions[:images] ? Array(instance_exec(record, &definitions[:images])).map { |location| absolute(location) } : [] + end + + def shard_size + @shard_size ||= definitions[:shard_size] || default_shard_size + end + + def default_shard_size + locale_count = configuration.locales.size + alternates = (locale_count > 1) ? locale_count + 1 : 0 + bytes_per_record = locale_count * (BYTES_PER_URL_ESTIMATE + alternates * BYTES_PER_ALTERNATE_ESTIMATE) + [ShardWriter::MAX_URLS_PER_FILE / locale_count, TARGET_BYTES_PER_FILE / bytes_per_record].min.clamp(1..) + end + + # Two updates in the same second must not look like one. + def precise(latest) + latest.respond_to?(:utc) ? latest.utc.iso8601(6) : latest.to_s + end + + def shard_sql + @shard_sql ||= begin + primary_key = model.primary_key + unless primary_key.is_a?(String) && model.type_for_attribute(primary_key).type == :integer + raise SourceError, "#{self.class.name} needs a scope whose model has an integer primary key" + end + # MySQL's / returns a decimal. + division = MYSQL_ADAPTERS.include?(model.connection_db_config.adapter) ? "DIV" : "/" + Arel.sql("#{model.quoted_table_name}.#{model.quoted_primary_key} #{division} #{shard_size}") + end + end + + def fingerprint_sql + column = definitions[:lastmod].is_a?(Symbol) ? definitions[:lastmod] : :updated_at + expression = definitions[:fingerprint] || + "#{model.quoted_table_name}.#{model.adapter_class.quote_column_name(column)}" + Arel.sql("MAX(#{expression})") + end + end +end diff --git a/lib/mapitaine/storage/active_storage.rb b/lib/mapitaine/storage/active_storage.rb new file mode 100644 index 0000000..2535425 --- /dev/null +++ b/lib/mapitaine/storage/active_storage.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Mapitaine + module Storage + # Sends the files through a service of config/storage.yml, without creating blobs: the sitemaps reuse the + # credentials of the app, in a bucket of their own when the service names one. + class ActiveStorage + # A service name of config/storage.yml, or a service object. + def initialize(service, prefix: nil) + @service = service + @prefix = prefix + end + + def read(name) + service.download(key(name)) + rescue ::ActiveStorage::FileNotFoundError + nil + end + + def write(name, source_path) + content_type = CONTENT_TYPE_BY_EXTENSION.fetch(File.extname(name)) + File.open(source_path, "rb") { |file| service.upload(key(name), file, content_type:) } + end + + def delete(name) + service.delete(key(name)) + end + + private + + # Looked up at the first call: an initializer runs before Active Storage has read config/storage.yml. + def service + @service = ::ActiveStorage::Blob.services.fetch(@service) if @service.is_a?(Symbol) || @service.is_a?(String) + @service + end + + def key(name) + @prefix ? File.join(@prefix, name) : name + end + end + end +end diff --git a/lib/mapitaine/storage/local.rb b/lib/mapitaine/storage/local.rb new file mode 100644 index 0000000..29f4051 --- /dev/null +++ b/lib/mapitaine/storage/local.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Mapitaine + module Storage + CONTENT_TYPE_BY_EXTENSION = { + ".gz" => "application/gzip", ".json" => "application/json", ".txt" => "text/plain", ".xml" => "application/xml" + }.freeze + + class Local + def initialize(directory) + @directory = directory.to_s + end + + def read(name) + path = File.join(@directory, name) + File.read(path) if File.exist?(path) + end + + # Copied beside the target, then renamed: a crawler never downloads half a file. + def write(name, source_path) + FileUtils.mkdir_p(@directory) + target = File.join(@directory, name) + partial = "#{target}.#{Process.pid}.partial" + FileUtils.cp(source_path, partial) + File.rename(partial, target) + end + + def delete(name) + FileUtils.rm_f(File.join(@directory, name)) + end + end + end +end diff --git a/lib/mapitaine/storage/null.rb b/lib/mapitaine/storage/null.rb new file mode 100644 index 0000000..ceb2263 --- /dev/null +++ b/lib/mapitaine/storage/null.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Mapitaine + module Storage + class Null + def read(_name) + end + + def write(_name, _source_path) + end + + def delete(_name) + end + end + end +end diff --git a/lib/mapitaine/tasks.rake b/lib/mapitaine/tasks.rake new file mode 100644 index 0000000..4714222 --- /dev/null +++ b/lib/mapitaine/tasks.rake @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +namespace :mapitaine do + report = lambda do |result| + puts "#{result.url_count} URLs: #{result.written.size} files written, #{result.kept.size} kept, " \ + "#{result.deleted.size} deleted, #{result.submitted} URLs sent to IndexNow" + result.warnings.each { |warning| warn " #{warning}" } + end + + desc "Rewrite the sitemap files whose records changed since the last build" + task build: :environment do + report.call(Mapitaine.build) + end + + desc "Rewrite every sitemap file" + task rebuild: :environment do + report.call(Mapitaine.build(full: true)) + end + + desc "Render every sitemap without publishing it, and fail on duplicate, foreign or future-dated URLs" + task check: :environment do + warnings = Mapitaine.check + abort warnings.join("\n") if warnings.any? + puts "Nothing to report" + end +end diff --git a/lib/mapitaine/version.rb b/lib/mapitaine/version.rb new file mode 100644 index 0000000..efd4014 --- /dev/null +++ b/lib/mapitaine/version.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +module Mapitaine + VERSION = "0.1.0" +end diff --git a/lib/mapitaine/xml.rb b/lib/mapitaine/xml.rb new file mode 100644 index 0000000..a84b442 --- /dev/null +++ b/lib/mapitaine/xml.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +module Mapitaine + module Xml + DECLARATION = %(\n) + URLSET_OPEN = %(#{DECLARATION}\n) + URLSET_CLOSE = "\n" + INDEX_OPEN = %(#{DECLARATION}\n) + INDEX_CLOSE = "\n" + # Google reads no more than 1,000 images per URL. + MAX_IMAGES_PER_URL = 1_000 + + module_function + + # One per language. Each lists every language of the page, itself included: search engines drop + # hreflang links that are not returned by the page they point to. + def url_entries(page, x_default:) + shared = "#{lastmod_tag(page.lastmod)}#{alternates(page, x_default)}#{images(page.images)}\n" + page.urls.map { |url| "#{escape(url)}#{shared}" } + end + + def sitemap_entry(url, lastmod) + "#{escape(url)}#{lastmod_tag(lastmod)}\n" + end + + def timestamp(value) + return if value.nil? + return value.utc.iso8601 if value.respond_to?(:utc) + + value.respond_to?(:iso8601) ? value.iso8601 : value.to_s + end + + def escape(text) + text.to_s.encode(xml: :text).gsub('"', """) + end + + def lastmod_tag(value) + stamp = timestamp(value) + stamp ? "#{escape(stamp)}" : "" + end + + def alternates(page, x_default) + return "" if page.url_by_locale.size < 2 + + links = page.url_by_locale.map { |locale, url| alternate(locale, url) } + links << alternate("x-default", page.url_by_locale[x_default]) if page.url_by_locale.key?(x_default) + links.join + end + + def alternate(hreflang, url) + %() + end + + def images(urls) + urls.first(MAX_IMAGES_PER_URL).map { |url| "#{escape(url)}" }.join + end + end +end diff --git a/mapitaine.gemspec b/mapitaine.gemspec new file mode 100644 index 0000000..6c11a5b --- /dev/null +++ b/mapitaine.gemspec @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +require_relative "lib/mapitaine/version" + +Gem::Specification.new do |spec| + spec.name = "mapitaine" + spec.version = Mapitaine::VERSION + spec.authors = ["Level UP Solutions"] + spec.summary = "XML sitemaps for Rails sites in many languages, rebuilt only where the content changed" + spec.description = "Each kind of page is a small class that names its records, its URLs and its languages. " \ + "Mapitaine writes the hreflang links itself, cuts the sitemap into files that keep the same " \ + "records from one build to the next, and rewrites only the files whose records changed." + spec.homepage = "https://github.com/levups/mapitaine" + spec.license = "MIT" + spec.required_ruby_version = ">= 3.3" + spec.metadata["source_code_uri"] = spec.homepage + spec.metadata["changelog_uri"] = "#{spec.homepage}/blob/main/CHANGELOG.md" + spec.metadata["documentation_uri"] = "https://levups.github.io/mapitaine/" + spec.metadata["bug_tracker_uri"] = "#{spec.homepage}/issues" + spec.metadata["rubygems_mfa_required"] = "true" + spec.files = Dir["lib/**/*.{rb,rake}", "LICENSE", "README.md", "CHANGELOG.md"] + spec.require_paths = ["lib"] + + spec.add_dependency "activesupport", ">= 8.0" +end diff --git a/script/highlight_docs.rb b/script/highlight_docs.rb new file mode 100644 index 0000000..4bf4463 --- /dev/null +++ b/script/highlight_docs.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +# Highlights every 
 block of docs/*.html with Rouge, in place.
+# Idempotent: it rebuilds each block from its text. --check writes nothing and exits 1 if a page would change.
+
+require "cgi"
+require "rouge"
+
+CODE_BLOCK = %r{(
)(.*?)(
)}m
+TOKEN_TAG = %r{]*>}
+FORMATTER = Rouge::Formatters::HTML.new
+
+def highlight(page, path)
+  page.gsub(CODE_BLOCK) do
+    opening, language, code, closing = Regexp.last_match.captures
+    lexer = Rouge::Lexer.find(language) || abort("#{path}: Rouge has no lexer for language-#{language}")
+    "#{opening}#{FORMATTER.format(lexer.lex(CGI.unescapeHTML(code.gsub(TOKEN_TAG, ""))))}#{closing}"
+  end
+end
+
+check = ARGV.include?("--check")
+stale_paths = Dir[File.expand_path("../docs/*.html", __dir__)].sort.select do |path|
+  page = File.read(path)
+  highlighted = highlight(page, path)
+  next false if highlighted == page
+
+  File.write(path, highlighted) unless check
+  true
+end
+
+relative_paths = stale_paths.map { |path| File.join("docs", File.basename(path)) }
+abort "Not highlighted: #{relative_paths.join(", ")}. Run bundle exec rake docs:highlight" if check && stale_paths.any?
+puts "Highlighted #{relative_paths.join(", ")}" if stale_paths.any?
diff --git a/test/build_test.rb b/test/build_test.rb
new file mode 100644
index 0000000..4d1e7a9
--- /dev/null
+++ b/test/build_test.rb
@@ -0,0 +1,202 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class BuildTest < MapitaineTest
+  def test_first_build_writes_a_file_per_shard_then_the_index_then_the_manifest
+    create_post(1)
+    create_post(12)
+    built = build
+
+    assert_equal %w[sitemap-post-00000.xml.gz sitemap-post-00001.xml.gz sitemap-page.xml.gz sitemap.xml sitemap-manifest.json],
+      storage.writes
+    assert_equal %w[sitemap-post-00000.xml.gz sitemap-post-00001.xml.gz sitemap-page.xml.gz], built.written
+    assert_equal 6, built.url_count
+    assert_empty built.warnings
+  end
+
+  def test_each_language_of_a_page_lists_all_of_them_and_the_default
+    create_post(1)
+    build
+
+    assert_equal <<~XML, storage.xml("sitemap-post-00000.xml.gz")
+      
+      
+      https://example.com/en/posts/12026-01-01T12:00:00Z
+      https://example.com/fr/posts/12026-01-01T12:00:00Z
+      
+    XML
+  end
+
+  def test_a_page_in_one_language_has_no_alternate_and_one_without_the_default_language_no_x_default
+    create_post(1, languages: "fr")
+    create_post(2, languages: "fr,de")
+    build
+    xml = storage.xml("sitemap-post-00000.xml.gz")
+
+    assert_includes xml, "https://example.com/fr/posts/12026-01-01T12:00:00Z"
+    assert_includes xml, %(hreflang="de" href="https://example.com/de/posts/2")
+    refute_includes xml, "x-default"
+  end
+
+  def test_a_path_is_completed_with_the_host_and_a_date_stays_a_date
+    build
+
+    assert_includes storage.xml("sitemap-page.xml.gz"), "https://example.com/en/about2026-01-15"
+  end
+
+  def test_the_index_lists_every_file_with_the_latest_change_it_holds
+    create_post(1)
+    create_post(2, updated_at: LONG_AGO + 3600)
+    build
+
+    assert_equal <<~XML, storage.xml("sitemap.xml")
+      
+      
+      https://example.com/sitemap-post-00000.xml.gz2026-01-01T13:00:00Z
+      https://example.com/sitemap-page.xml.gz2026-01-15
+      
+    XML
+  end
+
+  def test_a_build_with_nothing_new_rewrites_no_sitemap_file
+    create_post(1)
+    create_post(12)
+    build
+    built = build
+
+    assert_empty built.written
+    assert_equal %w[sitemap-post-00000.xml.gz sitemap-post-00001.xml.gz sitemap-page.xml.gz], built.kept
+    assert_equal %w[sitemap.xml sitemap-manifest.json], storage.writes
+    refute_predicate built, :changed?
+  end
+
+  def test_an_updated_record_rewrites_its_shard_only
+    create_post(1)
+    create_post(12)
+    build
+    Post.find(12).update!(title: "Edited", updated_at: LONG_AGO + 60)
+    built = build
+
+    assert_equal %w[sitemap-post-00001.xml.gz], built.written
+    assert_includes storage.xml("sitemap.xml"), "sitemap-post-00001.xml.gz2026-01-01T12:01:00Z"
+  end
+
+  def test_update_all_and_deletions_are_seen_without_callbacks
+    create_post(1)
+    create_post(2)
+    create_post(12)
+    build
+    Post.where(id: 12).update_all(updated_at: LONG_AGO + 60)
+    Post.where(id: 2).delete_all
+    built = build
+
+    assert_equal %w[sitemap-post-00000.xml.gz sitemap-post-00001.xml.gz], built.written
+    refute_includes storage.xml("sitemap-post-00000.xml.gz"), "posts/2"
+  end
+
+  def test_a_shard_left_without_records_loses_its_file
+    create_post(1)
+    create_post(12)
+    build
+    Post.find(12).update!(published: false)
+    built = build
+
+    assert_equal %w[sitemap-post-00001.xml.gz], built.deleted
+    assert_equal %w[sitemap-post-00001.xml.gz], storage.deletions
+    refute_includes storage.xml("sitemap.xml"), "sitemap-post-00001"
+    assert_predicate built, :changed?
+  end
+
+  def test_a_full_build_rewrites_everything
+    create_post(1)
+    build
+    built = build(full: true)
+
+    assert_equal %w[sitemap-post-00000.xml.gz sitemap-page.xml.gz], built.written
+  end
+
+  def test_a_changed_configuration_rewrites_everything
+    create_post(1)
+    build
+    Mapitaine.configuration.x_default = :fr
+    built = build
+
+    assert_equal %w[sitemap-post-00000.xml.gz sitemap-page.xml.gz], built.written
+  end
+
+  def test_an_unreadable_manifest_starts_over
+    create_post(1)
+    build
+    storage.content_by_name["sitemap-manifest.json"] = "{"
+    built = build
+
+    assert_equal %w[sitemap-post-00000.xml.gz sitemap-page.xml.gz], built.written
+  end
+
+  def test_a_source_that_is_not_a_relation_is_rewritten_when_its_content_changes
+    build
+    Mapitaine.configuration.sources = [PostSitemap, Class.new(PageSitemap) { scope { %i[about about] } }]
+    built = build
+
+    assert_equal %w[sitemap-page.xml.gz], built.written
+  end
+
+  def test_the_scope_may_be_ordered
+    create_post(1)
+    ordered = Class.new(PostSitemap) { scope { Post.order(:title) } }
+    Mapitaine.configuration.sources = [ordered]
+
+    assert_equal %w[sitemap-post-00000.xml.gz], build.written
+  end
+
+  def test_sitemaps_served_from_elsewhere_are_listed_there
+    create_post(1)
+    Mapitaine.configuration.sitemaps_url = "https://cdn.example.com/sitemaps/"
+    build
+
+    assert_includes storage.xml("sitemap.xml"), "https://cdn.example.com/sitemaps/sitemap-post-00000.xml.gz"
+  end
+
+  def test_special_characters_are_escaped_and_images_listed
+    create_post(1, title: %(Tom & "Jerry" <3))
+    with_images = Class.new(PostSitemap) do
+      url { |post, locale| "/#{locale}/posts?title=#{post.title}" }
+      images { |post| ["/images/#{post.id}.jpg", "https://cdn.example.com/#{post.id}.jpg?a=1&b=2"] }
+    end
+    Mapitaine.configuration.sources = [with_images]
+    build
+    xml = storage.xml("sitemap-post-00000.xml.gz")
+
+    assert_includes xml, "https://example.com/en/posts?title=Tom & "Jerry" <3"
+    assert_includes xml, "https://example.com/images/1.jpg" \
+                         "https://cdn.example.com/1.jpg?a=1&b=2"
+  end
+
+  def test_a_lastmod_block_is_fingerprinted_by_the_expression_it_names
+    create_post(1)
+    reviewed = Class.new(PostSitemap) do
+      lastmod { |post| [post.updated_at, post.reviewed_at].compact.max }
+      fingerprint "COALESCE(posts.reviewed_at, posts.updated_at)"
+    end
+    Mapitaine.configuration.sources = [reviewed]
+    build
+    Post.where(id: 1).update_all(reviewed_at: LONG_AGO + 7200)
+    built = build
+
+    assert_equal %w[sitemap-post-00000.xml.gz], built.written
+    assert_includes storage.xml("sitemap-post-00000.xml.gz"), "2026-01-01T14:00:00Z"
+  end
+
+  def test_a_build_is_instrumented
+    create_post(1)
+    events = []
+    subscriber = ActiveSupport::Notifications.subscribe(/mapitaine/) { |event| events << [event.name, event.payload] }
+    built = build
+
+    assert_equal ["shard.mapitaine", {source: "post", shard: "0"}], events.first
+    assert_equal ["build.mapitaine", {full: false, result: built}], events.last
+  ensure
+    ActiveSupport::Notifications.unsubscribe(subscriber)
+  end
+end
diff --git a/test/check_test.rb b/test/check_test.rb
new file mode 100644
index 0000000..4ba05f4
--- /dev/null
+++ b/test/check_test.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class CheckTest < MapitaineTest
+  def test_a_sound_site_has_nothing_to_report_and_nothing_is_written
+    create_post(1)
+
+    assert_empty Mapitaine.check
+    assert_empty storage.content_by_name
+  end
+
+  def test_a_url_on_another_host_a_duplicate_and_a_future_lastmod_are_reported
+    create_post(1, updated_at: Time.utc(2999, 1, 1))
+    create_post(12)
+    wrong = Class.new(PostSitemap) { url { |post, locale| (post.id == 1) ? "https://example.org/#{locale}" : "/same" } }
+    Mapitaine.configuration.sources = [wrong]
+
+    assert_equal [
+      "post: https://example.org/en is outside https://example.com",
+      "post: https://example.org/fr is outside https://example.com",
+      "post: https://example.org/en has a lastmod in the future (2999-01-01T00:00:00Z)",
+      "post: https://example.com/same is listed twice"
+    ], Mapitaine.check.map(&:to_s)
+  end
+
+  def test_a_build_reports_what_it_renders
+    create_post(1, updated_at: Time.utc(2999, 1, 1))
+
+    assert_equal 1, build.warnings.size
+    assert_empty build.warnings
+  end
+end
diff --git a/test/configuration_test.rb b/test/configuration_test.rb
new file mode 100644
index 0000000..b0f7764
--- /dev/null
+++ b/test/configuration_test.rb
@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class ConfigurationTest < MapitaineTest
+  def test_the_host_loses_its_trailing_slash_and_must_be_a_url
+    Mapitaine.configuration.host = "https://example.com/"
+
+    assert_equal "https://example.com", Mapitaine.configuration.host
+    ["example.com", "ftp://example.com", "https://", nil, "not a url"].each do |host|
+      assert_raises(Mapitaine::ConfigurationError, host.inspect) { Mapitaine.configuration.host = host }
+    end
+  end
+
+  def test_a_build_without_a_host_says_so
+    Mapitaine.reset_configuration
+    Mapitaine.configuration.storage = storage
+
+    assert_equal "Mapitaine.configuration.host is not set", assert_raises(Mapitaine::ConfigurationError) { Mapitaine.build }.message
+  end
+
+  def test_locales_default_to_the_languages_of_the_app
+    Mapitaine.reset_configuration
+
+    assert_equal I18n.available_locales, Mapitaine.configuration.locales
+  end
+
+  def test_sources_default_to_every_class_with_a_scope_once_loaded
+    loaded = false
+    Mapitaine.configuration.sources = nil
+    Mapitaine.configuration.load_sources = -> { loaded = true }
+    sources = Mapitaine.configuration.sources
+
+    assert loaded
+    assert_includes sources, PostSitemap
+    refute_includes sources, Mapitaine::Source
+    assert_equal sources.map(&:slug).sort, sources.map(&:slug)
+  end
+
+  def test_the_filename_names_every_file
+    Mapitaine.configuration.filename = "map"
+    create_post(1)
+    build
+
+    assert_equal %w[map-manifest.json map-page.xml.gz map-post-00000.xml.gz map.xml], storage.content_by_name.keys.sort
+  end
+end
diff --git a/test/index_now_test.rb b/test/index_now_test.rb
new file mode 100644
index 0000000..bab81e5
--- /dev/null
+++ b/test/index_now_test.rb
@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class IndexNowTest < MapitaineTest
+  Response = Data.define(:code)
+
+  def setup
+    super
+    Mapitaine.configuration.index_now_key = "0123456789abcdef"
+    @requests = requests = []
+    @response_code = "200"
+    test = self
+    @original_post = Net::HTTP.method(:post)
+    replace_post do |uri, body, headers|
+      raise test.network_error if test.network_error
+
+      requests << [uri.to_s, JSON.parse(body), headers]
+      Response.new(code: test.response_code)
+    end
+  end
+
+  def teardown
+    replace_post(&@original_post)
+  end
+
+  def replace_post(&)
+    silence_warnings { Net::HTTP.define_singleton_method(:post, &) }
+  end
+
+  attr_accessor :network_error, :response_code
+
+  def test_the_first_build_publishes_the_key_and_submits_nothing
+    create_post(1)
+    built = build
+
+    assert_equal "0123456789abcdef", storage.content_by_name["0123456789abcdef.txt"]
+    assert_empty @requests
+    assert_equal 0, built.submitted
+  end
+
+  def test_later_builds_submit_the_urls_changed_since_the_last_one
+    create_post(1)
+    create_post(2)
+    build
+    Post.find(2).update!(title: "Edited", updated_at: Time.now)
+    built = build
+
+    assert_equal [["https://api.indexnow.org/indexnow",
+      {"host" => "example.com", "key" => "0123456789abcdef",
+       "urlList" => %w[https://example.com/en/posts/2 https://example.com/fr/posts/2]},
+      {"Content-Type" => "application/json; charset=utf-8"}]], @requests
+    assert_equal 2, built.submitted
+  end
+
+  def test_a_key_hosted_elsewhere_is_named_and_not_published
+    Mapitaine.configuration.index_now_key_location = "https://example.com/keys/indexnow.txt"
+    create_post(1)
+    build
+    Post.find(1).update!(title: "Edited", updated_at: Time.now)
+    build
+
+    refute storage.content_by_name.key?("0123456789abcdef.txt")
+    assert_equal "https://example.com/keys/indexnow.txt", @requests.dig(0, 1, "keyLocation")
+  end
+
+  def test_a_refusal_counts_no_url
+    create_post(1)
+    build
+    Post.find(1).update!(title: "Edited", updated_at: Time.now)
+    self.response_code = "403"
+
+    assert_equal 0, build.submitted
+  end
+
+  def test_an_unreachable_endpoint_is_a_warning_and_the_sitemaps_are_published
+    create_post(1)
+    build
+    Post.find(1).update!(title: "Edited", updated_at: Time.now)
+    self.network_error = Errno::ECONNREFUSED
+    built = build
+
+    assert_equal ["indexnow: https://api.indexnow.org/indexnow could not be reached (Errno::ECONNREFUSED)"], built.warnings.map(&:to_s)
+    assert_equal %w[sitemap-post-00000.xml.gz], built.written
+  end
+
+  def test_urls_go_in_batches_of_ten_thousand
+    batches = []
+    transport = lambda do |_uri, body, _headers|
+      batches << JSON.parse(body)["urlList"].size
+      Response.new(code: "202")
+    end
+    index_now = Mapitaine::IndexNow.new(host: "https://example.com", key: "key", transport:)
+
+    assert_equal 10_001, index_now.submit(Array.new(10_001) { |number| "https://example.com/#{number}" })
+    assert_equal [10_000, 1], batches
+  end
+
+  def test_check_submits_nothing
+    create_post(1)
+    build
+    Post.find(1).update!(title: "Edited", updated_at: Time.now)
+    Mapitaine.check
+
+    assert_empty @requests
+  end
+end
diff --git a/test/shard_writer_test.rb b/test/shard_writer_test.rb
new file mode 100644
index 0000000..50fd2a5
--- /dev/null
+++ b/test/shard_writer_test.rb
@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class ShardWriterTest < Minitest::Test
+  def write(pages, **limits)
+    Dir.mktmpdir do |directory|
+      writer = Mapitaine::ShardWriter.new(directory:, basename: "sitemap-post-00000", **limits)
+      pages.each { |entries, lastmod| writer.add(entries, lastmod) }
+      yield writer.close.files, writer
+    end
+  end
+
+  def entry(name)
+    "https://example.com/#{name}\n"
+  end
+
+  def test_a_shard_over_the_url_limit_continues_in_a_second_file_and_pages_are_not_cut
+    pages = [[[entry("a/en"), entry("a/fr")], "2026-01-01"], [[entry("b/en"), entry("b/fr")], "2026-01-03"],
+      [[entry("c/en")], "2026-01-02"]]
+
+    write(pages, max_urls: 3) do |files|
+      assert_equal %w[sitemap-post-00000.xml.gz sitemap-post-00000-2.xml.gz], files.map(&:name)
+      assert_equal [2, 3], files.map(&:url_count)
+      assert_equal %w[2026-01-01 2026-01-03], files.map(&:lastmod)
+      assert_equal "#{Mapitaine::Xml::URLSET_OPEN}#{entry("a/en")}#{entry("a/fr")}\n", Zlib.gunzip(File.binread(files.first.path))
+    end
+  end
+
+  def test_a_shard_over_the_byte_limit_continues_in_a_second_file
+    envelope = Mapitaine::Xml::URLSET_OPEN.bytesize + Mapitaine::Xml::URLSET_CLOSE.bytesize
+    pages = [[[entry("a")], nil], [[entry("b")], nil], [[entry("c")], nil]]
+
+    write(pages, max_bytes: envelope + entry("a").bytesize * 2) do |files|
+      assert_equal [2, 1], files.map(&:url_count)
+      assert_equal [nil, nil], files.map(&:lastmod)
+    end
+  end
+
+  def test_no_page_no_file
+    write([]) { |files| assert_empty files }
+  end
+
+  def test_the_digest_follows_the_content
+    digests = [["a"], ["a"], ["b"]].map { |names| write([[names.map { |name| entry(name) }, nil]]) { |_files, writer| writer.content_digest } }
+
+    assert_equal digests[0], digests[1]
+    refute_equal digests[0], digests[2]
+  end
+end
diff --git a/test/source_test.rb b/test/source_test.rb
new file mode 100644
index 0000000..0c0b7bd
--- /dev/null
+++ b/test/source_test.rb
@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class SourceTest < MapitaineTest
+  def test_the_slug_comes_from_the_class_name_or_is_given
+    named = Class.new(PostSitemap) { slug "blog-posts" }
+
+    assert_equal "post", PostSitemap.slug
+    assert_equal "post", Class.new(PostSitemap).slug
+    assert_equal "blog-posts", named.slug
+    assert_equal "sitemap-blog-posts-00003", named.new(Mapitaine.configuration).basename("3")
+  end
+
+  def test_a_slug_that_would_leave_the_directory_is_refused
+    error = assert_raises(Mapitaine::SourceError) { Class.new(PostSitemap) { slug "../etc" }.new(Mapitaine.configuration) }
+
+    assert_includes error.message, %("../etc" must be lowercase letters)
+  end
+
+  def test_a_subclass_changes_its_own_definitions_only
+    Class.new(PostSitemap) { shard_size 500 }
+
+    assert_equal 10, PostSitemap.definitions[:shard_size]
+  end
+
+  def test_shards_hold_fewer_records_as_languages_multiply
+    sizes = [%i[en], %i[en fr], Array.new(24) { |number| :"l#{number}" }].map do |locales|
+      Mapitaine.configuration.locales = locales
+      Class.new(PostSitemap) { definitions.delete(:shard_size) }.new(Mapitaine.configuration).send(:shard_size)
+    end
+
+    assert_equal [50_000, 25_000, 554], sizes
+  end
+
+  def test_a_model_without_an_integer_primary_key_is_refused
+    Mapitaine.configuration.sources = [Class.new(PostSitemap) { scope { Token.all } }]
+
+    assert_includes assert_raises(Mapitaine::SourceError) { build }.message, "integer primary key"
+  end
+
+  def test_a_record_without_a_url_in_a_language_is_left_out_of_it
+    create_post(1)
+    english_only = Class.new(PostSitemap) { url { |post, locale| post_url(post, locale:) if locale == :en } }
+    pages = english_only.new(Mapitaine.configuration).each_page("0").to_a
+
+    assert_equal [{en: "https://example.com/en/posts/1"}], pages.map(&:url_by_locale)
+  end
+
+  def test_route_helpers_use_the_configured_host_port_and_scheme
+    Mapitaine.configuration.host = "http://localhost:3000/"
+    create_post(1)
+
+    assert_equal "http://localhost:3000/en/posts/1", PostSitemap.new(Mapitaine.configuration).each_page("0").first.urls.first
+  end
+end
diff --git a/test/storage_test.rb b/test/storage_test.rb
new file mode 100644
index 0000000..dbd11d5
--- /dev/null
+++ b/test/storage_test.rb
@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require "test_helper"
+require "active_storage"
+require "active_storage/service/disk_service"
+
+class StorageTest < Minitest::Test
+  def with_file(content)
+    Dir.mktmpdir do |directory|
+      path = File.join(directory, "source")
+      File.write(path, content)
+      yield path, directory
+    end
+  end
+
+  def test_local_writes_reads_and_deletes_and_leaves_no_partial_file
+    with_file("") do |path, directory|
+      public_directory = File.join(directory, "public")
+      storage = Mapitaine::Storage::Local.new(public_directory)
+
+      assert_nil storage.read("sitemap.xml")
+      storage.write("sitemap.xml", path)
+
+      assert_equal "", storage.read("sitemap.xml")
+      assert_equal %w[sitemap.xml], Dir.children(public_directory)
+
+      storage.delete("sitemap.xml")
+      storage.delete("sitemap.xml")
+
+      assert_empty Dir.children(public_directory)
+    end
+  end
+
+  def test_active_storage_goes_through_the_service_under_the_prefix
+    with_file("") do |path, directory|
+      service = ActiveStorage::Service::DiskService.new(root: File.join(directory, "storage"))
+      storage = Mapitaine::Storage::ActiveStorage.new(service, prefix: "sitemaps")
+
+      assert_nil storage.read("sitemap.xml")
+      storage.write("sitemap.xml", path)
+
+      assert_equal "", storage.read("sitemap.xml")
+      assert service.exist?("sitemaps/sitemap.xml")
+
+      storage.delete("sitemap.xml")
+
+      assert_nil storage.read("sitemap.xml")
+    end
+  end
+
+  def test_active_storage_tells_the_service_the_content_type
+    uploads = []
+    service = Object.new
+    service.define_singleton_method(:upload) { |key, _io, **options| uploads << [key, options] }
+    with_file("") { |path| Mapitaine::Storage::ActiveStorage.new(service).write("sitemap-post-00000.xml.gz", path) }
+
+    assert_equal [["sitemap-post-00000.xml.gz", {content_type: "application/gzip"}]], uploads
+  end
+end
diff --git a/test/test_helper.rb b/test/test_helper.rb
new file mode 100644
index 0000000..9d03d0d
--- /dev/null
+++ b/test/test_helper.rb
@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+require "minitest/autorun"
+require "action_dispatch"
+require "active_record"
+require "active_support/core_ext/kernel/reporting"
+require "mapitaine"
+
+ActiveRecord::Base.establish_connection(adapter: "sqlite3", database: ":memory:")
+ActiveRecord::Schema.verbose = false
+ActiveRecord::Schema.define do
+  create_table :posts do |table|
+    table.string :title, null: false
+    table.string :languages, null: false, default: "en,fr"
+    table.boolean :published, null: false, default: true
+    table.datetime :reviewed_at
+    table.timestamps
+  end
+
+  create_table :tokens, id: :string do |table|
+    table.timestamps
+  end
+end
+
+class Post < ActiveRecord::Base
+  scope :published, -> { where(published: true) }
+end
+
+class Token < ActiveRecord::Base
+end
+
+ROUTES = ActionDispatch::Routing::RouteSet.new
+ROUTES.draw do
+  scope "(:locale)" do
+    resources :posts, only: :show
+    get "about", to: "pages#about", as: :about
+  end
+end
+Mapitaine::Source.include(ROUTES.url_helpers)
+
+class PostSitemap < Mapitaine::Source
+  scope { Post.published }
+  url { |post, locale| post_url(post, locale:) }
+  locales { |post| post.languages.split(",").map(&:to_sym) }
+  shard_size 10
+end
+
+class PageSitemap < Mapitaine::Source
+  scope { %i[about] }
+  url { |page, locale| public_send(:"#{page}_path", locale:) }
+  lastmod { Date.new(2026, 1, 15) }
+end
+
+class MemoryStorage
+  attr_reader :content_by_name, :writes, :deletions
+
+  def initialize
+    @content_by_name = {}
+    @writes = []
+    @deletions = []
+  end
+
+  def read(name)
+    content_by_name[name]
+  end
+
+  def write(name, source_path)
+    writes << name
+    content_by_name[name] = File.binread(source_path)
+  end
+
+  def delete(name)
+    deletions << name
+    content_by_name.delete(name)
+  end
+
+  def xml(name)
+    content = content_by_name.fetch(name)
+    name.end_with?(".gz") ? Zlib.gunzip(content) : content
+  end
+
+  def forget_calls
+    writes.clear
+    deletions.clear
+  end
+end
+
+class MapitaineTest < Minitest::Test
+  LONG_AGO = Time.utc(2026, 1, 1, 12)
+
+  attr_reader :storage
+
+  def setup
+    Post.delete_all
+    Mapitaine.reset_configuration
+    @storage = MemoryStorage.new
+    Mapitaine.configure do |config|
+      config.host = "https://example.com"
+      config.locales = %i[en fr]
+      config.x_default = :en
+      config.storage = storage
+      config.sources = [PostSitemap, PageSitemap]
+    end
+  end
+
+  def create_post(id, **attributes)
+    Post.create!(id:, title: "Post #{id}", created_at: LONG_AGO, updated_at: LONG_AGO, **attributes)
+  end
+
+  def build(...)
+    storage.forget_calls
+    Mapitaine.build(...)
+  end
+end